From a8fcaf8231c351010f980592c3d2cc090d5b4126 Mon Sep 17 00:00:00 2001 From: Yufei Date: Mon, 21 Sep 2026 18:32:33 -0700 Subject: [PATCH 1/6] Require complete core documents in ontology mappings --- .github/workflows/converter-ontology-ci.yml | 4 +- .github/workflows/validation-ci.yml | 4 + .../converter/ossie_to_spec/converter.py | 1 + .../converter/palantir_to_ossie/converter.py | 2 +- .../converter/spec_to_ossie/converter.py | 1 + .../ontology/src/ossie_ontology/model.py | 10 +- .../ontology/src/ossie_ontology/spec.py | 7 +- .../ontology/tests/fixtures/flights.yaml | 1 + .../flights_roundtrip.yaml | 1 + .../ontology/tests/test_ossie_parser.py | 40 ++++++ .../ontology/tests/test_palantir_converter.py | 17 +++ core-spec/spec.md | 7 +- examples/flights.yaml | 1 + ontology/ontology.json | 2 +- ontology/ontology.md | 34 +++++ python/README.md | 5 +- validation/tests/test_ontology.py | 129 ++++++++++++++++++ validation/tests/test_validate.py | 4 +- 18 files changed, 253 insertions(+), 17 deletions(-) create mode 100644 validation/tests/test_ontology.py diff --git a/.github/workflows/converter-ontology-ci.yml b/.github/workflows/converter-ontology-ci.yml index 2abd3de4..b3fe56a4 100644 --- a/.github/workflows/converter-ontology-ci.yml +++ b/.github/workflows/converter-ontology-ci.yml @@ -25,6 +25,7 @@ on: paths: - 'converters/ontology/**' - 'core-spec/**' + - 'ontology/**' - 'examples/**' - '.github/workflows/converter-ontology-ci.yml' pull_request: @@ -32,6 +33,7 @@ on: paths: - 'converters/ontology/**' - 'core-spec/**' + - 'ontology/**' - 'examples/**' - '.github/workflows/converter-ontology-ci.yml' @@ -64,4 +66,4 @@ jobs: - name: Unit Tests working-directory: converters/ontology run: | - uv run pytest \ No newline at end of file + uv run pytest diff --git a/.github/workflows/validation-ci.yml b/.github/workflows/validation-ci.yml index 00122a06..5c129923 100644 --- a/.github/workflows/validation-ci.yml +++ b/.github/workflows/validation-ci.yml @@ -26,6 +26,8 @@ on: - "validation/**" - "python/**" - "core-spec/**" + - "ontology/**" + - "examples/flights.yaml" - "examples/tpcds_semantic_model.yaml" - ".github/workflows/validation-ci.yml" pull_request: @@ -34,6 +36,8 @@ on: - "validation/**" - "python/**" - "core-spec/**" + - "ontology/**" + - "examples/flights.yaml" - "examples/tpcds_semantic_model.yaml" - ".github/workflows/validation-ci.yml" diff --git a/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py b/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py index f6841f41..b63bf9a4 100644 --- a/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py +++ b/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py @@ -144,6 +144,7 @@ def _convert_relationship(rel: Relationship) -> SpecRelationship: def _convert_semantic_model(semantic_model: SemanticModel) -> SpecSemanticModel: return SpecSemanticModel( + version=semantic_model.version, name=semantic_model.name, description=semantic_model.description, ai_context=semantic_model.ai_context, diff --git a/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/converter.py b/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/converter.py index c87ef7ac..f1b606c7 100644 --- a/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/converter.py +++ b/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/converter.py @@ -151,7 +151,7 @@ def convert( schema_name: str = "palantir", ) -> OssieOntology: ontology = OntologyComponent() - model = OssieOntology(name="Palantir model", ontology=ontology, version="0.1.0") + model = OssieOntology(name="Palantir model", ontology=ontology, version="0.2.0.dev0") semantic_model = SemanticModel(name="Palantir semantic model") diff --git a/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py b/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py index 0b45ea8a..75992421 100644 --- a/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py +++ b/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py @@ -226,6 +226,7 @@ def _convert_relationship( def _convert_semantic_model(self, lm_spec: SpecSemanticModel) -> SemanticModel: semantic_model = SemanticModel( + version=lm_spec.version, name=lm_spec.name, description=lm_spec.description, ai_context=lm_spec.ai_context, diff --git a/converters/ontology/src/ossie_ontology/model.py b/converters/ontology/src/ossie_ontology/model.py index 2f9d1e72..c681a8b6 100644 --- a/converters/ontology/src/ossie_ontology/model.py +++ b/converters/ontology/src/ossie_ontology/model.py @@ -685,8 +685,8 @@ def custom_extensions(self) -> list[CustomExtension]: class SemanticModel: - """Bundle of datasets, join paths and metrics. One or more SemanticModels - can feed a single OntologyMapping (see spec).""" + """Versioned core semantic model embedded in a single OntologyMapping.""" + _version: str _name: str _description: str | None _ai_context: AiContext | None @@ -704,7 +704,9 @@ def __init__( description: str | None = None, ai_context: AiContext | None = None, custom_extensions: list[CustomExtension] | None = None, + version: str = "0.2.0.dev0", ): + self._version = version self._name = name self._description = description self._ai_context = ai_context @@ -716,6 +718,10 @@ def __init__( self._join_path_name_map = {} self._metric_name_map = {} + @property + def version(self) -> str: + return self._version + @property def name(self) -> str: return self._name diff --git a/converters/ontology/src/ossie_ontology/spec.py b/converters/ontology/src/ossie_ontology/spec.py index e61ce52a..86ee7217 100644 --- a/converters/ontology/src/ossie_ontology/spec.py +++ b/converters/ontology/src/ossie_ontology/spec.py @@ -175,11 +175,8 @@ class Metric(OssieObject): class SemanticModel(OssieObject): - """Model contents: datasets plus the join paths and metrics defined over them. - - OntologyMapping embeds these contents under ``semantic_model``; standalone - core documents place them at the root alongside document metadata. - """ + """Complete core document embedded under OntologyMapping.semantic_model.""" + version: Literal["0.2.0.dev0"] name: str description: str | None = None ai_context: AiContext | None = None diff --git a/converters/ontology/tests/fixtures/flights.yaml b/converters/ontology/tests/fixtures/flights.yaml index 803ff73d..b3fb5a13 100644 --- a/converters/ontology/tests/fixtures/flights.yaml +++ b/converters/ontology/tests/fixtures/flights.yaml @@ -540,6 +540,7 @@ ontology: ontology_mappings: - name: flights_mapping semantic_model: + version: 0.2.0.dev0 name: Flights semantic model datasets: - name: RUNWAY diff --git a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml index 805fadef..4c025815 100644 --- a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml +++ b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml @@ -610,6 +610,7 @@ ontology: ontology_mappings: - name: flights_mapping semantic_model: + version: 0.2.0.dev0 name: Flights semantic model datasets: - name: RUNWAY diff --git a/converters/ontology/tests/test_ossie_parser.py b/converters/ontology/tests/test_ossie_parser.py index f8e36e03..7bacbecf 100644 --- a/converters/ontology/tests/test_ossie_parser.py +++ b/converters/ontology/tests/test_ossie_parser.py @@ -25,6 +25,7 @@ import pytest import yaml +from pydantic import ValidationError from ossie_ontology.converter.ossie_to_spec.converter import ( OssieToSpecConverter, @@ -49,6 +50,45 @@ def test_parse_returns_model_with_metadata(flights_model): assert flights_model.description == "Ontology of flights into and out of airports." +def test_embedded_core_document_version_survives_roundtrip(flights_model, tmp_path): + assert all( + mapping.semantic_model.version == "0.2.0.dev0" + for mapping in flights_model.ontology_mappings + ) + + exported = OssieToSpecConverter.convert(flights_model).dump_yaml() + document = yaml.safe_load(exported) + assert all( + mapping["semantic_model"]["version"] == "0.2.0.dev0" + for mapping in document["ontology_mappings"] + ) + + path = tmp_path / "roundtrip.yaml" + path.write_text(exported) + reparsed = OssieParser().parse(path) + assert all( + mapping.semantic_model.version == "0.2.0.dev0" + for mapping in reparsed.ontology_mappings + ) + + +@pytest.mark.parametrize("version", ["missing", "0.1.0", "0.2.0", None, 2]) +def test_parse_rejects_invalid_embedded_version(flights_path, tmp_path, version): + document = yaml.safe_load(flights_path.read_text()) + model = document["ontology_mappings"][0]["semantic_model"] + if version == "missing": + del model["version"] + else: + model["version"] = version + path = tmp_path / "invalid.yaml" + path.write_text(yaml.safe_dump(document)) + + with pytest.raises(ValidationError) as exc: + OssieParser().parse(path) + + assert exc.value.errors()[0]["loc"] == ("ontology_mappings", 0, "semantic_model", "version") + + def test_parse_returns_populated_ontology(flights_model): ontology = flights_model.ontology # Built-in concepts (String, Integer, Decimal, ...) are always present on top diff --git a/converters/ontology/tests/test_palantir_converter.py b/converters/ontology/tests/test_palantir_converter.py index 3edd4980..e7a37f8a 100644 --- a/converters/ontology/tests/test_palantir_converter.py +++ b/converters/ontology/tests/test_palantir_converter.py @@ -277,6 +277,23 @@ def _identifier_columns(model: OssieOntology, concept_name: str) -> list[str]: return columns +# ----- document metadata ------------------------------------------------ + +def test_generated_semantic_model_has_core_document_version(tmp_path: Path): + model = _convert( + tmp_path, + [_object_type("widget", "Widget", status="active")], + datasets=[_dataset("widget", ["widget_id"])], + ) + + document = OssieToSpecConverter.convert(model).dump_dict() + + assert document["version"] == "0.2.0.dev0" + [mapping] = document["ontology_mappings"] + assert mapping["semantic_model"]["version"] == "0.2.0.dev0" + assert mapping["semantic_model"]["datasets"] + + # ----- object type statuses --------------------------------------------- def test_default_policy_admits_active_endorsed_and_intermediary(tmp_path: Path): diff --git a/core-spec/spec.md b/core-spec/spec.md index dce68361..d3b6a15b 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -151,9 +151,10 @@ custom extensions; never silently select only the first model or overwrite a fil when splitting a document. The reusable `$defs/SemanticModel` schema still describes model contents without -standalone document metadata. In particular, an ontology map continues to embed -those contents under its `semantic_model` property. This standalone document -change does not rename or flatten that ontology property. +document metadata. An [ontology map](../ontology/ontology.md#ontology-mappings) +embeds a complete core document under its `semantic_model` property, referencing +the root schema and requiring its own `version: 0.2.0.dev0`. Keep that property +and add `version` to each embedded model when migrating earlier ontology maps. --- diff --git a/examples/flights.yaml b/examples/flights.yaml index 803ff73d..b3fb5a13 100644 --- a/examples/flights.yaml +++ b/examples/flights.yaml @@ -540,6 +540,7 @@ ontology: ontology_mappings: - name: flights_mapping semantic_model: + version: 0.2.0.dev0 name: Flights semantic model datasets: - name: RUNWAY diff --git a/ontology/ontology.json b/ontology/ontology.json index 47a00e03..60a316de 100644 --- a/ontology/ontology.json +++ b/ontology/ontology.json @@ -297,7 +297,7 @@ "description": "Human-readable description of this ontology map" }, "semantic_model": { - "$ref": "https://raw.githubusercontent.com/apache/ossie/main/core-spec/ossie-schema.json#/$defs/SemanticModel" + "$ref": "https://raw.githubusercontent.com/apache/ossie/main/core-spec/ossie-schema.json" }, "concept_mappings": { "type": "array", diff --git a/ontology/ontology.md b/ontology/ontology.md index 94002a5c..5135d716 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -385,6 +385,39 @@ Ontology mappings declare how to map the values of fields at the logical level t in the ontology. Just as ontologies are partitioned by concept, ontology maps partition into concept mappings that group by some concept. +Each entry in `ontology_mappings` embeds one complete core semantic model document +under `semantic_model`. This property references the root of the +[core JSON Schema](../core-spec/ossie-schema.json), identified by +`https://github.com/apache/ossie/core-spec/ossie-schema.json`, rather than its +`$defs/SemanticModel` definition. The embedded document must include `version`, +`name`, and a non-empty `datasets` array, just like a standalone core document. +The ontology's top-level `version` does not supply the embedded model's version. + +For example, a minimal ontology with a semantic model is: + +```yaml +version: 0.2.0.dev0 +name: sales_ontology +ontology: + - concept: Order + type: EntityType +ontology_mappings: + - name: sales_mapping + semantic_model: + version: 0.2.0.dev0 + name: sales_analytics + datasets: + - name: orders + source: sales.public.orders + concept_mappings: [] +``` + +**Migration:** This is a breaking change to the unreleased `0.2.0.dev0` schema. +Add `version: 0.2.0.dev0` to every existing +`ontology_mappings[*].semantic_model`, preserving the model's contents. Keep the +`semantic_model` property; do not flatten it into the ontology map or wrap it in +an array. Validate each embedded model against the core root schema. + ### Concept mappings Each concept mapping declares how to populate a concept with objects and how to populate the relationships @@ -588,6 +621,7 @@ though `Store` plays a role in three of the relationships. ## Version History - **0.2.0.dev0** (2026-05-29): Basic support for ontologies and logical schema mappings + - Breaking: ontology maps embed complete core documents, requiring `version` in each `semantic_model`. - Core ontology structure: Concepts, relationships, and business rules (requires and derived_by) - Schema mappings from one or more logical models into an ontology diff --git a/python/README.md b/python/README.md index 43f9e73b..bcc54a3c 100644 --- a/python/README.md +++ b/python/README.md @@ -29,8 +29,9 @@ flat shape. The former `semantic_model` wrapper is rejected. Unwrap old single-model documents and split multi-model documents into separate files, preserving model contents and setting `version` in each file before loading them. -`OssieSemanticModel` remains available for embedded models, such as ontology -components, which do not include document metadata. +`OssieSemanticModel` remains available for model contents without document +metadata. Ontology maps embed complete core documents, including `version`; +use `OssieDocument` for those embedded documents. ## Development diff --git a/validation/tests/test_ontology.py b/validation/tests/test_ontology.py new file mode 100644 index 00000000..00aee8cc --- /dev/null +++ b/validation/tests/test_ontology.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Validate ontology mappings against the local core document schema.""" + +import json +from pathlib import Path + +import pytest +import yaml +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +_REPO = Path(__file__).parents[2] + + +@pytest.fixture +def ontology_validator() -> Draft202012Validator: + core = json.loads((_REPO / "core-spec/ossie-schema.json").read_text()) + ontology = json.loads((_REPO / "ontology/ontology.json").read_text()) + Draft202012Validator.check_schema(core) + Draft202012Validator.check_schema(ontology) + resource = Resource.from_contents(core) + # Resolve both the core schema's canonical ID and the retrieval URL used by + # ontology references locally. Registry never fetches over HTTP. + registry = Registry().with_resources( + [ + (core["$id"], resource), + ( + "https://raw.githubusercontent.com/apache/ossie/main/core-spec/ossie-schema.json", + resource, + ), + ] + ) + return Draft202012Validator(ontology, registry=registry) + + +def _ontology(semantic_model: object) -> dict: + return { + "version": "0.2.0.dev0", + "name": "sales", + "ai_context": {"instructions": "Use for sales analysis"}, + "ontology": [{"concept": "Order", "type": "EntityType"}], + "ontology_mappings": [ + { + "name": "sales_mapping", + "semantic_model": semantic_model, + "concept_mappings": [], + } + ], + } + + +def _semantic_model() -> dict: + return { + "version": "0.2.0.dev0", + "name": "sales_model", + "datasets": [{"name": "orders", "source": "sales.public.orders"}], + } + + +def test_ontology_accepts_complete_core_document(ontology_validator): + ontology_validator.validate(_ontology(_semantic_model())) + + +@pytest.mark.parametrize("required_property", ["version", "name", "datasets"]) +def test_embedded_model_requires_core_document_properties( + ontology_validator, required_property +): + model = _semantic_model() + del model[required_property] + + errors = list(ontology_validator.iter_errors(_ontology(model))) + + assert len(errors) == 1 + assert errors[0].validator == "required" + assert required_property in errors[0].message + assert list(errors[0].absolute_path) == ["ontology_mappings", 0, "semantic_model"] + + +@pytest.mark.parametrize("version", ["0.1.0", "0.2.0", "", None, 2]) +def test_embedded_model_rejects_invalid_version(ontology_validator, version): + model = _semantic_model() + model["version"] = version + + errors = list(ontology_validator.iter_errors(_ontology(model))) + + assert errors + assert all( + list(error.absolute_path) == ["ontology_mappings", 0, "semantic_model", "version"] + for error in errors + ) + + +@pytest.mark.parametrize( + "model", + [ + {**_semantic_model(), "datasets": []}, + {**_semantic_model(), "unexpected": True}, + {"semantic_model": _semantic_model()}, + [_semantic_model()], + ], +) +def test_embedded_model_rejects_invalid_document_shapes(ontology_validator, model): + assert not ontology_validator.is_valid(_ontology(model)) + + +def test_flights_embedded_models_validate_as_core_documents(ontology_validator): + flights = yaml.safe_load((_REPO / "examples/flights.yaml").read_text()) + + assert flights["ontology_mappings"] + for mapping in flights["ontology_mappings"]: + # Validate the embedded core documents without expanding this test to + # unrelated ontology concept and relationship constraints. + ontology_validator.validate(_ontology(mapping["semantic_model"])) diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 8583c53c..bbb1ad7d 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -73,8 +73,8 @@ def test_rejects_empty_root_datasets(core_schema: dict) -> None: assert errors == ["[Schema] datasets: [] should be non-empty"] -def test_embedded_semantic_model_does_not_require_document_version(core_schema: dict) -> None: - # Ontology components reference this definition without a document envelope. +def test_semantic_model_definition_does_not_require_document_version(core_schema: dict) -> None: + # The reusable contents definition remains independent of document metadata. embedded_schema = { "$ref": "#/$defs/SemanticModel", "$defs": core_schema["$defs"], From 40d9fdc040191378b3959a0149027801a1c311d1 Mon Sep 17 00:00:00 2001 From: Yufei Date: Mon, 21 Sep 2026 18:35:55 -0700 Subject: [PATCH 2/6] Simplify ontology document and migration guidance --- core-spec/spec.md | 8 +++----- ontology/ontology.md | 27 +++++++-------------------- python/README.md | 5 ++--- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index d3b6a15b..2212c50a 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -150,11 +150,9 @@ model array cannot produce a valid model document. Preserve model contents and custom extensions; never silently select only the first model or overwrite a file when splitting a document. -The reusable `$defs/SemanticModel` schema still describes model contents without -document metadata. An [ontology map](../ontology/ontology.md#ontology-mappings) -embeds a complete core document under its `semantic_model` property, referencing -the root schema and requiring its own `version: 0.2.0.dev0`. Keep that property -and add `version` to each embedded model when migrating earlier ontology maps. +For [ontology maps](../ontology/ontology.md#ontology-mappings), keep the +`semantic_model` property and add `version: 0.2.0.dev0` inside it. Each embedded +model must be a complete core document. --- diff --git a/ontology/ontology.md b/ontology/ontology.md index 5135d716..e068e7dc 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -385,22 +385,11 @@ Ontology mappings declare how to map the values of fields at the logical level t in the ontology. Just as ontologies are partitioned by concept, ontology maps partition into concept mappings that group by some concept. -Each entry in `ontology_mappings` embeds one complete core semantic model document -under `semantic_model`. This property references the root of the -[core JSON Schema](../core-spec/ossie-schema.json), identified by -`https://github.com/apache/ossie/core-spec/ossie-schema.json`, rather than its -`$defs/SemanticModel` definition. The embedded document must include `version`, -`name`, and a non-empty `datasets` array, just like a standalone core document. -The ontology's top-level `version` does not supply the embedded model's version. - -For example, a minimal ontology with a semantic model is: +Each mapping's `semantic_model` is a complete +[core document](../core-spec/spec.md#semantic-model), with its own `version`, +`name`, and at least one dataset. For example: ```yaml -version: 0.2.0.dev0 -name: sales_ontology -ontology: - - concept: Order - type: EntityType ontology_mappings: - name: sales_mapping semantic_model: @@ -412,11 +401,9 @@ ontology_mappings: concept_mappings: [] ``` -**Migration:** This is a breaking change to the unreleased `0.2.0.dev0` schema. -Add `version: 0.2.0.dev0` to every existing -`ontology_mappings[*].semantic_model`, preserving the model's contents. Keep the -`semantic_model` property; do not flatten it into the ontology map or wrap it in -an array. Validate each embedded model against the core root schema. +**Breaking change in `0.2.0.dev0` (unreleased):** Add `version: 0.2.0.dev0` inside +each existing `semantic_model`, keeping its other fields in place. The ontology's +top-level `version` does not replace this field. ### Concept mappings @@ -621,7 +608,7 @@ though `Store` plays a role in three of the relationships. ## Version History - **0.2.0.dev0** (2026-05-29): Basic support for ontologies and logical schema mappings - - Breaking: ontology maps embed complete core documents, requiring `version` in each `semantic_model`. + - Breaking: each ontology map's `semantic_model` now requires its own `version`. - Core ontology structure: Concepts, relationships, and business rules (requires and derived_by) - Schema mappings from one or more logical models into an ontology diff --git a/python/README.md b/python/README.md index bcc54a3c..61e89e8b 100644 --- a/python/README.md +++ b/python/README.md @@ -29,9 +29,8 @@ flat shape. The former `semantic_model` wrapper is rejected. Unwrap old single-model documents and split multi-model documents into separate files, preserving model contents and setting `version` in each file before loading them. -`OssieSemanticModel` remains available for model contents without document -metadata. Ontology maps embed complete core documents, including `version`; -use `OssieDocument` for those embedded documents. +Use `OssieDocument` for models embedded in ontology maps; they require `version`. +`OssieSemanticModel` provides the model fields without `version`. ## Development From 38fa60cc09c3cc494475c84822901a917f3c1c51 Mon Sep 17 00:00:00 2001 From: Yufei Date: Mon, 21 Sep 2026 18:47:21 -0700 Subject: [PATCH 3/6] Preserve the core document version in OrionBelt ontology exports --- .github/workflows/converter-orionbelt-ci.yml | 2 ++ converters/orionbelt/README.md | 4 ++++ .../orionbelt/ossie_obml_ontology_mapping_analysis.md | 2 +- converters/orionbelt/src/ossie_orionbelt/ontology.py | 1 - converters/orionbelt/tests/test_ossie_converter_ontology.py | 6 +++++- 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/converter-orionbelt-ci.yml b/.github/workflows/converter-orionbelt-ci.yml index 0b3aebf6..9c8e95e9 100644 --- a/.github/workflows/converter-orionbelt-ci.yml +++ b/.github/workflows/converter-orionbelt-ci.yml @@ -25,6 +25,7 @@ on: paths: - 'converters/orionbelt/**' - 'core-spec/**' + - 'ontology/**' - 'examples/**' - '.github/workflows/converter-orionbelt-ci.yml' pull_request: @@ -32,6 +33,7 @@ on: paths: - 'converters/orionbelt/**' - 'core-spec/**' + - 'ontology/**' - 'examples/**' - '.github/workflows/converter-orionbelt-ci.yml' diff --git a/converters/orionbelt/README.md b/converters/orionbelt/README.md index 15e590b8..313fb371 100644 --- a/converters/orionbelt/README.md +++ b/converters/orionbelt/README.md @@ -35,6 +35,10 @@ Ossie documents contain one model directly at the root, with `version`, `name`, `datasets`, and optional model properties. Legacy `semantic_model` wrappers (arrays or objects) are rejected. +Ontology exports embed a complete core document under +`ontology_mappings[].semantic_model`, including `version`. For older exports, +add `version: 0.2.0.dev0` inside that model. + ## Requirements - Python 3.12+ diff --git a/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md b/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md index 8e4ebd0e..a2762957 100644 --- a/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md +++ b/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md @@ -68,7 +68,7 @@ ontology_mappings: | `join.joinType` | `Relationship.multiplicity` | `many-to-one`→`ManyToOne`, `one-to-one`→`OneToOne` | | `column.primaryKey` | entity `object_mappings[].expression` | `.`; identifies the entity | | `join.columnsFrom` (FK) | `link_mappings[].object_mapping.expression` | `.`; binds the relationship to its far role | -| whole core model | `ontology_mappings[].semantic_model` | embedded from `OBMLtoOssie.convert()` without standalone document metadata (`version`, `dialects`, `vendors`) | +| whole core model | `ontology_mappings[].semantic_model` | complete output from `OBMLtoOssie.convert()`, including `version` | `
` is the final identifier of the dataset `source` (e.g. `db.schema.t` → `t`), falling back to the dataset name when `source` has no dotted physical table. diff --git a/converters/orionbelt/src/ossie_orionbelt/ontology.py b/converters/orionbelt/src/ossie_orionbelt/ontology.py index 0dfac93c..e5ee6897 100644 --- a/converters/orionbelt/src/ossie_orionbelt/ontology.py +++ b/converters/orionbelt/src/ossie_orionbelt/ontology.py @@ -106,7 +106,6 @@ def convert(self) -> dict: ai_instructions=self.ai_instructions, ) sem_model = core_conv.convert() - sem_model.pop("version", None) self.warnings.extend(core_conv.warnings) data_objects = self.obml.get("dataObjects", {}) or {} diff --git a/converters/orionbelt/tests/test_ossie_converter_ontology.py b/converters/orionbelt/tests/test_ossie_converter_ontology.py index 64933665..93068895 100644 --- a/converters/orionbelt/tests/test_ossie_converter_ontology.py +++ b/converters/orionbelt/tests/test_ossie_converter_ontology.py @@ -101,13 +101,17 @@ def test_embeds_core_semantic_model(self) -> None: omap = doc["ontology_mappings"][0] assert omap["name"] == "sales_map" assert "semantic_model" in omap - assert not {"version", "dialects", "vendors"} & set(omap["semantic_model"]) + assert omap["semantic_model"]["version"] == "0.2.0.dev0" + assert not {"dialects", "vendors"} & set(omap["semantic_model"]) assert omap["semantic_model"]["name"] == "sales" assert {d["name"] for d in omap["semantic_model"]["datasets"]} == { "Customers", "Products", "Orders", } + result = conv.validate_ossie(omap["semantic_model"]) + assert result.valid, result.schema_errors + result.semantic_errors + assert not result.semantic_warnings def test_concept_mappings_bind_keys_and_fks(self) -> None: doc = conv.OBMLtoOssieOntology(_OBML, model_name="sales").convert() From 9fa8cffbd8c2220ea77c201b0bc6e85c1bcca813 Mon Sep 17 00:00:00 2001 From: Yufei Date: Tue, 22 Sep 2026 10:42:42 -0700 Subject: [PATCH 4/6] Doc changes --- core-spec/spec.md | 3 +-- ontology/ontology.md | 4 ---- python/README.md | 3 +-- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 2212c50a..c1bc8e57 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -150,8 +150,7 @@ model array cannot produce a valid model document. Preserve model contents and custom extensions; never silently select only the first model or overwrite a file when splitting a document. -For [ontology maps](../ontology/ontology.md#ontology-mappings), keep the -`semantic_model` property and add `version: 0.2.0.dev0` inside it. Each embedded +For [ontology maps](../ontology/ontology.md#ontology-mappings), each embedded model must be a complete core document. --- diff --git a/ontology/ontology.md b/ontology/ontology.md index e068e7dc..fc2775be 100644 --- a/ontology/ontology.md +++ b/ontology/ontology.md @@ -401,10 +401,6 @@ ontology_mappings: concept_mappings: [] ``` -**Breaking change in `0.2.0.dev0` (unreleased):** Add `version: 0.2.0.dev0` inside -each existing `semantic_model`, keeping its other fields in place. The ontology's -top-level `version` does not replace this field. - ### Concept mappings Each concept mapping declares how to populate a concept with objects and how to populate the relationships diff --git a/python/README.md b/python/README.md index 61e89e8b..88a3b7f4 100644 --- a/python/README.md +++ b/python/README.md @@ -29,8 +29,7 @@ flat shape. The former `semantic_model` wrapper is rejected. Unwrap old single-model documents and split multi-model documents into separate files, preserving model contents and setting `version` in each file before loading them. -Use `OssieDocument` for models embedded in ontology maps; they require `version`. -`OssieSemanticModel` provides the model fields without `version`. +Use `OssieDocument` for models embedded in ontology maps. ## Development From d9597964cb888aae442859267d63b73c3d6e1541 Mon Sep 17 00:00:00 2001 From: Yufei Date: Tue, 22 Sep 2026 11:04:33 -0700 Subject: [PATCH 5/6] Doc changes --- converters/orionbelt/README.md | 4 ---- converters/orionbelt/ossie_obml_ontology_mapping_analysis.md | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/converters/orionbelt/README.md b/converters/orionbelt/README.md index 313fb371..15e590b8 100644 --- a/converters/orionbelt/README.md +++ b/converters/orionbelt/README.md @@ -35,10 +35,6 @@ Ossie documents contain one model directly at the root, with `version`, `name`, `datasets`, and optional model properties. Legacy `semantic_model` wrappers (arrays or objects) are rejected. -Ontology exports embed a complete core document under -`ontology_mappings[].semantic_model`, including `version`. For older exports, -add `version: 0.2.0.dev0` inside that model. - ## Requirements - Python 3.12+ diff --git a/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md b/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md index a2762957..5d39e182 100644 --- a/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md +++ b/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md @@ -68,7 +68,7 @@ ontology_mappings: | `join.joinType` | `Relationship.multiplicity` | `many-to-one`→`ManyToOne`, `one-to-one`→`OneToOne` | | `column.primaryKey` | entity `object_mappings[].expression` | `
.`; identifies the entity | | `join.columnsFrom` (FK) | `link_mappings[].object_mapping.expression` | `.`; binds the relationship to its far role | -| whole core model | `ontology_mappings[].semantic_model` | complete output from `OBMLtoOssie.convert()`, including `version` | +| whole core model | `ontology_mappings[].semantic_model` | complete output from `OBMLtoOssie.convert()`| `
` is the final identifier of the dataset `source` (e.g. `db.schema.t` → `t`), falling back to the dataset name when `source` has no dotted physical table. From 582a9285d639fe38c8f1cf2232ae30bc9204f55d Mon Sep 17 00:00:00 2001 From: Yufei Date: Thu, 24 Sep 2026 15:50:35 -0700 Subject: [PATCH 6/6] Fix ontology validation and semantic model constraints --- .../ontology/src/ossie_ontology/model.py | 4 ++ .../ontology/src/ossie_ontology/spec.py | 2 +- .../ontology/tests/test_ossie_parser.py | 17 +++++ .../ontology/tests/test_semantic_model.py | 62 +++++++++++++++++++ validation/tests/test_validate.py | 62 ++++++++++++++++++- validation/validate.py | 26 ++++++-- 6 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 converters/ontology/tests/test_semantic_model.py diff --git a/converters/ontology/src/ossie_ontology/model.py b/converters/ontology/src/ossie_ontology/model.py index c681a8b6..21c72228 100644 --- a/converters/ontology/src/ossie_ontology/model.py +++ b/converters/ontology/src/ossie_ontology/model.py @@ -706,6 +706,10 @@ def __init__( custom_extensions: list[CustomExtension] | None = None, version: str = "0.2.0.dev0", ): + if version != "0.2.0.dev0": + raise ValueError( + f"Unsupported semantic model version {version!r}; expected '0.2.0.dev0'" + ) self._version = version self._name = name self._description = description diff --git a/converters/ontology/src/ossie_ontology/spec.py b/converters/ontology/src/ossie_ontology/spec.py index 86ee7217..f2a59012 100644 --- a/converters/ontology/src/ossie_ontology/spec.py +++ b/converters/ontology/src/ossie_ontology/spec.py @@ -180,7 +180,7 @@ class SemanticModel(OssieObject): name: str description: str | None = None ai_context: AiContext | None = None - datasets: list[Dataset] = Field(default_factory=list) + datasets: list[Dataset] = Field(min_length=1) relationships: list[JoinPath] = Field(default_factory=list) metrics: list[Metric] = Field(default_factory=list) custom_extensions: list[CustomExtension] = Field(default_factory=list) diff --git a/converters/ontology/tests/test_ossie_parser.py b/converters/ontology/tests/test_ossie_parser.py index 7bacbecf..d23808a2 100644 --- a/converters/ontology/tests/test_ossie_parser.py +++ b/converters/ontology/tests/test_ossie_parser.py @@ -89,6 +89,23 @@ def test_parse_rejects_invalid_embedded_version(flights_path, tmp_path, version) assert exc.value.errors()[0]["loc"] == ("ontology_mappings", 0, "semantic_model", "version") +@pytest.mark.parametrize("missing", [True, False], ids=["missing", "empty"]) +def test_parse_rejects_missing_or_empty_datasets(flights_path, tmp_path, missing): + document = yaml.safe_load(flights_path.read_text()) + model = document["ontology_mappings"][0]["semantic_model"] + if missing: + del model["datasets"] + else: + model["datasets"] = [] + path = tmp_path / "invalid.yaml" + path.write_text(yaml.safe_dump(document)) + + with pytest.raises(ValidationError) as exc: + OssieParser().parse(path) + + assert exc.value.errors()[0]["loc"] == ("ontology_mappings", 0, "semantic_model", "datasets") + + def test_parse_returns_populated_ontology(flights_model): ontology = flights_model.ontology # Built-in concepts (String, Integer, Decimal, ...) are always present on top diff --git a/converters/ontology/tests/test_semantic_model.py b/converters/ontology/tests/test_semantic_model.py new file mode 100644 index 00000000..66f9f05a --- /dev/null +++ b/converters/ontology/tests/test_semantic_model.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Core document constraints at runtime construction and export.""" + +import pytest +from pydantic import ValidationError + +from ossie_ontology.converter.ossie_to_spec.converter import OssieToSpecConverter +from ossie_ontology.model import ( + Dataset, + OntologyComponent, + OntologyMapping, + OssieOntology, + SemanticModel, +) + + +@pytest.mark.parametrize("version", ["0.1.0", "0.2.0", "", None, 2]) +def test_rejects_unsupported_version_at_construction(version): + with pytest.raises(ValueError, match="Unsupported semantic model version"): + SemanticModel(name="sales", version=version) + + +@pytest.mark.parametrize("version_args", [{}, {"version": "0.2.0.dev0"}]) +def test_requires_datasets_at_export_but_allows_incremental_construction(version_args): + model = SemanticModel(name="sales", **version_args) + ontology = OssieOntology( + name="sales", version="0.2.0.dev0", ontology=OntologyComponent() + ) + ontology.add_ontology_mapping( + OntologyMapping( + name="sales_mapping", ontology=ontology.ontology, semantic_model=model + ) + ) + + with pytest.raises(ValidationError) as exc: + OssieToSpecConverter.convert(ontology) + assert exc.value.errors()[0]["loc"] == ("datasets",) + + model.add_dataset(Dataset(name="orders", source="sales.public.orders", fields=[])) + exported = OssieToSpecConverter.convert(ontology).dump_dict() + + assert exported["ontology_mappings"][0]["semantic_model"] == { + "version": "0.2.0.dev0", + "name": "sales", + "datasets": [{"name": "orders", "source": "sales.public.orders"}], + } diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index bbb1ad7d..94dac610 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -16,6 +16,7 @@ # under the License. import json +import urllib.request from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path @@ -67,6 +68,35 @@ def test_accepts_a_single_root_model(core_schema: dict) -> None: assert _VALIDATE.validate_schema(document, core_schema) == [] +@pytest.fixture +def offline(monkeypatch): + def reject_request(*args, **kwargs): + pytest.fail("Schema validation must not make HTTP requests") + + monkeypatch.setattr(urllib.request, "urlopen", reject_request) + monkeypatch.setattr("jsonschema.validators.urlopen", reject_request, raising=False) + + +@pytest.mark.parametrize("uri", [ + "https://github.com/apache/ossie/core-spec/ossie-schema.json", + "https://raw.githubusercontent.com/apache/ossie/main/core-spec/ossie-schema.json", +]) +@pytest.mark.parametrize("fragment, document", [ + ("", _document([_ORDERS], [])), + ("#/$defs/AIContext", {"instructions": "Use for sales analysis"}), +]) +def test_core_schema_references_resolve_offline(offline, uri, fragment, document): + assert _VALIDATE.validate_schema(document, {"$ref": uri + fragment}) == [] + + +def test_unresolvable_reference_is_a_validation_error(offline): + uri = "https://example.invalid/unknown-schema.json" + + errors = _VALIDATE.validate_schema({}, {"$ref": uri}) + + assert errors == [f"[Schema] Cannot resolve schema reference: {uri}"] + + def test_rejects_empty_root_datasets(core_schema: dict) -> None: errors = _VALIDATE.validate_schema(_document([], []), core_schema) @@ -294,10 +324,13 @@ def test_skips_malformed_flat_unique_keys() -> None: @pytest.fixture def run_validator(tmp_path, monkeypatch, capsys): - def run(document): + def run(document, schema_path=None): model_path = tmp_path / "model.json" model_path.write_text(json.dumps(document)) - monkeypatch.setattr(_VALIDATE.sys, "argv", [str(_VALIDATE_PATH), str(model_path)]) + args = [str(_VALIDATE_PATH), str(model_path)] + if schema_path is not None: + args.extend(["--schema", str(schema_path)]) + monkeypatch.setattr(_VALIDATE.sys, "argv", args) with pytest.raises(SystemExit) as caught: _VALIDATE.main() return caught.value.code, capsys.readouterr().out @@ -305,6 +338,31 @@ def run(document): return run +@pytest.mark.parametrize("version, expected_exit", [("0.2.0.dev0", 0), ("0.1.0", 1)]) +def test_ontology_cli_validates_embedded_documents_offline( + run_validator, offline, version, expected_exit +): + model = _document([_ORDERS], []) + model["version"] = version + document = { + "version": "0.2.0.dev0", + "name": "sales", + "ai_context": {"instructions": "Use for sales analysis"}, + "ontology": [{"concept": "Order", "type": "EntityType"}], + "ontology_mappings": [{"semantic_model": model, "concept_mappings": []}], + } + + exit_code, output = run_validator( + document, Path(__file__).parents[2] / "ontology/ontology.json" + ) + + assert exit_code == expected_exit + if expected_exit == 0: + assert "Validation PASSED" in output + else: + assert "ontology_mappings -> 0 -> semantic_model -> version" in output + + @pytest.mark.parametrize("target", [ "missing_customers", "Warning: missing_customers", diff --git a/validation/validate.py b/validation/validate.py index 99b390c6..8e7af189 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -50,6 +50,8 @@ try: import yaml from jsonschema import Draft202012Validator + from referencing import Registry, Resource + from referencing.exceptions import Unresolvable from yaml.constructor import ConstructorError except ImportError: print("Missing dependencies. Install with:") @@ -146,12 +148,26 @@ def _check_unique_keys(self, node: yaml.Node, visited: set) -> None: def validate_schema(data: dict, schema: dict) -> list[str]: - """Validate against JSON Schema.""" - validator = Draft202012Validator(schema) + """Validate against JSON Schema, resolving core references locally.""" + core_path = Path(__file__).parent.parent / "core-spec" / "ossie-schema.json" + core = json.loads(core_path.read_text()) + resource = Resource.from_contents(core) + # Ontology references use the raw URL; also register the canonical schema ID. + registry = Registry().with_resources([ + (core["$id"], resource), + ( + "https://raw.githubusercontent.com/apache/ossie/main/core-spec/ossie-schema.json", + resource, + ), + ]) + validator = Draft202012Validator(schema, registry=registry) errors = [] - for error in validator.iter_errors(data): - path = " -> ".join(str(p) for p in error.absolute_path) if error.absolute_path else "(root)" - errors.append(f"[Schema] {path}: {error.message}") + try: + for error in validator.iter_errors(data): + path = " -> ".join(str(p) for p in error.absolute_path) if error.absolute_path else "(root)" + errors.append(f"[Schema] {path}: {error.message}") + except Unresolvable as error: + errors.append(f"[Schema] Cannot resolve schema reference: {error.ref}") return errors