From 89c5fde8a97f494ac0845eff7a54f38228511fba Mon Sep 17 00:00:00 2001 From: Mike Carlo Date: Mon, 5 Oct 2026 09:34:49 -0500 Subject: [PATCH 1/4] fix(schema): update custom_extensions data type to object for structured vendor-specific data --- core-spec/ossie-schema.json | 4 +-- core-spec/spec.md | 64 +++++++++++++++++++++---------------- core-spec/spec.yaml | 10 +++--- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index a8d2fcbd..ab988cbc 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -85,8 +85,8 @@ "$ref": "#/$defs/Vendor" }, "data": { - "type": "string", - "description": "JSON string containing vendor-specific data" + "type": "object", + "description": "Vendor-specific data as a structured object. Keys are vendor-defined and values may be any JSON value, including nested objects and arrays. No keys are reserved." } }, "required": ["vendor_name", "data"], diff --git a/core-spec/spec.md b/core-spec/spec.md index f43fc1af..61ea59d3 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -119,7 +119,9 @@ relationships: [] metrics: [] custom_extensions: - vendor_name: DBT - data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + data: + project_name: tpcds_analytics + models_path: models/semantic ``` The same document structure in JSON: @@ -209,7 +211,8 @@ datasets: fields: [] custom_extensions: - vendor_name: DBT - data: '{"materialized": "table"}' + data: + materialized: table ``` --- @@ -459,16 +462,27 @@ expression: ## Custom Extensions -Custom extensions allow vendors to add platform-specific metadata without breaking core compatibility. Each extension includes a vendor name and arbitrary JSON data. +Custom extensions allow vendors to add platform-specific metadata without breaking core compatibility. Each extension includes a vendor name and a structured `data` object. ### Schema ```yaml custom_extensions: - vendor_name: string # Free-form string identifying the vendor - data: string # JSON string containing vendor-specific data + data: object # Structured vendor-specific data ``` +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `vendor_name` | string | Yes | Free-form identifier for the vendor or organization that owns the extension | +| `data` | object | Yes | Vendor-specific key/value data. May be empty (`{}`) | + +The keys inside `data` are defined by the vendor; the core specification does not +reserve or require any key. Values may be any JSON value: strings, numbers, +booleans, `null`, arrays, or nested objects. Prefer an existing core field when a +property already has a home in the specification, and use `data` only for genuine +vendor-specific or otherwise unmapped residue. + ### Vendor Names The `vendor_name` field is a free-form string, allowing any vendor or organization to @@ -496,46 +510,41 @@ The following are well-known examples: ```yaml - vendor_name: SNOWFLAKE - data: '{ - "warehouse": "ANALYTICS_WH", - "database": "PROD", - "schema": "PUBLIC" - }' + data: + warehouse: ANALYTICS_WH + database: PROD + schema: PUBLIC ``` **Salesforce Extension:** ```yaml - vendor_name: SALESFORCE - data: '{ - "tableau_workbook_id": "sales_dashboard", - "einstein_enabled": true, - "crm_sync": { - "enabled": true, - "sync_frequency": "daily" - } - }' + data: + tableau_workbook_id: sales_dashboard + einstein_enabled: true + crm_sync: + enabled: true + sync_frequency: daily ``` **DBT Extension:** ```yaml - vendor_name: DBT - data: '{ - "project_name": "analytics", - "materialized": "table", - "tags": ["daily", "core"] - }' + data: + project_name: analytics + materialized: table + tags: [daily, core] ``` **Databricks Extension:** ```yaml - vendor_name: Databricks - data: '{ - "default_catalog": "finance", - "default_schema": "gold" - }' + data: + default_catalog: finance + default_schema: gold ``` --- @@ -639,7 +648,8 @@ metrics: custom_extensions: - vendor_name: SNOWFLAKE - data: '{"warehouse": "ANALYTICS_WH"}' + data: + warehouse: ANALYTICS_WH ``` --- diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 5b2011a6..aee61f9d 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -99,7 +99,7 @@ metrics: [] # Allows vendors to add custom metadata without breaking core compatibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor - data: string + data: object # Structured vendor-specific data; keys are vendor-defined --- # Logical Dataset Schema @@ -150,7 +150,7 @@ datasets: # Optional: Vendor-specific attributes for extensibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor - data: string + data: object # Structured vendor-specific data; keys are vendor-defined --- # Relationship Schema @@ -191,7 +191,7 @@ relationships: # Optional: Vendor-specific attributes for extensibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor - data: string + data: object # Structured vendor-specific data; keys are vendor-defined --- # Fields Schema @@ -246,7 +246,7 @@ fields: # Optional: Vendor-specific attributes for extensibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor - data: string + data: object # Structured vendor-specific data; keys are vendor-defined --- # Metrics Schema @@ -283,4 +283,4 @@ metrics: # Optional: Vendor-specific attributes for extensibility custom_extensions: - vendor_name: string # Free-form string identifying the vendor - data: string + data: object # Structured vendor-specific data; keys are vendor-defined From f76f904f4078a556ca2968593f4d10c03aad25d9 Mon Sep 17 00:00:00 2001 From: Mike Carlo Date: Mon, 5 Oct 2026 09:35:27 -0500 Subject: [PATCH 2/4] fix: update custom_extensions to use structured objects for vendor-specific metadata --- docs/index.md | 4 ++-- examples/tpcds_semantic_model.yaml | 28 +++++++++++++--------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docs/index.md b/docs/index.md index 5860ac68..7cebf9b8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -245,7 +245,7 @@ Use the [validation script](../validation/validate.py) included in the repositor YAML is more human-readable and easier to author by hand, which is important for a specification that teams may edit directly. The Ossie JSON Schema is available for programmatic validation, and converters can work with either format. **How does Ossie handle vendor-specific features?** -Through `custom_extensions`. Each vendor can store arbitrary JSON metadata in extension blocks tagged with their vendor name. This metadata is preserved during round-trip conversions and ignored by tools that don't understand it — ensuring that no information is lost. +Through `custom_extensions`. Each vendor can store structured metadata (any keys and nested values it chooses) in extension blocks tagged with their vendor name. This metadata is preserved during round-trip conversions and ignored by tools that don't understand it — ensuring that no information is lost. **Can metrics reference multiple datasets?** Yes. Metrics are defined at the semantic model level (not within a dataset) and can reference fields from multiple datasets. @@ -321,7 +321,7 @@ A practical guide for organizations looking to adopt Ossie. | **Metric** | A quantitative measure computed by aggregating data across one or more datasets (e.g., total revenue, average order value). Metrics are defined at the semantic model level. | | **Relationship** | A foreign key connection between two datasets, defining how they can be joined. Relationships are always many-to-one (from the referencing dataset to the referenced dataset). | | **Dialect** | A specific SQL or expression language variant (e.g., `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`). Ossie supports multiple dialects so expressions can be tailored to each platform. | -| **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as a JSON string. Extensions allow platforms to carry additional information without modifying the core specification. | +| **Custom Extension** | Vendor-specific metadata attached to any Ossie construct as a structured object with vendor-defined keys. Extensions allow platforms to carry additional information without modifying the core specification. | | **AI Context** | Optional annotations on any Ossie construct (model, dataset, field, relationship, metric) that provide additional context for AI tools — including natural language instructions, synonyms, and example queries. | | **Converter** | A tool that translates between the Ossie format and a specific vendor's semantic model format. Converters come in pairs: import (vendor → Ossie) and export (Ossie → vendor). | | **Hub-and-Spoke** | The architectural pattern used by Ossie, where the specification acts as the central format (hub) and vendor converters act as spokes, avoiding the need for point-to-point integrations. | diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index dce43439..159263ed 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -652,20 +652,18 @@ metrics: custom_extensions: - vendor_name: SALESFORCE - data: | - { - "tableau_workbook_id": "tpcds_retail_dashboard", - "einstein_enabled": true, - "crm_sync": { - "enabled": true, - "sync_frequency": "daily", - "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" - }, - "tableau_semantics": { - "published": true, - "version": "0.1.1" - } - } + data: + tableau_workbook_id: tpcds_retail_dashboard + einstein_enabled: true + crm_sync: + enabled: true + sync_frequency: daily + customer_mapping: "customer.c_customer_id -> Account.AccountNumber" + tableau_semantics: + published: true + version: "0.1.1" - vendor_name: DBT - data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + data: + project_name: tpcds_analytics + models_path: models/semantic From 250e2cbf1d8f8b2a1ae82bd0938eb269191e5693 Mon Sep 17 00:00:00 2001 From: Mike Carlo Date: Mon, 5 Oct 2026 09:41:17 -0500 Subject: [PATCH 3/4] fix: update OssieCustomExtension to use structured objects for vendor-specific metadata --- python/src/ossie/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index f4098be0..f700f726 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -91,12 +91,12 @@ class OssieAIContextObject(BaseModel): class OssieCustomExtension(BaseModel): - """Vendor-specific metadata as a serialized JSON string.""" + """Vendor-specific metadata as a structured object with vendor-defined keys.""" model_config = ConfigDict(frozen=True) vendor_name: str - data: str + data: dict[str, Any] class OssieDialectExpression(BaseModel): From 2de12d6342f4821b071380a9dff1281892615e7d Mon Sep 17 00:00:00 2001 From: Mike Carlo Date: Mon, 5 Oct 2026 09:41:31 -0500 Subject: [PATCH 4/4] fix: update custom extensions to use object data structure for vendor-specific metadata --- python/tests/test_models.py | 2 +- python/tests/test_serialization.py | 24 ++++++++++++++++++++++-- python/tests/test_validation.py | 14 ++++++++++++-- validation/tests/test_validate.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 5bdb237a..f4af361e 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -130,7 +130,7 @@ def test_document_serialization_preserves_flat_model_and_metadata( data.update( description="A portable model", ai_context="Use the event timestamp", - custom_extensions=[{"vendor_name": "SIGMA", "data": '{"id":"model-1"}'}], + custom_extensions=[{"vendor_name": "SIGMA", "data": {"id": "model-1"}}], relationships=[ { "name": "event_link", diff --git a/python/tests/test_serialization.py b/python/tests/test_serialization.py index 6af7af4a..544d1ddd 100644 --- a/python/tests/test_serialization.py +++ b/python/tests/test_serialization.py @@ -74,15 +74,35 @@ def test_to_ossie_yaml_includes_all_dialects(document_data: dict) -> None: def test_to_ossie_yaml_includes_custom_extension_vendor(document_data: dict) -> None: document_data["custom_extensions"] = [ - {"vendor_name": "DATABRICKS", "data": '{"id":"model-1"}'} + {"vendor_name": "DATABRICKS", "data": {"id": "model-1"}} ] document = OssieDocument.model_validate(document_data) parsed = yaml.safe_load(document.to_ossie_yaml()) assert parsed["custom_extensions"] == [ - {"vendor_name": "DATABRICKS", "data": '{"id":"model-1"}'} + {"vendor_name": "DATABRICKS", "data": {"id": "model-1"}} ] +def test_custom_extension_data_round_trips_nested_values(document_data: dict) -> None: + data = { + "text": '{"legacy": true}', + "warehouse": "ANALYTICS_WH", + "some_unmapped_flag": True, + "retries": 3, + "unset": None, + "tags": ["daily", None], + "nested": {"enabled": True, "inner": {"value": None}}, + } + document_data["custom_extensions"] = [{"vendor_name": "SNOWFLAKE", "data": data}] + document = OssieDocument.model_validate(document_data) + + from_yaml = yaml.safe_load(document.to_ossie_yaml()) + from_json = json.loads(document.to_ossie_json()) + + assert from_yaml["custom_extensions"][0]["data"] == data + assert from_json["custom_extensions"][0]["data"] == data + + def test_to_ossie_yaml_excludes_none(document_data: dict) -> None: document = OssieDocument.model_validate(document_data) output = document.to_ossie_yaml() diff --git a/python/tests/test_validation.py b/python/tests/test_validation.py index e98fb925..d4232361 100644 --- a/python/tests/test_validation.py +++ b/python/tests/test_validation.py @@ -130,7 +130,7 @@ def test_semantic_model_missing_datasets() -> None: def test_custom_extension_missing_vendor_name() -> None: with pytest.raises(ValidationError): - OssieCustomExtension(data="{}") + OssieCustomExtension(data={}) def test_custom_extension_missing_data() -> None: @@ -138,6 +138,16 @@ def test_custom_extension_missing_data() -> None: OssieCustomExtension(vendor_name="ASF") +@pytest.mark.parametrize("data", ['{"id": "model-1"}', ["id"], 1, None]) +def test_custom_extension_rejects_non_object_data(data: object) -> None: + with pytest.raises(ValidationError): + OssieCustomExtension(vendor_name="ASF", data=data) + + +def test_custom_extension_accepts_empty_object_data() -> None: + assert OssieCustomExtension(vendor_name="ASF", data={}).data == {} + + def test_dialect_expression_missing_dialect() -> None: with pytest.raises(ValidationError): OssieDialectExpression(expression="order_id") @@ -203,7 +213,7 @@ def test_frozen_ai_context_object() -> None: def test_frozen_custom_extension() -> None: - custom_ext = OssieCustomExtension(vendor_name="ASF", data="{}") + custom_ext = OssieCustomExtension(vendor_name="ASF", data={}) with pytest.raises(ValidationError): custom_ext.vendor_name = "other" diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 3cc8a9b9..c6013bb3 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -225,6 +225,34 @@ def test_schema_rejects_empty_relationship_endpoints(core_schema: dict) -> None: ] +@pytest.mark.parametrize( + "data", + [ + {}, + {"warehouse": "ANALYTICS_WH", "some_unmapped_flag": True}, + {"text": '{"legacy": true}', "nested": {"tags": ["a", None], "n": 1.5}}, + ], +) +def test_custom_extension_data_accepts_objects(core_schema: dict, data: dict) -> None: + document = _document([dict(_ORDERS)], []) + document["custom_extensions"] = [{"vendor_name": "SNOWFLAKE", "data": data}] + document["datasets"][0]["custom_extensions"] = [{"vendor_name": "DBT", "data": data}] + + assert _VALIDATE.validate_schema(document, core_schema) == [] + + +@pytest.mark.parametrize("data", ['{"warehouse": "ANALYTICS_WH"}', ["warehouse"], 1, True, None]) +def test_custom_extension_data_rejects_non_objects(core_schema: dict, data: object) -> None: + document = _document([dict(_ORDERS)], []) + document["custom_extensions"] = [{"vendor_name": "SNOWFLAKE", "data": data}] + + errors = _VALIDATE.validate_schema(document, core_schema) + + assert len(errors) == 1 + assert errors[0].startswith("[Schema] custom_extensions -> 0 -> data:") + assert "is not of type 'object'" in errors[0] + + def test_sql_checks_traverse_root_fields_and_metrics(monkeypatch: pytest.MonkeyPatch) -> None: seen = []