diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1c0cb250..f331b2e7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -76,6 +76,12 @@ jobs: with: fetch-depth: 0 # Needed for setuptools_scm + - name: Determine cutout schema version + # Cached cutouts are only valid for the schema version they were created with + run: | + echo "CUTOUT_SCHEMA_VERSION=$(sed -n 's/^CUTOUT_SCHEMA_VERSION = \([0-9]*\).*/\1/p' atlite/schema.py)" >> $GITHUB_ENV + shell: bash + - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }} uses: actions/setup-python@v7 with: @@ -86,7 +92,7 @@ jobs: uses: actions/cache@v6 with: path: ${{ env.CACHE_PATH }} - key: retrieved-cutouts-${{ env.today }} + key: retrieved-cutouts-schema${{ env.CUTOUT_SCHEMA_VERSION }}-${{ env.today }} enableCrossOsArchive: true - name: Download package diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 2f48f25e..18dbaf61 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -20,6 +20,13 @@ Upcoming Release **Features** * Add support for Python 3.14. +* Feat: Cutouts data format (schema) is now versioned using a the attribute ``atlite_cutout_schema_version``. + Newly created cutouts carry the version recommended by the ``atlite`` version used to create them (version 1 at the moment) + Old cutouts are still supported but need to be migrated with + ``python -m atlite.migrate --output `` or + ``atlite.migrate.migrate_cutout(path, output=new_path)``. + Cutouts that have an incompatible data schema will in the future be rejected and raise a ``IncompatibleCutoutError``. + See :doc:`migrating_cutouts` for details (https://github.com/PyPSA/atlite/pull/533). **Bug fixes** diff --git a/atlite/cutout.py b/atlite/cutout.py index 8aa13df6..89538514 100644 --- a/atlite/cutout.py +++ b/atlite/cutout.py @@ -66,6 +66,13 @@ compute_intersectionmatrix, get_coords, ) +from atlite.schema import ( + CUTOUT_SCHEMA_VERSION, + SCHEMA_VERSION_ATTR, + check_schema_version, + read_schema_version, + schema_version_from_attrs, +) from atlite.utils import CachedAttribute logger = logging.getLogger(__name__) @@ -159,8 +166,16 @@ def __init__(self, path: PathLike, **cutoutparams: Any) -> None: If required arguments are missing when building a new cutout. ValueError If ``bounds`` has an invalid format. + atlite.schema.IncompatibleCutoutError + If the cutout's schema version is not supported by this version of + atlite. - """ + Warns + ----- + atlite.schema.OutdatedCutoutWarning + If the cutout has an older, still supported schema version. + + """ # noqa: DOC502 (IncompatibleCutoutError is raised by check_schema_version) path = Path(path).with_suffix(".nc") chunks = cutoutparams.pop("chunks", {"time": 100}) if isinstance(chunks, dict): @@ -171,6 +186,7 @@ def __init__(self, path: PathLike, **cutoutparams: Any) -> None: # Three cases. First, cutout exists -> take the data. # Second, data is given -> take it. Third, else -> build a new cutout if path.is_file(): + check_schema_version(read_schema_version(path), path) data = xr.open_dataset(str(path)) data = data.chunk(chunks) data.attrs.update(storable_chunks) @@ -182,6 +198,7 @@ def __init__(self, path: PathLike, **cutoutparams: Any) -> None: ) elif "data" in cutoutparams: data = cutoutparams.pop("data") + check_schema_version(schema_version_from_attrs(data.attrs)) else: logger.info("Building new cutout %s", path) @@ -216,6 +233,7 @@ def __init__(self, path: PathLike, **cutoutparams: Any) -> None: coords = get_coords(x, y, time, **cutoutparams) attrs = { + SCHEMA_VERSION_ATTR: CUTOUT_SCHEMA_VERSION, "module": module, "prepared_features": [], **storable_chunks, @@ -232,6 +250,11 @@ def __init__(self, path: PathLike, **cutoutparams: Any) -> None: self.path = path self.data = data + @property + def schema_version(self) -> int: + """Schema version of the cutout's file format.""" + return schema_version_from_attrs(self.data.attrs) + @property def name(self) -> str: """Name of the cutout.""" @@ -434,9 +457,22 @@ def merge( merged : Cutout Merged cutout. + Raises + ------ + ValueError + If the cutouts have different schema versions. + """ assert isinstance(other, Cutout) + if self.schema_version != other.schema_version: + raise ValueError( + f"Cannot merge cutouts with different schema versions " + f"({self.schema_version} and {other.schema_version}). " + "Migrate both to the current version first, then merge. " + "Use `python -m atlite.migrate` or `atlite.migrate.migrate_cutout`." + ) + if path is None: path = mktemp( prefix=f"{self.path.stem}-", diff --git a/atlite/data.py b/atlite/data.py index 2064a8dd..3012e8a8 100644 --- a/atlite/data.py +++ b/atlite/data.py @@ -22,6 +22,10 @@ from numpy import atleast_1d from atlite.datasets import modules as datamodules +from atlite.schema import ( + SCHEMA_VERSION_ATTR, + require_current_schema_version, +) if TYPE_CHECKING: from collections.abc import Callable, Iterable, Sequence @@ -252,8 +256,13 @@ def cutout_prepare( If ``tmpdir`` is None. FileNotFoundError If ``tmpdir`` does not exist. + atlite.schema.IncompatibleCutoutError + If features is to be added to an existing cutout that is not the recommended + schema version of the installed versino of ``atlite``. + RuntimeError + If the data of a module sets the cutout's schema version attribute. - """ + """ # noqa: DOC502 (IncompatibleCutoutError is raised by require_current_schema_version) if dask_kwargs is None: dask_kwargs = {} @@ -264,6 +273,8 @@ def cutout_prepare( logger.info("Cutout already prepared.") return cutout + require_current_schema_version(cutout.schema_version, cutout.path) + if tmpdir is None: raise ValueError("tmpdir cannot be None") temp_dir_path = Path(tmpdir) @@ -296,6 +307,10 @@ def cutout_prepare( monthly_requests=monthly_requests, concurrent_requests=concurrent_requests, ) + if SCHEMA_VERSION_ATTR in ds.attrs: + raise RuntimeError( + f"Data of module '{module}' must not contain attribute '{SCHEMA_VERSION_ATTR}'." + ) prepared |= set(missing_features) cutout.data.attrs.update({"prepared_features": list(prepared)}) diff --git a/atlite/migrate.py b/atlite/migrate.py new file mode 100644 index 00000000..eb2c1c70 --- /dev/null +++ b/atlite/migrate.py @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Contributors to atlite +# +# SPDX-License-Identifier: MIT +""" +Explicit migration of cutouts to the current schema version. + +Write the migrated cutout to a new file and keep the original:: + + python -m atlite.migrate path/to/cutout.nc --output path/to/migrated.nc + +or, from Python:: + + from atlite.migrate import migrate_cutout + + migrate_cutout("path/to/cutout.nc", output="path/to/migrated.nc") + +Overwriting the original file requires ``--in-place`` (``in_place=True``). + +Each migration step upgrades a cutout file by exactly one schema version and is +registered in :data:`MIGRATIONS` under the version it upgrades from. +""" + +from __future__ import annotations + +import argparse +import logging +import shutil +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +import xarray as xr + +from atlite.schema import ( + CUTOUT_SCHEMA_VERSION, + SCHEMA_VERSION_ATTR, + IncompatibleCutoutError, + read_schema_version, + require_supported_schema_version, + schema_version_from_attrs, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from atlite._types import PathLike + +logger = logging.getLogger(__name__) + + +def _migrate_v0_to_v1(path: Path) -> None: + """Migrate a cutout file in place from schema version 0 to 1.""" + # v1 only adds the version attribute + # Appending a dataset without variables updates the meta attributes in the header and leaves the data untouched + xr.Dataset(attrs={SCHEMA_VERSION_ATTR: 1}).to_netcdf(path, mode="a") + + +# Maps the schema version a step upgrades from to the step itself +MIGRATIONS: dict[int, Callable[[Path], None]] = { + 0: _migrate_v0_to_v1, +} + + +def migrate_cutout( + path: PathLike, output: PathLike | None = None, *, in_place: bool = False +) -> None: + """ + Migrate a cutout file to the current schema version. + + Exactly one of ``output`` and ``in_place=True`` must be given. + + Parameters + ---------- + path : str or Path + Path of the cutout to migrate. + output : str or Path, optional + Path where to write the migrated cutout to. Must not exist yet. + in_place : bool, default False + Overwrite the original cutout with the migrated cutout. + + Raises + ------ + ValueError + If neither or both of ``output`` and ``in_place=True`` are given. + IncompatibleCutoutError + If the file is not an atlite cutout or its schema version cannot be + migrated by this atlite version. + FileExistsError + If ``output`` already exists. + + """ + # Overwriting the original must be an explicit choice + if (output is None) == (not in_place): + raise ValueError("Specify either an output file or in_place=True") + + path = Path(path) + with xr.open_dataset(path) as ds: + attrs = dict(ds.attrs) + # Files without a version attribute are treated as v0 cutouts, so make sure + # it is a cutout at all before stamping a version on it + if "module" not in attrs: + raise IncompatibleCutoutError( + f"'{path}' does not appear to be an atlite cutout and is missing the attribute 'module'. " + f"Aborting." + ) + version = schema_version_from_attrs(attrs) + require_supported_schema_version(version, path) + + if output is None: + target = path + else: + output = Path(output) + if output.exists(): + raise FileExistsError(f"Output file '{output}' already exists.") + # Migrate a hidden copy next to the output, so `output` only appears + # once it is complete and the original is not touched before, avoiding + # corruption on failed upgrades + target = output.with_name(f".{output.name}.migration") + shutil.copyfile(path, target) + + try: + # Actual migration loop. Call all migration functions in order + for current in range(version, CUTOUT_SCHEMA_VERSION): + logger.info( + "Migrating '%s' from schema version %d to %d.", + path, + current, + current + 1, + ) + MIGRATIONS[current](target) + except BaseException: + # Also on KeyboardInterrupt: don't leave a half-migrated copy behind + if target != path: + target.unlink(missing_ok=True) + raise + + if output is not None: + target.rename(output) + + +def main(argv: Sequence[str] | None = None) -> int: + """ + Migrate a cutout from the command line. + + Parameters + ---------- + argv : sequence of str, optional + Command line arguments, defaults to ``sys.argv[1:]``. + + Returns + ------- + int + Exit code. + + """ + parser = argparse.ArgumentParser( + prog="python -m atlite.migrate", + description="Migrate an atlite cutout to the current schema version " + f"({CUTOUT_SCHEMA_VERSION}).", + ) + parser.add_argument("cutout", type=Path, help="Path to the cutout file.") + # Mirrors migrate_cutout: exactly one destination is required + destination = parser.add_mutually_exclusive_group(required=True) + destination.add_argument( + "-o", "--output", type=Path, help="Write to this file, keep the original." + ) + destination.add_argument( + "--in-place", action="store_true", help="Overwrite the original file." + ) + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") + + try: + version = read_schema_version(args.cutout) + migrate_cutout(args.cutout, args.output, in_place=args.in_place) + except (IncompatibleCutoutError, OSError) as err: + # Expected failures: report them without a traceback + logger.error("%s", err) + return 1 + + logger.info( + "'%s' has schema version %d (was %d).", + args.output or args.cutout, + CUTOUT_SCHEMA_VERSION, + version, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/atlite/schema.py b/atlite/schema.py new file mode 100644 index 00000000..dadac1d4 --- /dev/null +++ b/atlite/schema.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: Contributors to atlite +# +# SPDX-License-Identifier: MIT +""" +Schema version handling for cutouts. + +Every cutout written by atlite carries an integer schema version in the global +attribute :data:`SCHEMA_VERSION_ATTR`. Legacy cutouts written before this attribute +was introduced are treated as having version ``0``. + +atlite does not migrates cutouts between versions automatically. +Loading a cutout with an older but supported version emits a :class:`OutdatedCutoutWarning`. +Loading a cutout with an unsupported version raises a :class:`IncompatibleCutoutError`. +Outdated cutouts need to be migrated explicitly with ``python -m atlite.migrate`` or +:func:`atlite.migrate.migrate_cutout`. +""" + +from __future__ import annotations + +import logging +import warnings +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import xarray as xr + +if TYPE_CHECKING: + from collections.abc import Mapping + + from atlite._types import PathLike + +logger = logging.getLogger(__name__) + +# Name of the attribute stored as netCDF attr +SCHEMA_VERSION_ATTR = "atlite_cutout_schema_version" + +# Version written to newly created cutouts +# Bump with integer version when the schema changes +CUTOUT_SCHEMA_VERSION = 1 + +# Versions this release of atlite can read +SUPPORTED_SCHEMA_VERSIONS = frozenset({0, 1}) + +# Version assumed for cutouts written before the schema number was introduced +# keep this for legacy compatability, even if no longer supported by the current version +LEGACY_SCHEMA_VERSION = 0 + + +class IncompatibleCutoutError(ValueError): + """Raised when a cutout has a schema version atlite cannot read.""" + + +class OutdatedCutoutWarning(FutureWarning): + """Emitted when a cutout has an older, still supported schema version.""" + + +def schema_version_from_attrs(attrs: Mapping[str, Any]) -> int: + """ + Get the cutout schema version from the global attributes. + + Parameters + ---------- + attrs : Mapping + Global attributes of a cutout, e.g. ``cutout.data.attrs``. + + Returns + ------- + int + The schema version or `LEGACY_SCHEMA_VERSION` if not present. + + """ + if SCHEMA_VERSION_ATTR not in attrs: + logger.debug( + "Cutout has no attribute '%s', assuming schema version %d.", + SCHEMA_VERSION_ATTR, + LEGACY_SCHEMA_VERSION, + ) + return LEGACY_SCHEMA_VERSION + return int(attrs[SCHEMA_VERSION_ATTR]) + + +def read_schema_version(path: PathLike) -> int: + """ + Read the cutout schema version from a file without loading its data. + + The dataset is opened lazily, so only metadata and the small coordinate + indexes are read. This is cheap even for large cutouts. + + Parameters + ---------- + path : str or Path + Path to the cutout file. + + Returns + ------- + int + The schema version, :data:`LEGACY_SCHEMA_VERSION` if not present. + + """ + with xr.open_dataset(Path(path)) as ds: + return schema_version_from_attrs(ds.attrs) + + +def _cutout_name(path: PathLike | None) -> str: + return "This cutout" if path is None else f"The cutout '{path}'" + + +def migration_instructions(path: PathLike | None = None) -> str: + """ + Build the instructions for migrating a cutout to the current schema version. + + Parameters + ---------- + path : str or Path, optional + Path of the cutout. Placeholders are used if not given. + + Returns + ------- + str + The instructions, including the shell and Python commands to migrate. + + """ + source = Path("cutout.nc") if path is None else Path(path) + output = source.with_name(f"{source.stem}-v{CUTOUT_SCHEMA_VERSION}{source.suffix}") + + return ( + "Migrate the cutout to the current version; " + "the original file is not changed.\n" + "\n" + " From the command line:\n" + f' python -m atlite.migrate "{source}" --output "{output}"\n' + "\n" + " From Python:\n" + " from atlite.migrate import migrate_cutout\n" + f" migrate_cutout({str(source)!r}, output={str(output)!r})\n" + "\n" + "To overwrite the original file, pass --in-place on the command " + "line or in_place=True in Python, without an output file." + ) + + +def outdated_cutout_message(version: int, path: PathLike | None = None) -> str: + """ + Build the message explaining that a cutout is outdated and must be migrated to be used. + + Parameters + ---------- + version : int + Schema version of the cutout. + path : str or Path, optional + Path of the cutout. Placeholders are used if not given. + + Returns + ------- + str + The message, including the shell and Python commands to migrate. + + """ + return ( + f"The cutout uses cutout schema version {version}. " + f"This version of atlite can only modify cutouts of version {CUTOUT_SCHEMA_VERSION}. " + f"Version {version} can still be read but not modified." + "\n" + migration_instructions(path) + ) + + +def require_current_schema_version(version: int, path: PathLike | None = None) -> None: + """ + Require a cutout to have the current schema version before it is modified. + + atlite only writes the current schema version. Modifying a cutout with an + older version would mix both versions in one file, so it has to be + migrated explicitly first. + + Parameters + ---------- + version : int + Schema version of the cutout. + path : str or Path, optional + Path of the cutout, used in messages. + + Raises + ------ + IncompatibleCutoutError + If ``version`` is not :data:`CUTOUT_SCHEMA_VERSION`. + + """ + if version != CUTOUT_SCHEMA_VERSION: + raise IncompatibleCutoutError( + f"This cutout uses cutout schema version {version}. " + f"This version of atlite can only modify cutouts of " + f"version {CUTOUT_SCHEMA_VERSION}.\n" + "\n" + migration_instructions(path) + ) + + +def require_supported_schema_version( + version: int, path: PathLike | None = None +) -> None: + """ + Require a cutout schema version to be supported by this atlite version. + + Parameters + ---------- + version : int + Schema version of the cutout. + path : str or Path, optional + Path of the cutout, used in messages. + + Raises + ------ + IncompatibleCutoutError + If ``version`` is not supported in `SUPPORTED_SCHEMA_VERSIONS`. + + """ + supported = ", ".join(map(str, sorted(SUPPORTED_SCHEMA_VERSIONS))) + + if version not in SUPPORTED_SCHEMA_VERSIONS: + if version > CUTOUT_SCHEMA_VERSION: + raise IncompatibleCutoutError( + f"Cutout has schema version {version}, which is newer than " + f"this version of atlite supports ({supported}). " + "Upgrade atlite to use this cutout." + ) + raise IncompatibleCutoutError( + f"Cutout has schema version {version}, which is no longer " + f"supported by this version of atlite ({supported}). " + f"Migrate or re-create the cutout." + ) + + +def check_schema_version(version: int, path: PathLike | None = None) -> None: + """ + Check whether a cutout schema version can be read by this atlite version. + + Parameters + ---------- + version : int + Schema version of the cutout. + path : str or Path, optional + Path of the cutout, used in messages. + + Raises + ------ + IncompatibleCutoutError + If ``version`` is not supported. + + Warns + ----- + OutdatedCutoutWarning + If ``version`` is supported but older than the current version. + + """ # noqa: DOC502 (IncompatibleCutoutError is raised by require_supported_schema_version) + require_supported_schema_version(version, path) + + if version < CUTOUT_SCHEMA_VERSION: + warnings.warn( + outdated_cutout_message(version, path), + OutdatedCutoutWarning, + stacklevel=3, + ) diff --git a/doc/index.rst b/doc/index.rst index 359d245d..8d01e0df 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -102,6 +102,7 @@ If you would like to cite the atlite software, please refer to `this paper + + SPDX-License-Identifier: CC-BY-4.0 + +################## +Migrating cutouts +################## + +The structure of cutout files can change between atlite releases. +Each version of ``atlite`` supports one or multiple cutout versions and to tell these +apart, every cutout carries a **schema version**. +Cutouts written the before schema versions were introduced do not have this attribute and +are treated as version ``0``. + +Migrating a cutout +================== + +Migration writes the migrated cutout to a new file and leaves the original file +unchanged. From the command line: + +.. code-block:: bash + + python -m atlite.migrate europe-2013-era5.nc --output europe-2013-era5-v1.nc + +Or from Python: + +.. code-block:: python + + from atlite.migrate import migrate_cutout + + migrate_cutout("europe-2013-era5.nc", output="europe-2013-era5-v1.nc") + +If the migration fails, no output file is written. + +To overwrite the original file instead, pass ``--in-place`` on the command line +or ``in_place=True`` in Python without an output file. + +Migration upgrades a cutout step by step from its schema version to the +current one. Where possible, a step only updates the file's metadata. Migrating +from version 0 to 1, for example, only adds the version attribute and does not +rewrite the data, so it is fast even for large cutouts. + +There is no guarantee that migration of cutouts is always possible. + +.. note:: + + A migrated cutout is not necessarily identical to a cutout freshly created + with the current atlite release. Migration only converts the structure of + the cutout file to the current schema version; it keeps the data as it was + originally prepared. Between the creation of the original cutout and today, + the processing in atlite and the upstream data sources (for example + reanalysis updates or corrections published by data providers) may have + changed. If you need data that is consistent with the current atlite + release and upstream data, create the cutout again instead of migrating it. + + +Existing schema versions and changes between them +================================================= + +======== ===================================================== +Version Changes +======== ===================================================== +0 Cutouts written before schema versions were introduced. +1 Adds the ``atlite_cutout_schema_version`` attribute. +======== ===================================================== + +Checking the schema version +=========================== + +The schema version of a cutout file can be read without loading its data: + +>>> from atlite.schema import read_schema_version +>>> read_schema_version("europe-2013-era5.nc") +0 + +For a loaded cutout, use the attribute `atlite.Cutout.schema_version`. + +What happens when a cutout is loaded +==================================== + +atlite never migrates cutouts implicitly. +Depending on the schema version of the cutout, loading it with :py:class:`atlite.Cutout` behaves as follows: + +* **Current version:** the cutout is loaded. +* **Older, still supported version:** the cutout is loaded, and an + :py:class:`atlite.schema.OutdatedCutoutWarning` is shown that includes the + commands to migrate it. Modifications to the cutout, e.g. using :py:meth:`atlite.Cutout.prepare` + are refused and need the cutout to be migrated first. +* **Unsupported version:** an :py:class:`atlite.schema.IncompatibleCutoutError` + is raised. Cutouts that are too new require a newer atlite release. Cutouts + that are too old have to be migrated with an older atlite release or created + again. \ No newline at end of file diff --git a/doc/ref_api.rst b/doc/ref_api.rst index ed37449e..045adcc1 100644 --- a/doc/ref_api.rst +++ b/doc/ref_api.rst @@ -14,6 +14,16 @@ Cutout :members: +Schema versions and migration +----------------------------- + +.. automodule:: atlite.schema + :members: + +.. automodule:: atlite.migrate + :members: migrate_cutout + + Data ------ diff --git a/test/test_schema.py b/test/test_schema.py new file mode 100644 index 00000000..a28e12fd --- /dev/null +++ b/test/test_schema.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: Contributors to atlite +# +# SPDX-License-Identifier: MIT +"""Tests for cutout schema versions and their migration.""" + +import warnings + +import numpy as np +import pandas as pd +import pytest +import xarray as xr + +from atlite import Cutout +from atlite.migrate import main, migrate_cutout +from atlite.schema import ( + CUTOUT_SCHEMA_VERSION, + SCHEMA_VERSION_ATTR, + IncompatibleCutoutError, + OutdatedCutoutWarning, + read_schema_version, +) + + +def write_cutout(path, version=None): + """ + Write a minimal cutout file. + + Returns + ------- + Path + ``path``, which has no schema attribute if ``version`` is None. + + """ + attrs = {"module": "era5", "prepared_features": ["height"], "dx": 0.25, "dy": 0.25} + if version is not None: + attrs[SCHEMA_VERSION_ATTR] = version + ds = xr.Dataset( + {"height": (("y", "x"), np.zeros((2, 3), dtype="float32"))}, + coords={ + "x": [0.0, 0.25, 0.5], + "y": [50.0, 50.25], + "time": pd.date_range("2013-01-01", periods=2, freq="h"), + }, + attrs=attrs, + ) + ds["height"].attrs.update(module="era5", feature="height") + ds.to_netcdf(path) + return path + + +def test_new_cutout_has_current_version(tmp_path): + """A newly created cutout carries the current schema version in memory and on disk.""" + cutout = Cutout( + tmp_path / "new", module="era5", bounds=(0, 50, 1, 51), time="2013-01-01" + ) + assert cutout.schema_version == CUTOUT_SCHEMA_VERSION + cutout.to_file() + assert read_schema_version(cutout.path) == CUTOUT_SCHEMA_VERSION + + +def test_current_version_loads_without_warning(tmp_path): + """Loading a cutout with the current schema version emits no outdated warning.""" + path = write_cutout(tmp_path / "current.nc", CUTOUT_SCHEMA_VERSION) + with warnings.catch_warnings(): + warnings.simplefilter("error", OutdatedCutoutWarning) + Cutout(path) + + +def test_missing_attribute_is_version_zero(tmp_path): + """A cutout without the schema version attribute is read as version 0.""" + path = write_cutout(tmp_path / "legacy.nc") + assert read_schema_version(path) == 0 + + +def test_outdated_version_warns_with_commands(tmp_path): + """Loading an outdated cutout warns and shows the shell and Python migration commands.""" + path = write_cutout(tmp_path / "legacy.nc") + output = tmp_path / "legacy-v1.nc" + with pytest.warns(OutdatedCutoutWarning) as record: + cutout = Cutout(path) + assert cutout.schema_version == 0 + + message = str(record[0].message) + assert f'python -m atlite.migrate "{path}" --output "{output}"' in message + assert "from atlite.migrate import migrate_cutout" in message + assert f"migrate_cutout({str(path)!r}, output={str(output)!r})" in message + + +@pytest.mark.parametrize("version", [CUTOUT_SCHEMA_VERSION + 1, -1]) +def test_unsupported_version_raises(tmp_path, version): + """Loading or migrating a cutout with an unsupported schema version raises an error.""" + path = write_cutout(tmp_path / "unsupported.nc", version) + with pytest.raises(IncompatibleCutoutError): + Cutout(path) + with pytest.raises(IncompatibleCutoutError): + migrate_cutout(path, in_place=True) + + +def test_migrate_in_place(tmp_path): + """In-place migration updates the schema version without changing the data.""" + path = write_cutout(tmp_path / "legacy.nc") + with xr.open_dataset(path) as ds: + expected = ds.load() + + with warnings.catch_warnings(): + warnings.simplefilter("error", OutdatedCutoutWarning) + migrate_cutout(path, in_place=True) + assert read_schema_version(path) == CUTOUT_SCHEMA_VERSION + with xr.open_dataset(path) as ds: + xr.testing.assert_identical(ds.drop_attrs(), expected.drop_attrs()) + + with warnings.catch_warnings(): + warnings.simplefilter("error", OutdatedCutoutWarning) + Cutout(path) + + +def test_migrate_to_output_keeps_original(tmp_path): + """Migrating to an output file keeps the original and never overwrites an existing output.""" + path = write_cutout(tmp_path / "legacy.nc") + output = tmp_path / "migrated.nc" + + assert main([str(path), "--output", str(output)]) == 0 + assert read_schema_version(path) == 0 + assert read_schema_version(output) == CUTOUT_SCHEMA_VERSION + assert sorted(p.name for p in tmp_path.iterdir()) == ["legacy.nc", "migrated.nc"] + + # Existing outputs are never overwritten + assert main([str(path), "--output", str(output)]) == 1 + + +def test_migrate_current_version_is_noop(tmp_path): + """Migrating a cutout that already has the current schema version leaves it unchanged.""" + path = write_cutout(tmp_path / "current.nc", CUTOUT_SCHEMA_VERSION) + migrate_cutout(path, in_place=True) + assert read_schema_version(path) == CUTOUT_SCHEMA_VERSION + + +def test_migrate_rejects_non_cutout(tmp_path): + """Migrating a netCDF file that is not an atlite cutout raises an error.""" + path = tmp_path / "other.nc" + xr.Dataset({"a": ("x", [1, 2])}).to_netcdf(path) + with pytest.raises(IncompatibleCutoutError, match="atlite cutout"): + migrate_cutout(path, in_place=True) + + +@pytest.mark.parametrize("kwargs", [{}, {"output": "out.nc", "in_place": True}]) +def test_migrate_requires_output_or_in_place(tmp_path, kwargs): + """migrate_cutout requires exactly one of an output file or in_place=True.""" + path = write_cutout(tmp_path / "legacy.nc") + with pytest.raises(ValueError, match="in_place=True"): + migrate_cutout(path, **kwargs) + assert read_schema_version(path) == 0 + + +def test_cli_requires_output_or_in_place(tmp_path): + """The command line requires exactly one of --output or --in-place.""" + path = write_cutout(tmp_path / "legacy.nc") + with pytest.raises(SystemExit): + main([str(path)]) + with pytest.raises(SystemExit): + main([str(path), "--in-place", "-o", "out.nc"]) + assert read_schema_version(path) == 0 + + assert main([str(path), "--in-place"]) == 0 + assert read_schema_version(path) == CUTOUT_SCHEMA_VERSION + + +def test_merge_requires_equal_versions(tmp_path): + """Merging cutouts with different schema versions raises an error.""" + current = Cutout(write_cutout(tmp_path / "current.nc", CUTOUT_SCHEMA_VERSION)) + with pytest.warns(OutdatedCutoutWarning): + legacy = Cutout(write_cutout(tmp_path / "legacy.nc")) + with pytest.raises(ValueError, match="schema versions"): + current.merge(legacy) + + +def test_prepare_refuses_outdated_version(tmp_path): + """Preparing features for an outdated cutout is refused and leaves the file unchanged.""" + path = write_cutout(tmp_path / "legacy.nc") + with pytest.warns(OutdatedCutoutWarning): + cutout = Cutout(path) + with pytest.raises(IncompatibleCutoutError, match="python -m atlite.migrate"): + cutout.prepare(tmpdir=tmp_path) + assert read_schema_version(path) == 0 + + +def test_prepare_allows_current_version(tmp_path): + """Preparing a cutout with the current schema version passes the version check.""" + path = write_cutout(tmp_path / "current.nc", CUTOUT_SCHEMA_VERSION) + # All features of the minimal cutout are prepared, so nothing is downloaded + Cutout(path).prepare(tmpdir=tmp_path, features="height") + + +def test_prepare_skips_check_for_prepared_cutout(tmp_path, monkeypatch): + """Calling prepare on a fully prepared outdated cutout returns without an error.""" + path = write_cutout(tmp_path / "legacy.nc") + with pytest.warns(OutdatedCutoutWarning): + cutout = Cutout(path) + # Pretend all features are prepared, so prepare returns before any writing + monkeypatch.setattr(Cutout, "prepared", property(lambda self: True)) + assert cutout.prepare(tmpdir=tmp_path) is cutout + + +def test_prepare_rejects_schema_attribute_in_source_data(tmp_path, monkeypatch): + """Source data that sets the schema version attribute makes prepare fail before writing.""" + path = write_cutout(tmp_path / "current.nc", CUTOUT_SCHEMA_VERSION) + cutout = Cutout(path) + + def get_features(cutout, module, features, **kwargs): + ds = cutout.data[["height"]].rename(height="wnd100m") + return ds.assign_attrs({SCHEMA_VERSION_ATTR: CUTOUT_SCHEMA_VERSION}) + + monkeypatch.setattr("atlite.data.get_features", get_features) + with pytest.raises(RuntimeError, match=SCHEMA_VERSION_ATTR): + cutout.prepare(tmpdir=tmp_path, features="wind") + assert sorted(p.name for p in tmp_path.iterdir()) == ["current.nc"]