From af4762b0163f5d81a44c5000782af93281b7bead Mon Sep 17 00:00:00 2001 From: George Hilley Date: Tue, 22 Sep 2026 07:20:31 -0700 Subject: [PATCH 1/2] Version 1.0: compiled kernels, TopoToolbox parity, and a GDAL installer Repairs roughly 70 defects, several of which prevented the package from importing at all on current NumPy and SciPy, and replaces the recursive basin traversals -- which needed a million-frame recursion limit and still overflowed on real DEMs -- with iterative kernels. The public interface is unchanged: every class, method and keyword still exists and still means what it did. A handful of results differ because they were wrong; those are listed in CHANGELOG.md under "Results that change". Depression filling now uses the Priority-Flood algorithms of Barnes, Lehman and Mulla (2014), compiled to C++ and roughly 50x faster, with an equivalent NumPy implementation in TopoAnalysis.fastops that produces identical numbers when the extension cannot be built. The compiled kernels release the GIL. Adds TopoAnalysis.topotoolbox, which reproduces TopoToolbox 2 (MATLAB) function by function, and docs/topotoolbox_parity.md, which tabulates the differing conventions and says where exact agreement is not achievable. Adds an installer for GDAL, the one dependency pip cannot supply. There is no GDAL wheel on PyPI at any version for any platform -- only an sdist that compiles the bindings against a C library which must already exist -- so "install GDAL" is two jobs that have to agree. install.py and the topoanalysis-gdal command supply both halves, choosing between an existing C library, conda, a prebuilt Windows wheel, the system package manager, or a from-source build of PROJ and GDAL into the virtualenv that needs no administrator rights. Two failure modes are detected explicitly: bindings built without numpy, where osgeo imports cleanly and ReadAsArray fails at run time, and a stale PROJ_LIB inherited from another installation. Tests: 350 passing with the C++ kernels, 347 with the NumPy kernels. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 236 ++ .gitignore | 38 +- CHANGELOG.md | 366 +++ Dockerfile | 43 + MANIFEST.in | 29 + Makefile | 74 + MovingWindow.py | 186 +- README.md | 188 ++ __init__.py | 156 + _cpp/flow_routing.hpp | 625 ++++ _cpp/module.cpp | 540 ++++ _cpp/priority_flood.hpp | 378 +++ _gdal_source.py | 562 ++++ analysis.py | 20 +- benchmark.py | 121 + cli.py | 179 ++ datasets.py | 21 +- dem.py | 5616 ++++++++++++++++++++++-------------- demMethods.py | 65 +- demRecursionTools.py | 23 +- docs/algorithms.md | 397 +++ docs/api.md | 1993 +++++++++++++ docs/generate_api.py | 232 ++ docs/guide.md | 282 ++ docs/index.md | 82 + docs/installation.md | 392 +++ docs/topotoolbox_parity.md | 249 ++ environment.yml | 27 +- fastops.py | 811 ++++++ gdal_setup.py | 1561 ++++++++++ gui_tools.py | 29 +- install.py | 560 ++++ kernels.py | 264 ++ plotting.py | 23 +- process_dem.py | 17 +- pyproject.toml | 119 + requirements-dev.txt | 16 + requirements.txt | 8 + setup.py | 71 + tests/conftest.py | 169 ++ tests/test_flow.py | 572 ++++ tests/test_gdal_setup.py | 688 +++++ tests/test_grid.py | 402 +++ tests/test_kernels.py | 365 +++ tests/test_modules.py | 319 ++ tests/test_regressions.py | 642 +++++ tests/test_terrain.py | 175 ++ tests/test_topotoolbox.py | 751 +++++ topotoolbox.py | 1039 +++++++ utils.py | 65 +- 50 files changed, 19415 insertions(+), 2371 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 CHANGELOG.md create mode 100644 Dockerfile create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 README.md create mode 100644 _cpp/flow_routing.hpp create mode 100644 _cpp/module.cpp create mode 100644 _cpp/priority_flood.hpp create mode 100644 _gdal_source.py create mode 100644 benchmark.py create mode 100644 cli.py create mode 100644 docs/algorithms.md create mode 100644 docs/api.md create mode 100644 docs/generate_api.py create mode 100644 docs/guide.md create mode 100644 docs/index.md create mode 100644 docs/installation.md create mode 100644 docs/topotoolbox_parity.md create mode 100644 fastops.py create mode 100644 gdal_setup.py create mode 100644 install.py create mode 100644 kernels.py create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/conftest.py create mode 100644 tests/test_flow.py create mode 100644 tests/test_gdal_setup.py create mode 100644 tests/test_grid.py create mode 100644 tests/test_kernels.py create mode 100644 tests/test_modules.py create mode 100644 tests/test_regressions.py create mode 100644 tests/test_terrain.py create mode 100644 tests/test_topotoolbox.py create mode 100644 topotoolbox.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6a51b7e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,236 @@ +name: tests + +on: + push: + branches: [master, main] + pull_request: + workflow_dispatch: + # The from-source GDAL build takes about half an hour, so it runs on a + # schedule rather than on every push -- but it has to run *somewhere*, + # because it is the route that is hardest to keep working. + schedule: + - cron: '0 5 * * 1' + +jobs: + # The main matrix: no GDAL, so it runs everywhere and proves that the + # library works without it. + test: + name: py${{ matrix.python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ['3.8', '3.11', '3.12'] + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install + working-directory: TopoAnalysis + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test,stats]" + + - name: Report the active backend + working-directory: TopoAnalysis + run: python -c "import TopoAnalysis; print(TopoAnalysis.backend())" + + - name: Fail if the C++ kernels did not build + working-directory: TopoAnalysis + run: python -c "import sys, TopoAnalysis; sys.exit(0 if TopoAnalysis.backend()=='c++' else 1)" + + - name: Test + working-directory: TopoAnalysis + run: python -m pytest -q + + - name: Test the pure-NumPy kernels + working-directory: TopoAnalysis + env: + TOPOANALYSIS_PURE_PYTHON: '1' + run: python -m pytest -q + + # A second job with GDAL from conda-forge, which exercises the file I/O + # paths the matrix above skips. + test-with-gdal: + name: with GDAL (ubuntu) + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + + - uses: conda-incubator/setup-miniconda@v3 + with: + environment-file: TopoAnalysis/environment.yml + activate-environment: topoanalysis + auto-activate-base: false + + - name: Install + working-directory: TopoAnalysis + run: python -m pip install -e ".[test]" + + - name: Test + working-directory: TopoAnalysis + run: python -m pytest -q + + # Prove the package still installs when there is no compiler: the + # extension is optional and the NumPy kernels must take over. + test-without-compiler: + name: no compiler + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install with the compiler hidden + working-directory: TopoAnalysis + env: + CC: /bin/false + CXX: /bin/false + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test]" + + - name: The NumPy kernels must be active + working-directory: TopoAnalysis + run: python -c "import sys, TopoAnalysis; print(TopoAnalysis.backend()); sys.exit(0 if TopoAnalysis.backend()=='python' else 1)" + + - name: Test + working-directory: TopoAnalysis + run: python -m pytest -q + + # The advertised one-command install, on every platform. GDAL is skipped + # here so the job stays fast; the jobs below cover it. + install-script: + name: install.py on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install everything from a clean checkout + working-directory: TopoAnalysis + run: python install.py --no-gdal + - name: The installed environment must work + working-directory: TopoAnalysis + shell: bash + run: | + if [ -f .venv/bin/python ]; then PY=.venv/bin/python; else PY=.venv/Scripts/python.exe; fi + "$PY" -c "import sys, TopoAnalysis; print(TopoAnalysis.backend()); sys.exit(0 if TopoAnalysis.backend()=='c++' else 1)" + "$PY" -m TopoAnalysis.gdal_setup doctor || true + + # Ubuntu's own GDAL is older than the version whose sdist declares numpy as + # a build dependency, so this job is the regression test for the + # gdal_array trap: installing the bindings the naive way here produces an + # osgeo that imports fine and cannot ReadAsArray. + gdal-from-system-packages: + name: GDAL from apt (the gdal_array trap) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install the GDAL C library only + run: | + sudo apt-get update + sudo apt-get install -y gdal-bin libgdal-dev + gdal-config --version + - name: Install TopoAnalysis + working-directory: TopoAnalysis + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test]" + - name: The bindings are missing, and the doctor should say so + working-directory: TopoAnalysis + run: | + python -m TopoAnalysis.gdal_setup doctor && exit 1 || true + - name: Install the bindings against the system library + working-directory: TopoAnalysis + run: python -m TopoAnalysis.gdal_setup install + - name: ReadAsArray must work, not merely import + run: | + python - <<'EOF' + import numpy as np + from osgeo import gdal, gdal_array # noqa: F401 + ds = gdal.GetDriverByName('MEM').Create('', 4, 3, 1, gdal.GDT_Float32) + ds.GetRasterBand(1).WriteArray(np.arange(12, dtype='float32').reshape(3, 4)) + assert ds.ReadAsArray().shape == (3, 4) + print('gdal', gdal.__version__, 'numpy bridge ok') + EOF + - name: Test, including the GDAL-marked cases + working-directory: TopoAnalysis + run: python -m pytest -q + + # The no-root route. Slow, so it is scheduled or triggered by hand. + gdal-from-source: + name: GDAL built from source + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Cache the source tarballs + uses: actions/cache@v4 + with: + path: ~/.cache/TopoAnalysis/downloads + key: gdal-sources-${{ hashFiles('TopoAnalysis/_gdal_source.py') }} + - name: Install everything, building GDAL from source + working-directory: TopoAnalysis + run: python install.py --gdal-strategy source + - name: Test against the GDAL we built + working-directory: TopoAnalysis + run: | + .venv/bin/python -m TopoAnalysis.gdal_setup doctor + .venv/bin/python -m pytest -q + + sdist: + name: sdist rebuilds the extension + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build and install from an sdist + working-directory: TopoAnalysis + run: | + python -m pip install --upgrade pip build + python -m build --sdist + python -m pip install dist/*.tar.gz + cd / + python -c "import sys, TopoAnalysis; print(TopoAnalysis.backend()); sys.exit(0 if TopoAnalysis.backend()=='c++' else 1)" diff --git a/.gitignore b/.gitignore index 614013b..dd67ad1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,36 @@ +# Build artefacts +build/ +dist/ +*.egg-info/ +*.so +*.pyd +*.o + +# Python +__pycache__/ +*.py[cod] +.pytest_cache/ +.coverage +htmlcov/ + +# Environments. install.py creates .venv here, because this directory is +# itself the package. +.venv/ +venv/ + +# Where a from-source GDAL build installs when it has nowhere better +share/topoanalysis/ + +# Docs +docs/_build/ + +# Editors +.idea/ +.vscode/ .metadata -*.pyc -.idea -.idea/* +.project +.pydevproject +.settings/ + +# macOS +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5bb9c9b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,366 @@ +# Changelog + +## 1.0.0 + +The first release with a test suite, an install story, and compiled kernels. + +The **public interface is unchanged**: every class, method and keyword that +existed before still exists and still means the same thing. What changed is +that a lot of it now works. + +--- + +## Results that change + +Read this section before comparing new output against old. Each item is a +place where the previous answer was wrong, so the new number differs on +purpose. + +### Chi (`Ksi`) + +The integrand was `(Ao / (A - Ao))**theta`; it is now `(Ao / A)**theta`. The +old form diverges as drainage area approaches the threshold `Ao`, so chi was +inflated near every channel head and the error propagated downstream. +Chi is defined as ∫(A₀/A)^θ dx (Perron & Royden 2013). + +### Chi (`Chi`) + +Chi at the outlet is now exactly 0. It used to be one cell-step of the +integrand, so every chi value in a basin was offset by that amount. + +`Chi` also accepts `trapezoid=True`, which averages the integrand across each +step instead of taking it at the upstream cell. That is the rule TopoToolbox +uses; the default remains the original left-endpoint rule. + +### Hillshade + +The aspect term was `atan2(dz/dy, dz/dx)` where the ESRI definition the code +claimed to implement is `atan2(dz/dy, -dz/dx)`. Illumination was mirrored +east–west: slopes that should have been lit were shadowed and vice versa. +Hillshades are now clipped to 0–255 and stored as `uint8` rather than +wrapping negative values around the byte into bright speckle. + +### D8 flow directions + +Three separate problems, all fixed by moving to a single shared kernel: + +- The windowed comparison stopped two columns and two rows short, so the + last row and the last column of every grid received **no flow direction at + all**. +- The diagonal distance factor was `1.41`, not `sqrt(2)`. +- Ties between equally steep neighbours resolved towards the north-east + because of the order the codes were written; they now resolve + deterministically in the order E, SE, S, SW, W, NW, N, NE. + +`FlowDirectionD8.update_flow_codes_in_mask`, the per-cell re-router, had the +scaling **inverted** — it divided the cardinal drops by 1.41 and left the +diagonals alone — which biased routing towards the diagonals. + +### Flow accumulation (`Area`) + +The accumulation order came from `flow_direction.sort()`. That returns an +elevation sort only when the grid was built from a filled DEM in the same +session; for a flow-direction grid **loaded from a file** it sorted the cells +by their direction *code*, which is meaningless, and the resulting areas were +wrong. Accumulation now derives its own topological order from the flow +network, so it is correct however the grid was obtained. + +The `mask` keyword had no effect: the test was `mask[i, j] is not None`, and +`__getitem__` returns the value, so it was true even for a masked-out cell. +Masks now actually mask. + +### `MainstemValleyArea` + +Its `__calcD8Area` override was name-mangled to +`_MainstemValleyArea__calcD8Area` while the inherited caller looked for +`_Area__calcD8Area`, so it never ran: the class silently behaved exactly +like `ValleyArea`. It now accumulates along the longest-flow-length path as +intended. (The method also compared a whole grid object against a scalar, +which would have raised the moment it did run.) + +### `FlowLength` direction codes + +`FlowLength` stored its main-stem directions in a **row-flipped** convention: +internally consistent, but contradicting the ArcGIS convention used by +`FlowDirectionD8` and by the rest of the library. It now uses the same +convention as everything else. + +**`*_directions` side-files written by earlier versions must be +regenerated.** `FlowLength.load` cannot detect which convention a file uses. + +### Recursive-profile `distance_scale` + +In the nested dictionaries from `map_values_to_recursive_list`, +`distance_scale` was written onto the *parent* node and overwritten by each +child in turn, so it ended up describing whichever child was visited last. +Chi profiles built from these dictionaries were wrong at every confluence +where a diagonal and a cardinal tributary met. Each node now carries the +step length from its own parent. + +The root node's `index` is now a plain `(row, col)` pair like every other +node's, instead of the nested `((row, col),)` the caller happened to pass. + +### Flexural deflection + +`(2 * 3.141)**4` is 0.2% low; it is now `(2 * np.pi)**4`. The wavenumber +axes were built from half-width ranges whose length did not have to match +the grid, so the spectral multiply could fail outright; they now come from +`np.fft.fftfreq`. + +### Georeferencing of `nx/ny/projection/geo_transform` grids + +`__populate_georef_info_using_geoTransform` read `ny` two lines before it +assigned it, so a grid built from an explicit geotransform picked up the +default of 0 and its `yllcenter` came out `ny * dx` too far north. Every +coordinate derived from it — `_rowscols_to_xy`, `_xy_to_rowscols`, +`extent()`, the ArcInfo ASCII header, outlets given in map coordinates — +inherited the error, and the same grid round-tripped through GDAL disagreed +with itself. The other four construction paths were always correct. + +### Geographic curvature + +`GeographicLaplacian` divided by `dx`, which on a lat/lon grid is in +**degrees**: curvatures came out about 10¹⁰ too large. It now uses the true +metric cell size, like every other geographic class. + +### Strahler stream order + +The order was applied one donor at a time, which makes a junction of orders +{2, 2, 3} come out as 4 or 3 depending on which donor arrives first — and +the two backends emit different (equally valid) topological orders, so they +disagreed on about 3% of networks. Junctions are now resolved as a whole: +the largest incoming order, plus one if two or more donors carry it. + +Note this also differs from TopoToolbox, whose incremental form has the same +order-dependence. + +### Smaller behavioural changes + +**`search_down_flow_direction_with_length`** now includes the terminal cell +of the path, and follows a path that merely *starts* on the grid perimeter. +The stop condition is "no downstream neighbour inside the grid" rather than +"this cell is on the perimeter", so a cell on the western edge draining east +is now followed instead of returning an empty path. + +**`FlowDirectionD8.divides()`** now evaluates the perimeter as well as the +interior. Border cells with no upstream neighbour are divides, and used to +be reported as 0 regardless. + +**`Chi(mask=...)`** treats the mask as a boolean gate on the integration +rather than multiplying the finished chi grid by the mask's *values*. The +old behaviour scaled chi by whatever the mask happened to contain, and +carried a hard-coded `area > 1e4` escape hatch. + +**`_xy_to_rowscols`** rejects a point exactly one cell beyond the eastern or +southern edge. It used to return the subscript `nx` or `ny`, which raised an +`IndexError` somewhere else entirely. + +**`FlowDirectionD8` caches its flow network** (`receivers`, +`topological_order`) because every downstream computation now reads it. +Writing a flow code with `fd[i, j] = ...` or `set_value_at_rowscols` drops +the cache, so hand edits take effect as they always did; call +`fd.invalidate_network()` after modifying `_griddata` directly. + +**`tile(..., tile_xpadding=0)`** no longer duplicates the edge tiles' data. +`griddata[:, -0:]` selects the whole array rather than nothing, so a zero +padding produced double-width edge tiles. + +**`resample`** warns when SciPy cannot provide the requested interpolation +order and it falls back to linear, instead of quietly changing the answer. + +**`Elevation.track_flow_downhill`** and +**`DiscreteFlowAccumulation`** stop on a `NaN` neighbour as before, but the +test is now a real `NaN` check: `None not in numpy_array` never matched, so +off-grid neighbours did not stop the walk the way the code intended. + +### Other corrected numerics + +| What | Was | Now | +|---|---|---| +| `assignBCs` south-east corner | copied the south-west corner | copies the south-east corner | +| `get_XY_matricies` | returned `n-1` coordinates per axis | returns `n` | +| `resample` | dropped `xllcenter` from the axis stop, and flipped the grid left-right unconditionally | keeps the south-west cell centre fixed; uses `RegularGridInterpolator` | +| `sort(mask=...)` | indexed the mask with positions in the sorted array | selects the sorted positions inside the mask | +| `average_over_distance` | kernel centred half a cell off; `ifftshift` where `fftshift` was needed | kernel centred on cell (0,0), no shift needed | +| `extent_of_data` | scanned rows for the column limits | scans the right axis for each | +| gradient / Laplacian over a length scale | divided by `(2N+1)dx`; float slice indices | divides by `2N·dx`; integer indices | +| `LocalRelief` window | `np.ogrid[-r:r]`, one cell short on the far side | symmetric about the centre | +| `LogArea` | declared `uint8`, truncating log₁₀(area) to integers | `float64` | +| `tile` geotransform | north edge one cell too far north | consistent with the reader | +| `CircularMovingWindow` | squared the row offset twice, giving a vertical band | a disc | +| `datasets` `slope_y` | multiplied the relief by a ramp, zeroing row 0 | adds a ramp | +| geographic cell area | `np.float64(range(n))` — a `TypeError` on modern NumPy | `np.arange` | +| ArcInfo ASCII reader | ignored `NODATA_value`, turning −9999 flags into elevations | maps it to `NaN` | +| ArcInfo ASCII writer | wrote the literal text `nan` | writes −9999 by default | + +--- + +## Crashes fixed + +These raised immediately on any current scientific-Python stack. + +**Removed upstream APIs.** `np.NAN`, `np.NaN`, `np.float`, +`scipy.ndimage.morphology`, `scipy.ndimage.filters`, +`scipy.interpolate.interp2d`, `np.int`, and `matplotlib.pyplot.hold` were all +in use. A regression test now fails if any of them reappears. + +**Python 3 iterators.** `zip()` objects were used as NumPy subscripts and +subscripted directly in `ValueGrid.set_value_at_indexes`, +`CrossDivideDChi`, `NormalizedCrossDivideDChi` and +`FlowDirectionD8.divides_for_outlets`. Several functions returned a `zip` +that callers iterated twice, getting nothing the second time. + +**Comparisons with `None`.** `bounds_of_basin_for_outlet` evaluated +`lon > bounds[0][1] or bounds[0][1] is None` — the comparison runs first, so +it raised on its own first iteration. `areas_between` compared `None` with a +float when a cell had no upstream neighbour. + +**Names that did not exist.** + +| Call | Problem | +|---|---| +| `PriorityQueueMixIn.randomize_subbasins_with_mask` | called `self.__flood`, mangled to `_PriorityQueueMixIn__flood`; the method is `_flood` | +| `MovingWindow.apply_moving_window` | called `self.__adjustKernel`; the method is `__adjust_kernel` | +| `ScaledRelief(flooded_dem=...)` | called `_create_from_flow_direction_flooded_dem_and_elevation`, never defined | +| `ChannelSlopeWithSmoothing(horizontal_interval=...)` | dispatched to `_create_from_elevation_flow_direction`, never defined | +| `BaseSpatialGrid.apply_moving_window` | `out_grid = None` then `out_grid._georef_info = ...` | +| `convert_rivertools_directions_to_arc` | `int()` of an array; referenced an undefined `self.noData` | +| `dem.plot(xlabel=...)` | the local name was only bound when the argument was *absent* | +| `utils.calc_ks_for_outlet` | `kwargs.pop('xo')` with no default | +| `MovingWindow` subclasses | read `self.window_radius`, which the constructor never set | +| `DiscreteFlowAccumulation._mean_pixel_dimension` | required a `kwargs['elevation']` its callers did not pass | +| `GeographicThetaFromChiWithSmoothing` | inherited `KsFromChiWithSmoothing`, so it fitted ks instead of theta | + +**Stack overflows.** `get_indexes_of_upstream_cells`, +`map_values_to_recursive_list`, `Chi.__recurse_chi`, +`RestoredElevation.__fill_upstream_points` and +`FlowLength.__get_upstream_indexes` were all recursive, which is why the +module raised the recursion limit to 1 000 000 at import — a limit CPython +segfaults long before reaching. All of them are now iterative, and the +`setrecursionlimit` call is gone. + +**Undercounting.** `self._n[rows, cols] += 1` with repeated subscripts +applies each increment only once; the profile-crossing counts in +`KsFromChiWithSmoothing` and `ThetaFromChiWithSmoothing` now use +`np.add.at`. + +--- + +## Thread safety + +The compiled kernels release the GIL, but eleven of the thirteen bindings +resolved their NumPy buffers *inside* the released region. +`py::array::request()` calls `PyObject_GetBuffer`, which touches reference +counts, so running a kernel from two threads at once leaked references and +could abort the interpreter. Every binding now resolves and validates its +buffers while the GIL is held and runs nothing but plain C++ afterwards. +`test_kernels_are_thread_safe` pins it down. + +The bindings also validate array sizes, so a mismatched `order` array raises +instead of reading past the end of it. + +## Added + +- **`install.py`** — one command that takes a fresh clone to a working + installation: it creates a virtual environment, installs the dependencies, + compiles the kernels, installs GDAL, and then proves the result works by + round-tripping a georeferenced GeoTIFF. Standard library only, so it runs + on a machine where nothing is set up yet. `--dry-run` reports what it would + do without doing it. +- **`TopoAnalysis.gdal_setup`** and the `topoanalysis-gdal` command — find, + diagnose and install GDAL, meaning *both* the C library and the Python + bindings. There is no GDAL wheel on PyPI at any version for any platform, + so pip alone can never finish the job. `doctor` reports what is installed + and what is wrong with it; `plan` shows what an install would do; `install` + does it, choosing between an existing C library, conda, a prebuilt Windows + wheel, the system package manager, or a from-source build. + + Two failure modes it exists to catch: + + - **The silent `gdal_array` trap.** GDAL sdists only declared numpy as a + build dependency from 3.9 on. Below that, pip's build isolation leaves + numpy out and the numpy bridge is quietly omitted — `osgeo` imports + cleanly and `Band.ReadAsArray()` fails at run time. Ubuntu 22.04 (GDAL + 3.4) and 24.04 (3.8) are both on the wrong side of that line. The + bindings are now installed with `--no-build-isolation` against an + interpreter that already has numpy. + - **Architecture mismatch**, such as a Homebrew arm64 GDAL under an x86_64 + Python running via Rosetta. This is detected before anything is built, + and the package manager that caused it is not asked to fix it. + - **A stale `PROJ_LIB` or `GDAL_DATA`.** Activating a conda environment + exports both, and they then outrank the paths compiled into every other + GDAL on the machine; a PROJ 9 library handed PROJ 8's `proj.db` refuses + every coordinate lookup. `doctor` names the offending variable and its + value instead of proposing a reinstall, which would not have helped. + + Repairing broken bindings rebuilds them rather than re-requesting them: + the usual fault leaves `osgeo` at exactly the right version, so plain + `pip install gdal==X.Y.Z` reports "already satisfied", and pip's wheel + cache holds the very build that is broken. + + `install.py` exits non-zero when GDAL was asked for and did not arrive, + while saying plainly that the rest of the installation is fine. `--no-gdal` + is how to say it was never wanted. +- **`TopoAnalysis._gdal_source`** — builds SQLite, PROJ and GDAL from pinned, + SHA-256-checked tarballs into a single prefix, by default the active + virtual environment. It needs only a compiler and CMake: no root, no + package manager, and no system `-dev` packages, because GDAL is built + against its own bundled libtiff, libgeotiff, libpng, libjpeg and zlib. + Pins can be moved with `TOPOANALYSIS_GDAL_SOURCE_*` environment variables, + which require a checksum alongside any new URL. +- **`TopoAnalysis.kernels`** — compute kernels in C++ (`TopoAnalysis._cpp`) + with an equivalent NumPy implementation in `TopoAnalysis.fastops`. The + test suite asserts the two agree exactly. Set + `TOPOANALYSIS_PURE_PYTHON=1` to force the NumPy path. +- **`TopoAnalysis.topotoolbox`** — functions that reproduce TopoToolbox 2 + results: `fillsinks`, `identifyflats`, `imposemin`, `flowobj`, `flowacc`, + `flowdistance`, `drainagebasins`, `streamorder`, `gradient8`, `arcslope`, + `hillshade`, `aspect`, `curvature`, `cellarea`, `localtopography`, + `chitransform`, `ksn`, `mchi`, and the referencing helpers. +- **`topoanalysis-process`** — a command-line pipeline. `--info` reports the + active backend. +- **`python -m TopoAnalysis.benchmark`** — times both backends. +- `FilledElevation(aggradation_slope=0.0)` for a flat fill, and a + `fill_report` attribute recording how much was filled. +- `FlowDirectionD8.receivers`, `.topological_order`, `.cycle_count`, + `.basin_mask()` and `.step_lengths()`, which expose the flow network + directly. +- `FlowDirectionD8(elevation=...)` as an alias for `flooded_dem=`, for DEMs + conditioned elsewhere. +- Every grid class now accepts the in-memory constructors `dx=..., grid=...` + and `nx=..., ny=..., dx=...`, not just `BaseSpatialGrid`. +- `BaseSpatialGrid.extent()`, `.coordinate_vectors()` and the correctly + spelled `.get_XY_matrices()` alias. +- A circular flow path in a hand-edited flow-direction grid is now reported + (`cycle_count`, and a warning) instead of producing silently wrong results. +- `identifyflats(..., output='all')` also returns the closed basins. +- `FilledElevation`'s flood reports which cells a `maximum_pit_depth` left + unfilled, so `clip_to_fill` still excises them — the idiom for masking out + quarries and lakes, which the traversal fix would otherwise have broken. +- `FlowDirectionD8.invalidate_network()`. + +## Changed + +- **GDAL is imported lazily.** The package imports, and most of it runs, + without GDAL installed. `gdal_is_available()` reports whether it is there. + GDAL exceptions are enabled, so a bad path raises instead of returning + `None` and failing three frames later. +- **statsmodels is imported lazily** and only by the classes that fit + regressions; it used to print a warning at import time. +- **rasterio is imported lazily** by `analysis.Quadrats.load_data`. +- Both import styles work: `from TopoAnalysis import dem` and, with the + package directory on `sys.path`, `import dem`. +- `save()` writes `NaN` as the GeoTIFF no-data value and flushes before + returning. +- `mosaicFolder` uses the GDAL Python API instead of shelling out to + `gdal_merge.py`, which is not on most installations' `PATH`. +- Progress reporting is quieter and accepts `verbose=False`. +- Bare `except:` clauses — which swallowed `KeyboardInterrupt` and real + programming errors, and were used for bounds checking in the hot loops — + are gone. +- The NumPy kernels reject a non-C-contiguous grid rather than filling a + throwaway copy and reporting success. The compiled kernels always did. +- `from TopoAnalysis import *` no longer names classes that were never + imported into the package namespace. +- The test suite runs from a checkout under any name — a GitHub zip unpacks + to `TopoAnalysis-main` — as well as from an installed copy. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..aac2c04 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +# A reproducible TopoAnalysis image. +# +# docker build -t topoanalysis . +# docker run --rm topoanalysis topoanalysis-gdal doctor +# docker run --rm -v "$PWD:/data" topoanalysis \ +# topoanalysis-process /data/srtm.tif -o /data/out +# +# The base image already carries a matched GDAL C library and Python +# bindings, which is the one thing that is genuinely hard to arrange. +# ``install.py`` therefore finds GDAL already working and does nothing to it; +# everything else it does normally. +FROM ghcr.io/osgeo/gdal:ubuntu-small-3.9.3 + +# build-essential and python3-dev are for the Priority-Flood kernels; without +# them the package still installs and falls back to its NumPy kernels. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential python3-dev python3-pip python3-venv \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/TopoAnalysis +COPY . . + +# --system because the image *is* the environment: there is nothing here to +# protect from a pip install, and a virtualenv would only hide the GDAL that +# the base image went to the trouble of providing. +# +# --break-system-packages because this base image is built on Ubuntu 24.04, +# whose Python carries the PEP 668 "externally managed" marker. That marker +# exists to stop pip from fighting the host's package manager; inside a +# single-purpose container there is no host to protect, and overriding it is +# the normal and intended thing to do. +RUN python3 install.py --system --break-system-packages --extras test + +# Fail the build rather than ship an image whose kernels silently did not +# compile or whose GDAL does not work. +RUN python3 -c "import sys, TopoAnalysis; \ + print('backend:', TopoAnalysis.backend()); \ + sys.exit(0 if TopoAnalysis.backend() == 'c++' else 1)" \ + && topoanalysis-gdal doctor + +WORKDIR /data +CMD ["topoanalysis-process", "--info"] diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9019295 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,29 @@ +# Sources needed to rebuild the C++ extension from an sdist. +include _cpp/*.hpp +include _cpp/*.cpp + +include pyproject.toml +include setup.py +include README.md +include CHANGELOG.md +include environment.yml +include requirements.txt +include requirements-dev.txt +include Makefile +include install.py +include Dockerfile + +recursive-include docs *.md +recursive-include docs *.py +recursive-include tests *.py + +# install.py creates .venv here, because this directory is the package. +prune .venv +prune venv +prune build +prune dist + +exclude .DS_Store +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] +recursive-exclude * .DS_Store diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ca57585 --- /dev/null +++ b/Makefile @@ -0,0 +1,74 @@ +# Common tasks. `make help` lists them. + +PYTHON ?= python3 +PIP ?= $(PYTHON) -m pip + +.PHONY: help bootstrap gdal gdal-doctor gdal-source install develop build test \ + test-python bench docs clean distclean lint + +help: + @echo "bootstrap everything from a fresh clone, GDAL included" + @echo "gdal install GDAL (C library and Python bindings)" + @echo "gdal-doctor report what GDAL is installed and what is wrong" + @echo "gdal-source build GDAL from source, no root or package manager" + @echo "install install the package and build the C++ kernels" + @echo "develop editable install for working on the source" + @echo "build compile the C++ kernels in place" + @echo "test run the test suite" + @echo "test-python run the test suite against the pure-NumPy kernels" + @echo "bench compare the two backends on a synthetic DEM" + @echo "docs regenerate docs/api.md from the docstrings" + @echo "clean remove build artefacts" + @echo "distclean also remove the compiled extension" + +# The one-command path: creates an environment, installs everything, +# installs GDAL, and checks the result by round-tripping a GeoTIFF. +bootstrap: + $(PYTHON) install.py + +# Run by path rather than as `python -m TopoAnalysis.gdal_setup`: this +# directory *is* the package, so the dotted form only resolves once the +# package is installed, and these targets need to work before that. +gdal: + $(PYTHON) gdal_setup.py install + +gdal-doctor: + $(PYTHON) gdal_setup.py doctor + +gdal-source: + $(PYTHON) gdal_setup.py install --strategy source + +install: + $(PIP) install . + @$(PYTHON) -c "import TopoAnalysis; print('backend:', TopoAnalysis.backend())" + +develop: + $(PIP) install -e ".[test]" + @$(PYTHON) -c "import TopoAnalysis; print('backend:', TopoAnalysis.backend())" + +build: + $(PYTHON) setup.py build_ext --inplace + +test: + $(PYTHON) -m pytest -q + +test-python: + TOPOANALYSIS_PURE_PYTHON=1 $(PYTHON) -m pytest -q + +bench: + $(PYTHON) -m TopoAnalysis.benchmark + +docs: + $(PYTHON) docs/generate_api.py + @echo "docs/api.md regenerated; the rest of docs/ is hand-written" + +lint: + $(PYTHON) -m compileall -q . >/dev/null && echo "all modules compile" + +clean: + rm -rf build dist *.egg-info .pytest_cache docs/_build + find . -name '__pycache__' -type d -prune -exec rm -rf {} + + find . -name '*.py[co]' -delete + +distclean: clean + rm -f _topoanalysis*.so _topoanalysis*.pyd diff --git a/MovingWindow.py b/MovingWindow.py index 2042f9a..1e3c780 100644 --- a/MovingWindow.py +++ b/MovingWindow.py @@ -1,88 +1,148 @@ -import error +"""Moving-window filters over a grid. + +A :class:`MovingWindow` applies a reducing function to the cells inside a +window centred on every cell. Subclasses choose the window shape; a +concrete class supplies the reduction as :attr:`~MovingWindow.function`:: + + class WindowMean(CircularMovingWindow): + function = staticmethod(np.mean) + + grid.apply_moving_window(WindowMean(window_radius=200.0)) + +Cells outside the grid, and no-data cells, are dropped from the window +rather than filled, so the function always sees real data. + +For the common reductions (min, max, mean, median, standard deviation) +:func:`TopoAnalysis.topotoolbox.localtopography` is far faster, because it +uses SciPy's separable filters instead of a Python loop. +""" + +from __future__ import annotations + import numpy as np +try: # pragma: no cover - exercised by whichever import style is in use + from . import error +except ImportError: # pragma: no cover + import error + + class MovingWindow(object): - + """Abstract base for moving-window filters. + + Parameters + ---------- + window_radius : float + Radius of the window in map units. ``window_dimension`` is accepted + as a synonym for backward compatibility. + """ + + #: The reduction applied to the values inside each window. function = None - + def __init__(self, *args, **kwargs): - if kwargs.get('window_dimension') is None: - raise error.InputError('Window dimension', 'is a required parameter') - self.window_dimension = kwargs.get('window_dimension') - + radius = kwargs.get('window_radius', kwargs.get('window_dimension')) + if radius is None: + raise error.InputError('Window radius', + 'window_radius is a required parameter') + # Both names are kept: window_dimension is what the original + # constructor took, window_radius is what _build_search_kernel reads. + self.window_radius = radius + self.window_dimension = radius + if self.__class__ is MovingWindow: raise error.Error('MovingWindow is an abstract base class') - - def _build_search_kernel(self, dx): - return None, None - - def __adjust_kernel(self, row,col,grid,searchKernelRows,searchKernelCols): - # Function to turn the rows and columns of the search kernel specified by searchKernelRows & ...Cols as just indices of the - # data that is actually within the grid - - #Get the current absolute positions within the grid - theseRows = searchKernelRows+row - theseCols = searchKernelCols+col - - #Do any points in searchKernel extend to before the start of the grid? or off the end? - gdIndices = (theseCols < grid.shape[1]) & (theseRows < grid.shape[0]) & (theseRows >= 0) & (theseCols >= 0) - - #Which rows are within the grid - goodRows = theseRows[gdIndices] - goodCols = theseCols[gdIndices] - - #Now that we are all within the grid, of those points within the grid, do any include - #Do any points in searchKernel include nans ? - gdIndices = np.logical_not(np.isnan(grid[goodRows, goodCols])) - - return goodRows[gdIndices], goodCols[gdIndices] + + def _build_search_kernel(self, dx): + """Relative ``(row, col)`` offsets of the window, as flat arrays.""" + raise NotImplementedError + + def _adjust_kernel(self, row, col, grid, searchKernelRows, searchKernelCols): + """Clip the window to the grid and drop no-data cells.""" + theseRows = searchKernelRows + row + theseCols = searchKernelCols + col + + inside = ((theseCols < grid.shape[1]) & (theseRows < grid.shape[0]) + & (theseRows >= 0) & (theseCols >= 0)) + + goodRows = theseRows[inside] + goodCols = theseCols[inside] + + valid = np.logical_not(np.isnan(grid[goodRows, goodCols])) + + return goodRows[valid], goodCols[valid] def apply_moving_window(self, grid, dx, dtype): - # Function to scan the moving window specified by Kernel across the dem, a numpy grid, and apply the specified function - # to each location. function is a method that returns a single value given any number of inputs, e.g. - # movingWindow(i) = function(dem[Kernel@i]) - - outgrid = np.zeros_like(grid, dtype = dtype) - - search_kernel_rows, search_kernel_cols = self._build_search_kernel(grid) - + """Run the window over every cell of ``grid``. + + Parameters + ---------- + grid : ndarray + 2-D data; ``NaN`` is no-data. + dx : float + Cell size, used to convert the window radius to cells. + dtype : numpy dtype + Type of the output array. + + Returns + ------- + ndarray + """ + if self.function is None: + raise error.Error( + '{0} has no bound function'.format(self.__class__.__name__)) + + outgrid = np.zeros_like(grid, dtype=dtype) + + # The kernel depends on the cell size, not on the data; the original + # passed the grid itself here, so window_radius/dx divided a float by + # an array. + search_kernel_rows, search_kernel_cols = self._build_search_kernel(dx) + for i in range(grid.shape[0]): for j in range(grid.shape[1]): - these_rows,these_cols = self.__adjustKernel(i,j,grid,search_kernel_rows,search_kernel_cols) #trim the searchKernel and return it as a list of indices - outgrid[i,j] = self.function(grid[these_rows, these_cols]) #apply the specified function to the dem values for this search location - + # The original called self.__adjustKernel, which does not + # exist -- the method is __adjust_kernel. + these_rows, these_cols = self._adjust_kernel( + i, j, grid, search_kernel_rows, search_kernel_cols) + if these_rows.size == 0: + outgrid[i, j] = np.nan if np.issubdtype(dtype, np.floating) else 0 + continue + outgrid[i, j] = self.function(grid[these_rows, these_cols]) + return outgrid + class RectangularMovingWindow(MovingWindow): - + """A square window of side ``2 * round(window_radius / dx) + 1`` cells.""" + def __init__(self, *args, **kwargs): super(RectangularMovingWindow, self).__init__(*args, **kwargs) if self.__class__ is RectangularMovingWindow: raise error.Error('RectangularMovingWindow has no bound function') - + def _build_search_kernel(self, dx): - # Function to build a search kernel that is either square or circular - - pxlRadius = int(round(self.window_radius/dx)) #Is this right... - relCoords = np.arange(1 + 2*pxlRadius)-pxlRadius #Relative coordinates of neighbors within this row, col distance - searchKernelRow, serchKernelCol = np.meshgrid(relCoords,relCoords) - return searchKernelRow.flatten(), serchKernelCol.flatten() - + pxlRadius = int(round(self.window_radius / dx)) + relCoords = np.arange(1 + 2 * pxlRadius) - pxlRadius + searchKernelRow, searchKernelCol = np.meshgrid(relCoords, relCoords) + return searchKernelRow.flatten(), searchKernelCol.flatten() + + class CircularMovingWindow(MovingWindow): - + """A disc-shaped window of radius ``round(window_radius / dx)`` cells.""" + def __init__(self, *args, **kwargs): super(CircularMovingWindow, self).__init__(*args, **kwargs) if self.__class__ is CircularMovingWindow: raise error.Error('CircularMovingWindow has no bound function') - + def _build_search_kernel(self, dx): - # Function to build a search kernel that is either square or circular - - pxlRadius = int(round(self.window_radius/dx)) #Is this right... - relCoords = np.arange(1 + 2*pxlRadius)-pxlRadius #Relative coordinates of neighbors within this row, col distance - searchKernelRow, serchKernelCol = np.meshgrid(relCoords,relCoords) - dists = np.sqrt(searchKernelRow**2 + searchKernelRow**2) - searchKernelRow = searchKernelRow[dists +cd TopoAnalysis +python3 install.py +``` + +That is the whole procedure. It creates a virtual environment, installs the +dependencies, compiles the kernels, installs **GDAL — both the C library and +the Python bindings** — and then checks the result by writing a georeferenced +GeoTIFF and reading it back. + +GDAL needs its own installer because pip cannot install it: there is no GDAL +wheel on PyPI for any platform at any version, only an sdist that compiles +against a C library which must already exist. `install.py` supplies both +halves, using conda, the system package manager, a prebuilt wheel on Windows, +or — when none of those is available — by building PROJ and GDAL from pinned, +checksummed source into the virtual environment, which needs no +administrator rights at all. + +```bash +topoanalysis-gdal # what GDAL is installed, and what is wrong with it +topoanalysis-gdal install # install or repair it +python3 install.py --dry-run # show the plan for this machine, change nothing +``` + +Neither GDAL nor a C++ compiler is required: without a compiler the library +falls back to NumPy kernels that produce identical numbers, and without GDAL +everything except file I/O works. See +[docs/installation.md](docs/installation.md) for platform notes, offline +installs, and troubleshooting. + +## A first analysis + +```python +from TopoAnalysis import Elevation, FilledElevation, FlowDirectionD8, Area, FlowLength + +dem = Elevation(gdal_filename='srtm.tif') +filled = FilledElevation(elevation=dem) # Priority-Flood +d8 = FlowDirectionD8(flooded_dem=filled) # steepest-descent D8 +area = Area(flow_direction=d8) # contributing area, m^2 +length = FlowLength(flow_direction=d8) # longest upstream path + +area.save('area.tif') +``` + +Grids are built **by keyword**, and which keywords you pass decides how the +grid is made: + +| What you pass | What you get | +|---|---| +| `gdal_filename='dem.tif'` | read from a file | +| `ai_ascii_filename=..., EPSGprojectionCode=32611` | read an ArcInfo ASCII grid | +| `dx=30.0, grid=array` | wrap an array already in memory | +| `nx=..., ny=..., projection=..., geo_transform=...` | an empty georeferenced grid | +| `flow_direction=d8`, `elevation=dem`, … | derive from another grid | + +Every class lists the combinations it accepts in +`required_inputs_and_actions`; passing something else raises an error that +tells you what the class does accept. + +## Chi and channel steepness + +```python +from TopoAnalysis import Chi, Ksi, ScaledRelief + +outlets = area.areas_greater_than(1e8) # big basins + +chi = Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e6, outlets=outlets) +ksi = Ksi(area=area, flow_direction=d8, flow_length=length, theta=0.45, Ao=1e6) +relief = ScaledRelief(flow_direction=d8, elevation=dem, flow_length=length, + Ao=1e6, theta=0.45, area=area) +``` + +`Chi` integrates upstream from the outlets you name; `Ksi` integrates along +longest-flow-length paths across the whole grid. Plotting `ScaledRelief` +against `Ksi` gives a chi-elevation plot whose slope is the steepness index. + +## Cross-checking against TopoToolbox + +```python +from TopoAnalysis import topotoolbox as tt + +zf = tt.fillsinks(dem) # flat fill, == GRIDobj/fillsinks +fd = tt.flowobj(dem) # == FLOWobj(DEM,'preprocess','fill') +a = tt.flowacc(fd) # == flowacc(FD) -- in CELLS, not m^2 +c = tt.chitransform(fd, a) # == chitransform(S,A), mn=0.45, a0=1e6 +k = tt.ksn(fd, dem, a) # == ksn(S,DEM,A) + +tt.gradient8(dem) # == GRIDobj/gradient8 +tt.curvature(dem, 'profc') # == GRIDobj/curvature +tt.hillshade(dem) # == GRIDobj/hillshade (cosine, not 0-255) +tt.aspect(dem) # == GRIDobj/aspect +``` + +The conventions differ between the two packages in a few places that matter — +drainage area in cells versus square metres, chi integrated with the +trapezoidal rule versus the left-endpoint rule, hillshade as a cosine versus +ESRI's 0–255. [docs/topotoolbox_parity.md](docs/topotoolbox_parity.md) +tabulates all of them and says where exact agreement is and is not +achievable. + +## Command line + +```bash +topoanalysis-process srtm.tif -o results --products filled,flowdir,area,length +topoanalysis-process --info # which backend is active, is GDAL present +``` + +## Performance + +`python -m TopoAnalysis.benchmark 1024` — a 1024 × 1024 DEM, about a million +cells, on a 2019 laptop: + +| Stage | C++ | NumPy | +|---|---:|---:| +| Priority-Flood fill | 0.10 s | 5.5 s | +| D8 flow directions | 0.02 s | 0.04 s | +| Flow accumulation | 0.01 s | 0.41 s | +| Flow length | 0.02 s | 0.83 s | +| Chi | 0.04 s | 2.6 s | +| **whole pipeline** | **0.20 s** | **9.4 s** | + +The compiled kernels release the GIL, so several grids can be processed in +parallel from threads. + +## What changed in 1.0 + +Version 1.0 repairs about 70 defects — several of which stopped the library +importing at all on current NumPy and SciPy — and replaces the recursive +basin traversals, which needed a million-frame recursion limit and still +overflowed on real DEMs, with iterative kernels. + +**The public interface is unchanged**: every class, method and keyword still +exists and still means what it did. A handful of results are *different +because they were wrong*; they are listed in [CHANGELOG.md](CHANGELOG.md) +under "Results that change". Read that section before comparing new output +with old. + +## Documentation + +- [Installation](docs/installation.md) — including offline and no-compiler cases +- [User guide](docs/guide.md) — the grid model, the analysis chain, worked examples +- [API reference](docs/api.md) — every public class and function +- [Algorithms](docs/algorithms.md) — what each computation actually does, and why +- [TopoToolbox parity](docs/topotoolbox_parity.md) — the correspondence table +- [Changelog](CHANGELOG.md) — fixed defects and behavioural changes + +## References + +Barnes, R., Lehman, C. and Mulla, D. (2014). Priority-flood: An optimal +depression-filling and watershed-labeling algorithm for digital elevation +models. *Computers & Geosciences* 62, 117–127. + +Perron, J.T. and Royden, L. (2013). An integral approach to bedrock river +profile analysis. *Earth Surface Processes and Landforms* 38, 570–576. + +Schwanghart, W. and Scherler, D. (2014). TopoToolbox 2 — MATLAB-based +software for topographic analysis and modeling in Earth surface sciences. +*Earth Surface Dynamics* 2, 1–7. + +Willett, S.D., McCoy, S.W., Perron, J.T., Goren, L. and Chen, C.-Y. (2014). +Dynamic reorganization of river basins. *Science* 343, 1248765. + +## Credits + +Originally written by Sam Johnstone (January 2015); maintained since by the +Stanford geomorphology group. diff --git a/__init__.py b/__init__.py index e69de29..bb50802 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,156 @@ +"""TopoAnalysis -- digital elevation model analysis for tectonic geomorphology. + +A typical analysis chains grid classes together, each one built from the +last:: + + from TopoAnalysis import Elevation, FilledElevation, FlowDirectionD8, Area + + dem = Elevation(gdal_filename='srtm.tif') + filled = FilledElevation(elevation=dem) + d8 = FlowDirectionD8(flooded_dem=filled) + area = Area(flow_direction=d8) + +Grids are constructed by keyword: which keywords you pass decides how the +grid is built. ``Elevation(gdal_filename=...)`` reads a file, +``Elevation(dx=..., grid=...)`` wraps an array already in memory, and +``Area(flow_direction=...)`` derives one grid from another. Each class +lists its accepted combinations in ``required_inputs_and_actions``. + +Sub-modules +----------- +:mod:`TopoAnalysis.dem` + The grid classes. Everything public is re-exported here. +:mod:`TopoAnalysis.topotoolbox` + Functions that reproduce TopoToolbox 2 (MATLAB) results exactly. +:mod:`TopoAnalysis.kernels` + The compute kernels, in C++ where available and NumPy otherwise. +:mod:`TopoAnalysis.plotting`, :mod:`TopoAnalysis.demRecursionTools` + Longitudinal and chi profiles, and steepness fitting along them. +:mod:`TopoAnalysis.datasets` + Synthetic landscapes for testing. + +Depression filling uses the Priority-Flood algorithms of Barnes, Lehman and +Mulla (2014); see :mod:`TopoAnalysis.kernels`. +""" + +from __future__ import annotations + +__version__ = "1.0.0" + +# The package is importable both as ``TopoAnalysis`` and, with this directory +# on sys.path, flat -- the way the original scripts in this repository load +# it. The same shim appears in every module here. +try: # pragma: no cover - exercised by whichever import style is in use + from . import error, fastops, gdal_setup, kernels # noqa: F401 + from . import dem +except ImportError: # pragma: no cover + # The relative form failed, so this file was executed without a parent + # package. That happens whenever something imports it by path under a + # name other than ``TopoAnalysis`` -- notably pytest, which names the + # module after the checkout *directory*, and so only gets a valid package + # name when the directory happens to be called TopoAnalysis. + # + # Falling back to flat imports only works if this directory is on the + # path, which used to mean "only when it is also the working directory". + # Put it there explicitly instead. Appended rather than prepended: these + # names only need to be findable, and nothing here should ever take + # precedence over the standard library or an installed package. + import os as _os + import sys as _sys + + _here = _os.path.dirname(_os.path.abspath(__file__)) + if _here not in _sys.path: + _sys.path.append(_here) + del _os, _sys, _here + + import error, fastops, gdal_setup, kernels # noqa: F401 + import dem + +#: Everything :mod:`TopoAnalysis.dem` exports through this package. Listed +#: explicitly rather than filtered out of ``dir(dem)`` so that a name +#: disappearing from ``dem`` breaks the import here, loudly, instead of +#: leaving ``__all__`` promising something that is not there. +_FROM_DEM = ( + "Area", + "AlongFlowSmoothing", + "BaseSpatialGrid", + "BaseSpatialShape", + "CalculationMixin", + "ChannelDownSlopeWithSmoothing", + "ChannelSlope", + "ChannelSlopeWithSmoothing", + "ChannelUpSlopeWithSmoothing", + "Chi", + "ChiScaledRelief", + "CrossDivideDChi", + "Deflection", + "DiscreteFlowAccumulation", + "Elevation", + "FilledElevation", + "FlowDirection", + "FlowDirectionD8", + "FlowLength", + "GDALMixin", + "GeographicArea", + "GeographicChi", + "GeographicDeflection", + "GeographicDiscreteFlowAccumulation", + "GeographicElevation", + "GeographicFlowLength", + "GeographicGridMixin", + "GeographicHillshade", + "GeographicKsFromChiWithSmoothing", + "GeographicKsi", + "GeographicLaplacian", + "GeographicMainstemValleyArea", + "GeographicMaxSlope", + "GeographicRestoredElevation", + "GeographicThetaFromChiWithSmoothing", + "GeographicValleyArea", + "Georef_info", + "Gradient", + "Hillshade", + "Ksi", + "KsFromChiWithSmoothing", + "Laplacian", + "LocalRelief", + "LogArea", + "MainstemValleyArea", + "Mask", + "MaxFlowLengthTrackingMixin", + "MaxSlope", + "MultiscaleCurvatureValleyWidth", + "NormalizedCrossDivideDChi", + "PriorityFillGrid", + "PriorityQueueMixIn", + "Relief", + "RestoredElevation", + "ScaledRelief", + "ScarpWavelet", + "ThetaFromChiWithSmoothing", + "ValleyArea", + "ValueGrid", + "gdal_is_available", + "mosaicFolder", + "plot", +) + +for _name in _FROM_DEM: + globals()[_name] = getattr(dem, _name) +del _name + + +def backend() -> str: + """Which kernel backend is in use: ``'c++'`` or ``'python'``.""" + return kernels.backend() + + +__all__ = sorted(_FROM_DEM) + [ + "__version__", + "backend", + "dem", + "error", + "fastops", + "gdal_setup", + "kernels", +] diff --git a/_cpp/flow_routing.hpp b/_cpp/flow_routing.hpp new file mode 100644 index 0000000..772f30e --- /dev/null +++ b/_cpp/flow_routing.hpp @@ -0,0 +1,625 @@ +// flow_routing.hpp -- D8 flow routing kernels +// +// These replace the per-cell Python loops and the deeply recursive traversals +// that used to dominate TopoAnalysis' runtime. Everything here is iterative, +// so there is no recursion limit to raise and no stack to overflow on large +// basins. +// +// Flow-direction codes follow the ArcGIS convention used throughout +// TopoAnalysis, with row 0 at the north edge of the grid: +// +// | 32 64 128 | | (i-1,j-1) (i-1,j) (i-1,j+1) | +// | 16 X 1 | == | (i ,j-1) X (i ,j+1) | +// | 8 4 2 | | (i+1,j-1) (i+1,j) (i+1,j+1) | +// +// A code of 0 means "no downstream neighbour" (a pit, an outlet, or no-data). + +#ifndef TOPOANALYSIS_FLOW_ROUTING_HPP +#define TOPOANALYSIS_FLOW_ROUTING_HPP + +#include +#include +#include +#include +#include +#include + +#include "priority_flood.hpp" + +namespace topoanalysis { + +// --------------------------------------------------------------------------- +// Flow direction +// --------------------------------------------------------------------------- + +// Steepest-descent D8 flow directions. +// +// The drop to each neighbour is divided by the centre-to-centre distance, so +// diagonal neighbours compete on gradient rather than on raw elevation drop. +// This is the standard D8 rule (O'Callaghan & Mark 1984) and matches +// TopoToolbox's ``gradient8``/FLOWobj steepest-neighbour selection. +// +// ``cellsize`` may be a single value (uniform grid) or a per-cell array of +// mean pixel dimensions, which is what geographic (lat/lon) grids need. +// Ties are broken by neighbour order (E, SE, S, SW, W, NW, N, NE), which is +// deterministic and reproducible. +inline void d8_flow_directions(const double* elevations, + std::size_t ny, + std::size_t nx, + const double* cellsize, // ny*nx, or nullptr + double uniform_cellsize, + std::uint8_t* out_codes) { + const std::int64_t rows = static_cast(ny); + const std::int64_t cols = static_cast(nx); + + for (std::int64_t i = 0; i < rows; ++i) { + for (std::int64_t j = 0; j < cols; ++j) { + const std::int64_t k = i * cols + j; + const double z = elevations[k]; + if (std::isnan(z)) { + out_codes[k] = 0; + continue; + } + const double de = cellsize ? cellsize[k] : uniform_cellsize; + + double best_slope = 0.0; + int best_dir = -1; + for (int d = 0; d < 8; ++d) { + const std::int64_t ni = i + kD8Di[d]; + const std::int64_t nj = j + kD8Dj[d]; + if (ni < 0 || nj < 0 || ni >= rows || nj >= cols) continue; + const double zn = elevations[ni * cols + nj]; + if (std::isnan(zn)) continue; + const double slope = (z - zn) / (de * kD8Dist[d]); + if (slope > best_slope) { + best_slope = slope; + best_dir = d; + } + } + out_codes[k] = (best_dir < 0) ? 0 : static_cast(1u << best_dir); + } + } +} + +// Translate a flow-direction code grid into an explicit receiver index per +// cell (-1 where flow leaves the grid or the cell has no receiver). +inline void d8_receivers(const std::uint8_t* codes, + std::size_t ny, + std::size_t nx, + std::int64_t* out_receivers) { + const std::int64_t rows = static_cast(ny); + const std::int64_t cols = static_cast(nx); + + for (std::int64_t i = 0; i < rows; ++i) { + for (std::int64_t j = 0; j < cols; ++j) { + const std::int64_t k = i * cols + j; + const std::uint8_t code = codes[k]; + out_receivers[k] = -1; + if (code == 0) continue; + for (int d = 0; d < 8; ++d) { + if (code != static_cast(1u << d)) continue; + const std::int64_t ni = i + kD8Di[d]; + const std::int64_t nj = j + kD8Dj[d]; + if (ni < 0 || nj < 0 || ni >= rows || nj >= cols) break; + out_receivers[k] = ni * cols + nj; + break; + } + } + } +} + +// --------------------------------------------------------------------------- +// Topological ordering +// --------------------------------------------------------------------------- + +// Order the cells so that every cell appears before the cell it drains into +// (Kahn's algorithm on the donor counts). This is the ordering flow +// accumulation needs, and unlike sorting by elevation it is derived from the +// flow network itself -- so it stays correct for flow-direction grids that +// were loaded from disk, hand-edited, or derived from a different DEM. +// +// Returns the number of cells that could NOT be ordered because they lie on a +// cycle. Those cells are appended at the end in index order so that callers +// still see every cell exactly once. +inline std::size_t d8_topological_order(const std::int64_t* receivers, + std::size_t n, + std::int64_t* out_order) { + std::vector indegree(n, 0); + for (std::size_t k = 0; k < n; ++k) { + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && static_cast(r) != k) { + ++indegree[static_cast(r)]; + } + } + + std::vector stack; + stack.reserve(n); + for (std::size_t k = 0; k < n; ++k) { + if (indegree[k] == 0) stack.push_back(static_cast(k)); + } + + std::size_t written = 0; + while (!stack.empty()) { + const std::int64_t k = stack.back(); + stack.pop_back(); + out_order[written++] = k; + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && r != k) { + if (--indegree[static_cast(r)] == 0) stack.push_back(r); + } + } + + const std::size_t unordered = n - written; + if (unordered > 0) { + // Cells on a cycle: emit them in index order so the output is still a + // permutation of all cells. + std::vector emitted(n, 0); + for (std::size_t p = 0; p < written; ++p) emitted[static_cast(out_order[p])] = 1; + for (std::size_t k = 0; k < n; ++k) { + if (!emitted[k]) out_order[written++] = static_cast(k); + } + } + return unordered; +} + +// --------------------------------------------------------------------------- +// Flow accumulation +// --------------------------------------------------------------------------- + +// Accumulate ``weights`` downstream. ``weights`` is normally the per-cell +// area (dx*dy, or the true spherical cell area for geographic grids); pass an +// array of ones to get the number of contributing cells, which is what +// TopoToolbox's ``flowacc`` returns by default. +// +// ``accumulate_from`` is an optional 0/1 mask: cells with a zero entry +// contribute nothing and do not pass their accumulated value downstream. +// This reproduces the ``evaluate_at`` option of the original Python code. +inline void d8_accumulate(const std::int64_t* receivers, + const std::int64_t* order, + const double* weights, + const std::uint8_t* accumulate_from, // may be nullptr + std::size_t n, + double* out_accumulation) { + for (std::size_t k = 0; k < n; ++k) { + const bool active = (accumulate_from == nullptr) || (accumulate_from[k] != 0); + out_accumulation[k] = active ? weights[k] : 0.0; + } + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + if (accumulate_from != nullptr && accumulate_from[k] == 0) continue; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + out_accumulation[r] += out_accumulation[k]; + } +} + +// --------------------------------------------------------------------------- +// Flow length +// --------------------------------------------------------------------------- + +// Longest upstream flow distance reaching each cell, plus the code of the +// upstream neighbour that supplied it. ``step_length`` is the length of the +// step from each cell to its receiver. +// +// ``out_from_codes`` records, for each cell, the direction code *pointing back +// at* the donor that lies on the longest path. ``FlowLength`` uses it to walk +// the main stem back upstream. +// +// When two donors tie on length the one with the lower flat index wins. That +// tie-break matters: without it the "main stem" would depend on which valid +// topological order happened to be used, and two runs of the same analysis +// could disagree. +inline void d8_flow_length(const std::int64_t* receivers, + const std::int64_t* order, + const double* step_length, + std::size_t ny, + std::size_t nx, + double* out_length, + std::uint8_t* out_from_codes) { + const std::size_t n = ny * nx; + const std::int64_t cols = static_cast(nx); + + std::vector from_index(n, -1); + for (std::size_t k = 0; k < n; ++k) out_length[k] = 0.0; + + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + const double candidate = out_length[k] + step_length[k]; + const std::int64_t incumbent = from_index[static_cast(r)]; + const bool better = candidate > out_length[r] || + (candidate == out_length[r] && (incumbent < 0 || k < incumbent)); + if (better) { + out_length[r] = candidate; + from_index[static_cast(r)] = k; + } + } + + if (!out_from_codes) return; + for (std::size_t r = 0; r < n; ++r) { + out_from_codes[r] = 0; + const std::int64_t k = from_index[r]; + if (k < 0) continue; + const std::int64_t di = (k / cols) - (static_cast(r) / cols); + const std::int64_t dj = (k % cols) - (static_cast(r) % cols); + for (int d = 0; d < 8; ++d) { + if (kD8Di[d] == di && kD8Dj[d] == dj) { + out_from_codes[r] = static_cast(1u << d); + break; + } + } + } +} + +// Propagate a value downstream, but only along the longest-flow-length path. +// ``Relief`` uses mode=Carry (each cell inherits the value of its main-stem +// donor) and ``Ksi`` uses mode=Sum (each cell adds its main-stem donor's +// value). ``gate`` optionally suppresses the transfer for cells where the +// gate is zero, which is how ``Relief`` implements its ``Ao`` cut-off. +enum class PropagateMode { Carry = 0, Sum = 1 }; + +inline void d8_propagate_along_main_stem(const std::int64_t* receivers, + const std::int64_t* order, + const std::uint8_t* main_stem_from_codes, + const std::uint8_t* gate, // may be nullptr + std::size_t ny, + std::size_t nx, + PropagateMode mode, + double* values) { + const std::size_t n = ny * nx; + const std::int64_t cols = static_cast(nx); + + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + + // Is k the donor that defines r's longest flow path? + const std::int64_t di = (k / cols) - (r / cols); + const std::int64_t dj = (k % cols) - (r % cols); + std::uint8_t code = 0; + for (int d = 0; d < 8; ++d) { + if (kD8Di[d] == di && kD8Dj[d] == dj) { + code = static_cast(1u << d); + break; + } + } + if (code == 0 || main_stem_from_codes[r] != code) continue; + if (gate != nullptr && gate[k] == 0) continue; + + if (mode == PropagateMode::Sum) { + values[r] += values[k]; + } else { + values[r] = values[k]; + } + } +} + +// --------------------------------------------------------------------------- +// Upstream traversal +// --------------------------------------------------------------------------- + +// Mark every cell that drains (directly or indirectly) into any of +// ``outlets``. Iterative, so basin size is limited only by memory. +inline void d8_upstream_mask(const std::int64_t* receivers, + std::size_t n, + const std::int64_t* outlets, + std::size_t n_outlets, + std::uint8_t* out_mask) { + // Build the donor adjacency in CSR form so the traversal is O(n). + std::vector counts(n + 1, 0); + for (std::size_t k = 0; k < n; ++k) { + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && static_cast(r) != k) { + ++counts[static_cast(r) + 1]; + } + } + for (std::size_t k = 0; k < n; ++k) counts[k + 1] += counts[k]; + std::vector donors(static_cast(counts[n])); + std::vector cursor(counts.begin(), counts.end() - 1); + for (std::size_t k = 0; k < n; ++k) { + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && static_cast(r) != k) { + donors[static_cast(cursor[static_cast(r)]++)] = + static_cast(k); + } + } + + std::fill(out_mask, out_mask + n, static_cast(0)); + std::vector stack; + for (std::size_t o = 0; o < n_outlets; ++o) { + const std::int64_t k = outlets[o]; + if (k < 0 || static_cast(k) >= n) continue; + if (out_mask[k]) continue; + out_mask[k] = 1; + stack.push_back(k); + } + while (!stack.empty()) { + const std::int64_t k = stack.back(); + stack.pop_back(); + for (std::int64_t p = counts[static_cast(k)]; + p < counts[static_cast(k) + 1]; ++p) { + const std::int64_t donor = donors[static_cast(p)]; + if (out_mask[donor]) continue; + out_mask[donor] = 1; + stack.push_back(donor); + } + } +} + +// --------------------------------------------------------------------------- +// Chi +// --------------------------------------------------------------------------- + +// Integrate the chi coordinate upstream from a set of outlets: +// +// chi(cell) = chi(receiver) + integrand * step_length(cell) +// +// with ``integrand = (A0 / A)^theta``. ``trapezoid`` switches from the +// left-endpoint rule used by the original TopoAnalysis recursion to the +// trapezoidal rule used by TopoToolbox's ``chitransform``: +// +// chi(cell) = chi(receiver) +// + 0.5 * ((A0/A_cell)^theta + (A0/A_recv)^theta) * step_length +// +// Cells outside the outlets' basins, and cells where the drainage area is +// not positive, are left as NaN. ``max_length`` (if > 0) stops the +// integration once the along-stream distance from the outlet exceeds it. +inline void d8_chi(const std::int64_t* receivers, + const std::int64_t* order, + const double* area, + const double* step_length, + std::size_t n, + const std::int64_t* outlets, + std::size_t n_outlets, + double A0, + double theta, + bool trapezoid, + double max_length, + const std::uint8_t* mask, // may be nullptr + double* out_chi, + double* out_distance) { + const double nan = std::numeric_limits::quiet_NaN(); + for (std::size_t k = 0; k < n; ++k) { + out_chi[k] = nan; + if (out_distance) out_distance[k] = nan; + } + for (std::size_t o = 0; o < n_outlets; ++o) { + const std::int64_t k = outlets[o]; + if (k < 0 || static_cast(k) >= n) continue; + if (mask != nullptr && mask[k] == 0) continue; + out_chi[k] = 0.0; + if (out_distance) out_distance[k] = 0.0; + } + + // ``order`` runs donors-before-receivers, so walking it backwards visits + // every receiver before its donors. + for (std::size_t p = n; p-- > 0;) { + const std::int64_t k = order[p]; + if (!std::isnan(out_chi[k])) continue; // an outlet, already seeded + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + if (std::isnan(out_chi[r])) continue; + if (mask != nullptr && mask[k] == 0) continue; + + const double a = area[k]; + if (!(a > 0.0)) continue; + + const double dl = step_length[k]; + if (max_length > 0.0 && out_distance) { + if (out_distance[r] + dl > max_length) continue; + } + + double integrand = std::pow(A0 / a, theta); + if (trapezoid) { + const double ar = area[r]; + const double integrand_r = (ar > 0.0) ? std::pow(A0 / ar, theta) : integrand; + integrand = 0.5 * (integrand + integrand_r); + } + out_chi[k] = out_chi[r] + integrand * dl; + if (out_distance) out_distance[k] = out_distance[r] + dl; + } +} + +// --------------------------------------------------------------------------- +// Distance to the outlet +// --------------------------------------------------------------------------- + +// Along-flow distance from each cell down to the end of its flow path. +// This is TopoToolbox's ``flowdistance(FD,'upstream')``: 0 at every outlet +// and at every cell with no receiver, increasing upstream. +inline void d8_downstream_distance(const std::int64_t* receivers, + const std::int64_t* order, + const double* step_length, + std::size_t n, + double* out_distance) { + std::fill(out_distance, out_distance + n, 0.0); + // Receivers before donors, so a cell's receiver is final when it is read. + for (std::size_t p = n; p-- > 0;) { + const std::int64_t k = order[p]; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + out_distance[k] = out_distance[r] + step_length[k]; + } +} + +// --------------------------------------------------------------------------- +// Minima imposition (carving) +// --------------------------------------------------------------------------- + +// Lower each receiver until the flow path descends at a gradient of at least +// ``sl``. This is TopoToolbox's ``imposemin``: unlike depression filling it +// only ever *lowers* cells, so the channel network is carved through +// obstructions rather than drowned by them. +// +// ``elevations`` is modified in place. +inline void d8_imposemin(const std::int64_t* receivers, + const std::int64_t* order, + const double* step_length, + std::size_t n, + double sl, + double* elevations) { + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + const double candidate = elevations[k] - sl * step_length[k]; + if (candidate < elevations[r]) elevations[r] = candidate; + } +} + +// --------------------------------------------------------------------------- +// Stream order +// --------------------------------------------------------------------------- + +// Strahler stream order over the cells marked in ``is_stream``. +// +// Order 1 at channel heads; where two or more streams of the highest incoming +// order meet, the order increases by one; a smaller tributary joining a +// larger stream leaves the order unchanged. Non-stream cells stay 0. +// +// The rule is applied to each junction as a whole -- the largest incoming +// order and how many donors carry it -- rather than donor by donor. The +// incremental form gives a different answer depending on the sequence the +// donors arrive in (orders {2, 2, 3} yield 4 or 3 according to whether the 3 +// is seen first), which would make the result depend on which valid +// topological order the backend produced. +inline void d8_strahler(const std::int64_t* receivers, + const std::int64_t* order, + const std::uint8_t* is_stream, + std::size_t n, + std::int32_t* out_order) { + std::fill(out_order, out_order + n, 0); + + std::vector best(n, 0); // largest incoming order + std::vector ties(n, 0); // how many donors carry it + + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + if (!is_stream[k]) continue; + + // Donors are all settled by now, so this cell's own order is final. + out_order[k] = (ties[static_cast(k)] >= 2) + ? best[static_cast(k)] + 1 + : std::max(best[static_cast(k)], 1); + + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + if (!is_stream[r]) continue; + + const std::size_t ru = static_cast(r); + if (out_order[k] > best[ru]) { + best[ru] = out_order[k]; + ties[ru] = 1; + } else if (out_order[k] == best[ru]) { + ++ties[ru]; + } + } +} + +// Shreve magnitude: the number of channel heads upstream of each cell. +inline void d8_shreve(const std::int64_t* receivers, + const std::int64_t* order, + const std::uint8_t* is_stream, + std::size_t n, + std::int32_t* out_order) { + std::vector has_donor(n, 0); + for (std::size_t k = 0; k < n; ++k) { + if (!is_stream[k]) continue; + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && r != static_cast(k) + && is_stream[r]) { + has_donor[static_cast(r)] = 1; + } + } + for (std::size_t k = 0; k < n; ++k) { + out_order[k] = (is_stream[k] && !has_donor[k]) ? 1 : 0; + } + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + if (!is_stream[k]) continue; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + if (!is_stream[r]) continue; + out_order[r] += out_order[k]; + } +} + +// --------------------------------------------------------------------------- +// Drainage basins +// --------------------------------------------------------------------------- + +// Label every cell with the index (1-based) of the outlet it drains to. +// Cells that reach no listed outlet are labelled 0. +inline void d8_drainage_basins(const std::int64_t* receivers, + const std::int64_t* order, + std::size_t n, + const std::int64_t* outlets, + std::size_t n_outlets, + const std::uint8_t* valid, // may be nullptr + std::int32_t* out_labels) { + std::fill(out_labels, out_labels + n, 0); + for (std::size_t o = 0; o < n_outlets; ++o) { + const std::int64_t k = outlets[o]; + if (k < 0 || static_cast(k) >= n) continue; + if (valid != nullptr && valid[k] == 0) continue; + out_labels[k] = static_cast(o + 1); + } + for (std::size_t p = n; p-- > 0;) { + const std::int64_t k = order[p]; + if (out_labels[k] != 0) continue; + if (valid != nullptr && valid[k] == 0) continue; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + out_labels[k] = out_labels[r]; + } +} + +// Label each cell with the id of the terminal cell (pit or edge outlet) that +// its flow path ends at. This is the ``drainagebasins(FD)`` behaviour of +// TopoToolbox when no outlets are supplied. +// +// A terminal cell only seeds a basin if something drains into it. A cell +// that is neither a giver nor a receiver -- an isolated no-data cell, say -- +// stays 0, which is also what TopoToolbox does (such a cell never appears in +// its edge list). +inline void d8_drainage_basins_all(const std::int64_t* receivers, + const std::int64_t* order, + const std::uint8_t* valid, // may be nullptr + std::size_t n, + std::int32_t* out_labels) { + std::fill(out_labels, out_labels + n, 0); + + std::vector has_donor(n, 0); + for (std::size_t k = 0; k < n; ++k) { + if (valid != nullptr && valid[k] == 0) continue; + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && r != static_cast(k)) { + has_donor[static_cast(r)] = 1; + } + } + + std::int32_t next_label = 0; + for (std::size_t k = 0; k < n; ++k) { + if (valid != nullptr && valid[k] == 0) continue; + const std::int64_t r = receivers[k]; + const bool terminal = + (r < 0 || static_cast(r) >= n || r == static_cast(k)); + if (terminal && has_donor[k]) { + out_labels[k] = ++next_label; + } + } + for (std::size_t p = n; p-- > 0;) { + const std::int64_t k = order[p]; + if (out_labels[k] != 0) continue; + if (valid != nullptr && valid[k] == 0) continue; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + out_labels[k] = out_labels[r]; + } +} + +} // namespace topoanalysis + +#endif // TOPOANALYSIS_FLOW_ROUTING_HPP diff --git a/_cpp/module.cpp b/_cpp/module.cpp new file mode 100644 index 0000000..b166574 --- /dev/null +++ b/_cpp/module.cpp @@ -0,0 +1,540 @@ +// module.cpp -- pybind11 bindings for the TopoAnalysis C++ kernels. +// +// Everything exposed here has a pure-NumPy counterpart in +// ``TopoAnalysis.fastops``; the extension is an accelerator, never a hard +// requirement. Arrays are accepted as C-contiguous float64/uint8/int64. +// +// One rule governs every function below: **resolve every buffer and validate +// every size while the GIL is still held, then release it and run nothing but +// plain C++**. ``py::array::request()`` calls ``PyObject_GetBuffer``, which +// touches reference counts; calling it inside a ``gil_scoped_release`` block +// corrupts the interpreter as soon as two threads do it at once. The +// ``Resolved*`` helpers exist so that rule is enforced by construction rather +// than remembered at each call site. + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "priority_flood.hpp" +#include "flow_routing.hpp" + +namespace py = pybind11; +using namespace topoanalysis; + +namespace { + +using F64Array = py::array_t; +using U8Array = py::array_t; +using I64Array = py::array_t; +using I32Array = py::array_t; + +// --------------------------------------------------------------------------- +// GIL-safe buffer resolution +// --------------------------------------------------------------------------- + +// Holds an array alive together with the raw pointer into it. Construct one +// while the GIL is held; use ``.data`` after releasing it. +template +struct Resolved { + Array array; + T* data = nullptr; + std::size_t size = 0; + std::vector shape; + + Resolved() = default; + + explicit Resolved(Array a) : array(std::move(a)) { + auto info = array.request(); + data = static_cast(info.ptr); + size = static_cast(info.size); + shape.assign(info.shape.begin(), info.shape.end()); + } + + bool present() const { return data != nullptr; } + + void require_2d(const char* name) const { + if (shape.size() != 2) { + throw std::invalid_argument(std::string(name) + " must be a 2-D array"); + } + } + + void require_size(std::size_t expected, const char* name) const { + if (size != expected) { + throw std::invalid_argument( + std::string(name) + " has " + std::to_string(size) + + " elements; expected " + std::to_string(expected)); + } + } + + std::size_t ny() const { return static_cast(shape[0]); } + std::size_t nx() const { return static_cast(shape[1]); } +}; + +using ResolvedF64 = Resolved; +using ResolvedConstF64 = Resolved; +using ResolvedU8 = Resolved; +using ResolvedI64 = Resolved; +using ResolvedI32 = Resolved; + +// Optional arguments that may be None. +ResolvedU8 optional_u8(const py::object& obj) { + if (obj.is_none()) return ResolvedU8(); + return ResolvedU8(obj.cast()); +} + +ResolvedI64 optional_i64(const py::object& obj) { + if (obj.is_none()) return ResolvedI64(); + return ResolvedI64(obj.cast()); +} + +ResolvedF64 optional_f64(const py::object& obj) { + if (obj.is_none()) return ResolvedF64(); + return ResolvedF64(obj.cast()); +} + +// An output array of the given 2-D shape, resolved up front. +ResolvedF64 make_f64(std::size_t ny, std::size_t nx) { + return ResolvedF64(F64Array({static_cast(ny), static_cast(nx)})); +} + +ResolvedU8 make_u8(std::size_t ny, std::size_t nx) { + return ResolvedU8(U8Array({static_cast(ny), static_cast(nx)})); +} + +ResolvedI32 make_i32(std::size_t ny, std::size_t nx) { + return ResolvedI32(I32Array({static_cast(ny), static_cast(nx)})); +} + +// --------------------------------------------------------------------------- +// Depression filling +// --------------------------------------------------------------------------- + +py::dict py_priority_flood(F64Array elevations, + py::object closed_obj, + py::object seeds_obj, + const std::string& mode, + double epsilon, + double cellsize, + double max_pit_depth, + bool track_visited, + bool reference_algorithm) { + ResolvedF64 z(std::move(elevations)); + z.require_2d("elevations"); + const std::size_t ny = z.ny(), nx = z.nx(), n = ny * nx; + + std::vector closed(n, 0); + ResolvedU8 closed_arg = optional_u8(closed_obj); + if (closed_arg.present()) { + closed_arg.require_size(n, "closed"); + closed.assign(closed_arg.data, closed_arg.data + n); + } + for (std::size_t k = 0; k < n; ++k) { + if (std::isnan(z.data[k])) closed[k] = 1; + } + + std::vector seeds; + ResolvedI64 seeds_arg = optional_i64(seeds_obj); + if (!seeds_arg.present()) { + seeds = default_seeds(z.data, ny, nx); + // Honour a caller-supplied mask: a seed that is masked out would + // otherwise silently disappear and leave the grid unflooded. + std::vector kept; + kept.reserve(seeds.size()); + for (std::int64_t k : seeds) { + if (!closed[static_cast(k)]) kept.push_back(k); + } + if (closed_arg.present() && kept.empty()) { + // The mask excluded the whole grid perimeter. Fall back to the + // boundary of the masked region itself. + for (std::size_t i = 0; i < ny; ++i) { + for (std::size_t j = 0; j < nx; ++j) { + const std::size_t k = i * nx + j; + if (closed[k]) continue; + bool boundary = (i == 0 || j == 0 || i + 1 == ny || j + 1 == nx); + for (int d = 0; d < 8 && !boundary; ++d) { + const std::int64_t ni = static_cast(i) + kD8Di[d]; + const std::int64_t nj = static_cast(j) + kD8Dj[d]; + if (ni < 0 || nj < 0 || static_cast(ni) >= ny || + static_cast(nj) >= nx) { + continue; + } + if (closed[static_cast(ni) * nx + + static_cast(nj)]) { + boundary = true; + } + } + if (boundary) kept.push_back(static_cast(k)); + } + } + } + seeds.swap(kept); + } else { + seeds.assign(seeds_arg.data, seeds_arg.data + seeds_arg.size); + } + + FloodOptions options; + if (mode == "epsilon") { + options.mode = FillMode::Epsilon; + } else if (mode == "flat") { + options.mode = FillMode::Flat; + } else { + throw std::invalid_argument("mode must be 'flat' or 'epsilon'"); + } + options.epsilon = epsilon; + options.cellsize = cellsize; + options.max_pit_depth = max_pit_depth; + options.track_visited = track_visited; + options.track_unfilled = max_pit_depth > 0.0; + + FloodResult result; + { + py::gil_scoped_release release; + result = reference_algorithm + ? priority_flood_original(z.data, ny, nx, closed, seeds, options) + : priority_flood(z.data, ny, nx, closed, seeds, options); + } + + py::dict out; + out["cells_filled"] = result.cells_filled; + out["max_fill_depth"] = result.max_fill_depth; + if (track_visited) { + ResolvedU8 visited = make_u8(ny, nx); + std::copy(result.visited.begin(), result.visited.end(), visited.data); + out["visited"] = visited.array; + } else { + out["visited"] = py::none(); + } + if (options.track_unfilled) { + ResolvedU8 unfilled = make_u8(ny, nx); + std::copy(result.unfilled.begin(), result.unfilled.end(), unfilled.data); + out["unfilled"] = unfilled.array; + } else { + out["unfilled"] = py::none(); + } + return out; +} + +// --------------------------------------------------------------------------- +// Flow routing +// --------------------------------------------------------------------------- + +U8Array py_flow_directions(F64Array elevations, py::object cellsize_obj, + double uniform_cellsize) { + ResolvedF64 z(std::move(elevations)); + z.require_2d("elevations"); + const std::size_t ny = z.ny(), nx = z.nx(); + + ResolvedF64 cellsize = optional_f64(cellsize_obj); + if (cellsize.present()) cellsize.require_size(ny * nx, "cellsize_grid"); + + ResolvedU8 out = make_u8(ny, nx); + { + py::gil_scoped_release release; + d8_flow_directions(z.data, ny, nx, cellsize.data, uniform_cellsize, out.data); + } + return out.array; +} + +I64Array py_receivers(U8Array codes) { + ResolvedU8 c(std::move(codes)); + c.require_2d("codes"); + const std::size_t ny = c.ny(), nx = c.nx(); + + ResolvedI64 out(I64Array({static_cast(ny), static_cast(nx)})); + { + py::gil_scoped_release release; + d8_receivers(c.data, ny, nx, out.data); + } + return out.array; +} + +py::tuple py_topological_order(I64Array receivers) { + ResolvedI64 recv(std::move(receivers)); + const std::size_t n = recv.size; + + ResolvedI64 out(I64Array(static_cast(n))); + std::size_t cycles = 0; + { + py::gil_scoped_release release; + cycles = d8_topological_order(recv.data, n, out.data); + } + return py::make_tuple(out.array, cycles); +} + +F64Array py_accumulate(I64Array receivers, I64Array order, F64Array weights, + py::object gate_obj) { + ResolvedI64 recv(std::move(receivers)); + ResolvedI64 ord(std::move(order)); + ResolvedF64 w(std::move(weights)); + const std::size_t n = recv.size; + w.require_size(n, "weights"); + ord.require_size(n, "order"); + + ResolvedU8 gate = optional_u8(gate_obj); + if (gate.present()) gate.require_size(n, "gate"); + + ResolvedF64 out(F64Array(w.shape)); + { + py::gil_scoped_release release; + d8_accumulate(recv.data, ord.data, w.data, gate.data, n, out.data); + } + return out.array; +} + +py::tuple py_flow_length(I64Array receivers, I64Array order, F64Array step_length) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedF64 step(std::move(step_length)); + step.require_size(n, "step_length"); + + ResolvedF64 length = make_f64(ny, nx); + ResolvedU8 from_codes = make_u8(ny, nx); + { + py::gil_scoped_release release; + d8_flow_length(recv.data, ord.data, step.data, ny, nx, length.data, from_codes.data); + } + return py::make_tuple(length.array, from_codes.array); +} + +F64Array py_propagate_along_main_stem(I64Array receivers, I64Array order, + U8Array main_stem_from_codes, py::object gate_obj, + F64Array values, const std::string& mode) { + if (mode != "sum" && mode != "carry") { + throw std::invalid_argument("mode must be 'carry' or 'sum'"); + } + + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedU8 codes(std::move(main_stem_from_codes)); + codes.require_size(n, "main_stem_from_codes"); + ResolvedF64 vals(std::move(values)); + vals.require_size(n, "values"); + ResolvedU8 gate = optional_u8(gate_obj); + if (gate.present()) gate.require_size(n, "gate"); + + ResolvedF64 out = make_f64(ny, nx); + std::copy(vals.data, vals.data + n, out.data); + { + py::gil_scoped_release release; + d8_propagate_along_main_stem(recv.data, ord.data, codes.data, gate.data, ny, nx, + mode == "sum" ? PropagateMode::Sum : PropagateMode::Carry, + out.data); + } + return out.array; +} + +U8Array py_upstream_mask(I64Array receivers, I64Array outlets) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(); + ResolvedI64 out_idx(std::move(outlets)); + + ResolvedU8 out = make_u8(ny, nx); + { + py::gil_scoped_release release; + d8_upstream_mask(recv.data, ny * nx, out_idx.data, out_idx.size, out.data); + } + return out.array; +} + +py::tuple py_chi(I64Array receivers, I64Array order, F64Array area, F64Array step_length, + I64Array outlets, double A0, double theta, bool trapezoid, double max_length, + py::object mask_obj) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedF64 a(std::move(area)); + a.require_size(n, "area"); + ResolvedF64 step(std::move(step_length)); + step.require_size(n, "step_length"); + ResolvedI64 out_idx(std::move(outlets)); + ResolvedU8 mask = optional_u8(mask_obj); + if (mask.present()) mask.require_size(n, "mask"); + + ResolvedF64 chi_out = make_f64(ny, nx); + ResolvedF64 distance = make_f64(ny, nx); + { + py::gil_scoped_release release; + d8_chi(recv.data, ord.data, a.data, step.data, n, out_idx.data, out_idx.size, + A0, theta, trapezoid, max_length, mask.data, chi_out.data, distance.data); + } + return py::make_tuple(chi_out.array, distance.array); +} + +F64Array py_downstream_distance(I64Array receivers, I64Array order, F64Array step_length) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedF64 step(std::move(step_length)); + step.require_size(n, "step_length"); + + ResolvedF64 out = make_f64(ny, nx); + { + py::gil_scoped_release release; + d8_downstream_distance(recv.data, ord.data, step.data, n, out.data); + } + return out.array; +} + +void py_imposemin(I64Array receivers, I64Array order, F64Array step_length, double sl, + py::array_t elevations) { + ResolvedI64 recv(std::move(receivers)); + const std::size_t n = recv.size; + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedF64 step(std::move(step_length)); + step.require_size(n, "step_length"); + + auto einfo = elevations.request(); + if (static_cast(einfo.size) != n) { + throw std::invalid_argument("elevations must match the receiver grid"); + } + double* z = static_cast(einfo.ptr); + + py::gil_scoped_release release; + d8_imposemin(recv.data, ord.data, step.data, n, sl, z); +} + +I32Array py_stream_order(I64Array receivers, I64Array order, U8Array is_stream, + const std::string& kind) { + if (kind != "strahler" && kind != "shreve") { + throw std::invalid_argument("kind must be 'strahler' or 'shreve'"); + } + + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedU8 streams(std::move(is_stream)); + streams.require_size(n, "is_stream"); + + ResolvedI32 out = make_i32(ny, nx); + { + py::gil_scoped_release release; + if (kind == "shreve") { + d8_shreve(recv.data, ord.data, streams.data, n, out.data); + } else { + d8_strahler(recv.data, ord.data, streams.data, n, out.data); + } + } + return out.array; +} + +I32Array py_drainage_basins(I64Array receivers, I64Array order, py::object outlets_obj, + py::object valid_obj) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedU8 valid = optional_u8(valid_obj); + if (valid.present()) valid.require_size(n, "valid"); + ResolvedI64 outlets = optional_i64(outlets_obj); + + ResolvedI32 out = make_i32(ny, nx); + { + py::gil_scoped_release release; + if (!outlets.present()) { + d8_drainage_basins_all(recv.data, ord.data, valid.data, n, out.data); + } else { + // `valid` applies here too; dropping it made the two backends + // disagree whenever both arguments were supplied. + d8_drainage_basins(recv.data, ord.data, n, outlets.data, outlets.size, + valid.data, out.data); + } + } + return out.array; +} + +} // namespace + +PYBIND11_MODULE(_topoanalysis, m) { + m.doc() = + "Compiled kernels for TopoAnalysis.\n\n" + "Depression filling implements the Priority-Flood family of algorithms of\n" + "Barnes, R., Lehman, C. and Mulla, D. (2014), 'Priority-flood: An optimal\n" + "depression-filling and watershed-labeling algorithm for digital elevation\n" + "models', Computers & Geosciences 62, 117-127."; + + m.attr("__algorithm_reference__") = + "Barnes, R., Lehman, C., Mulla, D. (2014). Priority-flood: An optimal " + "depression-filling and watershed-labeling algorithm for digital elevation " + "models. Computers & Geosciences 62, 117-127. doi:10.1016/j.cageo.2013.04.024"; + + m.def("priority_flood", &py_priority_flood, py::arg("elevations").noconvert(), + py::arg("closed") = py::none(), py::arg("seeds") = py::none(), + py::arg("mode") = "flat", py::arg("epsilon") = 0.0, py::arg("cellsize") = 1.0, + py::arg("max_pit_depth") = 0.0, py::arg("track_visited") = false, + py::arg("reference_algorithm") = false, + "Fill depressions in place using Priority-Flood (Barnes et al. 2014)."); + + m.def("flow_directions", &py_flow_directions, py::arg("elevations"), + py::arg("cellsize_grid") = py::none(), py::arg("cellsize") = 1.0, + "Steepest-descent D8 flow directions as ArcGIS codes."); + + m.def("receivers", &py_receivers, py::arg("codes"), + "Flat receiver index for each cell (-1 where flow leaves the grid)."); + + m.def("topological_order", &py_topological_order, py::arg("receivers"), + "Donors-before-receivers ordering; returns (order, n_cells_on_cycles)."); + + m.def("accumulate", &py_accumulate, py::arg("receivers"), py::arg("order"), + py::arg("weights"), py::arg("gate") = py::none(), + "Accumulate weights downstream along the D8 network."); + + m.def("flow_length", &py_flow_length, py::arg("receivers"), py::arg("order"), + py::arg("step_length"), + "Longest upstream flow distance and the main-stem donor direction codes."); + + m.def("propagate_along_main_stem", &py_propagate_along_main_stem, py::arg("receivers"), + py::arg("order"), py::arg("main_stem_from_codes"), py::arg("gate"), py::arg("values"), + py::arg("mode"), "Carry or sum values downstream along the longest flow path."); + + m.def("upstream_mask", &py_upstream_mask, py::arg("receivers"), py::arg("outlets"), + "Mask of every cell draining to any of the given outlets."); + + m.def("chi", &py_chi, py::arg("receivers"), py::arg("order"), py::arg("area"), + py::arg("step_length"), py::arg("outlets"), py::arg("A0"), py::arg("theta"), + py::arg("trapezoid") = false, py::arg("max_length") = 0.0, + py::arg("mask") = py::none(), "Integrate chi upstream from the given outlets."); + + m.def("drainage_basins", &py_drainage_basins, py::arg("receivers"), py::arg("order"), + py::arg("outlets") = py::none(), py::arg("valid") = py::none(), + "Label drainage basins, either for given outlets or for every terminal cell."); + + m.def("downstream_distance", &py_downstream_distance, py::arg("receivers"), py::arg("order"), + py::arg("step_length"), + "Along-flow distance from each cell to the end of its flow path."); + + m.def("imposemin", &py_imposemin, py::arg("receivers"), py::arg("order"), + py::arg("step_length"), py::arg("sl"), py::arg("elevations").noconvert(), + "Carve elevations in place so flow paths descend at gradient >= sl."); + + m.def("stream_order", &py_stream_order, py::arg("receivers"), py::arg("order"), + py::arg("is_stream"), py::arg("kind") = "strahler", + "Strahler or Shreve stream order over the marked stream cells."); +} diff --git a/_cpp/priority_flood.hpp b/_cpp/priority_flood.hpp new file mode 100644 index 0000000..92670ed --- /dev/null +++ b/_cpp/priority_flood.hpp @@ -0,0 +1,378 @@ +// priority_flood.hpp -- Priority-Flood depression filling +// +// Implements the Priority-Flood family of depression-filling algorithms +// described in +// +// Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal +// depression-filling and watershed-labeling algorithm for digital +// elevation models." Computers & Geosciences 62, 117-127. +// doi:10.1016/j.cageo.2013.04.024 +// +// Three variants from that paper are provided: +// +// * Algorithm 1 -- Priority-Flood. Every cell passes through the priority +// queue. O(n log n). Kept as a reference implementation used to test +// the faster variants. +// * Algorithm 2 -- Improved Priority-Flood. Cells that need no filling are +// handled through a FIFO ("pit") queue instead of the priority queue, +// which reduces the practical cost to O(n) for most DEMs. This is the +// default. +// * Algorithm 4 -- Priority-Flood+epsilon. As Algorithm 2, but filled +// cells are raised by a small increment so that the filled surface has a +// strictly monotonic downhill path and contains no flat areas. +// +// The implementation is self-contained (standard library only) and templated +// on the elevation type so it can be instantiated for float and double. +// +// Conventions +// ----------- +// * Grids are row-major, ``ny`` rows by ``nx`` columns; index ``k = i*nx + j``. +// * ``NaN`` marks no-data. No-data cells are never filled and never enqueued. +// * Neighbourhood is D8 (8-connected), matching Barnes et al. and TopoToolbox. + +#ifndef TOPOANALYSIS_PRIORITY_FLOOD_HPP +#define TOPOANALYSIS_PRIORITY_FLOOD_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace topoanalysis { + +// D8 neighbour offsets. Order matches the ArcGIS code order used elsewhere in +// TopoAnalysis: E, SE, S, SW, W, NW, N, NE. +constexpr int kD8Di[8] = {0, 1, 1, 1, 0, -1, -1, -1}; +constexpr int kD8Dj[8] = {1, 1, 0, -1, -1, -1, 0, 1}; +// Centre-to-centre distance multiples for the same ordering. +const double kD8Dist[8] = {1.0, 1.4142135623730951, 1.0, 1.4142135623730951, + 1.0, 1.4142135623730951, 1.0, 1.4142135623730951}; + +enum class FillMode { + Flat = 0, // Algorithm 2: fill depressions to a flat surface + Epsilon = 1, // Algorithm 4: fill with a small upslope increment +}; + +// A priority-queue entry. ``order`` makes the ordering total, so ties are +// broken first-in-first-out. Without it std::priority_queue would resolve +// ties arbitrarily and the filled surface would not be reproducible. +template +struct PqCell { + T z; + std::int64_t order; + std::int64_t idx; +}; + +template +struct PqGreater { + bool operator()(const PqCell& a, const PqCell& b) const { + if (a.z != b.z) return a.z > b.z; + return a.order > b.order; + } +}; + +// Options controlling a Priority-Flood run. +struct FloodOptions { + FillMode mode = FillMode::Flat; + + // Epsilon increment. Interpreted as a *slope*: a cell is raised to + // parent_elevation + epsilon * distance_to_parent + // where the distance is in the same units as ``cellsize``. A value of 0 + // with ``mode == Epsilon`` falls back to the smallest representable + // increment (std::nextafter), which is what Barnes et al. Algorithm 4 + // prescribes. + double epsilon = 0.0; + double cellsize = 1.0; + + // Depressions deeper than this are left unfilled (0 or negative disables + // the check). Cells inside such a depression keep their original + // elevation but are still traversed, so the rest of the grid is reached. + double max_pit_depth = 0.0; + + // Record which cells the flood reached. + bool track_visited = false; + + // Record which cells a depth limit left unfilled. Without this the + // caller cannot tell a cell the flood never reached from one it reached + // but deliberately declined to raise. + bool track_unfilled = false; +}; + +struct FloodResult { + std::vector visited; // empty unless options.track_visited + std::vector unfilled; // empty unless options.track_unfilled + std::size_t cells_filled = 0; + double max_fill_depth = 0.0; +}; + +namespace detail { + +template +inline bool is_nodata(T v) { + return std::isnan(v); +} + +// Raise ``value`` by the smallest amount that is representable at that +// magnitude. Used by Algorithm 4 when no explicit epsilon is supplied. +template +inline T next_up(T value) { + return std::nextafter(value, std::numeric_limits::infinity()); +} + +} // namespace detail + +// --------------------------------------------------------------------------- +// Algorithm 2 of Barnes et al. (2014), with the epsilon variant of Algorithm 4 +// folded in. ``elevations`` is modified in place. +// +// ``closed`` must be a pre-initialised ny*nx byte array: a non-zero entry +// marks a cell that must never be processed (no-data, or outside a caller +// supplied mask). ``seeds`` lists the flat indices the flood starts from; +// they are usually the grid perimeter plus every valid cell adjacent to +// no-data, but callers may supply outlet cells instead. +// --------------------------------------------------------------------------- +template +FloodResult priority_flood(T* elevations, + std::size_t ny, + std::size_t nx, + std::vector& closed, + const std::vector& seeds, + const FloodOptions& options) { + if (ny == 0 || nx == 0) return FloodResult{}; + const std::size_t n = ny * nx; + if (closed.size() != n) { + throw std::invalid_argument("closed mask has the wrong size"); + } + + FloodResult result; + if (options.track_visited) result.visited.assign(n, 0); + if (options.track_unfilled) result.unfilled.assign(n, 0); + + std::priority_queue, std::vector>, PqGreater> open; + std::queue pit; + + std::int64_t order = 0; + for (std::int64_t k : seeds) { + if (k < 0 || static_cast(k) >= n) continue; + if (closed[static_cast(k)]) continue; + if (detail::is_nodata(elevations[k])) continue; + closed[static_cast(k)] = 1; + open.push(PqCell{elevations[k], order++, k}); + } + + const bool use_epsilon = (options.mode == FillMode::Epsilon); + const bool limit_depth = options.max_pit_depth > 0.0; + // The FIFO shortcut is only sound when every cell it holds sits at exactly + // the current flood level. Two options break that invariant: an epsilon + // fill raises each cell by its own increment, and a depth limit re-queues + // unfilled cells *below* the current level. In either case the priority + // queue handles everything, i.e. the run degrades to Algorithm 1. + const bool use_pit = !use_epsilon && !limit_depth; + + while (!open.empty() || !pit.empty()) { + std::int64_t cur; + T cur_z; + // Draining the FIFO first is what makes Algorithm 2 faster than + // Algorithm 1 while producing an identical surface: cells already at + // or below the current flood level cannot be reordered by the + // priority queue. When both queues hold the same elevation the + // priority queue wins, preserving Algorithm 1's visit order. + if (!pit.empty() && !open.empty() && open.top().z <= elevations[pit.front()]) { + cur = open.top().idx; + cur_z = open.top().z; + open.pop(); + } else if (!pit.empty()) { + cur = pit.front(); + pit.pop(); + cur_z = elevations[cur]; + } else { + cur = open.top().idx; + cur_z = open.top().z; + open.pop(); + } + + if (options.track_visited) result.visited[static_cast(cur)] = 1; + + const std::size_t i = static_cast(cur) / nx; + const std::size_t j = static_cast(cur) % nx; + + for (int d = 0; d < 8; ++d) { + const std::int64_t ni = static_cast(i) + kD8Di[d]; + const std::int64_t nj = static_cast(j) + kD8Dj[d]; + if (ni < 0 || nj < 0 || static_cast(ni) >= ny || + static_cast(nj) >= nx) { + continue; + } + const std::int64_t nk = ni * static_cast(nx) + nj; + if (closed[static_cast(nk)]) continue; + if (detail::is_nodata(elevations[nk])) { + closed[static_cast(nk)] = 1; + continue; + } + closed[static_cast(nk)] = 1; + + T target; + if (use_epsilon) { + if (options.epsilon > 0.0) { + target = static_cast(cur_z + options.epsilon * options.cellsize * kD8Dist[d]); + if (target <= cur_z) target = detail::next_up(cur_z); + } else { + target = detail::next_up(cur_z); + } + } else { + target = cur_z; + } + + if (elevations[nk] < target) { + const double depth = static_cast(target) - static_cast(elevations[nk]); + if (limit_depth && depth > options.max_pit_depth) { + // Leave this depression alone, but keep traversing so the + // terrain beyond it is still reached. + if (options.track_unfilled) result.unfilled[static_cast(nk)] = 1; + open.push(PqCell{elevations[nk], order++, nk}); + continue; + } + elevations[nk] = target; + ++result.cells_filled; + if (depth > result.max_fill_depth) result.max_fill_depth = depth; + if (use_pit) { + pit.push(nk); + } else { + open.push(PqCell{elevations[nk], order++, nk}); + } + } else { + open.push(PqCell{elevations[nk], order++, nk}); + } + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// Algorithm 1 of Barnes et al. (2014). Every cell goes through the priority +// queue. Retained because it is the simplest statement of the method and +// therefore a good oracle for testing the improved variant. +// --------------------------------------------------------------------------- +template +FloodResult priority_flood_original(T* elevations, + std::size_t ny, + std::size_t nx, + std::vector& closed, + const std::vector& seeds, + const FloodOptions& options) { + if (ny == 0 || nx == 0) return FloodResult{}; + const std::size_t n = ny * nx; + if (closed.size() != n) { + throw std::invalid_argument("closed mask has the wrong size"); + } + + FloodResult result; + if (options.track_visited) result.visited.assign(n, 0); + if (options.track_unfilled) result.unfilled.assign(n, 0); + + std::priority_queue, std::vector>, PqGreater> open; + std::int64_t order = 0; + for (std::int64_t k : seeds) { + if (k < 0 || static_cast(k) >= n) continue; + if (closed[static_cast(k)]) continue; + if (detail::is_nodata(elevations[k])) continue; + closed[static_cast(k)] = 1; + open.push(PqCell{elevations[k], order++, k}); + } + + const bool use_epsilon = (options.mode == FillMode::Epsilon); + const bool limit_depth = options.max_pit_depth > 0.0; + + while (!open.empty()) { + const PqCell c = open.top(); + open.pop(); + if (options.track_visited) result.visited[static_cast(c.idx)] = 1; + + const std::size_t i = static_cast(c.idx) / nx; + const std::size_t j = static_cast(c.idx) % nx; + + for (int d = 0; d < 8; ++d) { + const std::int64_t ni = static_cast(i) + kD8Di[d]; + const std::int64_t nj = static_cast(j) + kD8Dj[d]; + if (ni < 0 || nj < 0 || static_cast(ni) >= ny || + static_cast(nj) >= nx) { + continue; + } + const std::int64_t nk = ni * static_cast(nx) + nj; + if (closed[static_cast(nk)]) continue; + if (detail::is_nodata(elevations[nk])) { + closed[static_cast(nk)] = 1; + continue; + } + closed[static_cast(nk)] = 1; + + T target; + if (use_epsilon) { + if (options.epsilon > 0.0) { + target = static_cast(c.z + options.epsilon * options.cellsize * kD8Dist[d]); + if (target <= c.z) target = detail::next_up(c.z); + } else { + target = detail::next_up(c.z); + } + } else { + target = c.z; + } + + if (elevations[nk] < target) { + const double depth = static_cast(target) - static_cast(elevations[nk]); + if (limit_depth && depth > options.max_pit_depth) { + if (options.track_unfilled) result.unfilled[static_cast(nk)] = 1; + open.push(PqCell{elevations[nk], order++, nk}); + continue; + } + elevations[nk] = target; + ++result.cells_filled; + if (depth > result.max_fill_depth) result.max_fill_depth = depth; + } + open.push(PqCell{elevations[nk], order++, nk}); + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// Default seed set: the grid perimeter plus every valid cell that touches a +// no-data cell. This matches TopoToolbox's ``fillsinks``, whose marker image +// keeps the original elevation on the perimeter and around NaN holes. +// --------------------------------------------------------------------------- +template +std::vector default_seeds(const T* elevations, + std::size_t ny, + std::size_t nx) { + std::vector seeds; + if (ny == 0 || nx == 0) return seeds; + + const std::int64_t rows = static_cast(ny); + const std::int64_t cols = static_cast(nx); + + for (std::int64_t i = 0; i < rows; ++i) { + for (std::int64_t j = 0; j < cols; ++j) { + const std::int64_t k = i * cols + j; + if (detail::is_nodata(elevations[k])) continue; + + bool is_seed = (i == 0 || j == 0 || i + 1 == rows || j + 1 == cols); + for (int d = 0; d < 8 && !is_seed; ++d) { + const std::int64_t ni = i + kD8Di[d]; + const std::int64_t nj = j + kD8Dj[d]; + if (ni < 0 || nj < 0 || ni >= rows || nj >= cols) continue; + if (detail::is_nodata(elevations[ni * cols + nj])) is_seed = true; + } + if (is_seed) seeds.push_back(k); + } + } + return seeds; +} + +} // namespace topoanalysis + +#endif // TOPOANALYSIS_PRIORITY_FLOOD_HPP diff --git a/_gdal_source.py b/_gdal_source.py new file mode 100644 index 0000000..65d0268 --- /dev/null +++ b/_gdal_source.py @@ -0,0 +1,562 @@ +"""Download, verify and build the GDAL C library from source. + +This is the last resort of :mod:`TopoAnalysis.gdal_setup`, for machines with +no conda, no system package manager and no administrator rights. Everything +here installs into a single prefix that the caller chooses -- normally the +active virtualenv -- so nothing outside it is touched and removing the prefix +undoes the whole thing. + +Three components, built in order, because each needs the one before it: + + sqlite3 only when the system copy is unusable. PROJ needs both the + library *and* the ``sqlite3`` command, which it runs during its + own build to assemble ``proj.db``. + PROJ mandatory for GDAL 3.x. Built without TIFF and curl support, + which are only needed for downloading and reading the optional + high-accuracy datum grids. + GDAL built against its own bundled copies of libtiff, libgeotiff, + libpng, libjpeg and zlib (``GDAL_USE_INTERNAL_LIBS``), so the + only external dependency left is PROJ. That is what makes an + unattended build realistic: no hunting for system ``-dev`` + packages. The result reads GeoTIFF, ArcInfo ASCII, ENVI, HFA + and the rest of the built-in drivers, which is everything + TopoAnalysis uses. + +The Python bindings are deliberately *not* built here. ``cmake`` would build +them for whichever interpreter it happens to find; :mod:`TopoAnalysis.gdal_setup` +installs them with pip instead, against the interpreter we actually care +about. + +Every tarball is pinned by SHA-256. The pins below were taken from bytes +checked against what the projects themselves publish -- GDAL's and PROJ's +``.md5`` sidecars, SQLite's ``PRODUCT`` line -- so a download that does not +match them is a download that should not be built. +""" + +from __future__ import annotations + +import hashlib +import os +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request + +__all__ = [ + "SOURCES", + "apply_environment_overrides", + "build_one", + "components_needed", + "default_cache_dir", + "missing_build_tools", + "system_sqlite_is_usable", +] + + +class BuildError(RuntimeError): + """A download, configure, compile or install step failed.""" + + +# --------------------------------------------------------------------------- +# What we build, and exactly which bytes +# --------------------------------------------------------------------------- + +SOURCES = { + "sqlite": { + "version": "3.53.4", + "url": "https://www.sqlite.org/2026/sqlite-autoconf-3530400.tar.gz", + "sha256": "0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c", + "minutes": 2, + }, + "proj": { + "version": "9.4.1", + "url": "https://download.osgeo.org/proj/proj-9.4.1.tar.gz", + "sha256": "ffe20170ee2b952207adf8a195e2141eab12cda181e49fdeb54425d98c7171d7", + "minutes": 6, + }, + "gdal": { + # Not merely "a recent GDAL": the version matters for this build. + # GDAL bundles its own zlib, libpng, libtiff and friends, and the + # copies shipped up to about 3.9 do not compile against a current + # Apple SDK -- both zlib's ``zutil.h`` and libpng's ``pngpriv.h`` + # take their "Mac OS classic" branch, because that is selected on + # ``TARGET_OS_MAC``, which modern SDKs define on every Apple + # platform. zlib then redefines ``fdopen`` to NULL and the system + # ``_stdio.h`` fails to parse. Later GDAL releases carry repaired + # copies, so the pin has to stay reasonably current. + "version": "3.13.3", + "url": "https://github.com/OSGeo/gdal/releases/download/v3.13.3/gdal-3.13.3.tar.gz", + "sha256": "5e0c388d83da2d686cc00a40272882432cdb54edff43d4af173e532844a0a0ea", + "minutes": 20, + }, +} + +#: Order matters: each entry links against the ones before it. +BUILD_ORDER = ("sqlite", "proj", "gdal") + +#: Seconds to wait for any single read from a download before giving up. +DOWNLOAD_TIMEOUT = 60.0 + +#: The floor declared by both pinned projects: PROJ 9.4's CMakeLists and +#: GDAL 3.13's ``GdalCMakeMinimumRequired.cmake`` each say 3.16. +MIN_CMAKE_VERSION = (3, 16) + + +def _cmake_version(cmake): + """``(major, minor)`` reported by ``cmake --version``, or None.""" + try: + result = subprocess.run( + [cmake, "--version"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + except OSError: + return None + if result.returncode != 0: + return None + text = result.stdout.decode("utf8", "replace") + match = re.search(r"(\d+)\.(\d+)", text) + return (int(match.group(1)), int(match.group(2))) if match else None + + +def apply_environment_overrides(): + """Let ``TOPOANALYSIS_GDAL_SOURCE_*`` environment variables move a pin. + + A pinned version is a dead end the day it stops being suitable -- a + platform that needs a newer GDAL, a site that mirrors its own tarballs. + Overriding one means supplying its checksum too, so the integrity check + cannot be turned off by accident:: + + TOPOANALYSIS_GDAL_SOURCE_GDAL_VERSION=3.12.4 + TOPOANALYSIS_GDAL_SOURCE_GDAL_URL=https://example.org/gdal-3.12.4.tar.gz + TOPOANALYSIS_GDAL_SOURCE_GDAL_SHA256=<64 hex digits> + """ + for component in SOURCES: + prefix = "TOPOANALYSIS_GDAL_SOURCE_%s_" % component.upper() + url = os.environ.get(prefix + "URL") + sha256 = os.environ.get(prefix + "SHA256") + version = os.environ.get(prefix + "VERSION") + if url or sha256: + if not (url and sha256): + raise BuildError( + "%sURL and %sSHA256 must be set together" % (prefix, prefix) + ) + SOURCES[component]["url"] = url + SOURCES[component]["sha256"] = sha256 + if version: + SOURCES[component]["version"] = version + + +def default_cache_dir(): + """Where downloaded tarballs are kept between runs.""" + root = os.environ.get("XDG_CACHE_HOME") or os.path.join( + os.path.expanduser("~"), ".cache" + ) + return os.path.join(root, "TopoAnalysis", "downloads") + + +# --------------------------------------------------------------------------- +# Prerequisites +# --------------------------------------------------------------------------- + + +def missing_build_tools(): + """Names of build tools that are needed for a source build and absent. + + Returns an empty list when the machine can build. ``ninja`` is not + required -- CMake falls back to makefiles -- so it is not listed. + """ + missing = [] + cmake = shutil.which("cmake") + if cmake is None: + missing.append("cmake") + else: + version = _cmake_version(cmake) + # Both PROJ 9 and GDAL 3.13 declare this floor. Checking it here + # turns a puzzling failure part-way through `cmake -S` into a + # sentence naming the version that is needed. + if version is not None and version < MIN_CMAKE_VERSION: + missing.append( + "cmake %d.%d or newer (found %s)" + % (MIN_CMAKE_VERSION[0], MIN_CMAKE_VERSION[1], + ".".join(str(part) for part in version)) + ) + # ninja is not an alternative here: no -G is passed, so CMake generates + # makefiles, and SQLite's tarball is a plain autoconf build that calls + # make directly. + if shutil.which("make") is None: + missing.append("make") + if _c_compiler() is None: + missing.append("a C compiler") + # PROJ and GDAL are C++17 projects, and a machine can genuinely have cc + # without a C++ compiler -- a bare gcc package without g++ is the usual + # way. Checking only for a C compiler would let such a machine start a + # build that CMake aborts a few seconds later. + if _cxx_compiler() is None: + missing.append("a C++ compiler") + return missing + + +def _find_compiler(variable, candidates): + """The compiler named by ``variable``, else the first of ``candidates``.""" + explicit = os.environ.get(variable) + if explicit: + # The variable may carry arguments ("ccache gcc"); only the program + # matters here. + return shutil.which(explicit.split()[0]) + for name in candidates: + found = shutil.which(name) + if found is not None: + return found + return None + + +def _c_compiler(): + """The C compiler to probe with, honouring ``CC`` as the toolchain does.""" + return _find_compiler("CC", ("cc", "gcc", "clang")) + + +def _cxx_compiler(): + """The C++ compiler, honouring ``CXX``.""" + return _find_compiler("CXX", ("c++", "g++", "clang++")) + + +def system_sqlite_is_usable(): + """True when the system SQLite can satisfy PROJ's build. + + PROJ needs three things, and a machine can easily have one or two of + them: the shared library, the ``sqlite3.h`` header (often packaged + separately as ``libsqlite3-dev``), and the ``sqlite3`` command-line shell, + which PROJ runs to build ``proj.db``. Checking the header by compiling + against it is the only honest test -- the file can be present but + unreachable from the compiler's default include path, which is the normal + situation inside a conda environment. + """ + if shutil.which("sqlite3") is None: + return False + compiler = _c_compiler() + if compiler is None: + return False + + program = ( + "#include \n" + "#if SQLITE_VERSION_NUMBER < 3011000\n" + '#error "PROJ needs SQLite 3.11 or newer"\n' + "#endif\n" + "int main(void) { return sqlite3_libversion_number() > 0 ? 0 : 1; }\n" + ) + with tempfile.TemporaryDirectory() as tmp: + source = os.path.join(tmp, "probe.c") + with open(source, "w") as handle: + handle.write(program) + # Compile only: linking would additionally need -lsqlite3 to resolve, + # and a missing library is a different (rarer) problem than a missing + # header. PROJ's CMake will find the library if the header is there. + result = subprocess.run( + [compiler, "-c", source, "-o", os.path.join(tmp, "probe.o")], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + return result.returncode == 0 + + +def components_needed(): + """Which of :data:`BUILD_ORDER` this machine actually has to build.""" + needed = list(BUILD_ORDER) + if system_sqlite_is_usable(): + needed.remove("sqlite") + return needed + + +# --------------------------------------------------------------------------- +# Download and unpack +# --------------------------------------------------------------------------- + + +def _download(url, sha256, cache_dir, log): + """Fetch ``url`` into ``cache_dir``, verifying its SHA-256. + + A cached file that fails the check is deleted and fetched again: that is + far more often a truncated download than anything sinister, and leaving it + in place would make every later run fail the same way. + """ + os.makedirs(cache_dir, exist_ok=True) + target = os.path.join(cache_dir, url.rsplit("/", 1)[-1]) + + if os.path.exists(target): + if _sha256(target) == sha256: + log(" using cached %s" % os.path.basename(target)) + return target + log(" cached %s failed its checksum; re-downloading" % os.path.basename(target)) + os.remove(target) + + log(" downloading %s" % url) + partial = target + ".part" + try: + # With no timeout a stalled connection blocks the install for ever, + # with nothing on screen to say why. This bounds each individual + # read, not the whole transfer, so a slow-but-alive mirror still + # works. + with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT) as response: + with open(partial, "wb") as handle: + shutil.copyfileobj(response, handle) + except Exception as exc: # network, DNS, TLS, HTTP -- all equally fatal here + if os.path.exists(partial): + os.remove(partial) + raise BuildError("could not download %s: %s" % (url, exc)) + + got = _sha256(partial) + if got != sha256: + os.remove(partial) + raise BuildError( + "checksum mismatch for %s\n expected %s\n got %s\n" + "Refusing to build code that is not the code we pinned." % (url, sha256, got) + ) + os.replace(partial, target) + return target + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _extract(tarball, into): + """Unpack ``tarball`` and return the directory it created.""" + with tarfile.open(tarball) as archive: + names = archive.getnames() + # ``filter="data"`` rejects absolute paths, parent-directory escapes, + # links out of the tree and device nodes. It is the default from + # Python 3.14; ask for it explicitly wherever it exists. + try: + archive.extractall(into, filter="data") + except TypeError: # pragma: no cover - Python < 3.11.4 + _check_members(archive, names) + archive.extractall(into) + roots = {name.split("/", 1)[0] for name in names if name and not name.startswith("/")} + if len(roots) != 1: + raise BuildError("unexpected archive layout in %s" % tarball) + return os.path.join(into, roots.pop()) + + +def _check_members(archive, names): # pragma: no cover - old Python only + """The safety half of ``filter='data'`` for interpreters that lack it.""" + for name in names: + if name.startswith("/") or os.path.isabs(name) or ".." in name.split("/"): + raise BuildError("refusing to extract unsafe path %r" % name) + for member in archive.getmembers(): + if member.isdev(): + raise BuildError("refusing to extract device node %r" % member.name) + + +# --------------------------------------------------------------------------- +# Running the build +# --------------------------------------------------------------------------- + + +def _build_env(prefix): + """Environment that makes each component find the ones built before it.""" + env = dict(os.environ) + bindir = os.path.join(prefix, "bin") + env["PATH"] = bindir + os.pathsep + env.get("PATH", "") + + libdir = os.path.join(prefix, "lib") + pkgconfig = os.pathsep.join( + [os.path.join(libdir, "pkgconfig"), os.path.join(prefix, "share", "pkgconfig")] + ) + existing = env.get("PKG_CONFIG_PATH") + env["PKG_CONFIG_PATH"] = pkgconfig + (os.pathsep + existing if existing else "") + + # So the freshly built tools (PROJ's own, and sqlite3) can run during the + # build before anything has been relinked. + loader = "DYLD_LIBRARY_PATH" if sys.platform == "darwin" else "LD_LIBRARY_PATH" + env[loader] = libdir + (os.pathsep + env[loader] if env.get(loader) else "") + return env + + +def _run(command, cwd, env, log, logfile): + """Run a build command, tee-ing its output to ``logfile``.""" + log(" $ %s" % " ".join(command)) + with open(logfile, "a") as handle: + handle.write("\n$ %s\n" % " ".join(command)) + handle.flush() + result = subprocess.run( + command, cwd=cwd, env=env, stdout=handle, stderr=subprocess.STDOUT + ) + if result.returncode != 0: + raise BuildError( + "%s failed (exit %d).\nThe full build log is at:\n %s" + % (command[0], result.returncode, logfile) + ) + + +def _jobs(requested): + if requested: + return max(1, int(requested)) + return max(1, (os.cpu_count() or 2)) + + +def _cmake_common(prefix): + """CMake settings shared by PROJ and GDAL.""" + settings = [ + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_INSTALL_PREFIX=" + prefix, + "-DCMAKE_PREFIX_PATH=" + prefix, + "-DBUILD_TESTING=OFF", + "-DBUILD_SHARED_LIBS=ON", + # Pin the library directory instead of letting GNUInstallDirs choose. + # On Fedora, RHEL and openSUSE it defaults to lib64 on 64-bit hosts, + # and everything else here -- the RPATH below, the loader path in + # _build_env, the -L handed to the bindings build -- assumes one + # known location. A private prefix has no distro convention to + # honour, so "lib" everywhere is both simpler and correct. + "-DCMAKE_INSTALL_LIBDIR=lib", + # Without an install RPATH the shared libraries only resolve each + # other when LD_LIBRARY_PATH happens to be set, which it will not be + # in the user's next shell. Baking the path in makes the prefix work + # on its own. + "-DCMAKE_INSTALL_RPATH=" + os.path.join(prefix, "lib"), + "-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON", + "-DCMAKE_INSTALL_NAME_DIR=" + os.path.join(prefix, "lib"), + ] + if sys.platform == "darwin": + # Build for the architecture the *interpreter* runs as. On an Apple + # Silicon machine running an x86_64 Python under Rosetta, the default + # would be arm64 and the bindings would refuse to link. + settings.append("-DCMAKE_OSX_ARCHITECTURES=" + platform.machine()) + return settings + + +def build_one(component, prefix, jobs=None, log=None, cache_dir=None, keep_build=False): + """Download, build and install one component into ``prefix``. + + Parameters + ---------- + component : {'sqlite', 'proj', 'gdal'} + prefix : str + Installation prefix. Created if it does not exist. + jobs : int, optional + Parallel compile jobs; defaults to the CPU count. + log : callable, optional + Receives progress lines. Defaults to printing. + cache_dir : str, optional + Where tarballs are cached; defaults to :func:`default_cache_dir`. + keep_build : bool + Leave the unpacked source tree behind, for debugging a failure. + + Returns + ------- + str + The path of the build log, which is worth showing the user whether or + not the build succeeded. + """ + if component not in SOURCES: + raise ValueError("unknown component %r" % component) + log = log or (lambda line: print(line, flush=True)) + cache_dir = cache_dir or default_cache_dir() + apply_environment_overrides() + spec = SOURCES[component] + prefix = os.path.abspath(prefix) + os.makedirs(prefix, exist_ok=True) + + logdir = os.path.join(prefix, "var", "log", "topoanalysis") + os.makedirs(logdir, exist_ok=True) + logfile = os.path.join(logdir, "build-%s.log" % component) + # Start each attempt with a clean log so a failure message points at the + # failure, not at the tail of some earlier successful run. + open(logfile, "w").close() + + log(" %s %s" % (component, spec["version"])) + tarball = _download(spec["url"], spec["sha256"], cache_dir, log) + + workdir = tempfile.mkdtemp(prefix="topoanalysis-%s-" % component) + try: + source = _extract(tarball, workdir) + env = _build_env(prefix) + builder = { + "sqlite": _build_sqlite, + "proj": _build_proj, + "gdal": _build_gdal, + }[component] + builder(source, prefix, env, _jobs(jobs), log, logfile) + log(" %s installed into %s" % (component, prefix)) + finally: + if keep_build: + log(" build tree kept at %s" % workdir) + else: + shutil.rmtree(workdir, ignore_errors=True) + return logfile + + +def _build_sqlite(source, prefix, env, jobs, log, logfile): + """SQLite's amalgamation tarball: plain configure/make/install. + + Only flags that both of SQLite's build systems understand are used here. + Releases up to 3.48 shipped classic autoconf; 3.49 and later ship + autosetup, which has no ``--enable-shared`` at all (shared is the default + and only the negative form exists). Passing the positive form would make + the newer releases fail on an unknown option. + """ + _run( + [ + os.path.join(source, "configure"), + "--prefix=" + prefix, + "--disable-static", + # PROJ needs the core library and the shell, nothing else. + "--disable-readline", + ], + source, + env, + log, + logfile, + ) + _run(["make", "-j%d" % jobs], source, env, log, logfile) + _run(["make", "install"], source, env, log, logfile) + + +def _build_proj(source, prefix, env, jobs, log, logfile): + build = os.path.join(source, "build") + os.makedirs(build, exist_ok=True) + settings = _cmake_common(prefix) + [ + # Both are only used for the optional high-accuracy grid files, which + # need a network at run time anyway. Leaving them out removes the + # need for libtiff and libcurl development packages. + "-DENABLE_TIFF=OFF", + "-DENABLE_CURL=OFF", + # The command-line tools are not needed by GDAL, but projinfo is + # useful enough when diagnosing a projection problem to keep. + "-DBUILD_PROJSYNC=OFF", + "-DBUILD_TESTING=OFF", + ] + _run(["cmake", "-S", source, "-B", build] + settings, source, env, log, logfile) + _run(["cmake", "--build", build, "-j", str(jobs)], source, env, log, logfile) + _run(["cmake", "--install", build], source, env, log, logfile) + + +def _build_gdal(source, prefix, env, jobs, log, logfile): + build = os.path.join(source, "build") + os.makedirs(build, exist_ok=True) + settings = _cmake_common(prefix) + [ + # The point of the whole exercise: use GDAL's vendored copies of + # libtiff, libgeotiff, libpng, libjpeg, libz and friends rather than + # requiring system development packages we have no way to install. + "-DGDAL_USE_INTERNAL_LIBS=ON", + "-DGDAL_USE_EXTERNAL_LIBS=OFF", + # PROJ is the exception: it is mandatory and we just built it. + "-DGDAL_USE_PROJ=ON", + "-DPROJ_ROOT=" + prefix, + # Bindings are pip's job, against the interpreter we mean to use. + "-DBUILD_PYTHON_BINDINGS=OFF", + "-DBUILD_JAVA_BINDINGS=OFF", + "-DBUILD_CSHARP_BINDINGS=OFF", + "-DGDAL_BUILD_OPTIONAL_DRIVERS=ON", + "-DOGR_BUILD_OPTIONAL_DRIVERS=ON", + ] + _run(["cmake", "-S", source, "-B", build] + settings, source, env, log, logfile) + _run(["cmake", "--build", build, "-j", str(jobs)], source, env, log, logfile) + _run(["cmake", "--install", build], source, env, log, logfile) diff --git a/analysis.py b/analysis.py index a2281ec..8459b82 100644 --- a/analysis.py +++ b/analysis.py @@ -1,10 +1,10 @@ -import numpy as np -import rasterio - -import matplotlib.pyplot as plt +"""Quadrat sampling of a raster.""" from itertools import product +import numpy as np +import matplotlib.pyplot as plt + class Quadrats(object): @@ -22,6 +22,18 @@ def __init__(self, filename=None, data=None, band=1, dx=None, dy=None): self.quadrats = [] def load_data(self, filename, band=1): + """Read a raster band with rasterio. + + rasterio is imported here rather than at module scope so that the + rest of this module works without it. + """ + try: + import rasterio + except ImportError as exc: # pragma: no cover - environment dependent + raise ImportError( + "Quadrats.load_data needs rasterio; install it with " + "`pip install rasterio`, or pass data=... instead." + ) from exc with rasterio.open(filename, 'r') as src: self.data = src.read(band) diff --git a/benchmark.py b/benchmark.py new file mode 100644 index 0000000..dc5b5fa --- /dev/null +++ b/benchmark.py @@ -0,0 +1,121 @@ +"""Compare the compiled and pure-NumPy kernels on a synthetic DEM. + +Run it with:: + + python -m TopoAnalysis.benchmark # 512 x 512 + python -m TopoAnalysis.benchmark 2048 # a bigger grid + python -m TopoAnalysis.benchmark 2048 --skip-python +""" + +from __future__ import annotations + +import argparse +import sys +import time + +import numpy as np + +try: # pragma: no cover - exercised by whichever import style is in use + from . import fastops, kernels +except ImportError: # pragma: no cover + import fastops + import kernels + + +def synthetic_dem(n, seed=1): + """A noisy dome: a radial network with plenty of small depressions.""" + rng = np.random.default_rng(seed) + y, x = np.mgrid[0:n, 0:n] / float(n) + z = (500.0 * np.exp(-((x - 0.5) ** 2 + (y - 0.5) ** 2) / 0.1) + + 25.0 * np.sin(12 * np.pi * x) * np.sin(12 * np.pi * y)) + return z + rng.normal(0.0, 2.0, (n, n)) + + +def _time(label, function, *args, **kwargs): + start = time.perf_counter() + result = function(*args, **kwargs) + elapsed = time.perf_counter() - start + print(' {0:<22s} {1:9.1f} ms'.format(label, elapsed * 1000.0)) + return result, elapsed + + +def run(module, z, cellsize): + """Time the full pipeline with one backend; returns the totals.""" + total = 0.0 + + filled = np.ascontiguousarray(z, dtype=np.float64) + _, dt = _time('fill (epsilon)', module.priority_flood, filled, + mode='epsilon', epsilon=1e-9, cellsize=cellsize) + total += dt + + codes, dt = _time('flow directions', module.flow_directions, filled, + cellsize=cellsize) + total += dt + + receivers, dt = _time('receivers', module.receivers, codes) + total += dt + + (order, _), dt = _time('topological order', module.topological_order, receivers) + total += dt + + weights = np.full(z.shape, cellsize ** 2) + area, dt = _time('accumulate', module.accumulate, receivers, order, weights) + total += dt + + step = np.full(z.shape, cellsize) + (length, main_stem), dt = _time('flow length', module.flow_length, + receivers, order, step) + total += dt + + outlets = np.flatnonzero(np.asarray(receivers).reshape(-1) < 0).astype(np.int64) + _, dt = _time('chi', module.chi, receivers, order, np.asarray(area), step, + outlets, 1e6, 0.45, True, 0.0, None) + total += dt + + print(' {0:<22s} {1:9.1f} ms'.format('TOTAL', total * 1000.0)) + return total + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog='python -m TopoAnalysis.benchmark', + description='Time the TopoAnalysis kernels on a synthetic DEM.') + parser.add_argument('size', nargs='?', type=int, default=512, + help='grid side length in cells (default: 512)') + parser.add_argument('--cellsize', type=float, default=30.0) + parser.add_argument('--skip-python', action='store_true', + help='do not time the pure-NumPy kernels') + args = parser.parse_args(argv) + + n = args.size + z = synthetic_dem(n) + print('{0} x {1} cells ({2:,} total), dx = {3}'.format( + n, n, n * n, args.cellsize)) + + if not kernels.have_extension(): + print('\nThe compiled extension is not built; only the NumPy kernels ' + 'are available.\nBuild it with `pip install -e .` from the ' + 'TopoAnalysis directory.') + print('\nNumPy kernels:') + run(fastops, z, args.cellsize) + return 0 + + print('\nC++ kernels:') + cpp_total = run(kernels, z, args.cellsize) + + if args.skip_python: + return 0 + if n > 1024: + print('\nSkipping the NumPy kernels: they would take minutes at this ' + 'size. Pass a smaller size to compare them.') + return 0 + + print('\nNumPy kernels:') + python_total = run(fastops, z, args.cellsize) + + print('\nSpeed-up: {0:.0f}x'.format(python_total / cpp_total)) + return 0 + + +if __name__ == '__main__': # pragma: no cover + sys.exit(main()) diff --git a/cli.py b/cli.py new file mode 100644 index 0000000..f8da17c --- /dev/null +++ b/cli.py @@ -0,0 +1,179 @@ +"""Command-line entry point: ``topoanalysis-process``. + +Runs the standard pipeline over a DEM and writes the derived grids as +GeoTIFFs beside it:: + + topoanalysis-process srtm.tif --outdir results --theta 0.45 --a0 1e6 + +Use ``topoanalysis-process --info`` to report which compute backend is +active and whether GDAL is available. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time + +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as _dem + from . import kernels + from .__init__ import __version__ +except ImportError: # pragma: no cover + import dem as _dem + import kernels + __version__ = "unknown" + + +#: Derived grids the pipeline can produce, in dependency order. +PRODUCTS = ('filled', 'flowdir', 'area', 'length', 'slope', 'hillshade', + 'curvature', 'chi', 'ksi', 'relief') + + +def _build_parser(): + parser = argparse.ArgumentParser( + prog='topoanalysis-process', + description='Derive flow routing and channel metrics from a DEM.') + parser.add_argument('dem', nargs='?', help='input DEM readable by GDAL') + parser.add_argument('-o', '--outdir', default='.', + help='directory for the output grids (default: .)') + parser.add_argument('-p', '--prefix', default=None, + help="output filename prefix (default: the DEM's basename)") + parser.add_argument('--products', default='filled,flowdir,area,length', + help='comma-separated subset of: ' + ','.join(PRODUCTS)) + parser.add_argument('--theta', type=float, default=0.45, + help='reference concavity for chi and ksi (default: 0.45)') + parser.add_argument('--a0', type=float, default=1e6, + help='reference drainage area in m^2 (default: 1e6)') + parser.add_argument('--min-area', type=float, default=1e6, + help='drainage area defining the channel network, m^2') + parser.add_argument('--geographic', action='store_true', + help='treat the grid as lat/lon and use spherical cell areas') + parser.add_argument('--flat-fill', action='store_true', + help='fill depressions to a flat surface (as TopoToolbox ' + 'does) instead of adding a small gradient') + parser.add_argument('--info', action='store_true', + help='report the active backend and exit') + parser.add_argument('--version', action='version', + version='TopoAnalysis {0}'.format(__version__)) + return parser + + +def _report_environment(): + print('TopoAnalysis {0}'.format(__version__)) + print(' compute backend : {0}'.format(kernels.backend())) + if not kernels.have_extension(): + print(' (build the C++ extension with `pip install -e .` ' + 'for a large speed-up)') + print(' GDAL available : {0}'.format(_dem.gdal_is_available())) + print(' fill algorithm : {0}'.format(kernels.PRIORITY_FLOOD_REFERENCE)) + + +def main(argv=None): + """Run the CLI. Returns a process exit status.""" + parser = _build_parser() + args = parser.parse_args(argv) + + if args.info: + _report_environment() + return 0 + + if not args.dem: + parser.error('a DEM is required unless --info is given') + + requested = [p.strip() for p in args.products.split(',') if p.strip()] + unknown = [p for p in requested if p not in PRODUCTS] + if unknown: + parser.error('unknown product(s): {0}'.format(', '.join(unknown))) + + os.makedirs(args.outdir, exist_ok=True) + prefix = args.prefix or os.path.splitext(os.path.basename(args.dem))[0] + + def out(name): + return os.path.join(args.outdir, '{0}_{1}.tif'.format(prefix, name)) + + elevation_cls = _dem.GeographicElevation if args.geographic else _dem.Elevation + area_cls = _dem.GeographicArea if args.geographic else _dem.Area + length_cls = _dem.GeographicFlowLength if args.geographic else _dem.FlowLength + ksi_cls = _dem.GeographicKsi if args.geographic else _dem.Ksi + + started = time.time() + + def step(label): + print('[{0:6.1f}s] {1}'.format(time.time() - started, label)) + sys.stdout.flush() + + step('reading {0}'.format(args.dem)) + elevation = elevation_cls(gdal_filename=args.dem) + print(' {0} x {1} cells, dx = {2}'.format( + elevation._georef_info.nx, elevation._georef_info.ny, elevation._georef_info.dx)) + + step('filling depressions') + fill_kwargs = {'elevation': elevation} + if args.flat_fill: + fill_kwargs['aggradation_slope'] = 0.0 + filled = _dem.FilledElevation(**fill_kwargs) + if 'filled' in requested: + filled.save(out('filled')) + + step('routing flow') + flow_direction = _dem.FlowDirectionD8(flooded_dem=filled) + if 'flowdir' in requested: + flow_direction.save(out('flowdir')) + + area = None + if {'area', 'chi', 'ksi', 'relief'} & set(requested): + step('accumulating drainage area') + area = area_cls(flow_direction=flow_direction) + if 'area' in requested: + area.save(out('area')) + + flow_length = None + if {'length', 'ksi', 'relief'} & set(requested): + step('measuring flow length') + flow_length = length_cls(flow_direction=flow_direction) + if 'length' in requested: + flow_length.save(out('length')) + + if 'slope' in requested: + step('computing slope') + slope_cls = _dem.GeographicMaxSlope if args.geographic else _dem.MaxSlope + slope_cls(elevation=elevation).save(out('slope')) + + if 'hillshade' in requested: + step('computing hillshade') + hs_cls = _dem.GeographicHillshade if args.geographic else _dem.Hillshade + hs_cls(elevation=elevation, azimuth=315, inclination=45).save(out('hillshade')) + + if 'curvature' in requested: + step('computing curvature') + lap_cls = _dem.GeographicLaplacian if args.geographic else _dem.Laplacian + lap_cls(elevation=elevation).save(out('curvature')) + + if 'chi' in requested: + step('integrating chi') + outlets = area.areas_greater_than(args.min_area) + edge_outlets = [xy for xy in outlets + if not flow_direction.get_flow_to_cell( + *flow_direction._xy_to_rowscols((xy,))[0])[2]] + _dem.Chi(area=area, flow_direction=flow_direction, theta=args.theta, + Ao=args.a0, outlets=edge_outlets or outlets).save(out('chi')) + + if 'ksi' in requested: + step('integrating ksi') + ksi_cls(area=area, flow_direction=flow_direction, flow_length=flow_length, + theta=args.theta, Ao=args.a0).save(out('ksi')) + + if 'relief' in requested: + step('computing scaled relief') + _dem.ScaledRelief(flow_direction=flow_direction, elevation=elevation, + flow_length=flow_length, Ao=args.a0, + theta=args.theta, area=area).save(out('relief')) + + step('done') + return 0 + + +if __name__ == '__main__': # pragma: no cover + raise SystemExit(main()) diff --git a/datasets.py b/datasets.py index 1280c4b..d0f3c74 100644 --- a/datasets.py +++ b/datasets.py @@ -6,7 +6,10 @@ from scipy.signal import sawtooth -from dem import Elevation +try: # pragma: no cover - exercised by whichever import style is in use + from .dem import Elevation +except ImportError: # pragma: no cover + from dem import Elevation def triangle_grid(ny, nx, width, amp=1, sig=0, slope_y=None): @@ -45,12 +48,11 @@ def triangle_grid(ny, nx, width, amp=1, sig=0, slope_y=None): if slope_y: y = np.linspace(0, ny, num=ny).reshape(ny, 1) X = np.tile(y, (1, nx)) - tilt = slope_y * X - triangle *= tilt + # Added, not multiplied: a slope is a ramp superimposed on the + # relief, whereas multiplying scaled the relief and zeroed row 0. + triangle = triangle + slope_y * X - out_obj = Elevation(nx = nx, ny = ny, dx = 1.0) - out_obj._griddata = triangle - return out_obj + return Elevation(dx=1.0, grid=triangle) def sinusoid_grid(ny, nx, width, amp=1, sig=0, slope_y=None): @@ -89,9 +91,6 @@ def sinusoid_grid(ny, nx, width, amp=1, sig=0, slope_y=None): if slope_y: y = np.linspace(0, ny, num=ny).reshape(ny, 1) X = np.tile(y, (1, nx)) - tilt = slope_y * X - sinusoid *= tilt + sinusoid = sinusoid + slope_y * X - out_obj = Elevation(nx = nx, ny = ny, dx = 1.0) - out_obj._griddata = sinusoid - return out_obj + return Elevation(dx=1.0, grid=sinusoid) diff --git a/dem.py b/dem.py index 15119b6..d685a33 100644 --- a/dem.py +++ b/dem.py @@ -1,30 +1,184 @@ -#Functions for demMethods calculations typical of geomorphology -#Developed by Sam Johnstone, January 2015, samuelj@stanford.edu , johnsamstone@gmail.com -#Not all functions are benchmarked -#If you somehow have this, and have never spoken to me, please reach out -#- I'd be curious to hear what you are doing (maybe I can even help with something!) - -from osgeo import gdal, ogr, osr # Used to load gis in files -import os # Used to join file paths, iteract with os in other ways.. -import glob # Used for finding files that I want to mosaic (by allowing wildcard searches of the filesystem) -import heapq # Used for constructing priority queue, which is used for filling dems -import numpy as np # Used for tons o stuff, keeping most data stored as numpy arrays -import subprocess # Used to run gdal_merge.py from the command line -from numpy import uint8, int8, float64 -from matplotlib import pyplot as plt +"""Raster grids and DEM analysis for tectonic geomorphology. + +Originally developed by Sam Johnstone (January 2015); maintained since by the +Stanford geomorphology group. + +Every raster is a :class:`BaseSpatialGrid` or one of its subclasses. Grids +are constructed by keyword, and the keywords you supply select how the grid is +built -- see :attr:`BaseSpatialGrid.required_inputs_and_actions`:: + + elevation = Elevation(gdal_filename='dem.tif') + filled = FilledElevation(elevation=elevation) + d8 = FlowDirectionD8(flooded_dem=filled) + area = Area(flow_direction=d8) + +Conventions +----------- +* ``_griddata`` is a 2-D NumPy array with row 0 at the **north** edge. +* ``NaN`` marks no-data in floating-point grids. +* D8 flow-direction codes follow the ArcGIS convention:: + + | 32 64 128 | + | 16 X 1 | + | 8 4 2 | + +* Depression filling uses the Priority-Flood algorithms of Barnes, Lehman and + Mulla (2014); see :mod:`TopoAnalysis.kernels`. +""" + +import glob # Used for finding files that I want to mosaic +import os +import subprocess # Used to run gdal_merge.py from the command line import sys -from scipy import stats -from . import error as Error +import warnings -try: - import statsmodels.api as sm -except: - print('Warning: no statsmodels present. If you want to compute steepness, you will need to install this.') +import numpy as np +from numpy import float64, uint8 +from matplotlib import pyplot as plt + +# These modules are importable both as a package (``from TopoAnalysis import +# dem``) and flat (``import dem`` with this directory on sys.path). The flat +# form is how the original scripts in this repository load them, so it has to +# keep working. +try: # pragma: no cover - exercised by whichever import style is in use + from . import error as Error + from . import kernels +except ImportError: # pragma: no cover + import error as Error + import kernels + + +class _LazyGDALModule(object): + """Defer ``from osgeo import ...`` until something actually touches GDAL. + + GDAL is only needed to read and write files. Importing it lazily keeps + the rest of the library usable -- and testable -- where GDAL is absent, + which is common because it cannot be installed reliably from PyPI. When + it really is needed and really is missing, the error says what to do + about it instead of surfacing as a bare ``ImportError`` at import time. + """ + + __slots__ = ("_submodule", "_module") + + def __init__(self, submodule): + self._submodule = submodule + self._module = None + + def _load(self): + if self._module is None: + # Point GDAL and PROJ at the data files of a prefix TopoAnalysis + # built for itself, if there is one. A no-op otherwise, and it + # never overrides a setting the user made. + try: + from . import gdal_setup + except ImportError: # pragma: no cover - flat import style + try: + import gdal_setup + except ImportError: + gdal_setup = None + if gdal_setup is not None: + gdal_setup.activate_environment() + + try: + from osgeo import gdal, ogr, osr + except ImportError as exc: # pragma: no cover - environment dependent + raise ImportError( + "TopoAnalysis needs the GDAL Python bindings to read and " + "write raster files, and they are not installed.\n" + "\n" + " topoanalysis-gdal install\n" + "\n" + "installs both the GDAL C library and the bindings, " + "choosing whichever route suits this machine; run " + "`topoanalysis-gdal doctor` first to see what it would " + "do. Grids built in memory (dx=..., grid=...) do not " + "require GDAL at all." + ) from exc + # Without this GDAL signals failure by returning None, which turns + # a bad file path into an AttributeError three frames later. + gdal.UseExceptions() + ogr.UseExceptions() + osr.UseExceptions() + self._module = {"gdal": gdal, "ogr": ogr, "osr": osr}[self._submodule] + return self._module + + def __getattr__(self, name): + return getattr(self._load(), name) + + +gdal = _LazyGDALModule("gdal") +ogr = _LazyGDALModule("ogr") +osr = _LazyGDALModule("osr") + + +def gdal_is_available(): + """True when the GDAL Python bindings can be imported.""" + try: + gdal._load() + except ImportError: + return False + return True + + +def _require_statsmodels(): + """Import statsmodels on demand; only the regression grids need it.""" + try: + import statsmodels.api as sm + except ImportError as exc: # pragma: no cover - depends on the environment + raise ImportError( + "Fitting channel steepness needs statsmodels. Install it with " + "`pip install statsmodels`." + ) from exc + return sm + + +SQRT2 = np.sqrt(2.0) + + +class _Progress(object): + """Percent-complete ticker for the long per-cell regression loops. + + Prints ``Percent completion... 10...20...`` on one line and finishes with + ``100``. Pass ``verbose=False`` to silence it. + """ + + def __init__(self, total, verbose=True, stream=None): + self.total = int(total) + self.verbose = verbose and self.total > 0 + self.stream = stream if stream is not None else sys.stdout + self.count = 0 + self.next_readout = 10 + if self.verbose: + self.stream.write('Percent completion...') + self.stream.flush() + + def tick(self): + if not self.verbose: + return + self.count += 1 + # Integer arithmetic: accumulating 1.0/total in a float drifted and + # could skip or repeat a readout. + percent = (100 * self.count) // self.total + while percent >= self.next_readout and self.next_readout < 100: + self.stream.write(str(self.next_readout) + '...') + self.stream.flush() + self.next_readout += 10 + + def done(self): + if not self.verbose: + return + self.stream.write('100\n') + self.stream.flush() -sys.setrecursionlimit(1000000) - class GDALMixin(object): + """Reading and writing rasters through GDAL. + + Mixed into :class:`BaseSpatialGrid`; nothing here is meant to be called + directly. Every method reaches GDAL through the module-level lazy + proxies, so the mixin can be present without GDAL being installed. + """ + def _get_projection_from_EPSG_projection_code(self, EPSGprojectionCode): # Get raster projection @@ -76,19 +230,33 @@ def _get_numpy_type_for_gdal_type(self, gdal_type): def _readGDALFile(self, filename, dtype): gdal_file = gdal.Open(filename) - geoTransform, nx, ny, data = self._read_GDAL_dataset(gdal_file, dtype) - gdal_file = None - return geoTransform, nx, ny, data - + try: + return self._read_GDAL_dataset(gdal_file, dtype) + finally: + # GDAL only flushes and releases the file handle when the last + # reference goes away, so drop it explicitly rather than waiting + # for the garbage collector. + gdal_file = None + + @staticmethod + def _read_band(band, dtype): + """Read one band, converting its no-data value to ``NaN``. + + Integer grids have no NaN to convert to, so their no-data value is + left as-is. + """ + data = band.ReadAsArray().astype(dtype) + nodata = band.GetNoDataValue() + if nodata is not None and np.issubdtype(np.dtype(dtype), np.floating): + if np.isnan(nodata): + pass # already NaN in the array + else: + data[data == nodata] = np.nan + return data + def _read_GDAL_dataset(self, gdal_dataset, dtype): band = gdal_dataset.GetRasterBand(1) - nodata = band.GetNoDataValue() - data = band.ReadAsArray().astype(dtype) - nodata_elements = np.where(data == nodata) - from numpy import uint8 - if dtype is not uint8: - data[nodata_elements] = np.NAN - + data = self._read_band(band, dtype) geoTransform = gdal_dataset.GetGeoTransform() nx = gdal_dataset.RasterXSize ny = gdal_dataset.RasterYSize @@ -106,6 +274,7 @@ def _getGeoRefInfo(self, gdalDataset): return nx, ny, projection, geoTransform def getDEMcoords(self, GdalData, dx): + """Cell-centre x and y coordinate lists for an open GDAL dataset.""" #Get grid size nx, ny = GdalData.RasterXSize, GdalData.RasterYSize @@ -121,30 +290,44 @@ def getDEMcoords(self, GdalData, dx): return xcoordinates, ycoordinates - def _create_gdal_representation_from_array(self, georef_info, GDALDRIVERNAME, array_data, dtype, outfile_path='name', dst_options = [], multiple_bands = False): - #A function to write the data in the numpy array arrayData into a georeferenced dataset of type - # specified by GDALDRIVERNAME, a string, options here: http://www.gdal.org/formats_list.html - # This is accomplished by copying the georeferencing information from an existing GDAL dataset, - # provided by createDataSetFromArray - - #Initialize new data - drvr = gdal.GetDriverByName(GDALDRIVERNAME) # Get the desired driver - bands = 1 if multiple_bands is False else len(array_data) - outRaster = drvr.Create(outfile_path, georef_info.nx, georef_info.ny, bands , self._get_gdal_type_for_numpy_type(dtype), dst_options) # Open the file - - #Write geographic information - outRaster.SetGeoTransform(georef_info.geoTransform) # Steal the coordinate system from the old dataset - if georef_info.projection != 0: - outRaster.SetProjection(georef_info.projection) # Steal the Projections from the old dataset - - #Write the array - if not multiple_bands: - outRaster.GetRasterBand(1).WriteArray(array_data) # Writes my array to the raster - else: - for i in range(len(array_data)): - print('writing band ' + str(i) + "/" + str(bands)) - outRaster.GetRasterBand(i+1).WriteArray(array_data[i]) - + def _create_gdal_representation_from_array(self, georef_info, GDALDRIVERNAME, array_data, dtype, + outfile_path='name', dst_options=None, + multiple_bands=False, verbose=False): + """Write a NumPy array out as a georeferenced GDAL dataset. + + Driver names are listed at https://gdal.org/drivers/raster/. The + returned dataset is still open; let it go out of scope (or set it to + ``None``) to flush it to disk. + """ + if dst_options is None: + dst_options = [] + + drvr = gdal.GetDriverByName(GDALDRIVERNAME) + if drvr is None: + raise Error.InputError( + 'GDAL driver', "no GDAL driver named '{0}'".format(GDALDRIVERNAME)) + + bands = len(array_data) if multiple_bands else 1 + outRaster = drvr.Create(outfile_path, georef_info.nx, georef_info.ny, bands, + self._get_gdal_type_for_numpy_type(dtype), dst_options) + + outRaster.SetGeoTransform(georef_info.geoTransform) + # An unset projection is stored as the integer 0 by Georef_info. + if georef_info.projection != 0 and georef_info.projection is not None: + outRaster.SetProjection(georef_info.projection) + + is_float = np.issubdtype(np.dtype(dtype), np.floating) + band_arrays = array_data if multiple_bands else [array_data] + for index, band_array in enumerate(band_arrays): + if verbose: + print('writing band {0}/{1}'.format(index + 1, bands)) + band = outRaster.GetRasterBand(index + 1) + if is_float: + # Recording NaN as the no-data value means other GIS software + # renders the holes as holes instead of as real elevations. + band.SetNoDataValue(float('nan')) + band.WriteArray(np.asarray(band_array)) + return outRaster def _clipRasterToRaster(self, input_gdal_dataset, clipping_gdal_dataset, dtype): @@ -252,38 +435,59 @@ def _getRasterGeoTransformFromAsciiRaster(self, fileName): return (xUL, dx, 0, yUL, 0, -dx), nx, ny - def _writeArcAsciiRaster(self, georef_info, outfile_path, np_array_data, nodata_value, format_string): - #A function to write the data stored in the numpy array npArrayData to a ArcInfo Text grid. Gdal doesn't - #allow creation of these types of data for whatever reason - + def _writeArcAsciiRaster(self, georef_info, outfile_path, np_array_data, nodata_value, + format_string): + """Write an ArcInfo ASCII grid. + + GDAL's AAIGrid driver cannot ``Create()`` -- only ``CreateCopy()`` -- + so the file is written directly. ``NaN`` cells are replaced by + ``nodata_value`` on the way out, because most GIS software does not + recognise the literal text ``nan`` in an ASCII grid. + """ + data = np.asarray(np_array_data) + if np.issubdtype(data.dtype, np.floating): + if nodata_value is None or (isinstance(nodata_value, float) and np.isnan(nodata_value)): + nodata_value = -9999.0 + data = np.where(np.isnan(data), nodata_value, data) + header = "ncols %s\n" % georef_info.nx header += "nrows %s\n" % georef_info.ny header += "xllcenter %s\n" % georef_info.xllcenter header += "yllcenter %s\n" % georef_info.yllcenter header += "cellsize %s\n" % georef_info.dx header += "NODATA_value %s" % nodata_value - - np.savetxt(outfile_path, np_array_data, header=header, fmt=format_string, comments='') - - def _asciiRasterToMemory(self, fileName): - # the geotransfrom structured as (xUL, dx, skewX, yUL, scewY, -dy) + np.savetxt(outfile_path, data, header=header, fmt=format_string, comments='') + + def _asciiRasterToMemory(self, fileName): + """Read an ArcInfo ASCII grid, honouring its ``NODATA_value``.""" + # The geotransform is (xUL, dx, skewX, yUL, skewY, -dy). It is parsed + # from the header rather than taken from GDAL so that grids written + # with xllcenter (rather than xllcorner) land in the right place. gt, nx, ny = self._getRasterGeoTransformFromAsciiRaster(fileName) - - #Open gdal dataset + ds = gdal.Open(fileName) - - #Prep output - memDrv = gdal.GetDriverByName('MEM') # Create a gdal driver in memory - dataOut = memDrv.CreateCopy('name', ds, 0) #Copy data to gdal driver - - data = dataOut.ReadAsArray() - dataOut = None - - data = self.dtype(data) + try: + # Ignoring NODATA_value here is what used to turn -9999 flags into + # real elevations several thousand metres below sea level. + data = self._read_band(ds.GetRasterBand(1), self.dtype) + finally: + ds = None + return gt, nx, ny, data class GeographicGridMixin(object): + """Spherical geometry for grids in degrees of latitude and longitude. + + Mix it in *before* the grid class to override the cell area and cell + dimension with their true, latitude-dependent values:: + + class GeographicArea(GeographicGridMixin, Area): + pass + + Every ``Geographic*`` class in this module is exactly that. + """ + def _getUTMZone(self, dataset): #Function to get the approximate UTM zone %NOTE: I need to check how east and west are handled... @@ -335,40 +539,56 @@ def _approximateDxFromGeographicData(self, geoTransform): return int(dTheta*metersPerDegree) #convert degrees to meters + #: Earth radius in metres used for spherical cell areas. TopoToolbox's + #: ``cellarea`` instead derives its radius from a total surface area of + #: 510 072 000 km^2, i.e. 6 371 047 m, which makes its areas larger by + #: about 1.5e-5 relative. Override this attribute to match exactly. + earth_radius = 6371.0 * 1000.0 + def _area_per_pixel(self, *args, **kwargs): + """Area of each cell in square metres, on a sphere. - if hasattr(self, '_GeographicGridMixin__app'): + Cells shrink towards the poles, so this varies with row but not with + column. The result is cached because the trigonometry is not free + and several derived grids ask for it. + """ + if getattr(self, '_GeographicGridMixin__app', None) is not None: return self._GeographicGridMixin__app - - #Returns a grid of the area of each pixel within the domain specified by the gdalDataset - - re = 6371.0 * 1000.0 #radius of the earth in meters - + + re = self.earth_radius + dLong = np.abs(self._georef_info.geoTransform[1]) dLat = np.abs(self._georef_info.geoTransform[5]) - - lats = self._georef_info.yllcenter + np.float64(range(self._georef_info.ny))*dLat - longs = self._georef_info.xllcenter + np.float64(range(self._georef_info.nx))*dLong - - #Get the size of pixels in the lat and long direction + + # np.float64(range(n)) is a TypeError on modern NumPy -- np.float64 is + # a scalar type, not an array constructor. + lats = self._georef_info.yllcenter + np.arange(self._georef_info.ny, dtype=float64) * dLat + longs = self._georef_info.xllcenter + np.arange(self._georef_info.nx, dtype=float64) * dLong [LONG, LAT] = np.meshgrid(longs, lats) LAT1 = np.radians(LAT - dLat / 2.0) LAT2 = np.radians(LAT + dLat / 2.0) LONG1 = np.radians(LONG - dLong / 2.0) LONG2 = np.radians(LONG + dLong / 2.0) - + + # lats ascends northwards; flip so row 0 is the north edge again. initialAreas = np.flipud(np.abs((re**2)*(LONG1 - LONG2)*(np.sin(LAT2) - np.sin(LAT1)))) - + self.__app = initialAreas return self.__app - + def _mean_pixel_dimension(self, *args, **kwargs): - + """Square root of the cell area: an equivalent isotropic cell size.""" return np.sqrt(self._area_per_pixel()) class CalculationMixin(object): + """Finite-difference terrain derivatives. + + Supplies the slope, curvature and boundary-padding helpers that + :class:`Elevation`, :class:`Hillshade` and :class:`MaxSlope` share. + """ + def _calcFiniteSlopes(self, grid, dx, nx, ny): # sx,sy = calcFiniteDiffs(elevGrid,dx) @@ -388,6 +608,11 @@ def _calcFiniteSlopes(self, grid, dx, nx, ny): return Sx, Sy def assignBCs(self, grid, nx, ny): + """Pad a grid by one cell, replicating the edge values. + + Returns an array two rows and two columns larger, so that a centred + difference can be taken everywhere including the perimeter. + """ # Pads the boundaries of a grid # Boundary condition pads the boundaries with equivalent values # to the data margins, e.g. x[-1,1] = x[1,1] @@ -406,11 +631,12 @@ def assignBCs(self, grid, nx, ny): Zbc[0, 0] = grid[0, 0] Zbc[0, -1] = grid[0, -1] Zbc[-1, 0] = grid[-1, 0] - Zbc[-1, -1] = grid[-1, 0] - + Zbc[-1, -1] = grid[-1, -1] + return Zbc def calcFiniteCurv(self, grid, dx): + """Laplacian from a 5-point centred stencil, same shape as the input.""" #C = calcFiniteCurv(elevGrid, dx) #calculates finite differnces in X and Y direction using the centered difference method. #Applies a boundary condition such that the size and location of the grids in is the same as that out. @@ -425,6 +651,11 @@ def calcFiniteCurv(self, grid, dx): return Cx+Cy def calcContourCurvature(self, grid,dx): + """Contour (plan) curvature: the curvature of the contour lines. + + Positive where contours are convex downslope, i.e. on noses; negative + in hollows. The one-cell border comes back as ``NaN``. + """ # kt = (fxx*fy^2 - 2*fxyfxfy + fyy*fx^2)/((fx^2 + fy^2)*sqrt((fx^2 + fy^2)+1) #Preallocate @@ -448,20 +679,41 @@ def calcContourCurvature(self, grid,dx): return Kt def calcAverageSlopeOfGridSubset(self, gridSubset,dx): + """Least-squares plane through a block of cells; returns its slopes. + + Returns ``(dz/dx, dz/dy)`` of the best-fit plane ``z = ax + by + c``. + """ ## Sx,Sy = calcAverageSlopeOfGridSubset(numpy matrix, dx) #Compute the average slope over a subset of a grid (or a whole grid if you're into that), #by fitting a plane to the elevation data stored in grid subset - nx,ny = len(gridSubset[1,:]), len(gridSubset[:,1]) + ny, nx = gridSubset.shape xs = (0.5+np.arange(nx))*dx - (nx*dx)/2.0 ys = (0.5+np.arange(ny))*dx - (ny*dx)/2.0 X,Y = np.meshgrid(xs,ys) #Fit a plane of the form z = ax + by + c, where beta = [a b c] - M=np.vstack((X.flatten(),Y.flatten(),np.ones((1,nx*ny)))).T - beta = np.linalg.lstsq(M,gridSubset.flatten())[0] - - return beta[0], beta[1] #Return the slope in the x and y directions respecticely + M=np.vstack((X.flatten(),Y.flatten(),np.ones(nx*ny))).T + beta = np.linalg.lstsq(M, gridSubset.flatten(), rcond=None)[0] + + return beta[0], beta[1] #Return the slope in the x and y directions respectively class Georef_info(object): + """Where a grid sits on the ground. + + Attributes + ---------- + geoTransform : tuple + The GDAL 6-tuple ``(x_origin, dx, 0, y_origin, 0, -dy)``, whose + origin is the *outer corner* of the north-west cell. + projection : str or int + WKT, or the integer 0 when no projection is set. + dx : float + Cell size in map units. Cells are assumed square. + xllcenter, yllcenter : float + Centre of the south-west cell. + nx, ny : int + Columns and rows. + """ + def __init__(self): self.geoTransform = 0 self.projection = 0 @@ -482,13 +734,23 @@ def __deepcopy__(self, memo): class BaseSpatialShape(object): - # Wrapper for GDAL shapes. + """A vector layer, used to cut rasters to a boundary. + + :: + + shape = BaseSpatialShape(shapefile_name='basin.shp') + mask = shape.createMaskFromShape(dem._georef_info, + dem._georef_info.projection, + gdal.GDT_Byte) + """ + def __init__(self, *args, **kwargs): if kwargs.get('shapefile_name') is None: raise Error.InputError('Input Error', 'Inputs not satisfied') self.shapedata = ogr.Open(kwargs.get('shapefile_name')) def createMaskFromShape(self, geoRefInfo, projection, dtype, noDataValue = 0): + """Rasterise the shape onto ``geoRefInfo``'s frame, 1 inside and 0 out.""" #Open Shapefile source_ds = self.shapedata @@ -507,7 +769,28 @@ def createMaskFromShape(self, geoRefInfo, projection, dtype, noDataValue = 0): class BaseSpatialGrid(GDALMixin): - + """A georeferenced raster, and the base class of every grid here. + + Construct one by keyword; the combination you pass selects how it is + built (see :attr:`required_inputs_and_actions`):: + + BaseSpatialGrid(gdal_filename='grid.tif') + BaseSpatialGrid(dx=30.0, grid=array) + BaseSpatialGrid(nx=100, ny=80, dx=30.0) # random values + BaseSpatialGrid(nx=100, ny=80, projection=wkt, geo_transform=gt) + + The data live in ``_griddata``, a 2-D NumPy array with row 0 at the + **north** edge and ``NaN`` for no-data; the georeferencing lives in + ``_georef_info`` (a :class:`Georef_info`). + + Indexing is by ``(row, col)`` and is forgiving: reading outside the grid + returns ``None`` and writing outside it is ignored, which is what the + neighbourhood traversals in this module rely on. + """ + + #: Keyword combinations this class accepts, each paired with the method + #: that builds the grid from them. Subclasses extend the list; the + #: entries here are available to every grid class. required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), @@ -527,57 +810,88 @@ def __deepcopy__(self, memo): return result def __init__(self, *args, **kwargs): - + super(BaseSpatialGrid,self).__init__() self._georef_info = Georef_info() self._sorted = False - + if len(kwargs.keys()) == 0: return - + evaluative_action = self.__get_evaluative_action(*args, **kwargs) - - if evaluative_action == None: - raise Error.InputError('Input Error', 'Inputs not satisfied') - - eval('self.' + evaluative_action + '(*args, **kwargs)') - + + if evaluative_action is None: + raise Error.InputError( + 'Input Error', + '{0} could not be built from keywords {1}. Supported ' + 'combinations are: {2}'.format( + self.__class__.__name__, + sorted(kwargs), + '; '.join(', '.join(keys) + for keys, _ in self.required_inputs_and_actions))) + + # Resolve the method by name rather than by building and executing a + # source string: the action comes from class data, but compiling text + # to call a method is both slower and a needless way to run code. + getattr(self, evaluative_action)(*args, **kwargs) + def __setitem__(self, key, value): + """Set a cell, ignoring writes that fall outside the grid. + + Out-of-range writes are silently dropped rather than raising: several + of the neighbourhood traversals in this module rely on being able to + write past an edge without checking first. + """ i, j = key - if i < 0 or j < 0 or i > self._georef_info.ny or j > self._georef_info.nx: + if i < 0 or j < 0 or i >= self._georef_info.ny or j >= self._georef_info.nx: return self._griddata[i, j] = value - + def __getitem__(self, key): + """Return a cell value, or ``None`` if the index is outside the grid. + + The ``None`` return -- rather than an ``IndexError`` -- is part of the + interface: neighbourhood code throughout this module tests the result + against ``None`` instead of bounds-checking every access. + """ i, j = key i = int(i) j = int(j) - - if i >= 0 and j >= 0 and i < self._georef_info.ny and j < self._georef_info.nx: - try: - return self._griddata[i, j] - except: - print('error: '+ str(i) + str(j)) - + + if 0 <= i < self._georef_info.ny and 0 <= j < self._georef_info.nx: + return self._griddata[i, j] + return None - + def __get_evaluative_action(self, *args, **kwargs): - - for required_input_set, evaluative_action in self.required_inputs_and_actions: - these_kw = set(kwargs.keys()) - required_kw = set(required_input_set) - if required_kw.issubset(these_kw): - return evaluative_action - + """Pick the constructor that the supplied keywords satisfy.""" + these_kw = set(kwargs.keys()) + + # Subclass-specific combinations win, then the generic ones every + # grid supports. Falling back to the base list is what lets + # `Mask(dx=..., grid=...)` and friends be built in memory without + # every subclass having to restate the common entries. + for source in (self.required_inputs_and_actions, + BaseSpatialGrid.required_inputs_and_actions): + for required_input_set, evaluative_action in source: + if set(required_input_set).issubset(these_kw): + return evaluative_action + return None def __populate_georef_info_using_geoTransform(self, nx, ny): - self._georef_info.dx = self._georef_info.geoTransform[1] - self._georef_info.xllcenter = self._georef_info.geoTransform[0]+self._georef_info.dx/2.0 - self._georef_info.yllcenter = self._georef_info.geoTransform[3]-(self._georef_info.dx*(self._georef_info.ny-0.5)) + """Derive dx and the south-west cell centre from the geotransform.""" + # nx and ny are assigned FIRST: yllcenter depends on ny, and reading + # self._georef_info.ny before setting it picked up the default of 0, + # which put the grid ny*dx too far north for anyone building one with + # nx/ny/projection/geo_transform. self._georef_info.nx = nx self._georef_info.ny = ny + self._georef_info.dx = self._georef_info.geoTransform[1] + self._georef_info.xllcenter = self._georef_info.geoTransform[0] + self._georef_info.dx / 2.0 + self._georef_info.yllcenter = (self._georef_info.geoTransform[3] + - self._georef_info.dx * (ny - 0.5)) def _create(self, *args, **kwargs): @@ -589,23 +903,44 @@ def _create(self, *args, **kwargs): else: self._griddata = kwargs.get('grid') + def _geoTransform_from_corner(self): + """Derive the GDAL geotransform from dx and the lower-left centre. + + Grids built in memory used to leave ``geoTransform`` at its default + of 0, which made ``save()`` fail with an unhelpful GDAL error. + """ + dx = self._georef_info.dx + return (self._georef_info.xllcenter - 0.5 * dx, + dx, + 0.0, + self._georef_info.yllcenter + (self._georef_info.ny - 0.5) * dx, + 0.0, + -dx) + def _create_random_grid(self, *args, **kwargs): self._georef_info.dx = kwargs['dx'] self._georef_info.nx = kwargs['nx'] self._georef_info.ny = kwargs['ny'] - self._georef_info.xllcenter = 0 - self._georef_info.yllcenter = 0 + self._georef_info.xllcenter = kwargs.get('xllcenter', 0.0) + self._georef_info.yllcenter = kwargs.get('yllcenter', 0.0) + self._griddata = np.zeros((self._georef_info.ny, self._georef_info.nx), + dtype=self.dtype) self._randomize_grid_values(*args, **kwargs) + self._georef_info.geoTransform = self._geoTransform_from_corner() def _create_from_grid(self, *args, **kwargs): self._georef_info.dx = kwargs['dx'] - (ny, nx) = kwargs['grid'].shape + grid = np.asarray(kwargs['grid']) + (ny, nx) = grid.shape self._georef_info.nx = nx self._georef_info.ny = ny - self._georef_info.xllcenter = 0 - self._georef_info.yllcenter = 0 - self._griddata = kwargs['grid'] - + self._georef_info.xllcenter = kwargs.get('xllcenter', 0.0) + self._georef_info.yllcenter = kwargs.get('yllcenter', 0.0) + self._griddata = grid + self._georef_info.geoTransform = self._geoTransform_from_corner() + if kwargs.get('projection') is not None: + self._georef_info.projection = kwargs['projection'] + def _randomize_grid_values(self, *args, **kwargs): if kwargs.get('mask') is not None: i = np.where(kwargs.get('mask')._griddata == 1) @@ -658,7 +993,10 @@ def _xy_to_rowscols(self, v): col = int(round((x-self._georef_info.xllcenter)/self._georef_info.dx)) row = int((self._georef_info.ny - 1) - round((y-self._georef_info.yllcenter)/self._georef_info.dx)) - if col > self._georef_info.nx or row > self._georef_info.ny or col < 0 or row < 0: + # ``>=``, not ``>``: a point one cell past the right or bottom + # edge used to come back as a valid subscript and then raise an + # IndexError somewhere else entirely. + if col >= self._georef_info.nx or row >= self._georef_info.ny or col < 0 or row < 0: l.append((None, None)) else: l.append((row,col)) @@ -678,113 +1016,209 @@ def _area_per_pixel(self, *args, **kwargs): def _mean_pixel_dimension(self, *args, **kwargs): return self._georef_info.dx * np.ones_like(self._griddata, dtype = float64) + def coordinate_vectors(self): + """``(x, y)`` coordinates of the cell centres, west-to-east and + south-to-north. + + Returns + ------- + (ndarray, ndarray) + ``x`` has ``nx`` entries and ``y`` has ``ny`` entries. Note that + ``y`` ascends northwards, i.e. ``y[0]`` is the *last* row of + ``_griddata``. + """ + info = self._georef_info + x = info.xllcenter + np.arange(info.nx) * info.dx + y = info.yllcenter + np.arange(info.ny) * info.dx + return x, y + def get_XY_matricies(self): - xllc, yllc, nx, ny, dx = (self._georef_info.xllcenter, self._georef_info.yllcenter, self._georef_info.nx, self._georef_info.ny, self._georef_info.dx) - x = np.arange(xllc, xllc+(nx-1)*dx, dx) - y = np.arange(yllc, yllc+(ny-1)*dx, dx) - return np.meshgrid(x,y) + """Meshgrid of cell-centre coordinates. - def resample(self, de, interpolation = 'quintic'): - return_grid = self.__class__() - return_grid._copy_info_from_grid(self, set_zeros=True) + The name's spelling is preserved for backward compatibility; + :meth:`get_XY_matrices` is the same method. + """ + # np.arange(start, start + (n-1)*dx, dx) yields n-1 points, not n, so + # the returned grids used to be one column and one row short. + x, y = self.coordinate_vectors() + return np.meshgrid(x, y) + + #: Correctly spelled alias for :meth:`get_XY_matricies`. + get_XY_matrices = get_XY_matricies + + def resample(self, de, interpolation='quintic'): + """Resample onto a grid of spacing ``de``. + + Parameters + ---------- + de : float + Target cell size, in the grid's own units. + interpolation : str + ``'nearest'``, ``'linear'``, ``'cubic'`` or ``'quintic'``. The + last two use spline interpolation of order 3 and 5. + + Returns + ------- + BaseSpatialGrid + A new grid of the same class. + """ import copy - geoTransform = (self._georef_info.geoTransform[0], - np.sign(self._georef_info.geoTransform[1])*de, - self._georef_info.geoTransform[2], - self._georef_info.geoTransform[3], - self._georef_info.geoTransform[4], - np.sign(self._georef_info.geoTransform[5])*de) - - georef = copy.deepcopy(self._georef_info) - georef.geoTransform = geoTransform + from scipy.interpolate import RegularGridInterpolator + + order = {'nearest': 0, 'linear': 1, 'cubic': 3, 'quintic': 5}.get(interpolation) + if order is None: + raise Error.InputError( + 'resample', "interpolation must be one of 'nearest', 'linear', " + "'cubic' or 'quintic'; got {0!r}".format(interpolation)) + + info = self._georef_info + georef = copy.deepcopy(info) georef.dx = de - georef.nx = int(np.round(self._georef_info.nx * self._georef_info.dx / de)) - georef.ny = int(np.round(self._georef_info.ny * self._georef_info.dx / de)) + georef.nx = max(int(np.round(info.nx * info.dx / de)), 1) + georef.ny = max(int(np.round(info.ny * info.dx / de)), 1) + # Keep the south-west cell centre fixed, then rebuild the transform + # from it -- the old code kept the source transform's origin and + # merely swapped in the new spacing, which shifted the whole grid. + georef.xllcenter = info.xllcenter + georef.yllcenter = info.yllcenter + georef.geoTransform = (georef.xllcenter - 0.5 * de, + de, + info.geoTransform[2] if info.geoTransform != 0 else 0.0, + georef.yllcenter + (georef.ny - 0.5) * de, + info.geoTransform[4] if info.geoTransform != 0 else 0.0, + -de) + + # Interpolate in south-to-north order so the y axis is increasing, + # which is what RegularGridInterpolator requires. + source = np.flipud(np.asarray(self._griddata, dtype=float64)) + xo = info.xllcenter + np.arange(info.nx) * info.dx + yo = info.yllcenter + np.arange(info.ny) * info.dx + + method = {0: 'nearest', 1: 'linear', 3: 'cubic', 5: 'quintic'}[order] + try: + interpolator = RegularGridInterpolator( + (yo, xo), source, method=method, bounds_error=False, fill_value=None) + except ValueError: + # 'cubic' and 'quintic' need SciPy >= 1.10 and enough points per + # axis. Fall back rather than fail, but say so: silently + # changing the interpolation order changes the answer. + warnings.warn( + "SciPy cannot do {0!r} interpolation on this grid (it needs " + "SciPy >= 1.10 and at least {1} points per axis); falling " + "back to 'linear'.".format(interpolation, order + 1), + RuntimeWarning, stacklevel=2) + interpolator = RegularGridInterpolator( + (yo, xo), source, method='linear', bounds_error=False, fill_value=None) + + xf = georef.xllcenter + np.arange(georef.nx) * de + yf = georef.yllcenter + np.arange(georef.ny) * de + YF, XF = np.meshgrid(yf, xf, indexing='ij') + resampled = interpolator(np.column_stack((YF.ravel(), XF.ravel()))) + resampled = resampled.reshape(georef.ny, georef.nx) + + return_grid = self.__class__() return_grid._georef_info = georef - xo = np.linspace(self._georef_info.xllcenter,(self._georef_info.nx-1)*(self._georef_info.dx), num = self._georef_info.nx) - yo = np.linspace(self._georef_info.yllcenter,(self._georef_info.ny-1)*(self._georef_info.dx), num = self._georef_info.ny) - xf = np.linspace(georef.xllcenter,(georef.nx-1)*georef.dx, num = georef.nx) - yf = np.linspace(georef.yllcenter,(georef.ny-1)*georef.dx, num = georef.ny) - from scipy.interpolate import interp2d - interp_grid = copy.deepcopy(self._griddata) - if geoTransform[1] > 0: - interp_grid = np.fliplr(interp_grid) - if geoTransform[5] < 0: - interp_grid = np.flipud(interp_grid) - f = interp2d(xo, yo, interp_grid, kind=interpolation) - return_grid._griddata = f(xf, yf) + return_grid._griddata = np.flipud(resampled).astype(self.dtype) return return_grid - + def location_in_grid(self, xo): + """True when ``xo = (x, y)`` lands on a cell holding real data. + + Real means: inside the grid, not ``NaN``, and not zero. + """ index = self._xy_to_rowscols((xo,))[0] return index[0] is not None and index[1] is not None and self[index[0], index[1]] is not None and self[index[0], index[1]] != 0 and not np.isnan(self[index[0], index[1]]) def tile(self, tile_xdim = 400, tile_ydim=400, tile_xpadding = 10, tile_ypadding = 10): - - num_xtiles = int(np.ceil(float(self._georef_info.nx) / float(tile_xdim))) - num_ytiles = int(np.ceil(float(self._georef_info.ny) / float(tile_ydim))) - + """Split the grid into overlapping tiles. + + Interior tiles overlap their neighbours by the padding width; tiles on + the outside of the grid are mirrored outwards instead, so every tile + has the same amount of context around its core. Analyses that need a + neighbourhood (filling, curvature at a scale) can then be run + tile-by-tile and reassembled with :meth:`remove_padding` followed by + :meth:`mosaic`. + + Returns + ------- + list + Tiles, each an independent grid of this class. + """ + dx = self._georef_info.dx + grid_nx, grid_ny = self._georef_info.nx, self._georef_info.ny + + num_xtiles = int(np.ceil(float(grid_nx) / float(tile_xdim))) + num_ytiles = int(np.ceil(float(grid_ny) / float(tile_ydim))) + tiles = [] - import copy - + for x_left in range(num_xtiles): for y_top in range(num_ytiles): - - left = x_left*tile_xdim-tile_xpadding if (x_left*tile_xdim-tile_xpadding) >= 0 else 0 - right = (x_left+1)*tile_xdim+tile_xpadding if ((x_left+1)*tile_xdim+tile_xpadding) <= (self._georef_info.nx-1) else self._georef_info.nx - top = y_top*tile_ydim-tile_ypadding if (y_top*tile_ydim-tile_ypadding) >= 0 else 0 - bottom = (y_top+1)*tile_ydim+tile_ypadding if ((y_top+1)*tile_ydim+tile_ypadding) <= (self._georef_info.ny-1) else self._georef_info.ny - - new_tile = self.__class__() - + + left = max(x_left * tile_xdim - tile_xpadding, 0) + right = min((x_left + 1) * tile_xdim + tile_xpadding, grid_nx) + top = max(y_top * tile_ydim - tile_ypadding, 0) + bottom = min((y_top + 1) * tile_ydim + tile_ypadding, grid_ny) + needs_left_padding = (x_left == 0) - needs_right_padding = (x_left == (num_xtiles-1)) - + needs_right_padding = (x_left == (num_xtiles - 1)) needs_top_padding = (y_top == 0) - needs_bottom_padding = (y_top == (num_ytiles-1)) - - griddata = self._griddata[top:bottom,left:right] - - xllcenter = self._georef_info.xllcenter + left*self._georef_info.dx - yllcenter = self._georef_info.yllcenter + (self._georef_info.ny - bottom)*self._georef_info.dx - nx = right-left + needs_bottom_padding = (y_top == (num_ytiles - 1)) + + griddata = self._griddata[top:bottom, left:right].copy() + + xllcenter = self._georef_info.xllcenter + left * dx + yllcenter = self._georef_info.yllcenter + (grid_ny - bottom) * dx + nx = right - left ny = bottom - top - - if needs_left_padding: - griddata = np.concatenate((np.fliplr(griddata[:,0:tile_xpadding]), griddata), axis = 1) - xllcenter -= tile_xpadding*self._georef_info.dx + + # Each `and tile_?padding` guard matters: with zero padding + # `griddata[:, -0:]` is the whole array, so the edge tiles + # came back with their data duplicated. + if needs_left_padding and tile_xpadding: + griddata = np.concatenate((np.fliplr(griddata[:, 0:tile_xpadding]), griddata), axis=1) + xllcenter -= tile_xpadding * dx nx += tile_xpadding - - if needs_right_padding: - griddata = np.concatenate((griddata, np.fliplr(griddata[:,-tile_xpadding:])), axis = 1) + + if needs_right_padding and tile_xpadding: + griddata = np.concatenate((griddata, np.fliplr(griddata[:, -tile_xpadding:])), axis=1) nx += tile_xpadding - - if needs_top_padding: - griddata = np.concatenate((np.flipud(griddata[0:tile_ypadding,:]), griddata), axis = 0) + + if needs_top_padding and tile_ypadding: + griddata = np.concatenate((np.flipud(griddata[0:tile_ypadding, :]), griddata), axis=0) ny += tile_ypadding - - if needs_bottom_padding: - griddata = np.concatenate((griddata, np.flipud(griddata[-tile_ypadding:,:])), axis = 0) + + if needs_bottom_padding and tile_ypadding: + griddata = np.concatenate((griddata, np.flipud(griddata[-tile_ypadding:, :])), axis=0) ny += tile_ypadding - yllcenter -= tile_ypadding*self._georef_info.dx - - new_tile._georef_info.geoTransform = (xllcenter - 0.5*self._georef_info.dx, self._georef_info.dx, 0, yllcenter + (ny + 0.5)*self._georef_info.dx, 0, -self._georef_info.dx) + yllcenter -= tile_ypadding * dx + + new_tile = self.__class__() + # (ny - 0.5), not (ny + 0.5): the north edge of the transform + # has to invert yllcenter = gt[3] - dx*(ny - 0.5), otherwise + # every tile is georeferenced one cell too far north. + new_tile._georef_info.geoTransform = ( + xllcenter - 0.5 * dx, dx, 0, yllcenter + (ny - 0.5) * dx, 0, -dx) + new_tile._georef_info.projection = self._georef_info.projection new_tile._georef_info.xllcenter = xllcenter new_tile._georef_info.yllcenter = yllcenter - new_tile._georef_info.dx = self._georef_info.dx + new_tile._georef_info.dx = dx new_tile._georef_info.nx = nx new_tile._georef_info.ny = ny new_tile._griddata = griddata - - tiles += [copy.deepcopy(new_tile)] - + + tiles.append(new_tile) + return tiles - + def remove_padding(self, xpadding = 10, ypadding = 10): - + """Strip the overlap that :meth:`tile` added, ready for :meth:`mosaic`.""" import copy unpadded = copy.deepcopy(self) - unpadded._griddata = unpadded._griddata[ypadding:-ypadding,xpadding:-xpadding] + # `[-0:]`-style slices select everything, so guard the zero case. + rows = slice(ypadding, -ypadding if ypadding else None) + cols = slice(xpadding, -xpadding if xpadding else None) + unpadded._griddata = unpadded._griddata[rows, cols] unpadded._georef_info.xllcenter += xpadding*unpadded._georef_info.dx unpadded._georef_info.yllcenter += ypadding*unpadded._georef_info.dx unpadded._georef_info.nx -= 2*xpadding @@ -794,92 +1228,161 @@ def remove_padding(self, xpadding = 10, ypadding = 10): return unpadded def sort(self, reverse=True, force = False, mask = None): + """Flat indices of the cells, ordered by value. + + Parameters + ---------- + reverse : bool + ``True`` (the default) returns highest-value first. + force : bool + Recompute even if a sort is already cached. + mask : BaseSpatialGrid, optional + Keep only cells where the mask equals 1. + + Returns + ------- + ndarray + Flat (C-order) indices into ``_griddata``. + + Notes + ----- + The result is cached, so passing a different ``mask`` on a later call + has no effect unless ``force=True`` is also given. + """ if force: self._sorted = False if not self._sorted: + sort_indexes = self._griddata.argsort(axis=None) if mask is not None: - sort_indexes = self._griddata.argsort(axis = None) - unraveled_sort_indexes = np.unravel_index(sort_indexes, mask.shape) - query_indexes = np.where(mask._griddata[unraveled_sort_indexes] == 1) - query_indexes_raveled = np.ravel_multi_index(query_indexes,mask.shape) - self._sort_indexes = sort_indexes[query_indexes_raveled] - else: - self._sort_indexes = self._griddata.argsort(axis = None) + # Select the sorted positions whose cell is inside the mask. + # The previous version fed positions-in-the-sorted-array to + # ravel_multi_index as if they were (row, col) subscripts, + # which produced indices unrelated to the mask. + mask_flat = np.asarray(mask._griddata).reshape(-1) + sort_indexes = sort_indexes[mask_flat[sort_indexes] == 1] + self._sort_indexes = sort_indexes self._sorted = True if reverse: return self._sort_indexes[::-1] else: return self._sort_indexes - + def apply_moving_window(self, moving_window): - out_grid = None - out_grid._georef_info = self._georef_info - out_grid._griddata = moving_window.apply_moving_window(self._griddata, self._georef_info.dx, self.dtype) + """Apply a :class:`~TopoAnalysis.MovingWindow.MovingWindow` to the grid.""" + out_grid = self.__class__() + out_grid._copy_info_from_grid(self, set_zeros=True) + out_grid._griddata = moving_window.apply_moving_window( + self._griddata, self._georef_info.dx, self.dtype) return out_grid - - def average_over_distance(self, distance, grid = None): - (nx, ny) = (self._georef_info.nx, self._georef_info.ny) - (dx, dy) = (self._georef_info.dx, self._georef_info.dx) - (xmin, xmax) = (self._georef_info.xllcenter, nx*dx+self._georef_info.xllcenter) - (ymin, ymax) = (self._georef_info.yllcenter, ny*dy+self._georef_info.yllcenter) - (xcenter, ycenter) = ((xmin+xmax) / 2.0, (ymin+ymax) / 2.0) - [x,y] = np.meshgrid(np.arange(xmin, xmax, dx),np.arange(ymin, ymax, dy)) - D = np.sqrt(np.power((x - xcenter),2) + np.power((y - ycenter), 2)) - template = (D <= distance).astype(float) - template /= np.sum(template[:]) + + def average_over_distance(self, distance, grid=None): + """Mean of the grid within ``distance`` of each cell. + + The averaging kernel is a disc of radius ``distance`` in map units, + normalised to unit sum, applied by FFT convolution. Edges wrap, as + they always do for an FFT convolution. + """ + info = self._georef_info + dx = info.dx if grid is None: grid = self._griddata - - from numpy.fft import fft2 as fft2 - from numpy.fft import ifft2 as ifft2 - from numpy.fft import ifftshift - - return np.real(ifftshift(ifft2(fft2(grid)*fft2(template)))) - + grid = np.asarray(grid, dtype=float64) + ny, nx = grid.shape + + # Build the kernel about index (0, 0) with wrap-around distances. + # That is exactly the origin a circular convolution assumes, so no + # fftshift is needed afterwards -- and unlike a centre-of-array + # kernel it stays correct for even grid dimensions. (Building the + # distances with np.arange(xmin, xmax, dx) could also produce + # nx +/- 1 samples and make the FFT multiply below fail outright.) + di = np.minimum(np.arange(ny), ny - np.arange(ny)) * dx + dj = np.minimum(np.arange(nx), nx - np.arange(nx)) * dx + D = np.hypot(di[:, None], dj[None, :]) + template = (D <= distance).astype(float64) + total = template.sum() + if total == 0: + raise Error.InputError( + 'average_over_distance', + 'distance {0} is smaller than one cell ({1})'.format(distance, dx)) + template /= total + + from numpy.fft import fft2, ifft2 + + return np.real(ifft2(fft2(grid) * fft2(template))) + def clip_to_bounds(self, bounds): + """Clip to ``((xmin, xmax), (ymin, ymax))``.""" extent = (bounds[0][0], bounds[0][1], bounds[1][0], bounds[1][1]) return self.clip_to_extent(extent) - + def clip_to_extent(self, extent): + """Clip to ``(xmin, xmax, ymin, ymax)``, keeping every cell inside it. + + Returns + ------- + BaseSpatialGrid + A new grid. The original is untouched. + """ import copy - return_grid = copy.deepcopy(self) - return_grid._georef_info = copy.deepcopy(self._georef_info) - lower_left = (extent[0]+self._georef_info.dx, extent[2]+self._georef_info.dx) - upper_right = (extent[1]-self._georef_info.dx, extent[3]-self._georef_info.dx) - idx = self._xy_to_rowscols((lower_left, upper_right)) - return_grid._griddata = return_grid._griddata[idx[1][0]+1:idx[0][0]+1,idx[0][1]:idx[1][1]] + + xmin, xmax, ymin, ymax = extent + info = self._georef_info + if xmax < xmin or ymax < ymin: + raise Error.InputError('clip_to_extent', + 'extent must be (xmin, xmax, ymin, ymax)') + + # Work in index space and clamp, rather than round-tripping through + # _xy_to_rowscols (which returns None outside the grid) and then + # nudging the result by a cell in each direction. + col_min = int(np.ceil((xmin - info.xllcenter) / info.dx - 1e-9)) + col_max = int(np.floor((xmax - info.xllcenter) / info.dx + 1e-9)) + row_from_y_min = int(np.ceil((ymin - info.yllcenter) / info.dx - 1e-9)) + row_from_y_max = int(np.floor((ymax - info.yllcenter) / info.dx + 1e-9)) + + col_min = max(col_min, 0) + col_max = min(col_max, info.nx - 1) + row_from_y_min = max(row_from_y_min, 0) + row_from_y_max = min(row_from_y_max, info.ny - 1) + + if col_max < col_min or row_from_y_max < row_from_y_min: + raise Error.InputError('clip_to_extent', + 'extent does not overlap the grid') + + # Row 0 is the north edge, so the largest y is the smallest row. + top = (info.ny - 1) - row_from_y_max + bottom = (info.ny - 1) - row_from_y_min + + return_grid = self.__class__() + return_grid._georef_info = copy.deepcopy(info) + return_grid._griddata = self._griddata[top:bottom + 1, col_min:col_max + 1].copy() return_grid._georef_info.nx = return_grid._griddata.shape[1] return_grid._georef_info.ny = return_grid._griddata.shape[0] - return_grid._georef_info.xllcenter = self._rowscols_to_xy((idx[0],))[0][0] - return_grid._georef_info.yllcenter = self._rowscols_to_xy((idx[0],))[0][1] - return_grid._georef_info.geoTransform = (return_grid._georef_info.xllcenter, return_grid._georef_info.geoTransform[1], return_grid._georef_info.geoTransform[2], return_grid._georef_info.yllcenter + (return_grid._georef_info.dx*(return_grid._georef_info.ny-1)), return_grid._georef_info.geoTransform[4], return_grid._georef_info.geoTransform[5],) - + return_grid._georef_info.xllcenter = info.xllcenter + col_min * info.dx + return_grid._georef_info.yllcenter = info.yllcenter + row_from_y_min * info.dx + return_grid._georef_info.geoTransform = return_grid._geoTransform_from_corner() + return return_grid - + def extent_of_data(self): - - top = 0 - bottom = self._georef_info.ny - 1 - left = 0 - right = self._georef_info.nx - 1 - - while np.sum(np.isnan(self._griddata[top,:])) == self._georef_info.nx: - top = top + 1 - - while np.sum(np.isnan(self._griddata[bottom,:])) == self._georef_info.nx: - bottom = bottom - 1 - - while np.sum(np.isnan(self._griddata[left,:])) == self._georef_info.ny: - left = left + 1 - - while np.sum(np.isnan(self._griddata[right,:])) == self._georef_info.ny: - right = right - 1 - - (ll, ur) = self._rowscols_to_xy(((bottom, left), (top, right))) + """``(xmin, xmax, ymin, ymax)`` of the smallest box holding all valid data.""" + valid = ~np.isnan(np.asarray(self._griddata, dtype=float64)) + if not valid.any(): + raise Error.InputError('extent_of_data', 'the grid is entirely no-data') + + rows = np.flatnonzero(valid.any(axis=1)) + # Columns need any() along axis 0; the original indexed rows for both, + # so a grid with empty left columns reported the wrong extent (and + # ran off the end for non-square grids). + cols = np.flatnonzero(valid.any(axis=0)) + top, bottom = int(rows[0]), int(rows[-1]) + left, right = int(cols[0]), int(cols[-1]) + + (ll, ur) = self._rowscols_to_xy(((bottom, left), (top, right))) return (ll[0], ur[0], ll[1], ur[1]) - - + + def clip_to_mask_grid(self, mask_grid): + """Reproject and resample this grid onto ``mask_grid``'s frame, in place.""" gdal_source = self._create_gdal_representation_from_array(self._georef_info, 'MEM', self._griddata, self.dtype) gdal_mask = mask_grid._create_gdal_representation_from_array(mask_grid._georef_info, 'MEM', mask_grid._griddata, mask_grid.dtype) gdal_clip = self._clipRasterToRaster(gdal_source, gdal_mask, self.dtype) @@ -887,861 +1390,1148 @@ def clip_to_mask_grid(self, mask_grid): self.__populate_georef_info_using_geoTransform(self._georef_info.nx, self._georef_info.ny) def clip_to_shapefile(self, shapefile): + """Cut the grid to a vector boundary, in place. + + .. warning:: + ``_clipRasterToShape`` is a stub, so this currently does nothing. + """ gdal_source = self._create_gdal_representation_from_array(self._georef_info, 'MEM', self._griddata, self.dtype) gdal_clip = shapefile.shapedata gdal_result = self._clipRasterToShape(gdal_source, gdal_clip) self._georef_info.geoTransform, self._georef_info.nx, self._georef_info.ny, self._griddata = self._read_GDAL_dataset(gdal_result, self.dtype) self.__populate_georef_info_using_geoTransform(self._georef_info.nx, self._georef_info.ny) - def calculate_gradient_over_length_scale(self,length_scale): - # sx,sy = calcFiniteDiffs(elevGrid,dx) - # calculates finite differences in X and Y direction using a finite difference - # kernel that extends N cells from the center. The width of the kernel is then - # 2N + 1 by 2N + 1. Applies a boundary condition such that the size and location - # of the grids in is the same as that out. However, the larger N is, the more NoData - #Will be around the edges . - - #Compute finite differences - elevGrid = self._griddata + def calculate_gradient_over_length_scale(self, length_scale): + """Centred finite-difference gradient measured over ``length_scale``. + + The stencil reaches ``N = ceil(length_scale / dx)`` cells either side + of the centre, so the two sampled cells are ``2*N*dx`` apart. A band + ``N`` cells wide around the grid edge comes back as ``NaN``. + + Returns + ------- + (ndarray, ndarray) + ``(dz/dx, dz/dy)`` with y increasing northwards. + """ + elevGrid = np.asarray(self._griddata, dtype=float64) dx = self._georef_info.dx - N = np.ceil(length_scale / dx) - - Sx = (elevGrid[N:-N, (2*N):] - elevGrid[N:-N, :-(2*N)])/(((2*N)+1)*dx) - Sy = (elevGrid[(2*N):,N:-N] - elevGrid[:-(2*N), N:-N])/(((2*N)+1)*dx) - - #Create two new arrays of the original DEMs size - SxPadded = np.empty(elevGrid.shape) - SxPadded[:] = np.NAN - SyPadded = np.empty(elevGrid.shape) - SyPadded[:] = np.NAN - - SyPadded[N:-N, N:-N] = Sy + # int(), not a bare np.ceil(): NumPy returns a float and Python 3 + # refuses to slice with one. + N = int(np.ceil(length_scale / dx)) + if N < 1: + raise Error.InputError('calculate_gradient_over_length_scale', + 'length_scale must be at least one cell') + if 2 * N >= min(elevGrid.shape): + raise Error.InputError('calculate_gradient_over_length_scale', + 'length_scale is larger than the grid') + + # The separation between the sampled cells is 2*N*dx, not (2*N+1)*dx. + span = 2.0 * N * dx + Sx = (elevGrid[N:-N, (2 * N):] - elevGrid[N:-N, :-(2 * N)]) / span + # Row index increases southwards, so negate to get dz/dy in map space. + Sy = -(elevGrid[(2 * N):, N:-N] - elevGrid[:-(2 * N), N:-N]) / span + + SxPadded = np.full(elevGrid.shape, np.nan) + SyPadded = np.full(elevGrid.shape, np.nan) SxPadded[N:-N, N:-N] = Sx - + SyPadded[N:-N, N:-N] = Sy + return SxPadded, SyPadded - + def calculate_laplacian_over_length_scale(self, length_scale): - #C = calcFiniteCurv(elevGrid, dx) - #calculates finite differnces in X and Y direction using the centered difference method. - #Applies a boundary condition such that the size and location of the grids in is the same as that out. - + """Laplacian measured over ``length_scale`` with a centred stencil. + + A band ``N = ceil(length_scale / dx)`` cells wide around the grid edge + comes back as ``NaN``. + """ dx = self._georef_info.dx - grid = self._griddata - winRad = np.ceil(length_scale / dx) - #Assign boundary conditions - Curv = np.zeros_like(grid)*np.nan - - #Compute finite differences - Cx = (grid[winRad:-winRad, (2*winRad):] - 2*grid[winRad:-winRad, winRad:-winRad] + grid[winRad:-winRad, :-(2*winRad)])/(2*dx*winRad)**2 - Cy = (grid[(2*winRad):, winRad:-winRad] - 2*grid[winRad:-winRad, winRad:-winRad] + grid[:-(2*winRad), winRad:-winRad])/(2*dx*winRad)**2 - - Curv[winRad:-winRad,winRad:-winRad] = Cx+Cy + grid = np.asarray(self._griddata, dtype=float64) + N = int(np.ceil(length_scale / dx)) + if N < 1: + raise Error.InputError('calculate_laplacian_over_length_scale', + 'length_scale must be at least one cell') + if 2 * N >= min(grid.shape): + raise Error.InputError('calculate_laplacian_over_length_scale', + 'length_scale is larger than the grid') + + Curv = np.full(grid.shape, np.nan) + h2 = (N * dx) ** 2 + centre = grid[N:-N, N:-N] + Cx = (grid[N:-N, (2 * N):] - 2 * centre + grid[N:-N, :-(2 * N)]) / h2 + Cy = (grid[(2 * N):, N:-N] - 2 * centre + grid[:-(2 * N), N:-N]) / h2 + + Curv[N:-N, N:-N] = Cx + Cy return Curv def principal_curvatures(self): - Zy, Zx = np.gradient(self._griddata, self._georef_info.dx) - Zxy, Zxx = np.gradient(Zx, self._georef_info.dx) - Zyy, _ = np.gradient(Zy, self._georef_info.dx) - - H = (Zx**2 + 1)*Zyy - 2*Zx*Zy*Zxy + (Zy**2 + 1)*Zxx - H = -H/(2*(Zx**2 + Zy**2 + 1)**(1.5)) - - K = (Zxx * Zyy - (Zxy ** 2)) / (1 + (Zx ** 2) + (Zy **2)) ** 2 - - k1 = H + np.sqrt(np.power(H,2) - K) - k2 = H - np.sqrt(np.power(H,2) - K) - - k1re = BaseSpatialGrid() + """Maximum and minimum principal curvature of the surface. + + Returns + ------- + (BaseSpatialGrid, BaseSpatialGrid) + ``(k1, k2)`` with ``k1 >= k2``. Positive curvature is convex-up. + """ + dx = self._georef_info.dx + grid = np.asarray(self._griddata, dtype=float64) + # np.gradient returns the derivative along axis 0 (rows) first, and + # rows increase southwards, so flip its sign to get dz/dy. + dz_drow, Zx = np.gradient(grid, dx) + Zy = -dz_drow + _, Zxx = np.gradient(Zx, dx) + dZy_drow, Zxy = np.gradient(Zy, dx) + Zyy = -dZy_drow + + p = Zx ** 2 + Zy ** 2 + H = (Zx ** 2 + 1) * Zyy - 2 * Zx * Zy * Zxy + (Zy ** 2 + 1) * Zxx + H = -H / (2 * (p + 1) ** 1.5) + + K = (Zxx * Zyy - Zxy ** 2) / (1 + p) ** 2 + + # H**2 - K is non-negative in exact arithmetic; clip so that rounding + # near an umbilic point does not turn both curvatures into NaN. + discriminant = np.sqrt(np.maximum(H ** 2 - K, 0.0)) + k1 = H + discriminant + k2 = H - discriminant + + k1re = BaseSpatialGrid() k1re._copy_info_from_grid(self, True) k2re = BaseSpatialGrid() k2re._copy_info_from_grid(self, True) k1re._griddata = k1 k2re._griddata = k2 - + return (k1re, k2re) - def plot(self, **kwargs): + def extent(self): + """``(xmin, xmax, ymin, ymax)`` of the grid's outer pixel edges. + + These are the bounds matplotlib's ``imshow(extent=...)`` expects. + """ + info = self._georef_info + half = 0.5 * info.dx + return (info.xllcenter - half, + info.xllcenter + (info.nx - 0.5) * info.dx, + info.yllcenter - half, + info.yllcenter + (info.ny - 0.5) * info.dx) + def plot(self, **kwargs): + """Display the grid with ``imshow``, georeferenced in map coordinates. + + Parameters + ---------- + interactive : bool + Leave the figure open without blocking (default ``True``). + colorbar : bool + Draw a colour bar (default ``True``). + ax : matplotlib Axes, optional + Draw into an existing axes instead of a new figure. + **kwargs + Passed through to ``imshow`` (``cmap``, ``vmin``, ``alpha``, ...). + + Returns + ------- + matplotlib.figure.Figure + """ interactive = kwargs.pop('interactive', True) colorbar = kwargs.pop('colorbar', True) - extent = [self._georef_info.xllcenter, self._georef_info.xllcenter+(self._georef_info.nx-0.5)*self._georef_info.dx, self._georef_info.yllcenter, self._georef_info.yllcenter+(self._georef_info.ny-0.5)*self._georef_info.dx] - fig = plt.figure() - plt.axis(extent) - if interactive: - plt.ion() - plt.show(block = False) + ax = kwargs.pop('ax', None) + kwargs.setdefault('extent', self.extent()) + + if ax is None: + fig = plt.figure() + ax = fig.gca() else: - plt.ioff() - im = plt.imshow(self._griddata, extent = extent, **kwargs) + fig = ax.figure + + im = ax.imshow(self._griddata, **kwargs) if colorbar: - plt.colorbar(im) - plt.draw() - plt.pause(0.001) + fig.colorbar(im, ax=ax) if interactive: - plt.show() + plt.ion() + plt.show(block=False) + plt.pause(0.001) return fig - + def _neighbourhood_indices(self, index, pixel_radius): + """Row/column indices of the in-bounds cells within ``pixel_radius``. + + Returns ``(rows, cols)`` as flat integer arrays. The disc is + inclusive of the radius and of the far edge -- the previous + ``range(i - r, i + r)`` stopped one cell short and let negative + indices wrap around to the opposite side of the grid. + """ + i, j = int(index[0]), int(index[1]) + r = int(np.ceil(pixel_radius)) + ny, nx = self._griddata.shape + + rows = np.arange(max(i - r, 0), min(i + r, ny - 1) + 1) + cols = np.arange(max(j - r, 0), min(j + r, nx - 1) + 1) + if rows.size == 0 or cols.size == 0: + return np.empty(0, dtype=int), np.empty(0, dtype=int) + + RR, CC = np.meshgrid(rows, cols, indexing='ij') + inside = np.hypot(RR - i, CC - j) <= pixel_radius + return RR[inside], CC[inside] + def find_nearest_cell_with_value(self, index, value, pixel_radius): + """Index of the cell within ``pixel_radius`` whose value is closest to ``value``. + + Returns ``(None, None)`` when the neighbourhood holds no valid data. + """ + rows, cols = self._neighbourhood_indices(index, pixel_radius) + if rows.size == 0: + return None, None + difference = np.abs(value - self._griddata[rows, cols]) + if np.all(np.isnan(difference)): + return None, None + best = int(np.nanargmin(difference)) + return int(rows[best]), int(cols[best]) - (i, j) = index - nearest = tuple() - minimum_difference = np.nan - for y in range(int(i-pixel_radius),int(i+pixel_radius)): - for x in range(int(j-pixel_radius),int(j+pixel_radius)): - if x is not None and y is not None and np.sqrt( (y-i)**2 + (x-j)**2) <= pixel_radius and x < self._griddata.shape[1] and y < self._griddata.shape[0]: - value_difference = np.abs(value - self._griddata[y,x]) - if np.isnan(minimum_difference) or minimum_difference > value_difference: - minimum_difference = value_difference - nearest = (y,x) - (y,x) = nearest - return y, x - def find_nearest_cell_with_value_greater_than(self, index, value, pixel_radius): + """Index of the cell within ``pixel_radius`` whose value is the smallest + one at or above ``value``. + + If no cell reaches ``value``, falls back to the cell closest to it + from below. Returns ``(None, None)`` when there is no valid data. + """ + rows, cols = self._neighbourhood_indices(index, pixel_radius) + if rows.size == 0: + return None, None + difference = self._griddata[rows, cols] - value + + above = difference >= 0 + if np.any(above & ~np.isnan(difference)): + candidates = np.where(above, difference, np.nan) + else: + candidates = np.where(~np.isnan(difference), -difference, np.nan) + if np.all(np.isnan(candidates)): + return None, None + best = int(np.nanargmin(candidates)) + return int(rows[best]), int(cols[best]) - (i, j) = index - nearest = tuple() - minimum_difference = np.nan - for y in range(int(i-pixel_radius),int(i+pixel_radius)): - for x in range(int(j-pixel_radius),int(j+pixel_radius)): - if x is not None and y is not None and np.sqrt( (y-i)**2 + (x-j)**2) <= pixel_radius and x < self._griddata.shape[1] and y < self._griddata.shape[0]: - value_difference = self._griddata[y,x] - value - if (np.isnan(minimum_difference) or (minimum_difference > value_difference)) and (value_difference >= 0): - minimum_difference = value_difference - nearest = (y,x) - if np.isnan(minimum_difference): - for y in range(int(i-pixel_radius),int(i+pixel_radius)): - for x in range(int(j-pixel_radius),int(j+pixel_radius)): - if x is not None and y is not None and np.sqrt( (y-i)**2 + (x-j)**2) <= pixel_radius and x < self._griddata.shape[1] and y < self._griddata.shape[0]: - value_difference = value - self._griddata[y,x] - if (np.isnan(minimum_difference) or (minimum_difference > value_difference)) : - minimum_difference = value_difference - nearest = (y,x) - (y,x) = nearest - return y, x - def find_nearest_cell_with_greatest_value(self, index, pixel_radius = 5): - - (i, j) = index - nearest = tuple() - maximum_value = np.nan - for y in range(int(i-pixel_radius),int(i+pixel_radius)): - for x in range(int(j-pixel_radius),int(j+pixel_radius)): - if np.sqrt( (y-i)**2 + (x-j)**2) <= pixel_radius: - if np.isnan(maximum_value) or maximum_value < self._griddata[y,x]: - maximum_value = self._griddata[y,x] - nearest = (y,x) - - (y,x) = nearest - return y, x - + """Index of the highest-valued cell within ``pixel_radius``. + + Returns ``(None, None)`` when the neighbourhood holds no valid data. + """ + rows, cols = self._neighbourhood_indices(index, pixel_radius) + if rows.size == 0: + return None, None + values = self._griddata[rows, cols] + if np.all(np.isnan(values)): + return None, None + best = int(np.nanargmax(values)) + return int(rows[best]), int(cols[best]) + def snap_locations_to_greatest_value(self, v, pixel_radius = 5): - - idxs = self._xy_to_rowscols(v) + """Move each ``(x, y)`` to the highest-valued cell within ``pixel_radius``. + + Locations for which no valid cell is found are dropped. + """ snap_idxs = list() - for idx in idxs: + for idx in self._xy_to_rowscols(v): + if idx[0] is None: + continue i, j = self.find_nearest_cell_with_greatest_value(idx, pixel_radius) - snap_idxs.append((i,j)) + if i is not None: + snap_idxs.append((i, j)) return self._rowscols_to_xy(snap_idxs) - + def snap_locations_to_closest_value(self, v, value, pixel_radius = 5): - idxs = self._xy_to_rowscols(v) + """Move each ``(x, y)`` to the nearby cell whose value best matches. + + ``value`` is a sequence with one target value per location. + Locations for which no valid cell is found are dropped. + """ snap_idxs = list() - values_with_idx = zip(idxs, value) - for (idx, value) in values_with_idx: - i, j = self.find_nearest_cell_with_value(idx, value, pixel_radius) - snap_idxs.append((i,j)) + for (idx, target) in zip(self._xy_to_rowscols(v), value): + if idx[0] is None: + continue + i, j = self.find_nearest_cell_with_value(idx, target, pixel_radius) + if i is not None: + snap_idxs.append((i, j)) return self._rowscols_to_xy(snap_idxs) - - def export_arc_e00_grid(self, filename): + def export_arc_e00_grid(self, filename): + """Write an ArcInfo E00 interchange grid.""" self._create_gdal_representation_from_array(self._georef_info, 'E00GRID', self._griddata, self.dtype, filename) def set_value_at_rowscols(self, value, rowscols): - rowscols_array = (np.array(list(zip(*rowscols))).T[:,0],np.array(list(zip(*rowscols))).T[:,1]) - self._griddata[rowscols_array] = value - + """Set ``value`` at each ``(row, col)`` in ``rowscols``.""" + rowscols = list(rowscols) + if not rowscols: + return + rows, cols = zip(*rowscols) + self._griddata[np.asarray(rows, dtype=int), np.asarray(cols, dtype=int)] = value + def save(self, filename): - - self._create_gdal_representation_from_array(self._georef_info, 'GTiff', self._griddata, self.dtype, filename, ['COMPRESS=LZW']) - - def write_to_ai(self, filename): - self._writeArcAsciiRaster(self._georef_info, filename, self._griddata, np.NAN, '%10.2f') - + """Write the grid to an LZW-compressed GeoTIFF.""" + dataset = self._create_gdal_representation_from_array( + self._georef_info, 'GTiff', self._griddata, self.dtype, filename, + ['COMPRESS=LZW']) + # Dropping the reference is what flushes GDAL's write cache; without + # it the file can be truncated if the process exits soon after. + dataset.FlushCache() + del dataset + + def write_to_ai(self, filename, nodata_value=-9999.0): + """Write the grid as an ArcInfo ASCII grid.""" + self._writeArcAsciiRaster(self._georef_info, filename, self._griddata, + nodata_value, '%10.2f') + def vectorize(self, filename): - - import uuid - tmpfilename = str(uuid.uuid4()) - self.save(tmpfilename) - gdal_dataset = gdal.Open(tmpfilename) - srcband = gdal_dataset.GetRasterBand(1) - - dst_layername = filename - drv = ogr.GetDriverByName("ESRI Shapefile") - dst_ds = drv.CreateDataSource( dst_layername + ".shp" ) - dst_layer = dst_ds.CreateLayer(dst_layername, srs = None ) - newField = ogr.FieldDefn('RASTER_VAL', ogr.OFTReal) - dst_layer.CreateField(newField) - gdal.Polygonize( srcband, None, dst_layer, 0, [], callback=None ) - os.remove(tmpfilename) - + """Polygonize the grid into an ESRI shapefile called ``filename.shp``. + + Each polygon carries the raster value in a ``RASTER_VAL`` field. + """ + import tempfile + + handle, tmpfilename = tempfile.mkstemp(suffix='.tif') + os.close(handle) + try: + self.save(tmpfilename) + gdal_dataset = gdal.Open(tmpfilename) + try: + srcband = gdal_dataset.GetRasterBand(1) + drv = ogr.GetDriverByName("ESRI Shapefile") + dst_ds = drv.CreateDataSource(filename + ".shp") + try: + dst_layer = dst_ds.CreateLayer(os.path.basename(filename), srs=None) + dst_layer.CreateField(ogr.FieldDefn('RASTER_VAL', ogr.OFTReal)) + gdal.Polygonize(srcband, None, dst_layer, 0, [], callback=None) + finally: + # The shapefile is only written when the data source is + # released; the original code left it dangling. + dst_ds = None + finally: + gdal_dataset = None + finally: + if os.path.exists(tmpfilename): + os.remove(tmpfilename) + + @classmethod + def _georef_from_dataset(cls, return_object, gdal_dataset): + """Populate a grid's georeferencing from an open GDAL dataset.""" + nx = gdal_dataset.RasterXSize + ny = gdal_dataset.RasterYSize + info = return_object._georef_info + info.geoTransform = gdal_dataset.GetGeoTransform() + info.projection = gdal_dataset.GetProjection() + info.dx = info.geoTransform[1] + info.xllcenter = info.geoTransform[0] + info.dx / 2.0 + info.yllcenter = info.geoTransform[3] - (info.dx * (ny - 0.5)) + info.nx = nx + info.ny = ny + return return_object + @classmethod def load(cls, filename): - + """Read a grid previously written by :meth:`save`.""" return_object = cls() - gdal_dataset = gdal.Open(filename) - band = gdal_dataset.GetRasterBand(1) - nodata = band.GetNoDataValue() - return_object._griddata = band.ReadAsArray().astype(cls.dtype) - if nodata is not None: - nodata_elements = np.where(return_object._griddata == nodata) - from numpy import uint8 - if cls.dtype is not uint8: - return_object._griddata[nodata_elements] = np.NAN - - - geoTransform = gdal_dataset.GetGeoTransform() - nx = gdal_dataset.RasterXSize - ny = gdal_dataset.RasterYSize - - return_object._georef_info.geoTransform = geoTransform - return_object._georef_info.dx = return_object._georef_info.geoTransform[1] - return_object._georef_info.xllcenter = return_object._georef_info.geoTransform[0]+return_object._georef_info.dx/2.0 - return_object._georef_info.yllcenter = return_object._georef_info.geoTransform[3]-(return_object._georef_info.dx*(ny-0.5)) - return_object._georef_info.nx = nx - return_object._georef_info.ny = ny - - - gdal_file = None + try: + return_object._griddata = cls._read_band(gdal_dataset.GetRasterBand(1), cls.dtype) + cls._georef_from_dataset(return_object, gdal_dataset) + finally: + gdal_dataset = None return return_object + @classmethod + def _mosaic_frame(cls, tiles): + """Georeferencing for the smallest grid covering every tile.""" + if not tiles: + raise Error.InputError('mosaic', 'no tiles to mosaic') + + dx = tiles[0]._georef_info.dx + xmin = min(t._georef_info.xllcenter for t in tiles) + ymin = min(t._georef_info.yllcenter for t in tiles) + xmax = max(t._georef_info.xllcenter + (t._georef_info.nx - 1) * t._georef_info.dx + for t in tiles) + ymax = max(t._georef_info.yllcenter + (t._georef_info.ny - 1) * t._georef_info.dx + for t in tiles) + + ny = int(round((ymax - ymin) / dx)) + 1 + nx = int(round((xmax - xmin) / dx)) + 1 + + info = Georef_info() + info.nx = nx + info.ny = ny + info.dx = dx + info.xllcenter = xmin + info.yllcenter = ymin + info.projection = tiles[0]._georef_info.projection + info.geoTransform = (xmin - 0.5 * dx, dx, 0, + ymin + (ny - 0.5) * dx, 0, -dx) + return info + + @staticmethod + def _tile_slice(info, tile): + """Where ``tile`` belongs inside a mosaic described by ``info``.""" + j_min = int(round((tile._georef_info.xllcenter - info.xllcenter) / info.dx)) + i_min = int(round((info.yllcenter + (info.ny - 1) * info.dx + - tile._georef_info.yllcenter + - (tile._georef_info.ny - 1) * tile._georef_info.dx) / info.dx)) + return (slice(i_min, i_min + tile._georef_info.ny), + slice(j_min, j_min + tile._georef_info.nx)) + @classmethod def mosaic(cls, tiles): - - xmin = np.nan - xmax = np.nan - ymin = np.nan - ymax = np.nan - - for tile in tiles: - - xmin = tile._georef_info.xllcenter if (np.isnan(xmin) or (xmin > tile._georef_info.xllcenter)) else xmin - ymin = tile._georef_info.yllcenter if (np.isnan(ymin) or (ymin > tile._georef_info.yllcenter)) else ymin - xmax = tile._georef_info.xllcenter+(tile._georef_info.nx-1)*tile._georef_info.dx if (np.isnan(xmax) or (xmax < tile._georef_info.xllcenter+(tile._georef_info.nx-1)*tile._georef_info.dx)) else xmax - ymax = tile._georef_info.yllcenter+(tile._georef_info.ny-1)*tile._georef_info.dx if (np.isnan(ymax) or (ymax < tile._georef_info.yllcenter+(tile._georef_info.ny-1)*tile._georef_info.dx)) else ymax - - ny = int(round((ymax - ymin) / tiles[0]._georef_info.dx)) + 1 - nx = int(round((xmax - xmin) / tiles[0]._georef_info.dx)) + 1 - + """Reassemble tiles produced by :meth:`tile` into a single grid. + + Tiles must share a cell size and be aligned on a common grid. Where + tiles overlap, the last one in ``tiles`` wins. + """ + tiles = list(tiles) + info = cls._mosaic_frame(tiles) + return_object = cls() - return_object._griddata = np.zeros((ny, nx)) - return_object._griddata[:] = np.nan - return_object._georef_info.nx = nx - return_object._georef_info.ny = ny - return_object._georef_info.dx = tiles[0]._georef_info.dx - return_object._georef_info.xllcenter = xmin - return_object._georef_info.yllcenter = ymin - return_object._georef_info.geoTransform = (return_object._georef_info.xllcenter - 0.5*return_object._georef_info.dx, return_object._georef_info.dx, 0, return_object._georef_info.yllcenter + (float(return_object._georef_info.ny-0.5))*return_object._georef_info.dx, 0, -return_object._georef_info.dx) - + return_object._georef_info = info + return_object._griddata = np.full((info.ny, info.nx), np.nan) + for tile in tiles: - j_min = int(round((tile._georef_info.xllcenter - return_object._georef_info.xllcenter) / return_object._georef_info.dx)) - j_max = j_min + tile._georef_info.nx - i_min = int(round((return_object._georef_info.yllcenter + (return_object._georef_info.ny-1)*return_object._georef_info.dx - tile._georef_info.yllcenter - (tile._georef_info.ny-1)*tile._georef_info.dx) / return_object._georef_info.dx)) - i_max = i_min + tile._georef_info.ny - - return_object._griddata[i_min:i_max, j_min:j_max] = tile._griddata - + rows, cols = cls._tile_slice(info, tile) + return_object._griddata[rows, cols] = tile._griddata + return return_object - + class ValueGrid(BaseSpatialGrid): - + """A plain raster of values, with a helper for scattered writes.""" + + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), (('dx', 'grid'), '_create_from_grid')) - + def set_value_at_indexes(self, indexes, value): - ij = zip(*indexes) - self._griddata[ij] = value - - + """Set ``value`` at each ``(row, col)`` pair in ``indexes``.""" + indexes = list(indexes) + if not indexes: + return + # zip() is a lazy iterator in Python 3; using one directly as a NumPy + # subscript raises rather than doing what it did in Python 2. + rows, cols = zip(*indexes) + self._griddata[np.asarray(rows, dtype=int), np.asarray(cols, dtype=int)] = value + + class FlowDirection(BaseSpatialGrid): + """Base class for flow-direction grids.""" pass class FlowDirectionD8(FlowDirection): - + """Single-direction (D8) flow directions. + + Values are ArcGIS direction codes; 0 means the cell has no downstream + neighbour (a pit, an outlet at the grid edge, or no-data):: + + | 32 64 128 | + | 16 X 1 | + | 8 4 2 | + + Build one from a filled DEM:: + + d8 = FlowDirectionD8(flooded_dem=FilledElevation(elevation=dem)) + + Direction is chosen by steepest *gradient*, so the drop to a diagonal + neighbour is divided by ``sqrt(2) * dx`` before being compared with the + cardinal ones. Ties go to the first direction in the order E, SE, S, SW, + W, NW, N, NE. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), - (('flooded_dem',), '_create_from_flooded_dem')) - - from numpy import uint8 + (('gdal_filename',), '_read_gdal'), + (('flooded_dem',), '_create_from_flooded_dem'), + (('elevation',), '_create_from_elevation')) + dtype = uint8 + def _create_from_elevation(self, *args, **kwargs): + """Route an already-hydrologically-correct DEM. + + Alias for :meth:`_create_from_flooded_dem`; use it when the DEM has + been conditioned elsewhere. + """ + kwargs = dict(kwargs) + kwargs['flooded_dem'] = kwargs.pop('elevation') + self._create_from_flooded_dem(*args, **kwargs) + def _create_from_flooded_dem(self, *args, **kwargs): flooded_dem = kwargs['flooded_dem'] self._copy_info_from_grid(flooded_dem) - self._griddata = np.zeros_like(flooded_dem._griddata, dtype = self.dtype) - - flooded_dem_ijp1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ijp1[:,0:-2] = (flooded_dem._griddata[:,0:-2] - flooded_dem._griddata[:, 1:-1]) - - flooded_dem_ip1jp1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ip1jp1[0:-2,0:-2] = (flooded_dem._griddata[0:-2,0:-2] - flooded_dem._griddata[1:-1, 1:-1]) / 1.41 - - flooded_dem_ip1j = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ip1j[0:-2,:] = (flooded_dem._griddata[0:-2,:] - flooded_dem._griddata[1:-1, :]) - - flooded_dem_ip1jm1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ip1jm1[0:-2,1:-1] = (flooded_dem._griddata[0:-2,1:-1] - flooded_dem._griddata[1:-1, 0:-2]) / 1.41 - - flooded_dem_ijm1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ijm1[:,1:-1] = (flooded_dem._griddata[:,1:-1] - flooded_dem._griddata[:, 0:-2]) - - flooded_dem_im1jm1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_im1jm1[1:-1,1:-1] = (flooded_dem._griddata[1:-1,1:-1] - flooded_dem._griddata[0:-2, 0:-2]) / 1.41 - - flooded_dem_im1j = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_im1j[1:-1,:] = (flooded_dem._griddata[1:-1,:] - flooded_dem._griddata[0:-2, :]) - - flooded_dem_im1jp1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_im1jp1[1:-1,0:-2] = (flooded_dem._griddata[1:-1,0:-2] - flooded_dem._griddata[0:-2, 1:-1]) / 1.41 - - min_sl = np.fmax(np.fmax(np.fmax(np.fmax(np.fmax(np.fmax(np.fmax(flooded_dem_ijp1, flooded_dem_ip1jp1), flooded_dem_ip1j), flooded_dem_ip1jm1), flooded_dem_ijm1), flooded_dem_im1jm1), flooded_dem_im1j), flooded_dem_im1jp1) - - self._griddata[flooded_dem_ijp1 == min_sl] = 1 - self._griddata[flooded_dem_ip1jp1 == min_sl] = 2 - self._griddata[flooded_dem_ip1j == min_sl] = 4 - self._griddata[flooded_dem_ip1jm1 == min_sl] = 8 - self._griddata[flooded_dem_ijm1 == min_sl] = 16 - self._griddata[flooded_dem_im1jm1 == min_sl] = 32 - self._griddata[flooded_dem_im1j == min_sl] = 64 - self._griddata[flooded_dem_im1jp1 == min_sl] = 128 - - self._griddata[np.isnan(flooded_dem._griddata)] = 0 - + + elevations = np.ascontiguousarray(flooded_dem._griddata, dtype=float64) + kernels.warn_if_slow(elevations.size) + + # A geographic grid's cell size varies with latitude, so ask the grid + # itself rather than assuming a single dx. + cellsize_grid = np.ascontiguousarray( + flooded_dem._mean_pixel_dimension(*args, **kwargs), dtype=float64) + + # The old hand-rolled version compared each cell only with a window + # that stopped two columns/rows short, so the last row and column + # never received a direction; it also scaled diagonals by 1.41 rather + # than sqrt(2), and resolved ties towards NE rather than + # deterministically. + self._griddata = np.asarray( + kernels.flow_directions(elevations, cellsize_grid), dtype=self.dtype) + self._griddata[np.isnan(elevations)] = 0 + mask = kwargs.get('mask') - self._sort_indexes = flooded_dem.sort(reverse = False, force = True, mask = mask) + self._sort_indexes = flooded_dem.sort(reverse=False, force=True, mask=mask) self._sorted = True - + self._invalidate_network() + + # -- flow network ----------------------------------------------------- + # + # ``receivers`` and ``topological_order`` are derived from the direction + # codes and cached. Everything that walks the network downstream uses + # them, which is what lets accumulation, flow length and chi run in a + # single linear pass instead of a per-cell Python loop. + + def _invalidate_network(self): + self._receivers = None + self._topological_order = None + self._cycle_count = 0 + + @property + def receivers(self): + """Flat index of the cell each cell drains to, or -1 for none.""" + if getattr(self, '_receivers', None) is None: + self._receivers = np.asarray(kernels.receivers(self._griddata), dtype=np.int64) + return self._receivers + + @property + def topological_order(self): + """Flat cell indices ordered so every donor precedes its receiver.""" + if getattr(self, '_topological_order', None) is None: + order, cycles = kernels.topological_order(self.receivers) + self._topological_order = np.asarray(order, dtype=np.int64) + self._cycle_count = int(cycles) + if cycles: + warnings.warn( + "{0} cells of this flow-direction grid lie on circular " + "flow paths and cannot be ordered; results that depend on " + "routing through them will be wrong. This usually means " + "the DEM was not filled before routing.".format(cycles), + RuntimeWarning, stacklevel=2) + return self._topological_order + + @property + def cycle_count(self): + """How many cells lie on circular flow paths (0 for a valid network).""" + self.topological_order # populate the cache + return self._cycle_count + + def step_lengths(self, pixel_dimension=None): + """Distance from each cell to its receiver, in map units.""" + if pixel_dimension is None: + pixel_dimension = self._mean_pixel_dimension() + return np.asarray(pixel_dimension, dtype=float64) * self.pixel_scale(float64) + + # -- keeping the cache honest ----------------------------------------- + # + # Everything downstream now reads `receivers`/`topological_order` rather + # than re-deriving routing from `_griddata` on every call, so any write + # to a flow code has to drop the cache. Editing a code and having the + # edit silently ignored would be far worse than the cost of rebuilding. + + def __setitem__(self, key, value): + super(FlowDirectionD8, self).__setitem__(key, value) + self._invalidate_network() + + def set_value_at_rowscols(self, value, rowscols): + super(FlowDirectionD8, self).set_value_at_rowscols(value, rowscols) + self._invalidate_network() + + def invalidate_network(self): + """Drop the cached receivers and ordering. + + Call this after modifying ``_griddata`` directly; the indexed write + paths do it for you. + """ + self._invalidate_network() + def __flow_code_for_position(self, flooded_dem, i, j): - - flow_code = 0 - min_diff = 0 - - if flooded_dem[i,j+1] is not None and ((flooded_dem[i, j] - flooded_dem[i,j+1])) / 1.41 > min_diff: - flow_code = 1 - min_diff = (flooded_dem[i, j] - flooded_dem[i,j+1]) / 1.41 - - if flooded_dem[i+1,j+1] is not None and ((flooded_dem[i, j] - flooded_dem[i+1,j+1])) > min_diff: - flow_code = 2 - min_diff = flooded_dem[i, j] - flooded_dem[i+1,j+1] - - if flooded_dem[i+1,j] is not None and ((flooded_dem[i, j] - flooded_dem[i+1,j])) / 1.41 > min_diff: - flow_code = 4 - min_diff = (flooded_dem[i, j] - flooded_dem[i+1,j]) / 1.41 - - if flooded_dem[i+1,j-1] is not None and ((flooded_dem[i, j] - flooded_dem[i+1,j-1])) > min_diff: - flow_code = 8 - min_diff = flooded_dem[i, j] - flooded_dem[i+1,j-1] - - if flooded_dem[i,j-1] is not None and ((flooded_dem[i, j] - flooded_dem[i,j-1])) / 1.41 > min_diff: - flow_code = 16 - min_diff = (flooded_dem[i, j] - flooded_dem[i,j-1]) / 1.41 - - if flooded_dem[i-1,j-1] is not None and ((flooded_dem[i, j] - flooded_dem[i-1,j-1])) > min_diff: - flow_code = 32 - min_diff = flooded_dem[i, j] - flooded_dem[i-1,j-1] - - if flooded_dem[i-1,j] is not None and ((flooded_dem[i, j] - flooded_dem[i-1,j]) / 1.41) > min_diff: - flow_code = 64 - min_diff = (flooded_dem[i, j] - flooded_dem[i-1,j]) / 1.41 - - if flooded_dem[i-1,j+1] is not None and ((flooded_dem[i, j] - flooded_dem[i-1,j+1])) > min_diff: - flow_code = 128 - min_diff = flooded_dem[i, j] - flooded_dem[i-1,j+1] - - return flow_code - + """Steepest-descent code for a single cell. + + Kept for :meth:`update_flow_codes_in_mask`, which re-routes a handful + of cells after their elevations change. + """ + z = flooded_dem[i, j] + if z is None or np.isnan(z): + return 0 + + best_code = 0 + best_slope = 0.0 + de = self._georef_info.dx + for d, (di, dj) in enumerate(zip(kernels.D8_DI, kernels.D8_DJ)): + neighbour = flooded_dem[i + int(di), j + int(dj)] + if neighbour is None or np.isnan(neighbour): + continue + # The scaling used to be inverted -- cardinal drops were divided + # by 1.41 and diagonal ones were not -- which biased routing + # towards the diagonals. + slope = (z - neighbour) / (de * kernels.D8_DIST[d]) + if slope > best_slope: + best_slope = slope + best_code = int(kernels.D8_CODES[d]) + return best_code + def get_flow_to_cell(self,i,j): - #Function to get the indices of the cell that is drained to based on the flow direction specified in fd - - iOut = None - jOut = None - isGood = False - - if self._griddata[i,j] == 1 and j+1 < self._georef_info.nx: - iOut = i - jOut = j+1 - elif self._griddata[i,j] == 2 and i+1 < self._georef_info.ny and j+1 < self._georef_info.nx: - iOut = i+1 - jOut = j+1 - elif self._griddata[i,j] == 4 and i+1 < self._georef_info.ny: - iOut = i+1 - jOut = j - elif self._griddata[i,j] == 8 and i+1 < self._georef_info.ny and j-1 >= 0: - iOut = i+1 - jOut = j-1 - elif self._griddata[i,j] == 16 and j-1 >= 0: - iOut = i - jOut = j-1 - elif self._griddata[i,j] == 32 and i-1 >= 0 and j-1 >= 0: - iOut = i-1 - jOut = j-1 - elif self._griddata[i,j] == 64 and i-1 >= 0: - iOut = i-1 - jOut = j - elif self._griddata[i,j] == 128 and i-1 >= 0 and j+1 < self._georef_info.nx: - iOut = i-1 - jOut = j+1 - - if not(iOut is None): - isGood = True - - return iOut, jOut, isGood + """Index of the cell that ``(i, j)`` drains into. + + Returns + ------- + (int or None, int or None, bool) + Row, column and a flag that is ``False`` when the cell has no + downstream neighbour inside the grid. + """ + code = self._griddata[i, j] + if code == 0: + return None, None, False + + for d in range(8): + if code != kernels.D8_CODES[d]: + continue + i_out = i + int(kernels.D8_DI[d]) + j_out = j + int(kernels.D8_DJ[d]) + if 0 <= i_out < self._georef_info.ny and 0 <= j_out < self._georef_info.nx: + return i_out, j_out, True + break + + return None, None, False def get_upstream_cell_indexes(self, i, j): - + """Indices of the immediate neighbours that drain into ``(i, j)``.""" options = list() - - if self[i, j+1] == 16: - options += [(i, j+1)] - - if self[i+1, j+1] == 32: - options += [(i+1, j+1)] - - if self[i+1, j] == 64: - options += [(i+1, j)] - - if self[i+1, j-1] == 128: - options += [(i+1, j-1)] - - if self[i, j-1] == 1: - options += [(i, j-1)] - - if self[i-1, j-1] == 2: - options += [(i-1, j-1)] - - if self[i-1, j] == 4: - options += [(i-1, j)] - - if self[i-1, j+1] == 8: - options += [(i-1, j+1)] - + for d in range(8): + i_up = i + int(kernels.D8_DI[d]) + j_up = j + int(kernels.D8_DJ[d]) + # A neighbour in direction d drains here if it points back the + # opposite way, i.e. four bit positions round the compass. + opposite = int(kernels.D8_CODES[(d + 4) % 8]) + if self[i_up, j_up] == opposite: + options.append((i_up, j_up)) return options - - def __get_flow_from_cell(self, i, j, max_recursion_depth=None, depth=0): - - i_source = [i] - j_source = [j] - if max_recursion_depth is not None and depth > max_recursion_depth: - return i_source, j_source + def __get_flow_from_cell(self, i, j, max_recursion_depth=None, depth=0): + """Every cell upstream of ``(i, j)``, including ``(i, j)`` itself. - recursion_depth = depth + 1 + Returns ``(rows, cols)`` as plain lists. ``max_recursion_depth`` + limits how many steps upstream the search goes; ``depth`` is accepted + for backward compatibility and gives the starting depth. + The traversal is iterative. The original recursive version needed + ``sys.setrecursionlimit(1000000)`` and still crashed the interpreter + on basins more than a few tens of thousands of cells long. + """ + if max_recursion_depth is None and depth == 0: + # No depth limit: let the compiled kernel do the whole basin in + # one linear pass over the flow network. + flat = np.int64(i) * self._georef_info.nx + np.int64(j) + mask = kernels.upstream_mask(self.receivers, np.array([flat], dtype=np.int64)) + rows, cols = np.nonzero(mask) + return rows.tolist(), cols.tolist() + + ny, nx = self._georef_info.ny, self._georef_info.nx + limit = None if max_recursion_depth is None else max_recursion_depth - depth + + i_source = [] + j_source = [] + seen = set() + stack = [((int(i), int(j)), 0)] + while stack: + (ci, cj), d = stack.pop() + if (ci, cj) in seen: + continue + seen.add((ci, cj)) + i_source.append(ci) + j_source.append(cj) + if limit is not None and d >= limit: + continue + for k in range(8): + ui = ci + int(kernels.D8_DI[k]) + uj = cj + int(kernels.D8_DJ[k]) + if not (0 <= ui < ny and 0 <= uj < nx): + continue + if self._griddata[ui, uj] == kernels.D8_CODES[(k + 4) % 8]: + stack.append(((ui, uj), d + 1)) - if self[i, j+1] == 16: - [i_append, j_append] = self.__get_flow_from_cell(i,j+1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i+1, j+1] == 32: - [i_append, j_append] = self.__get_flow_from_cell(i+1,j+1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i+1, j] == 64: - [i_append, j_append] = self.__get_flow_from_cell(i+1,j,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i+1, j-1] == 128: - [i_append, j_append] = self.__get_flow_from_cell(i+1,j-1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i, j-1] == 1: - [i_append, j_append] = self.__get_flow_from_cell(i,j-1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i-1, j-1] == 2: - [i_append, j_append] = self.__get_flow_from_cell(i-1,j-1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i-1, j] == 4: - [i_append, j_append] = self.__get_flow_from_cell(i-1,j,max_recursion_depth = max_recursion_depth,depth = recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i-1, j+1] == 8: - [i_append, j_append] = self.__get_flow_from_cell(i-1,j+1,max_recursion_depth = max_recursion_depth,depth = recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - return i_source, j_source def __map_flow_from_cell(self, index, **kwargs): - + """Build the nested upstream dictionary rooted at ``index``. + + Each node is a dict with: + + ``index`` + the ``(row, col)`` of the cell; + ``distance_scale`` + 1 or sqrt(2), the step length from this node's *parent* in cell + widths -- 1 at the root; + ``next`` + the list of upstream child nodes, absent at a headwater; + plus one entry per keyword grid, holding that grid's value at the cell. + + ``assume_valid_routing=False`` guards against cycles in a + flow-direction grid that was not derived from a filled DEM. + + Notes + ----- + ``distance_scale`` used to be written onto the *parent* node and + overwritten by each child in turn, so it ended up describing whichever + child happened to be visited last. Chi profiles built from these + dictionaries were wrong wherever a confluence had a diagonal and a + cardinal tributary. + """ if len(index) == 1: - (i, j) = index[0] + root = tuple(index[0]) else: - (i, j) = index - - return_dict = dict() - - return_dict['index'] = index - for arg in kwargs: - if arg != 'assume_valid_routing': - return_dict[arg] = kwargs[arg][i,j] - - return_dict['next'] = [] - return_dict['distance_scale'] = 1.0 - - if self[i, j+1] == 16: - if kwargs.get('assume_valid_routing', True) or not self.visited[i, j+1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i, j+1] = True - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell((i,j+1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i+1, j+1] == 32: - if kwargs.get('assume_valid_routing', True) or not self.visited[i+1, j+1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i+1, j+1] = True - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell((i+1,j+1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i+1, j] == 64: - if kwargs.get('assume_valid_routing', True) or not self.visited[i+1, j]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i+1, j] = True - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell((i+1,j), **kwargs) - return_dict['next'].append(child_dict) - - if self[i+1, j-1] == 128: - if kwargs.get('assume_valid_routing', True) or not self.visited[i+1, j-1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i+1, j-1] = True - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell((i+1,j-1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i, j-1] == 1: - if kwargs.get('assume_valid_routing', True) or not self.visited[i, j-1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i, j-1] = True - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell((i,j-1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i-1, j-1] == 2: - if kwargs.get('assume_valid_routing', True) or not self.visited[i-1, j-1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i-1, j-1] = True - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell((i-1,j-1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i-1, j] == 4: - if kwargs.get('assume_valid_routing', True) or not self.visited[i-1, j]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i-1, j] = True - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell((i-1,j), **kwargs) - return_dict['next'].append(child_dict) - - if self[i-1, j+1] == 8: - if kwargs.get('assume_valid_routing', True) or not self.visited[i-1, j+1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i-1, j+1] = True - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell((i-1,j+1), **kwargs) - return_dict['next'].append(child_dict) - - if len(return_dict.get('next', 0)) == 0: - return_dict.pop('next') - - return return_dict + root = tuple(index) + + assume_valid_routing = kwargs.get('assume_valid_routing', True) + value_grids = {name: grid for name, grid in kwargs.items() + if name != 'assume_valid_routing'} + ny, nx = self._georef_info.ny, self._georef_info.nx + + def make_node(cell, scale): + i, j = cell + node = {'index': cell, 'distance_scale': scale} + for name, grid in value_grids.items(): + node[name] = grid[i, j] + return node + + # Every node carries `index` as a plain (row, col) pair, the root + # included. It used to keep the nested form the caller passed, so + # consumers had to special-case the root -- and those that did not + # failed to unpack it. + root_node = make_node(root, 1.0) + # Explicit stack of (node, cell) pairs instead of recursion. + stack = [(root_node, root)] + while stack: + node, (i, j) = stack.pop() + children = [] + for k in range(8): + ui = i + int(kernels.D8_DI[k]) + uj = j + int(kernels.D8_DJ[k]) + if not (0 <= ui < ny and 0 <= uj < nx): + continue + if self._griddata[ui, uj] != kernels.D8_CODES[(k + 4) % 8]: + continue + if not assume_valid_routing: + if self.visited[ui, uj]: + continue + self.visited[ui, uj] = True + child = make_node((ui, uj), float(kernels.D8_DIST[k])) + children.append(child) + stack.append((child, (ui, uj))) + if children: + node['next'] = children + + return root_node def update_flow_codes_in_mask(self, *args, **kwargs): - + """Re-derive flow codes inside a mask after the DEM has changed.""" flooded_dem = args[0] mask = args[1] - + indexes = np.where(mask._griddata == 1) for (i,j) in zip(indexes[0],indexes[1]): flow_code = self.__flow_code_for_position(flooded_dem, i, j) self._griddata[i,j] = flow_code - + self._invalidate_network() + + def basin_mask(self, outlets): + """Boolean array marking every cell draining to any of ``outlets``. + + ``outlets`` is a sequence of ``(x, y)`` map coordinates. + """ + flat = [] + for (row, col) in self._xy_to_rowscols(outlets): + if row is None: + continue + flat.append(row * self._georef_info.nx + col) + mask = kernels.upstream_mask(self.receivers, + np.array(flat, dtype=np.int64)) + return np.asarray(mask).astype(bool) + def divides_for_outlets(self, outlet1, outlet2): - basin1 = BaseSpatialGrid() - basin1._copy_info_from_grid(self, True) - - ind = self.get_indexes_of_upstream_cells_for_location(outlet1[0], outlet1[1]) - i,j = zip(*ind) - basin1._griddata[i,j] = 1 - - basin2 = BaseSpatialGrid() - basin2._copy_info_from_grid(self, True) - - ind = self.get_indexes_of_upstream_cells_for_location(outlet2[0], outlet2[1]) - i,j = zip(*ind) - basin2._griddata[i,j] = 1 - - from scipy.ndimage.morphology import binary_dilation as dilate - - basin3 = BaseSpatialGrid() - basin3._copy_info_from_grid(self, True) - basin3._griddata = basin1._griddata + dilate(basin2._griddata) - - basin4 = BaseSpatialGrid() - basin4._copy_info_from_grid(self, True) - basin4._griddata = basin2._griddata + dilate(basin1._griddata) - - (i, j) = np.where(basin3._griddata == 2) - ind1 = zip(i,j) - - (i,j) = np.where(basin4._griddata == 2) - ind2 = zip(i,j) - - return ind1, ind2 - + """Cells on each side of the divide shared by two basins. + + Returns + ------- + (tuple, tuple) + ``(cells_in_basin1, cells_in_basin2)``, each a tuple of + ``(row, col)`` pairs lying against the shared divide. + """ + from scipy.ndimage import binary_dilation, generate_binary_structure + + in_basin1 = self.basin_mask((outlet1,)) + in_basin2 = self.basin_mask((outlet2,)) + + # An 8-connected structure, so cells meeting only at a corner still + # count as facing each other across the divide. + structure = generate_binary_structure(2, 2) + ind1 = np.argwhere(in_basin1 & binary_dilation(in_basin2, structure=structure)) + ind2 = np.argwhere(in_basin2 & binary_dilation(in_basin1, structure=structure)) + + # Materialise the pairs: the old version returned zip objects, which + # are single-use and silently came back empty on a second read. + return (tuple(map(tuple, ind1)), tuple(map(tuple, ind2))) + def locations_of_paired_hollows(self, outlet1, outlet2, area, Ao=1E5): - - def map_down_to_hollow(ind, fd): - (i,j) = ind - (next_i, next_j, _) = fd.get_flow_to_cell(i,j) - if((area[i,j] < Ao) and (area[next_i, next_j] >= Ao)): - return next_i, next_j - else: - return map_down_to_hollow((next_i, next_j), fd) - + """Channel heads reached by flowing down from a shared divide. + + Walks downstream from each divide cell until drainage area first + exceeds ``Ao``, and returns the distinct cells where that happens on + each side. + """ + def map_down_to_hollow(ind): + """Follow flow down until area first reaches Ao; None if never.""" + (i, j) = ind + seen = set() + while True: + if (i, j) in seen: + return None # circular routing + seen.add((i, j)) + (next_i, next_j, good) = self.get_flow_to_cell(i, j) + if not good: + return None + here = area[i, j] + there = area[next_i, next_j] + if here is None or there is None: + return None + if here < Ao <= there: + return next_i, next_j + (i, j) = (next_i, next_j) + ind1, ind2 = self.divides_for_outlets(outlet1, outlet2) - h1 = set() - for ind in ind1: - h1.add(map_down_to_hollow(ind, self)) - h2 = set() - for ind in ind2: - h2.add(map_down_to_hollow(ind, self)) - - return tuple(h1), tuple(h2) - - + h1 = {h for h in (map_down_to_hollow(ind) for ind in ind1) if h is not None} + h2 = {h for h in (map_down_to_hollow(ind) for ind in ind2) if h is not None} + + return tuple(sorted(h1)), tuple(sorted(h2)) + def get_indexes_of_upstream_cells(self, i, j): - - i,j = self.__get_flow_from_cell(i,j) - return zip(i,j) - + """Tuple of ``(row, col)`` for every cell upstream of ``(i, j)``.""" + rows, cols = self.__get_flow_from_cell(i, j) + # A tuple rather than a zip object: callers iterate the result more + # than once, and a zip is exhausted after the first pass. + return tuple(zip(rows, cols)) + def get_indexes_of_upstream_cells_for_location(self, x, y): - + """As :meth:`get_indexes_of_upstream_cells`, from a map coordinate.""" v = ((x, y), ) ((i, j),) = self._xy_to_rowscols(v) + if i is None: + return tuple() return self.get_indexes_of_upstream_cells(i, j) def search_down_flow_direction_with_length(self, start, search_length = np.inf): - l = list() + """Follow flow downstream from ``start``, recording distance travelled. + + Stops when flow leaves the grid, at a pit, once ``search_length`` is + covered, or if the path revisits a cell. The terminal cell is + included in the result. + + Returns + ------- + (tuple, tuple) + ``(cells, distances)``, where ``cells`` are ``(row, col)`` pairs + and ``distances`` are the along-path distances at each cell. + """ + cells = list() length_list = list() - ((row,col),) = self._xy_to_rowscols((start,)) + ((row, col),) = self._xy_to_rowscols((start,)) + if row is None: + return tuple(), tuple() + + # A set for the revisit test: the original scanned the whole path + # list at every step, making long flow paths quadratic. The edge + # test is now `no in-grid receiver` rather than `on the perimeter`, + # so a path that merely starts on the edge still gets followed. + seen = set() length = 0.0 - while not (row == 0 or row == self._georef_info.ny-1 or col == 0 or col == self._georef_info.nx - 1 or \ - (row,col) in l) and (length < search_length): - l.append((row, col)) + while True: + if (row, col) in seen or length >= search_length: + break + seen.add((row, col)) + cells.append((row, col)) length_list.append(length) - row_n,col_n,inBounds = self.get_flow_to_cell(row, col) - length += self._georef_info.dx * (1.0 if ((row_n == row) or (col_n == col)) else 1.414) - (row, col) = (row_n, col_n) + row_n, col_n, inBounds = self.get_flow_to_cell(row, col) if not inBounds: break - return tuple(l), tuple(length_list) + diagonal = (row_n != row) and (col_n != col) + length += self._georef_info.dx * (SQRT2 if diagonal else 1.0) + (row, col) = (row_n, col_n) + + return tuple(cells), tuple(length_list) def search_down_flow_direction(self, start, search_length = np.inf): + """The cells along the flow path from ``start``, without distances.""" return self.search_down_flow_direction_with_length(start, search_length=search_length)[0] def search_down_flow_direction_from_rowscols_location(self, start, return_rowscols = False, search_length = np.inf): - + """Follow flow downstream from a ``(row, col)`` start. + + Returns map coordinates unless ``return_rowscols`` is true. + """ rcs = self.search_down_flow_direction(self._rowscols_to_xy(((start),))[0], search_length = search_length) return rcs if return_rowscols else self._rowscols_to_xy(rcs) def search_down_flow_direction_from_xy_location(self, start, return_rowscols=False, search_length=np.inf): + """Follow flow downstream from an ``(x, y)`` start. + Returns map coordinates unless ``return_rowscols`` is true. + """ rcs = self.search_down_flow_direction(start, search_length=search_length) return rcs if return_rowscols else self._rowscols_to_xy(rcs) - def convert_rivertools_directions_to_arc(self): - # Function to convert river tools flow directions to arcGisFlowDirections - # ArcGIS convention - # |i-1,j-1 i-1,j i-1,j+1| |32 64 128| - # |i,j-1 i,j i,j+1| |16 X 1 | - # |i+1,j-1 i+1,j i+1,j+1| |8 4 2 | - # In river tools convention the 1 is in the top right - - convertedFlowDir = int(np.log2(self._griddata)) - convertedFlowDir -= 1 - convertedFlowDir[convertedFlowDir == -1] = 7 - convertedFlowDir = 2**convertedFlowDir - convertedFlowDir[self._griddata == self.noData] = self.noData - self._griddata = convertedFlowDir - + def convert_rivertools_directions_to_arc(self, no_data=0): + """Convert RiverTools flow codes in place to the ArcGIS convention. + + RiverTools numbers the same eight directions but starts from the top + right, so every code is rotated by one position:: + + ArcGIS | 32 64 128 | + | 16 X 1 | + | 8 4 2 | + """ + codes = np.asarray(self._griddata) + valid = codes > 0 + # np.log2 of an array cannot be wrapped in int(); use integer + # arithmetic on the exponent instead, and take the no-data value as an + # argument rather than reading an attribute that never existed. + exponent = np.zeros(codes.shape, dtype=np.int16) + exponent[valid] = np.round(np.log2(codes[valid].astype(float64))).astype(np.int16) + exponent -= 1 + exponent[exponent == -1] = 7 + converted = np.zeros(codes.shape, dtype=self.dtype) + converted[valid] = (2 ** exponent[valid]).astype(self.dtype) + converted[codes == no_data] = no_data + self._griddata = converted + self._invalidate_network() + def pixel_scale(self, dtype = np.float32): - + """Step-length multiplier for each cell: 1 cardinal, sqrt(2) diagonal.""" dim = np.ones_like(self._griddata, dtype = dtype) - dim[np.where((self._griddata == 32) | (self._griddata == 128) | (self._griddata == 2) | (self._griddata == 8))] = 1.41421356 + diagonal = np.isin(self._griddata, [2, 8, 32, 128]) + dim[diagonal] = SQRT2 return dim - + def map_values_to_recursive_list(self, outlet, **kwargs): - + """Nested dictionary of the basin upstream of ``outlet``. + + Pass grids as keywords to sample them at every cell, e.g. + ``map_values_to_recursive_list(outlet, elevation=dem, area=area)``. + See :meth:`__map_flow_from_cell` for the node layout. + """ v = (outlet, ) (ij_outlet, ) = self._xy_to_rowscols(v) + if ij_outlet[0] is None: + raise Error.InputError('map_values_to_recursive_list', + 'outlet {0} lies outside the grid'.format(outlet)) if kwargs.get('assume_valid_routing', True) is False: self.visited = np.zeros_like(self._griddata).astype(bool) return self.__map_flow_from_cell((ij_outlet,), **kwargs) - def bounds_of_basin_for_outlet(self, outlet): - + """``((xmin, xmax), (ymin, ymax))`` of the basin draining to ``outlet``. + + ``outlet`` is given as ``(y, x)`` -- latitude then longitude -- which + is the order the rest of this method's callers use. + """ (lat, lon) = outlet indexes = self.get_indexes_of_upstream_cells_for_location(lon, lat) - xy = self._rowscols_to_xy(indexes) - bounds = [[None, None], [None, None]] - for (lon, lat) in xy: - if lon > bounds[0][1] or bounds[0][1] is None: - bounds[0][1] = lon - if lon < bounds[0][0] or bounds[0][0] is None: - bounds[0][0] = lon - if lat > bounds[1][1] or bounds[1][1] is None: - bounds[1][1] = lat - if lat < bounds[1][0] or bounds[1][0] is None: - bounds[1][0] = lat - return ((bounds[0][0], bounds[0][1]), (bounds[1][0], bounds[1][1])) - + xy = self._rowscols_to_xy(indexes) + if not xy: + raise Error.InputError('bounds_of_basin_for_outlet', + 'no cells drain to {0}'.format(outlet)) + # Comparing a float with None raises in Python 3, and `x > None or + # None is None` evaluates the comparison first, so the original could + # not survive its own first iteration. + lons = [p[0] for p in xy] + lats = [p[1] for p in xy] + return ((min(lons), max(lons)), (min(lats), max(lats))) + def divides(self): - + """Grid marking drainage divides: cells with no upstream neighbour.""" divides = BaseSpatialGrid() divides._copy_info_from_grid(self, True) - - ip1j = self._griddata[2:-1,1:-2] - im1j = self._griddata[0:-3,1:-2] - ijp1 = self._griddata[1:-2,2:-1] - ijm1 = self._griddata[1:-2,0:-3] - - ip1jp1 = self._griddata[2:-1, 2:-1] - ip1jm1 = self._griddata[2:-1, 0:-3] - im1jp1 = self._griddata[0:-3, 2:-1] - im1jm1 = self._griddata[0:-3, 0:-3] - - divides._griddata[1:-2,1:-2] = ((ijm1 != 1) & (im1jm1 != 2) & (im1j != 4) & (im1jp1 != 8) & (ijp1 != 16) & (ip1jp1 != 32) & (ip1j != 64) & (ip1jm1 != 128)) + + codes = self._griddata + # A cell is a divide when none of its eight neighbours drains into it. + is_divide = np.ones(codes.shape, dtype=bool) + for d in range(8): + di, dj = int(kernels.D8_DI[d]), int(kernels.D8_DJ[d]) + opposite = kernels.D8_CODES[(d + 4) % 8] + neighbour = np.zeros(codes.shape, dtype=codes.dtype) + src_i = slice(max(di, 0), codes.shape[0] + min(di, 0)) + src_j = slice(max(dj, 0), codes.shape[1] + min(dj, 0)) + dst_i = slice(max(-di, 0), codes.shape[0] + min(-di, 0)) + dst_j = slice(max(-dj, 0), codes.shape[1] + min(-dj, 0)) + neighbour[dst_i, dst_j] = codes[src_i, src_j] + is_divide &= (neighbour != opposite) + + divides._griddata = is_divide.astype(divides.dtype) return divides - + def paired_divides(self, mask = None): - + """Pair each divide cell with the cell directly across the divide. + + The partner is the reflection of the divide cell's downstream + neighbour through the cell itself, i.e. the cell one step *up* the + opposite hillslope. + + Returns + ------- + tuple + ``((xy_from, xy_to), ...)`` in map coordinates. + """ divides = self.divides() if mask is not None: - i = np.where((divides._griddata == 1) & (mask._griddata > 0)) + selected = np.where((divides._griddata == 1) & (mask._griddata > 0)) else: - i = np.where(divides._griddata == 1) + selected = np.where(divides._griddata == 1) - rcs = zip(i[0].tolist(),i[1].tolist()) - pairs = list() - for (i,j) in rcs: - - (i_next, j_next, good) = self.get_flow_to_cell(i,j) - - if good and i_next is not None and j_next is not None: - i_next = 2*i - i_next - j_next = 2*j - j_next - - if i_next >= 0 and i_next < self._georef_info.ny and j_next >= 0 and j_next < self._georef_info.nx: - (xy, ) = self._rowscols_to_xy(((i,j),)) - (xy_next, ) = self._rowscols_to_xy(((i_next, j_next),)) - pairs.append( (xy, xy_next)) - + for (i, j) in zip(selected[0].tolist(), selected[1].tolist()): + + (i_next, j_next, good) = self.get_flow_to_cell(i, j) + + if good: + i_opposite = 2*i - i_next + j_opposite = 2*j - j_next + + if (0 <= i_opposite < self._georef_info.ny + and 0 <= j_opposite < self._georef_info.nx): + (xy, ) = self._rowscols_to_xy(((i, j),)) + (xy_next, ) = self._rowscols_to_xy(((i_opposite, j_opposite),)) + pairs.append((xy, xy_next)) + return tuple(pairs) - + class Elevation(CalculationMixin, BaseSpatialGrid): + """A DEM. + + Adds the terrain-derivative helpers of :class:`CalculationMixin` to the + base raster, plus the edge detection that depression filling starts from. + """ def findDEMedge(self): - #Pad the data so that we can take a windowed max - original = np.zeros((self._georef_info.ny, self._georef_info.nx)) - i = np.where(np.isnan(self._griddata)) - original[i] = 1 - - from scipy.ndimage.morphology import binary_dilation as dilation - from scipy.ndimage import generate_binary_structure as generate_binary_structure - structure = generate_binary_structure(2, 2) - dilated = dilation(original, structure = structure, border_value = 1).astype(int) - edges = dilated - original - - return np.where(edges) - - def outlets_at_coastlines(self): - - import scipy.ndimage.morphology as morph - - mask1 = BaseSpatialGrid() - mask2 = BaseSpatialGrid() - mask3 = BaseSpatialGrid() - mask1._copy_info_from_grid(self, True) - mask1.dtype = np.uint8 - mask1._griddata = mask1._griddata.astype(np.uint8) - mask1._griddata[(self._griddata > 0)] = 1 - mask2._copy_info_from_grid(mask1, False) - mask3._copy_info_from_grid(mask1, False) - mask2._griddata = morph.binary_dilation(mask2._griddata, iterations = 3).astype(uint8) - mask3._griddata = morph.binary_erosion(mask3._griddata, iterations = 3).astype(uint8) - mask3._griddata = (mask3._griddata.astype(float) + mask2._griddata.astype(float)).astype(uint8) - i = np.where(mask3._griddata == 1) - rc = zip(i[0].tolist(),i[1].tolist()) - return self._rowscols_to_xy(rc) - + """Cells where water can leave the DEM. + + Those are the grid perimeter plus every valid cell touching a no-data + cell -- the same seed set TopoToolbox's ``fillsinks`` uses. Priority + flooding starts here. + + Returns + ------- + (ndarray, ndarray) + ``(rows, cols)``, as :func:`numpy.where` returns. + """ + # Delegates to the kernel so that the cells the flood is seeded with + # are, by construction, exactly the cells reported here. + flat = kernels.default_seeds(np.asarray(self._griddata, dtype=float64)) + return np.unravel_index(flat, (self._georef_info.ny, self._georef_info.nx)) + + def outlets_at_coastlines(self, iterations=3): + """Map coordinates of the cells forming the land/sea boundary. + + Land is taken to be every cell with a positive elevation. The + coastline is the band left over when the land mask is dilated and + eroded by ``iterations`` cells and the two are compared. + """ + from scipy.ndimage import binary_dilation, binary_erosion + + land = np.asarray(self._griddata) > 0 + dilated = binary_dilation(land, iterations=iterations) + eroded = binary_erosion(land, iterations=iterations) + # Exactly one of the two is set only in the fringe between them. + coastline = dilated.astype(np.int16) + eroded.astype(np.int16) == 1 + + rows, cols = np.nonzero(coastline) + return self._rowscols_to_xy(list(zip(rows.tolist(), cols.tolist()))) + def track_flow_downhill(self, starting_point, maximum_pit_depth = 20): - - adjust = [(-1, -1), (0, -1), (1,-1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)] - + """Walk downhill from ``starting_point``, hopping over shallow pits. + + At each step the lowest unvisited neighbour is taken, provided it is + no more than ``maximum_pit_depth`` above the current cell. That lets + the path climb out of small closed depressions instead of stopping in + them, without filling the DEM. + + Returns + ------- + (tuple, tuple, tuple) + ``(xy, distance, elevation)`` along the path. + """ + adjust = [(-1, -1), (0, -1), (1, -1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)] + (ij, ) = self._xy_to_rowscols((starting_point,)) - visited = (ij, ) - ij_a = [(ij[0] + x[0], ij[1] + x[1]) for x in adjust] - e_a = np.array([self[i[0], i[1]] for i in ij_a]) - new = True - while None not in e_a and np.sum(np.isnan(e_a)) == 0 and new: - sls = np.argsort(e_a) - new = False - for sl in sls: + if ij[0] is None: + raise Error.InputError('track_flow_downhill', + 'starting point lies outside the grid') + + def neighbour_elevations(cell): + """Elevations of the eight neighbours; NaN where off-grid.""" + # Building this with `self[...] or None` produced an object-dtype + # array, and `None not in array` then silently never matched. + values = np.empty(8, dtype=float64) + for k, (di, dj) in enumerate(adjust): + value = self[cell[0] + di, cell[1] + dj] + values[k] = np.nan if value is None else value + return values + + visited = [tuple(ij)] + seen = {tuple(ij)} + e_a = neighbour_elevations(ij) + + while not np.any(np.isnan(e_a)): + here = self[ij[0], ij[1]] + moved = False + for sl in np.argsort(e_a): this_ij = (ij[0] + adjust[sl][0], ij[1] + adjust[sl][1]) - if (this_ij not in visited) and ( (e_a[sl] - self[ij[0], ij[1]]) <= maximum_pit_depth): - visited += (this_ij, ) - ij = this_ij - new = True - break - if new: - ij_a = [(ij[0] + x[0], ij[1] + x[1]) for x in adjust] - e_a = np.array([self[i[0], i[1]] for i in ij_a]) + if this_ij in seen: + continue + if (e_a[sl] - here) > maximum_pit_depth: + continue + visited.append(this_ij) + seen.add(this_ij) + ij = this_ij + moved = True + break + if not moved: + break + e_a = neighbour_elevations(ij) + xy = self._rowscols_to_xy(visited) - this_xy = xy[0] - l = tuple() - e = tuple() - last_l = 0 - for txy in xy: - (ij, ) = self._xy_to_rowscols((txy,)) - e += (self[ij[0], ij[1]], ) - this_dl = np.sqrt(np.power(txy[0] - this_xy[0], 2) + np.power(txy[1] - this_xy[1], 2)) - last_l = last_l + this_dl - l += (last_l, ) - this_xy = txy - return xy, l, e + elevations = tuple(self[i, j] for (i, j) in visited) + distances = [0.0] + for previous, current in zip(xy[:-1], xy[1:]): + step = np.hypot(current[0] - previous[0], current[1] - previous[1]) + distances.append(distances[-1] + step) + return xy, tuple(distances), elevations class GeographicElevation(GeographicGridMixin, Elevation): + """A DEM on a latitude/longitude grid.""" + pass class Gradient(BaseSpatialGrid): + """The two components of the topographic gradient. + + :: + + gradient = Gradient(elevation=dem) + gradient._gx, gradient._gy + + Stored as two bands rather than one, so direction survives ``save()``. + :meth:`average_gradient` smooths both components over a length scale, + which is how a regional aspect is obtained. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), @@ -1755,6 +2545,7 @@ def _create_from_elevation(self, *args, **kwargs): [self._gy, self._gx] = np.gradient(elevation._griddata,dx) def average_gradient(self, distance): + """Both gradient components averaged over a disc of radius ``distance``.""" outgrid = Gradient() outgrid._copy_info_from_grid(self, True) outgrid._gy = self.average_over_distance(distance, grid = self._gy) @@ -1766,7 +2557,7 @@ def save(self, filename): @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -1774,7 +2565,7 @@ def get_band(gdal_dataset, band_number): if nodata is not None: nodata_elements = np.where(grid == nodata) if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -1822,6 +2613,14 @@ def plot(self, **kwargs): plt.show(block=True) class ScarpWavelet(BaseSpatialGrid): + """Template-matched scarp amplitude, age, orientation and signal-to-noise. + + Holds the four-band output of a scarp template search. Load one with + :meth:`load`; the bands become ``_A``, ``_kt``, ``_orientation`` and + ``_SNR``. :meth:`template_window` needs the separate ``scarplet`` + package. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), @@ -1829,7 +2628,7 @@ class ScarpWavelet(BaseSpatialGrid): @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -1837,7 +2636,7 @@ def get_band(gdal_dataset, band_number): if nodata is not None: nodata_elements = np.where(grid == nodata) if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -1864,10 +2663,16 @@ def get_band(gdal_dataset, band_number): return return_object def load_elevation(self, filename): + """Attach the DEM the templates were matched against.""" self.elevation = Elevation.load(filename) def plot_orientations(self, *args, **kwargs): - + """Draw scarp orientations over a hillshade, faded by signal-to-noise. + + Pass ``elevation`` and ``distance`` to have the regional slope + direction removed first, so what is drawn is the scarp orientation + *relative to* the hillslope it sits on. + """ # pop inputs, create those that are needed: elevation = kwargs.pop('elevation', None) @@ -1935,8 +2740,14 @@ def plot_orientations(self, *args, **kwargs): def calculate_valid_orientations(self, window_size, age, - num=46, + num=46, discard_max_min=True): + """Range of template orientations that stay inside the valid data. + + Returns ``(max_orientation, min_orientation)`` in radians, ``NaN`` + where no orientation fits. Cells within ``window_size`` of an edge + are always ``NaN``. + """ data = self.valid_data() max = -np.inf * np.ones_like(data) min = np.inf * np.ones_like(data) @@ -1985,6 +2796,10 @@ def _convolve_mask(self, data, window_size, age, orientation): return count def template_window(self, window_size, age, orientation, use_pixels=False): + """The scarp template mask at one size, age and orientation. + + Needs the separate ``scarplet`` package. + """ nx = self._georef_info.nx ny = self._georef_info.ny if use_pixels: @@ -1999,6 +2814,7 @@ def template_window(self, window_size, age, orientation, use_pixels=False): return window def valid_data(self): + """1 where the attached DEM has usable, above-sea-level data; NaN elsewhere.""" if self.elevation is None: raise AttributeError('No elevation data! Use load_elevation first') valid = np.ones_like(self.elevation._griddata) @@ -2007,493 +2823,723 @@ def valid_data(self): return valid class LocalRelief(BaseSpatialGrid): - + """Elevation range within a circular window. + + :: + + relief = LocalRelief(elevation=dem, pixel_radius=25) + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation','pixel_radius'), '_create_from_elevation_and_radius')) - + def _create_from_elevation_and_radius(self, *args, **kwargs): + from scipy.ndimage import maximum_filter, minimum_filter + elevation = kwargs['elevation'] - pixel_radius = kwargs['pixel_radius'] - self._copy_info_from_grid(elevation,True) - - import scipy.ndimage.filters as filters - y,x = np.ogrid[-pixel_radius:pixel_radius, -pixel_radius:pixel_radius] - footprint = x*x + y*y <= pixel_radius*pixel_radius - maximum = np.zeros_like(elevation._griddata) - minimum = np.zeros_like(elevation._griddata) - filters.maximum_filter(elevation._griddata, footprint = footprint, output = maximum) - filters.minimum_filter(elevation._griddata, footprint = footprint, output = minimum) + pixel_radius = int(kwargs['pixel_radius']) + self._copy_info_from_grid(elevation, True) + + # np.ogrid[-r:r] yields 2r samples, so the window used to be one cell + # short on the far side and therefore off-centre. + y, x = np.ogrid[-pixel_radius:pixel_radius + 1, -pixel_radius:pixel_radius + 1] + footprint = x * x + y * y <= pixel_radius * pixel_radius + + data = np.asarray(elevation._griddata, dtype=float64) + maximum = maximum_filter(data, footprint=footprint) + minimum = minimum_filter(data, footprint=footprint) self._griddata = maximum - minimum - + class Mask(BaseSpatialGrid): - + """0/1 grid marking the cells that drain to a set of outlets. + + :: + + mask = Mask(flow_direction=d8, outlets=[(x, y), ...]) + """ + dtype = uint8 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction','outlets'), '_create_from_flow_direction_and_outlets')) - + def _create_from_flow_direction_and_outlets(self, *args, **kwargs): flow_direction = kwargs['flow_direction'] outlets = kwargs['outlets'] self._copy_info_from_grid(flow_direction, True) - for (outlet_x, outlet_y) in outlets: - indexes = flow_direction.get_indexes_of_upstream_cells_for_location(outlet_x, outlet_y) - for index in indexes: - self[index[0],index[1]] = 1 + # One linear pass over the flow network for all the outlets at once, + # rather than a separate upstream walk per outlet. + self._griddata = flow_direction.basin_mask(outlets).astype(self.dtype) def perform_opening(self, structure = None, iterations = 1): - from scipy.ndimage.morphology import binary_opening - self._griddata = binary_opening(self._griddata, iterations = iterations, structure = structure) + """Morphological opening: erode then dilate, removing specks.""" + from scipy.ndimage import binary_opening + self._griddata = binary_opening( + self._griddata, iterations=iterations, structure=structure).astype(self.dtype) def perform_erosion(self, structure = None, iterations = 1): - from scipy.ndimage.morphology import binary_erosion - self._griddata = binary_erosion(self._griddata, iterations = iterations, structure = structure) + """Morphological erosion: shrink the mask by ``iterations`` cells.""" + from scipy.ndimage import binary_erosion + self._griddata = binary_erosion( + self._griddata, iterations=iterations, structure=structure).astype(self.dtype) class LogArea(BaseSpatialGrid): - - dtype = uint8 - + """Base-10 logarithm of drainage area.""" + + # float64, not uint8: log10(area) is a real number, and declaring it as a + # byte meant `save()` wrote a grid of 6s and 7s. + dtype = float64 + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('area',), '_create_from_area')) - + def _create_from_area(self, *args, **kwargs): area = kwargs['area'] self._copy_info_from_grid(area, True) - self._griddata = np.log10(area._griddata) - + data = np.asarray(area._griddata, dtype=float64) + with np.errstate(divide='ignore', invalid='ignore'): + self._griddata = np.where(data > 0, np.log10(np.where(data > 0, data, 1.0)), np.nan) + class Hillshade(CalculationMixin, BaseSpatialGrid): - + """Shaded relief, following the ESRI hillshade definition. + + :: + + hs = Hillshade(elevation=dem, azimuth=315, inclination=45) + + ``azimuth`` is the compass bearing of the light source in degrees + (0 = north, 90 = east) and ``inclination`` its height above the horizon. + Values run 0-255. + """ + dtype = uint8 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation','azimuth', 'inclination'), '_create_from_elevation')) - + def _create_from_elevation(self, *args, **kwargs): elevation = kwargs['elevation'] az = kwargs['azimuth'] elev = kwargs['inclination'] - + self._copy_info_from_grid(elevation) self.calcHillshade(az, elev) - - def calcHillshade(self,az,elev): - #Hillshade = calcHillshade(elevGrid,az,elev) - #Esri calculation for generating a hillshade, elevGrid is expected to be a numpy array - - # Convert angular measurements to radians - - azRad, elevRad = (360 - az + 90)*np.pi/180, (90-elev)*np.pi/180 - Sx, Sy = self._calcFiniteSlopes(self._griddata, self._mean_pixel_dimension(), self._georef_info.nx, self._georef_info.ny) # Calculate slope in X and Y directions - - AspectRad = np.arctan2(Sy, Sx) # Angle of aspect - SmagRad = np.arctan(np.sqrt(Sx**2 + Sy**2)) # magnitude of slope in radians - - self._griddata = 255.0 * ((np.cos(elevRad) * np.cos(SmagRad)) + (np.sin(elevRad)* np.sin(SmagRad) * np.cos(azRad - AspectRad))) + + def calcHillshade(self, az, elev, z_factor=1.0): + """Fill the grid with an ESRI-style hillshade. + + Parameters + ---------- + az : float + Illumination azimuth, degrees clockwise from north. + elev : float + Illumination altitude above the horizon, in degrees. + z_factor : float + Vertical exaggeration. + """ + azRad = (360.0 - az + 90.0) * np.pi / 180.0 + zenithRad = (90.0 - elev) * np.pi / 180.0 + + # Sx is dz/dx (positive east); Sy is dz/drow (positive south), which + # is the same sign convention ESRI's [dz/dy] uses. + Sx, Sy = self._calcFiniteSlopes(self._griddata, self._mean_pixel_dimension(), + self._georef_info.nx, self._georef_info.ny) + Sx = Sx * z_factor + Sy = Sy * z_factor + + # ESRI's aspect is atan2(dz/dy, -dz/dx). Without the negation the + # illumination came out mirrored east-west. + AspectRad = np.arctan2(Sy, -Sx) + SmagRad = np.arctan(np.sqrt(Sx**2 + Sy**2)) + + shade = 255.0 * ((np.cos(zenithRad) * np.cos(SmagRad)) + + (np.sin(zenithRad) * np.sin(SmagRad) + * np.cos(azRad - AspectRad))) + # Slopes facing away from the light give a negative value; ESRI + # clamps those to 0 rather than letting them wrap round the byte. + self._griddata = np.clip(np.nan_to_num(shade), 0.0, 255.0).astype(self.dtype) class GeographicHillshade(GeographicGridMixin, Hillshade): + """Hillshade on a latitude/longitude grid.""" + pass class MaxSlope(CalculationMixin, BaseSpatialGrid): - + """Magnitude of the topographic gradient, from centred differences. + + :: + + slope = MaxSlope(elevation=dem) + + Values are dimensionless (rise over run); take ``arctan`` for degrees. + """ + dtype = float64 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation',), '_create_from_elevation')) - + def _create_from_elevation(self, *args, **kwargs): elevation = kwargs['elevation'] - + self._copy_info_from_grid(elevation) self.calcSlope() - + def calcSlope(self): - - Sx, Sy = self._calcFiniteSlopes(self._griddata, self._mean_pixel_dimension(), self._georef_info.nx, self._georef_info.ny) # Calculate slope in X and Y directions + """Replace the grid with the gradient magnitude of its own values.""" + Sx, Sy = self._calcFiniteSlopes(self._griddata, self._mean_pixel_dimension(), + self._georef_info.nx, self._georef_info.ny) self._griddata = np.sqrt(Sx**2 + Sy**2) - + class GeographicMaxSlope(GeographicGridMixin, MaxSlope): - + """Gradient magnitude on a latitude/longitude grid.""" + pass class Laplacian(CalculationMixin, BaseSpatialGrid): - + """Laplacian of elevation -- positive in valleys, negative on ridges. + + :: + + curvature = Laplacian(elevation=dem) + + Uses SciPy's 5-point stencil, scaled by ``1 / dx**2``, so the units are + inverse length. + """ + dtype = float64 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation',), '_create_from_elevation')) - + def _create_from_elevation(self, *args, **kwargs): from scipy.ndimage import laplace elevation = kwargs['elevation'] - + self._copy_info_from_grid(elevation) - self._griddata = laplace(elevation._griddata) / np.power(elevation._georef_info.dx,2) - + # _mean_pixel_dimension(), not _georef_info.dx: on a geographic grid + # dx is in DEGREES, and dividing by it gave a curvature ten orders of + # magnitude too large. The mixin overrides this with the true metric + # cell size; for a projected grid the two are the same number. + cell = self._mean_pixel_dimension(*args, **kwargs) + self._griddata = (laplace(np.asarray(elevation._griddata, dtype=float64)) + / np.power(cell, 2)) + class GeographicLaplacian(GeographicGridMixin, Laplacian): - + """Laplacian of elevation on a latitude/longitude grid.""" + pass class PriorityQueueMixIn(object): - + """Depression filling by Priority-Flood. + + Mix into a grid class to give it :meth:`_flood`, which removes closed + depressions so that every cell has a downhill path to an outlet. + + The algorithms are those of + + Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal + depression-filling and watershed-labeling algorithm for digital + elevation models." *Computers & Geosciences* 62, 117-127. + + and run in :mod:`TopoAnalysis.kernels`, in C++ where the extension is + built and in NumPy otherwise. + + Attributes + ---------- + aggradation_slope : float + Gradient of the increment added while filling. The default of 1e-12 + makes filled surfaces very slightly convergent, which gives D8 routing + something to follow across what would otherwise be a dead flat + plateau. Set it to 0 for the flat fill that TopoToolbox's + ``fillsinks`` produces. + """ + aggradation_slope = 1E-12 class priorityQueue: - #Implements a priority queue using heapq. Python has a priority queue module built in, but it - # is not stably sorted (meaning that two items who are tied in priority are treated arbitrarily, as opposed to being - # returned on a first in first out basis). This circumvents that by keeping a count on the items inserted and using that - # count as a secondary priority - + """Stably-ordered priority queue. + + Retained because external code constructs it directly. The filling + routines no longer use it -- they call the compiled kernel instead -- + so it is kept purely for compatibility. + """ + def __init__(self): - # A counter and the number of items are stored separately to ensure that items remain stably sorted and to - # keep track of the size of the queue (so that we can check if its empty, which will be useful will iterating - # through the queue) + import heapq + self._heapq = heapq self.__pq = [] self.__counter = 0 self.__nItems = 0 - + def get(self): - #Remove an item and its priority from the queue - priority, count, item = heapq.heappop(self.__pq) + """Pop the lowest-priority item; returns ``(priority, item)``.""" + priority, count, item = self._heapq.heappop(self.__pq) self.__nItems -= 1 return priority, item - + def put(self, priority, item): - #Add an item to the priority queue + """Insert ``item``. Equal priorities come back out in FIFO order.""" self.__counter += 1 self.__nItems += 1 - entry = [priority, self.__counter, item] - heapq.heappush(self.__pq, entry) - + self._heapq.heappush(self.__pq, [priority, self.__counter, item]) + def isEmpty(self): return self.__nItems == 0 def randomize_subbasins_with_mask(self, *args, **kwargs): + """Replace elevations inside a mask with random values and re-flood. + + Used to generate synthetic drainage networks with the same outlets as + a real one. + """ mask = args[0] outlets = args[1] - - self.__flood(mask = mask, outlets = outlets, randomize = True) - + + # `self.__flood` inside this class mangles to _PriorityQueueMixIn__flood, + # which does not exist -- the method is _flood. + self._flood(mask=mask, outlets=outlets, randomize=True) + def _flood(self, *args, **kwargs): - # dem is a numpy array of elevations to be flooded, aggInc is the minimum amount to increment elevations by moving upstream - # use priority flood algorithm described in Barnes et al., 2013 - # Priority-Flood: An Optimal Depression-Filling and Watershed-Labeling Algorithm for Digital Elevation Models - # NOTE: They have another algorithm to make this more efficient, but to use that and a slope makes things more - # complicated - - #Create a grid to keep track of which cells have been filled - closed = np.zeros_like(self._griddata) - i = np.where(np.isnan(self._griddata)) - closed[i] = 1 - #using_mask = False - - if kwargs.get('mask') is not None: - closed = (kwargs.get('mask')._griddata != 1).astype(int) - if kwargs.get('outlets') is not None: - v = self._xy_to_rowscols(kwargs.get('outlets')) - #using_mask = True - edgeRows = [a[0] for a in v] - edgeCols = [a[1] for a in v] - else: - #Add all the edge cells to the priority queue, mark those cells as draining (not closed) - edgeRows, edgeCols = self.findDEMedge() - + """Fill depressions in ``self._griddata``, in place. + + Keyword arguments + ----------------- + mask : BaseSpatialGrid, optional + Only cells where the mask equals 1 take part. + outlets : sequence of (x, y), optional + Flood inwards from these points instead of from the DEM edge. + randomize : bool, optional + Randomise elevations inside the mask before flooding. + maximum_pit_depth : float, optional + Leave depressions deeper than this unfilled. + binary_result : bool, optional + Replace the grid with a 0/1 map of the cells the flood reached. + clip_to_fill : bool, optional + Set cells the flood never reached to ``NaN``. + + Returns + ------- + dict + ``{'cells_filled': int, 'max_fill_depth': float, + 'visited': ndarray|None}``. + """ + elevations = np.ascontiguousarray(self._griddata, dtype=float64) + # Kept for the clip_to_fill/maximum_pit_depth comparison below. + original = elevations.copy() if kwargs.get('clip_to_fill') else None + + mask = kwargs.get('mask') + closed = None + if mask is not None: + closed = (np.asarray(mask._griddata) != 1).astype(np.uint8) + if kwargs.get('randomize') is True: - self._randomize_grid_values(mask = kwargs['mask']) - - should_randomize_priority_queue = False - - if kwargs.get('randomize') is not None: - if kwargs.get('randomize') == True: - should_randomize_priority_queue = True - - priority_queue = FilledElevation.priorityQueue() # priority queue to sort filling operation - - for i in range(len(edgeCols)): - row, col = edgeRows[i], edgeCols[i] - - closed[row, col] = True - priority_queue.put(self._griddata[row, col], (row, col)) # store the indices as a vector of row column, in the priority queue prioritized by the dem value - - if kwargs.get('binary_result') is True or kwargs.get('clip_to_fill') is True: - visited = np.zeros_like(self._griddata, dtype = np.float) - - #While there is anything left in the priority queue, continue to fill holes - while not priority_queue.isEmpty(): - - priority, (row, col) = priority_queue.get() - - if kwargs.get('binary_result') is True or kwargs.get('clip_to_fill') is True: - visited[row, col] = 1.0 - - elevation = max(priority, self[row,col]) - - neighborRows, neighborCols, dxMults = self._getNeighborIndices(row, col) - - dxs = self._georef_info.dx * dxMults - - #Look through the upstream neighbors - for i in range(len(neighborCols)): - should_fill = True - - if not closed[neighborRows[i], neighborCols[i]]: - #If this was a hole (lower than the cell downstream), fill it - if self._griddata[neighborRows[i], neighborCols[i]] <= elevation: - - if kwargs.get('maximum_pit_depth'): - fill_depth = elevation - self._griddata[neighborRows[i], neighborCols[i]] - if fill_depth > kwargs.get('maximum_pit_depth'): - should_fill = False - - if should_fill: - self._griddata[neighborRows[i], neighborCols[i]] = elevation + self.aggradation_slope*dxs[i] - - closed[neighborRows[i], neighborCols[i]] = True - if should_fill: - if should_randomize_priority_queue: - # Use elevation as primary priority, random value for tie-breaking - priority = self._griddata[neighborRows[i], neighborCols[i]] + np.random.rand(1)[0] * 1e-10 - else: - priority = self._griddata[neighborRows[i], neighborCols[i]] - priority_queue.put(priority, [neighborRows[i], neighborCols[i]]) - + self._griddata = elevations + self._randomize_grid_values(mask=mask) + elevations = np.ascontiguousarray(self._griddata, dtype=float64) + if original is not None: + original = elevations.copy() + + seeds = None + outlets = kwargs.get('outlets') + if outlets is not None: + flat = [] + for (row, col) in self._xy_to_rowscols(outlets): + if row is None: + continue + flat.append(row * self._georef_info.nx + col) + if not flat: + raise Error.InputError('_flood', 'none of the outlets lie inside the grid') + seeds = np.array(flat, dtype=np.int64) + + want_visited = bool(kwargs.get('binary_result') or kwargs.get('clip_to_fill')) + max_pit_depth = kwargs.get('maximum_pit_depth') or 0.0 + + kernels.warn_if_slow(elevations.size) + result = kernels.priority_flood( + elevations, + closed=closed, + seeds=seeds, + # A non-zero aggradation slope is Barnes et al.'s Algorithm 4 + # (Priority-Flood+epsilon); zero gives the flat fill of + # Algorithm 2, which is what TopoToolbox's fillsinks returns. + mode='epsilon' if self.aggradation_slope > 0 else 'flat', + epsilon=self.aggradation_slope, + cellsize=self._georef_info.dx, + max_pit_depth=max_pit_depth, + track_visited=want_visited, + ) + + self._griddata = elevations.astype(self.dtype, copy=False) + + visited = result['visited'] if kwargs.get('binary_result'): - self._griddata = visited - if kwargs.get('clip_to_fill') is True: - self._griddata[visited == 0] = np.NAN - - - + self._griddata = np.asarray(visited, dtype=self.dtype) + elif kwargs.get('clip_to_fill') is True: + self._griddata = self._griddata.astype(float64, copy=True) + self._griddata[np.asarray(visited) == 0] = np.nan + if max_pit_depth: + # A depth limit leaves whole depressions unfilled that the + # flood nonetheless traverses, so `visited` no longer marks + # them. Compare against an unlimited fill: the cells that + # would have risen but did not are exactly the depressions + # that were declined, which is what the old implementation + # excised here. + unlimited = np.ascontiguousarray( + np.array(original, dtype=float64, copy=True)) + kernels.priority_flood( + unlimited, + closed=None if closed is None else closed.copy(), + seeds=seeds, + mode='epsilon' if self.aggradation_slope > 0 else 'flat', + epsilon=self.aggradation_slope, + cellsize=self._georef_info.dx) + declined = unlimited > self._griddata + 1e-12 + self._griddata[declined] = np.nan + + return result + class PriorityFillGrid(PriorityQueueMixIn, BaseSpatialGrid): - + """Cells of a mask that are connected to a set of outlets. + + Floods outwards from ``outlets`` through the cells where ``mask`` is 1 + and returns a 0/1 grid of everything reached. Used to grow a valley + network from a curvature mask, keeping only the parts that connect to a + real channel. + """ + dtype = uint8 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('outlets', 'mask'), '_create_from_outlets_and_mask')) - + def _create_from_outlets_and_mask(self, *args, **kwargs): - + mask = kwargs['mask'] outlets = kwargs['outlets'] + kwargs = dict(kwargs) kwargs['randomize'] = False kwargs['binary_result'] = True - self._copy_info_from_grid(mask,True) - rcs = self._xy_to_rowscols(outlets) - for rc in rcs: - self._griddata[rc[0],rc[1]] = 100 - i = np.where(np.isnan(mask._griddata)) - self._griddata[i] = 0 - i = np.where(mask._griddata != 1) - self._griddata[i] = 0 + + self._copy_info_from_grid(mask, True) + # Flooding needs a surface to descend; give the outlets a high value + # so the flood spreads from them through the rest of the mask. + self._griddata = np.zeros((self._georef_info.ny, self._georef_info.nx), + dtype=float64) + for (row, col) in self._xy_to_rowscols(outlets): + if row is not None: + self._griddata[row, col] = 100.0 + self._griddata[np.asarray(mask._griddata) != 1] = 0.0 + self._flood(*args, **kwargs) - + self._griddata = np.asarray(self._griddata, dtype=self.dtype) + class FilledElevation(PriorityQueueMixIn, Elevation): - - + """A DEM with its closed depressions removed. + + :: + + filled = FilledElevation(elevation=dem) + + Filling uses Priority-Flood (Barnes et al. 2014). By default the filled + surface carries a gradient of + :attr:`PriorityQueueMixIn.aggradation_slope` across former depressions so + that D8 routing has a direction to follow; pass + ``aggradation_slope=0`` -- or set the class attribute -- for the flat fill + that TopoToolbox's ``fillsinks`` produces. + + Other keywords (``mask``, ``outlets``, ``maximum_pit_depth``, + ``clip_to_fill``) are passed to :meth:`PriorityQueueMixIn._flood`. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation',), '_create_from_elevation')) - + def _create_from_elevation(self, *args, **kwargs): elevation = kwargs['elevation'] self._copy_info_from_grid(elevation) - self._flood(*args, **kwargs) - + if 'aggradation_slope' in kwargs: + # Per-instance override, so one call can ask for a flat fill + # without changing the default for every other grid. + self.aggradation_slope = kwargs['aggradation_slope'] + self.fill_report = self._flood(*args, **kwargs) + + class Area(BaseSpatialGrid): - + """D8 contributing (drainage) area. + + :: + + area = Area(flow_direction=d8) + + Values are in squared map units: each cell contributes its own area + (``dx**2``, or the true spherical cell area for a + :class:`GeographicArea`) plus everything upstream of it. + + To count contributing *cells* instead -- which is what TopoToolbox's + ``flowacc`` returns -- divide by the per-cell area, or pass + ``weights=np.ones(...)``. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction',), '_create_from_flow_direction'), (('dx', 'grid'), '_create_from_grid')) def _create_from_flow_direction(self, *args, **kwargs): - + flow_dir = kwargs['flow_direction'] self._copy_info_from_grid(flow_dir, True) - #if flow_dir.__class__ == FlowDirectionD8: - self.__calcD8Area(*args, **kwargs) - - def __calcD8Area(self, *args, **kwargs): - - # I am returning area and flowDirections but NOTE!I might be in danger the way I handle this... - # Flow directions are indices of arrays which may be different sizes, this is not truly giving me flow directions - # Because in the getNeighbor function I don't return values at edges.... might want to think about improving this, - # although it should work in this application - # Calculate the D8 drainage area for the numpy array representing a demMethods in elevGrid, assumes that elevGrid has already been filled - # Assigns BCs to deal with edges - these should be changed to handle flow differently, currently it will not force flow through the edges - + self._calc_d8_area(*args, **kwargs) + + def _calc_d8_area(self, *args, **kwargs): + """Accumulate per-cell weights down the D8 network. + + Keyword arguments + ----------------- + flow_direction : FlowDirectionD8 + weights : ndarray, optional + Per-cell contribution. Defaults to the cell area. + mask, evaluate_at : BaseSpatialGrid, optional + Cells where these are 0 contribute nothing and pass nothing on. + """ flow_dir = kwargs['flow_direction'] + kernels.warn_if_slow(flow_dir._griddata.size) + + weights = kwargs.get('weights') + if weights is None: + weights = self._area_per_pixel(*args, **kwargs) + weights = np.ascontiguousarray(weights, dtype=float64) + + gate = None + for gate_name in ('evaluate_at', 'mask'): + gate_grid = kwargs.get(gate_name) + if gate_grid is None: + continue + # `mask[i, j] is not None` was always true -- __getitem__ returns + # the value, not None, for a masked-out cell -- so a supplied + # mask never actually excluded anything. + this_gate = (np.asarray(gate_grid._griddata) != 0).astype(np.uint8) + gate = this_gate if gate is None else (gate & this_gate) + + # Ordering comes from the flow network itself rather than from an + # elevation sort, so it stays correct for a flow-direction grid + # loaded from disk, where `sort()` would have ordered cells by their + # direction *code*. + receivers = flow_dir.receivers + order = flow_dir.topological_order + self._griddata = np.asarray( + kernels.accumulate(receivers, order, weights, gate)) + + # Preserved under its original name-mangled spelling for any external + # caller that reached in for it. + _Area__calcD8Area = _calc_d8_area - if kwargs.get('sorted_indexes') is not None: - idcs = kwargs.get('sorted_indexes') - else: - idcs = flow_dir.sort() - - area = self._area_per_pixel(*args, **kwargs) # area of a pixel - - [ind_i, ind_j] = np.unravel_index(idcs, flow_dir._griddata.shape) - - has_mask = kwargs.get('mask') is not None - has_evaluate_at = kwargs.get('evaluate_at') is not None - - if has_evaluate_at: - area[kwargs['evaluate_at']._griddata == 0] = 0 - - - for i, j in zip(ind_i, ind_j): # Loop through all the data in sorted order - i_next, j_next, is_good = flow_dir.get_flow_to_cell(i,j) - if is_good and not has_mask: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - area[i_next, j_next] += area[i,j] - elif is_good and kwargs['mask'][i, j] is not None: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - area[i_next, j_next] += area[i,j] - - self._griddata = area # Return non bc version of area - def areas_greater_than(self, min_area): - ij_cols = np.where(self._griddata >= min_area) - ij = zip(ij_cols[0].tolist(), ij_cols[1].tolist()) - return self._rowscols_to_xy(ij) - + """Map coordinates of every cell whose area is at least ``min_area``.""" + rows, cols = np.where(self._griddata >= min_area) + return self._rowscols_to_xy(list(zip(rows.tolist(), cols.tolist()))) + def areas_between(self, fd, min_area, max_area): + """Channel heads: cells in an area band whose donors are all below it. + + These are the points where drainage area first crosses ``min_area``, + so they mark the upstream ends of the channel network. + """ ij_out = [] - ij_cols = np.where((self._griddata >= min_area) & (self._griddata <= max_area)) - for (i,j) in zip(ij_cols[0].tolist(), ij_cols[1].tolist()): - options = fd.get_upstream_cell_indexes(i,j) - max_a = None - for (i_p, j_p) in options: - if max_a is None or (self._griddata[i_p, j_p] > max_a): - max_a = self._griddata[i_p, j_p] + rows, cols = np.where((self._griddata >= min_area) & (self._griddata <= max_area)) + for (i, j) in zip(rows.tolist(), cols.tolist()): + options = fd.get_upstream_cell_indexes(i, j) + # No upstream neighbour at all means this cell *is* a source, so + # it qualifies. The original compared None < min_area and raised. + if not options: + ij_out.append((i, j)) + continue + max_a = max(self._griddata[i_p, j_p] for (i_p, j_p) in options) if max_a < min_area: - ij_out += [(i,j)] + ij_out.append((i, j)) return self._rowscols_to_xy(ij_out) - - + + class GeographicArea(GeographicGridMixin, Area): + """Drainage area on a latitude/longitude grid, using true cell areas.""" + pass -class ValleyArea(Area): - - required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), - (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), - (('flow_direction', 'area', 'laplace', 'valley_laplace_value', 'min_area_value'), '_create_from_flow_direction_and_valley_slope_and_area')) +class _ValleyMaskMixin(object): + """Shared construction of a connected valley-floor mask. + + Cells whose Laplacian exceeds ``valley_laplace_value`` are convergent + (valley-like). Those that are connected to a cell with more than + ``min_area_value`` of drainage area form the valley network; isolated + convergent patches on hillslopes are discarded. + """ + + def _valley_mask(self, *args, **kwargs): + from scipy.ndimage import binary_dilation, binary_erosion - def _create_from_flow_direction_and_valley_slope_and_area(self, *args, **kwargs): - mask = Mask() mask._copy_info_from_grid(kwargs['laplace'], True) - mask._griddata[kwargs['laplace']._griddata >= kwargs['valley_laplace_value'] ] = 1 + mask._griddata[kwargs['laplace']._griddata >= kwargs['valley_laplace_value']] = 1 + outlets = kwargs['area'].areas_greater_than(kwargs['min_area_value']) - pfg = PriorityFillGrid(mask = mask, outlets = outlets) - import scipy.ndimage.morphology as morph - if kwargs.get('iterations') is None or kwargs.get('iterations') == 0: - pass - else: - iterations = kwargs['iterations'] - pfg._griddata = morph.binary_dilation(pfg._griddata, iterations = iterations) - pfg._griddata = morph.binary_erosion(pfg._griddata, iterations = iterations) - - kwargs['evaluate_at'] = pfg + pfg = PriorityFillGrid(mask=mask, outlets=outlets) + + iterations = kwargs.get('iterations') or 0 + if iterations: + # Closing: dilate then erode, to bridge one-cell gaps in the + # valley network without growing it overall. + pfg._griddata = binary_dilation(pfg._griddata, iterations=iterations) + pfg._griddata = binary_erosion( + pfg._griddata, iterations=iterations).astype(pfg.dtype) + return pfg + + def _create_from_flow_direction_and_valley_slope_and_area(self, *args, **kwargs): + kwargs = dict(kwargs) + kwargs['evaluate_at'] = self._valley_mask(*args, **kwargs) self._create_from_flow_direction(*args, **kwargs) - - + + +class ValleyArea(_ValleyMaskMixin, Area): + """Drainage area accumulated only over valley-floor cells. + + :: + + va = ValleyArea(flow_direction=d8, area=area, laplace=laplacian, + valley_laplace_value=0.01, min_area_value=1e6) + """ + + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), + (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), + (('gdal_filename',), '_read_gdal'), + (('flow_direction', 'area', 'laplace', 'valley_laplace_value', 'min_area_value'), '_create_from_flow_direction_and_valley_slope_and_area')) + + class GeographicValleyArea(GeographicGridMixin, ValleyArea): - pass - -class MainstemValleyArea(Area): - + """Valley-floor area on a latitude/longitude grid.""" + + pass + + +class MainstemValleyArea(_ValleyMaskMixin, Area): + """Valley-floor area accumulated along the main stem only. + + Where :class:`ValleyArea` sums every tributary, this follows only the + longest flow path into each cell, so the result is the valley area of the + trunk stream rather than of the whole basin. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('area', 'laplace', 'valley_laplace_value', 'min_area_value', 'flow_direction'), '_create_from_flow_direction_and_valley_slope_and_area')) - def _create_from_flow_direction_and_valley_slope_and_area(self, *args, **kwargs): - - mask = Mask() - mask._copy_info_from_grid(kwargs['laplace'], True) - mask._griddata[kwargs['laplace']._griddata >= kwargs['valley_laplace_value'] ] = 1 - outlets = kwargs['area'].areas_greater_than(kwargs['min_area_value']) - pfg = PriorityFillGrid(mask = mask, outlets = outlets) - import scipy.ndimage.morphology as morph - if kwargs.get('iterations') is None or kwargs.get('iterations') == 0: - pass - else: - iterations = kwargs['iterations'] - pfg._griddata = morph.binary_dilation(pfg._griddata, iterations = iterations) - pfg._griddata = morph.binary_erosion(pfg._griddata, iterations = iterations) - - kwargs['evaluate_at'] = pfg - self._create_from_flow_direction(*args, **kwargs) - - def __calcD8Area(self, *args, **kwargs): - + def _calc_d8_area(self, *args, **kwargs): + """Accumulate down the longest-flow-length path only. + + Notes + ----- + The original version of this method was named ``__calcD8Area``, so + Python mangled it to ``_MainstemValleyArea__calcD8Area`` while the + inherited caller looked for ``_Area__calcD8Area``. It was therefore + never called, and ``MainstemValleyArea`` silently behaved exactly like + :class:`ValleyArea`. It also compared a whole grid object against a + scalar, which would have raised as soon as it did run. + """ flow_dir = kwargs['flow_direction'] + kernels.warn_if_slow(flow_dir._griddata.size) + + dA = np.ascontiguousarray(self._area_per_pixel(*args, **kwargs), dtype=float64) + step = flow_dir.step_lengths(self._mean_pixel_dimension(*args, **kwargs)) + + gate = None + for gate_name in ('evaluate_at', 'mask'): + gate_grid = kwargs.get(gate_name) + if gate_grid is not None: + this_gate = (np.asarray(gate_grid._griddata) != 0).astype(np.uint8) + gate = this_gate if gate is None else (gate & this_gate) + if gate is not None: + dA = np.where(gate != 0, dA, 0.0) + + receivers = flow_dir.receivers + order = flow_dir.topological_order + _, main_stem = kernels.flow_length(receivers, order, + np.ascontiguousarray(step, dtype=float64)) + + self._griddata = np.asarray( + kernels.propagate_along_main_stem(receivers, order, main_stem, gate, dA, 'sum')) + + _Area__calcD8Area = _calc_d8_area + - if kwargs.get('sorted_indexes') is not None: - idcs = kwargs.get('sorted_indexes') - else: - idcs = flow_dir.sort() - - dA = self._area_per_pixel(*args, **kwargs) - dx = self._mean_pixel_dimension(*args, **kwargs) * flow_dir.pixel_scale() - - length = np.zeros_like(dA) - - [ind_i, ind_j] = np.unravel_index(idcs, flow_dir._griddata.shape) - - has_mask = kwargs.get('mask') is not None - has_evaluate_at = kwargs.get('evaluate_at') is not None - - if has_evaluate_at: - dA[kwargs['evaluate_at']._griddata == 0] = 0 - - for i, j in zip(ind_i, ind_j): # Loop through all the data in sorted order - i_next, j_next, is_good = flow_dir.get_flow_to_cell(i,j) - - if is_good: - next_l = length[i,j] + dx[i,j] - if next_l > length[i_next, j_next]: - length[i_next, j_next] = next_l - if not has_mask: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - self._griddata[i_next, j_next] = dA[i_next,j_next] + self._griddata[i,j] - elif kwargs['mask'][i, j] is not None: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - self._griddata[i_next, j_next] = dA[i_next,j_next] + self._griddata[i,j] - elif kwargs['area'] <= kwargs['min_area_value']: - if not has_mask: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - self._griddata[i_next, j_next] += self._griddata[i,j] - elif kwargs['mask'][i, j] is not None: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - self._griddata[i_next, j_next] += self._griddata[i,j] - class GeographicMainstemValleyArea(GeographicGridMixin, MainstemValleyArea): + """Main-stem valley area on a latitude/longitude grid.""" + pass -class AlongFlowSmoothing(object): +def _chi_profile_along_path(points, area_grid, elevation_grid, de, theta): + """Chi and elevation sampled along an ordered list of cells. + + ``points`` is a 2 x n integer array of rows and columns running + downstream to upstream. The integral uses the trapezoidal rule over the + along-path distance, with diagonal steps counted as sqrt(2) cell widths. + + Returns + ------- + (ndarray, ndarray) + ``(chi, elevation)``, both length n, with ``chi[0] == 0``. + """ + n = points.shape[1] + adjustment = np.ones(n) + if n > 2: + diagonal = ((points[0, 1:-1] != points[0, 2:]) + & (points[1, 1:-1] != points[1, 2:])) + adjustment[np.flatnonzero(diagonal) + 1] += (SQRT2 - 1.0) + + area_profile = area_grid[points[0], points[1]] + elevation_profile = elevation_grid[points[0], points[1]] + de_profile = de[points[0], points[1]] + + chi_profile = np.zeros_like(elevation_profile, dtype=float64) + if n > 1: + integrand = 0.5 * (np.power(area_profile[1:], -theta) + + np.power(area_profile[:-1], -theta)) + step = 0.5 * (de_profile[1:] + de_profile[:-1]) * adjustment[1:] + chi_profile[1:] = np.cumsum(integrand * step) + return chi_profile, elevation_profile + + +class AlongFlowSmoothing(object): + """Collect the cells within a window along each flow path. + + Mixed into the ``*WithSmoothing`` classes, which fit a model over that + window at every cell. The window is set either by drop + (``vertical_interval``) or by distance (``horizontal_interval``), and + extends both upstream and downstream of the cell being evaluated. + """ + def _find_points_along_path(self, de, **kwargs): area = kwargs['area'] @@ -2572,6 +3618,24 @@ def find_points_along_path(this_i, this_j): return find_points_along_path class KsFromChiWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): + """Channel steepness fitted to chi--elevation in a moving window. + + :: + + ks = KsFromChiWithSmoothing(elevation=dem, area=area, + flow_direction=d8, theta=0.45, + vertical_interval=50.0) + + At every channel cell, chi and elevation are extracted over a window + along the flow path and a line through the origin is fitted; its slope + is k_s. ``save()`` writes seven bands: ks, the number of profiles + crossing each cell, mean squared error, sum of squares, r-squared, + p-value, and the number of points in the regression. + + Needs ``statsmodels``. Pass ``verbose=False`` to silence the progress + ticker. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), @@ -2580,13 +3644,15 @@ class KsFromChiWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): ) def _create_from_elevation_area_flow_direction(self, *args, **kwargs): - + + sm = _require_statsmodels() + elevation = kwargs['elevation'] area = kwargs['area'] theta = kwargs['theta'] area_threshold = kwargs.get('area_threshold', 0) de = area._mean_pixel_dimension() - + self._copy_info_from_grid(elevation) self._griddata = np.zeros_like(elevation._griddata) self._n = np.zeros_like(self._griddata).astype(int) @@ -2599,58 +3665,45 @@ def _create_from_elevation_area_flow_direction(self, *args, **kwargs): find_points_along_path = self._find_points_along_path(de, **kwargs) - def calc_ks(i,j): - points = find_points_along_path(i, j) - if points is not None: - pts = list(zip(*(points))) - points = np.array(pts).astype(int) - adjustment = np.ones((len(points[0]))) - i = np.where((points[0,1:-1] != points[0,2:]) & (points[1,1:-1] != points[1,2:])) - adjustment[i[0]+1] += 0.414 - area_profile = area._griddata[points[0],points[1]] - elevation_profile = elevation._griddata[points[0], points[1]] - de_profile = de[points[0], points[1]] - chi_profile = np.zeros_like(elevation_profile) - chi_profile[1:] = np.cumsum(0.25*(np.power(area_profile[1:], -theta) + np.power(area_profile[0:-1], -theta)) * (de_profile[1:] + de_profile[0:-1])*adjustment[1:]) - X = np.array(chi_profile) - y = np.array(elevation_profile) - elevation_profile[0] - model = sm.OLS(y, X) - res = model.fit() - SS = res.ssr - return res.params[0], SS / float(len(chi_profile)), SS, res.rsquared, points, res.pvalues[0], len(chi_profile) - - else: - - return np.nan, np.nan, np.nan, np.nan, [[],[]], np.nan, 0 - - i = np.where(~np.isnan(area._griddata) & ~np.isnan(elevation._griddata) & (area._griddata > area_threshold)) - ij = list(zip(i[0],i[1])) - totalnumber = len(ij) - counter = 0.0 - next_readout = 0.1 - sys.stdout.write('Percent completion...') - sys.stdout.flush() - for (i,j) in ij: - self._griddata[i,j], self._mse[i,j], self._ss[i,j], self._r2[i,j], pts, self._pval[i,j], self._n_regression[i,j] = calc_ks(i,j) - self._n[pts[0], pts[1]] += 1 - counter += 1.0 / totalnumber - if counter > next_readout: - sys.stdout.write(str(int(next_readout*100)) + "...") - sys.stdout.flush() - next_readout += 0.1 - - sys.stdout.write('Percent completion...') - sys.stdout.flush() - sys.stdout.write('100') - sys.stdout.flush() - + def calc_ks(i, j): + points = find_points_along_path(i, j) + if points is None: + return np.nan, np.nan, np.nan, np.nan, [[], []], np.nan, 0 + + points = np.array(list(zip(*points))).astype(int) + chi_profile, elevation_profile = _chi_profile_along_path( + points, area._griddata, elevation._griddata, de, theta) + + X = np.array(chi_profile) + y = np.array(elevation_profile) - elevation_profile[0] + res = sm.OLS(y, X).fit() + SS = res.ssr + return (res.params[0], SS / float(len(chi_profile)), SS, res.rsquared, + points, res.pvalues[0], len(chi_profile)) + + rows, cols = np.where(~np.isnan(area._griddata) & ~np.isnan(elevation._griddata) + & (area._griddata > area_threshold)) + ij = list(zip(rows, cols)) + progress = _Progress(len(ij), kwargs.get('verbose', True)) + for (i, j) in ij: + (self._griddata[i, j], self._mse[i, j], self._ss[i, j], self._r2[i, j], + pts, self._pval[i, j], self._n_regression[i, j]) = calc_ks(i, j) + # np.add.at, not `+=`: with repeated subscripts a fancy-indexed + # `+=` applies each increment only once, so cells crossed by + # several profiles were undercounted. + if len(pts[0]): + np.add.at(self._n, (pts[0], pts[1]), 1) + progress.tick() + progress.done() + + def save(self, filename): self._create_gdal_representation_from_array(self._georef_info, 'GTiff', [self._griddata, self._n, self._mse, self._ss, self._r2, self._pval, self._n_regression], self.dtype, filename, ['COMPRESS=LZW', 'BIGTIFF=YES'], multiple_bands=True) @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -2659,7 +3712,7 @@ def get_band(gdal_dataset, band_number): nodata_elements = np.where(grid == nodata) from numpy import uint8 if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -2690,6 +3743,16 @@ def get_band(gdal_dataset, band_number): class ThetaFromChiWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): + """Concavity fitted to chi--elevation in a moving window. + + As :class:`KsFromChiWithSmoothing`, but ``theta`` is optimised at each + cell rather than held fixed: the value returned is the one that makes + the chi--elevation relation most nearly linear. The search is bounded + at +/-10. + + Needs ``statsmodels``. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',), '_create'), (('ai_ascii_filename', 'EPSGprojectionCode'), '_read_ai'), (('gdal_filename',), '_read_gdal'), @@ -2701,6 +3764,10 @@ class ThetaFromChiWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): def _create_from_elevation_area_flow_direction(self, *args, **kwargs): + from scipy.optimize import fmin + + sm = _require_statsmodels() + elevation = kwargs['elevation'] area = kwargs['area'] min_area = kwargs['min_area'] @@ -2717,72 +3784,53 @@ def _create_from_elevation_area_flow_direction(self, *args, **kwargs): self._griddata[:] = np.nan find_points_along_path = self._find_points_along_path(de, **kwargs) + empty = (np.nan, np.nan, np.nan, np.nan, [[], []], np.nan, 0) def calc_theta(i, j): points = find_points_along_path(i, j) - if points is not None: - pts = list(zip(*(points))) - points = np.array(pts).astype(int) - adjustment = np.ones((len(points[0]))) - i = np.where((points[0, 1:-1] != points[0, 2:]) & (points[1, 1:-1] != points[1, 2:])) - adjustment[i[0] + 1] += 0.414 - area_profile = area._griddata[points[0], points[1]] - elevation_profile = elevation._griddata[points[0], points[1]] - de_profile = de[points[0], points[1]] - - def r2_for_theta(theta): - if theta > 10 or theta < -10: - return np.inf - chi_profile = np.zeros_like(elevation_profile) - chi_profile[1:] = np.cumsum( - 0.25 * (np.power(area_profile[1:], -theta) + np.power(area_profile[0:-1], -theta)) * ( - de_profile[1:] + de_profile[0:-1]) * adjustment[1:]) - X = np.array(chi_profile) - y = np.array(elevation_profile) - elevation_profile[0] - model = sm.OLS(y, X) - res = model.fit() - return res.ssr - from scipy.optimize import fmin - try: - (theta_bf, funval, iter, funcalls, warnflag) = fmin(r2_for_theta, np.array([0.5]), (), 1E-5, 1E-5, 100, 200, True, False, 0, None) - chi_profile = np.zeros_like(elevation_profile) - chi_profile[1:] = np.cumsum( - 0.25 * (np.power(area_profile[1:], -theta_bf) + np.power(area_profile[0:-1], -theta_bf)) * ( - de_profile[1:] + de_profile[0:-1]) * adjustment[1:]) - X = np.array(chi_profile) - y = np.array(elevation_profile) - elevation_profile[0] - model = sm.OLS(y, X) - res = model.fit() - SS = res.ssr - return theta_bf, SS / float(len(chi_profile)), SS, res.rsquared, points, res.pvalues[0], len(chi_profile) - except: - return np.nan, np.nan, np.nan, np.nan, [[], []], np.nan, 0 - else: + if points is None: + return empty - return np.nan, np.nan, np.nan, np.nan, [[],[]], np.nan, 0 + points = np.array(list(zip(*points))).astype(int) - i = np.where((area._griddata != 0) & ~np.isnan(area._griddata) & ~np.isnan(elevation._griddata) & (area._griddata > min_area)) - ij = list(zip(i[0], i[1])) - totalnumber = len(ij) - counter = 0.0 - next_readout = 0.1 - sys.stdout.write('Percent completion...') - sys.stdout.flush() - for (i, j) in ij: - self._griddata[i, j], self._mse[i, j], self._ss[i, j], self._r2[i, j], pts, self._pval[i, j], \ - self._n_regression[i, j] = calc_theta(i, j) - self._n[pts[0], pts[1]] += 1 - counter += 1.0 / totalnumber - if counter > next_readout: - sys.stdout.write(str(int(next_readout * 100)) + "...") - sys.stdout.flush() - next_readout += 0.1 + def fit_for_theta(theta): + chi_profile, elevation_profile = _chi_profile_along_path( + points, area._griddata, elevation._griddata, de, theta) + X = np.array(chi_profile) + y = np.array(elevation_profile) - elevation_profile[0] + return sm.OLS(y, X).fit(), chi_profile - sys.stdout.write('Percent completion...') - sys.stdout.flush() - sys.stdout.write('100') - sys.stdout.flush() + def ssr_for_theta(theta): + theta = float(np.atleast_1d(theta)[0]) + if theta > 10 or theta < -10: + return np.inf + return fit_for_theta(theta)[0].ssr + + try: + theta_bf, funval, iterations, funcalls, warnflag = fmin( + ssr_for_theta, np.array([0.5]), xtol=1E-5, ftol=1E-5, + maxiter=100, maxfun=200, full_output=True, disp=False) + res, chi_profile = fit_for_theta(float(theta_bf[0])) + SS = res.ssr + return (float(theta_bf[0]), SS / float(len(chi_profile)), SS, + res.rsquared, points, res.pvalues[0], len(chi_profile)) + except (ValueError, np.linalg.LinAlgError, ZeroDivisionError, FloatingPointError): + # A bare `except:` here also swallowed KeyboardInterrupt and + # genuine programming errors. + return empty + + rows, cols = np.where((area._griddata != 0) & ~np.isnan(area._griddata) + & ~np.isnan(elevation._griddata) & (area._griddata > min_area)) + ij = list(zip(rows, cols)) + progress = _Progress(len(ij), kwargs.get('verbose', True)) + for (i, j) in ij: + (self._griddata[i, j], self._mse[i, j], self._ss[i, j], self._r2[i, j], + pts, self._pval[i, j], self._n_regression[i, j]) = calc_theta(i, j) + if len(pts[0]): + np.add.at(self._n, (pts[0], pts[1]), 1) + progress.tick() + progress.done() def save(self, filename): @@ -2793,7 +3841,7 @@ def save(self, filename): @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -2802,7 +3850,7 @@ def get_band(gdal_dataset, band_number): nodata_elements = np.where(grid == nodata) from numpy import uint8 if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -2837,22 +3885,32 @@ def get_band(gdal_dataset, band_number): class ChannelSlopeWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): + """Channel gradient measured over a window along the flow path. + + Far less noisy than a cell-to-cell slope, which on a real DEM is mostly + vertical quantisation. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',), '_create'), (('ai_ascii_filename', 'EPSGprojectionCode'), '_read_ai'), (('gdal_filename',), '_read_gdal'), (('elevation', 'area', 'flow_direction', 'vertical_interval'), '_create_from_elevation_area_flow_direction'), (('elevation', 'area', 'flow_direction', 'horizontal_interval'), - '_create_from_elevation_flow_direction'), + '_create_from_elevation_area_flow_direction'), ) scale_factor = 1.0 def points_along_path(self, points, i, j): + """Which part of the window to measure over; the whole of it here.""" return points def calc_channel_slope(self, i, j, elevation, de, find_points_along_path): + """Gradient across the window centred on ``(i, j)``. + Returns ``NaN`` where the window runs off the end of the network. + """ points = find_points_along_path(i, j) if points is None or len(points) == 0: @@ -2895,68 +3953,85 @@ def _create_from_elevation_area_flow_direction(self, *args, **kwargs): find_points_along_path = self._find_points_along_path(de, **kwargs) - i = np.where((area._griddata != 0) & ~np.isnan(area._griddata) & ~np.isnan(elevation._griddata) & (area._griddata > min_area)) - ij = list(zip(i[0], i[1])) - totalnumber = len(ij) - counter = 0.0 - next_readout = 0.1 - sys.stdout.write('Percent completion...') - sys.stdout.flush() + rows, cols = np.where((area._griddata != 0) & ~np.isnan(area._griddata) + & ~np.isnan(elevation._griddata) & (area._griddata > min_area)) + ij = list(zip(rows, cols)) + progress = _Progress(len(ij), kwargs.get('verbose', True)) for (i, j) in ij: self._griddata[i, j] = self.calc_channel_slope(i, j, elevation, de, find_points_along_path) - counter += 1.0 / totalnumber - if counter > next_readout: - sys.stdout.write(str(int(next_readout * 100)) + "...") - sys.stdout.flush() - next_readout += 0.1 - sys.stdout.write('Percent completion...') - sys.stdout.flush() - sys.stdout.write('100') - sys.stdout.flush() + progress.tick() + progress.done() class ChannelDownSlopeWithSmoothing(ChannelSlopeWithSmoothing): + """Channel gradient over the downstream half of the window.""" + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',), '_create'), (('ai_ascii_filename', 'EPSGprojectionCode'), '_read_ai'), (('gdal_filename',), '_read_gdal'), (('elevation', 'area', 'flow_direction', 'vertical_interval'), '_create_from_elevation_area_flow_direction'), (('elevation', 'area', 'flow_direction', 'horizontal_interval'), - '_create_from_elevation_flow_direction'), + '_create_from_elevation_area_flow_direction'), ) scale_factor = 2.0 def points_along_path(self, points, i, j): + """Only the downstream half of the window.""" position_of_center = list([ind[0] for ind in zip(range(len(points)),points) if (ind[1][0] == i and ind[1][1] == j)])[0] return points[0:position_of_center] if len(points[0:position_of_center]) > 0 else None class ChannelUpSlopeWithSmoothing(ChannelSlopeWithSmoothing): + """Channel gradient over the upstream half of the window.""" + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',), '_create'), (('ai_ascii_filename', 'EPSGprojectionCode'), '_read_ai'), (('gdal_filename',), '_read_gdal'), (('elevation', 'area', 'flow_direction', 'vertical_interval'), '_create_from_elevation_area_flow_direction'), (('elevation', 'area', 'flow_direction', 'horizontal_interval'), - '_create_from_elevation_flow_direction'), + '_create_from_elevation_area_flow_direction'), ) scale_factor = 2.0 def points_along_path(self, points, i, j): + """Only the upstream half of the window.""" position_of_center = list([ind[0] for ind in zip(range(len(points)),points) if (ind[1][0] == i and ind[1][1] == j)])[0] return points[position_of_center:] if len(points[position_of_center:]) > 0 else None class GeographicKsFromChiWithSmoothing(GeographicGridMixin, KsFromChiWithSmoothing): + """Windowed steepness fit on a latitude/longitude grid.""" + pass -class GeographicThetaFromChiWithSmoothing(GeographicGridMixin, KsFromChiWithSmoothing): +class GeographicThetaFromChiWithSmoothing(GeographicGridMixin, ThetaFromChiWithSmoothing): + """Windowed concavity fit on a latitude/longitude grid.""" + pass class MultiscaleCurvatureValleyWidth(BaseSpatialGrid): + """Valley width from the scale at which curvature is most negative. + + A local quadratic is fitted at a range of window sizes; the width + recorded at each cell is the window that minimises the smaller principal + curvature, i.e. the scale at which the valley is best resolved. The + curvature itself is kept in ``_minC``. + + :: + + width = MultiscaleCurvatureValleyWidth( + elevation=dem, area=area, area_cutoff=1e6, + min_width=30.0, max_width=600.0) + + Pass ``use_dask=True`` to evaluate the scales in parallel. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), (('elevation', 'area', 'area_cutoff', 'max_width', 'min_width'), '_create_from_inputs'),) class Utilities(object): + """The quadratic-fit machinery, as classmethods so dask can pickle it.""" @classmethod def _calc_inv_G_for_kernel(cls, X, Y, N, fix_center=False): @@ -2984,17 +4059,24 @@ def _calc_inv_G_for_kernel(cls, X, Y, N, fix_center=False): @classmethod def _convolve(cls, X, Y, Z, K, fix_center=False): + """Cross-correlate the elevation grid with the five moment kernels. - from numpy.fft import fft2 as fft2 - from numpy.fft import ifft2 as ifft2 - from numpy import flipud as flipud - from numpy import fliplr as fliplr - from numpy.fft import ifftshift + Each returned array is the windowed sum of ``z * basis`` with the + centre cell's own contribution removed, which is what the normal + equations for the local quadratic fit need. + """ + from numpy.fft import fft2, ifft2, fftshift Xt = X Yt = Y - - FZ = fft2(fliplr(fliplr(Z._griddata))) + data = np.asarray(Z._griddata, dtype=float64) + + # The kernels are symmetric about the grid centre, so correlation + # and convolution coincide and the elevation grid can be used as + # is. The original wrote fliplr(fliplr(...)), which is the + # identity -- presumably a typo for a 180-degree rotation, and + # harmless only because of that symmetry. + FZ = fft2(data) FX1 = fft2(np.power(Xt, 2)) FX2 = fft2(np.power(Yt, 2)) FX3 = fft2(Xt * Yt) @@ -3002,13 +4084,20 @@ def _convolve(cls, X, Y, Z, K, fix_center=False): FX5 = fft2(Yt) FX6 = fft2(K) - g = np.real(ifftshift(ifft2(FZ * FX1))) - np.sum(np.power(Xt, 2)) * Z._griddata - h = np.real(ifftshift(ifft2(FZ * FX2))) - np.sum(np.power(Yt, 2)) * Z._griddata - i = np.real(ifftshift(ifft2(FZ * FX3))) - np.sum(Xt * Yt) * Z._griddata - j = np.real(ifftshift(ifft2(FZ * FX4))) - np.sum(Xt) * Z._griddata - k = np.real(ifftshift(ifft2(FZ * FX5))) - np.sum(Yt) * Z._griddata + # fftshift, not ifftshift: the kernels are centred in the array, + # so the convolution output must be rolled forward by half the + # grid. For the odd-sized grids this class constructs the two + # differ by one cell, which displaced every fitted coefficient. + def correlate(kernel_hat): + return np.real(fftshift(ifft2(FZ * kernel_hat))) + + g = correlate(FX1) - np.sum(np.power(Xt, 2)) * data + h = correlate(FX2) - np.sum(np.power(Yt, 2)) * data + i = correlate(FX3) - np.sum(Xt * Yt) * data + j = correlate(FX4) - np.sum(Xt) * data + k = correlate(FX5) - np.sum(Yt) * data if not fix_center: - l = np.real(ifftshift(ifft2(FZ * FX6))) - np.sum(K) * Z._griddata + l = correlate(FX6) - np.sum(K) * data else: l = None @@ -3068,14 +4157,15 @@ def _create_kernel(cls, Z, de): return X * K, Y * K, N, K @classmethod - def _Cmin_for_scale(cls, Z, de, fix_center=False): - + def _Cmin_for_scale(cls, Z, de, fix_center=False, verbose=True): + """Smaller principal curvature of the quadratic fitted at one scale.""" X, Y, N, K = cls._create_kernel(Z, de) H = cls._calc_inv_G_for_kernel(X, Y, N, fix_center) g, h, i, j, k, l = cls._convolve(X, Y, Z, K, fix_center) Cmin = cls._Cmin(H, g, h, i, j, k, l, fix_center) - sys.stdout.write('scale ' + str(de) + '\n') - sys.stdout.flush() + if verbose: + sys.stdout.write('scale ' + str(de) + '\n') + sys.stdout.flush() return Cmin, de @classmethod @@ -3116,7 +4206,10 @@ def _elevation_fit_for_location(cls, x, y, elevation, de, fix_center = False): (nx, ny, dx) = (Z._georef_info.nx, Z._georef_info.ny, Z._georef_info.dx) (xllcenter, yllcenter) = (Z._georef_info.xllcenter, Z._georef_info.yllcenter) - [X,Y] = np.meshgrid(np.arange(xllcenter+dx/2, xllcenter + (nx-0.5)*dx, dx),np.arange(yllcenter+dx/2, yllcenter + (ny-0.5)*dx, dx)) + # arange over a half-open interval of exactly n*dx yields n-1 samples + # as often as n; build the axes from the cell count instead. + [X, Y] = np.meshgrid(xllcenter + dx / 2 + np.arange(nx) * dx, + yllcenter + dx / 2 + np.arange(ny) * dx) X -= xa Y -= ya @@ -3135,7 +4228,11 @@ def _create_from_inputs(self, *args, **kwargs): # Condition inputs to ensure that grids produce square convolution matrices: - (area_cutoff, max_width, min_width, normalize, fix_center, use_dask) = (kwargs['area_cutoff'], kwargs['max_width'], kwargs['min_width'], kwargs.get('normalize', False), kwargs.get('fix_center', False), kwargs.get('use_dask', False)) + (area_cutoff, max_width, min_width, normalize, fix_center, use_dask) = ( + kwargs['area_cutoff'], kwargs['max_width'], kwargs['min_width'], + kwargs.get('normalize', False), kwargs.get('fix_center', False), + kwargs.get('use_dask', False)) + verbose = kwargs.get('verbose', True) if use_dask: from dask import compute, delayed @@ -3175,7 +4272,8 @@ def _create_from_inputs(self, *args, **kwargs): if use_dask: from functools import partial - wrapper = partial(self.Utilities._Cmin_for_scale, Z, fix_center = fix_center) + wrapper = partial(self.Utilities._Cmin_for_scale, Z, + fix_center=fix_center, verbose=verbose) tasks = [delayed(wrapper)(s) for s in scales] results = compute(*tasks) for result in results: @@ -3188,7 +4286,8 @@ def _create_from_inputs(self, *args, **kwargs): ind += 1 else: for scale in scales: - minC, _ = self.Utilities._Cmin_for_scale(Z, scale, fix_center) + minC, _ = self.Utilities._Cmin_for_scale(Z, scale, fix_center, + verbose=verbose) if normalize: minC *= scale i = np.where(minC < g_minC) @@ -3205,11 +4304,13 @@ def _create_from_inputs(self, *args, **kwargs): self._minC = g_minC[start_y_index:, start_x_index:] def remove_padding(self, xpadding = 10, ypadding = 10): - + """Strip tile overlap from both the widths and the curvatures.""" import copy unpadded = copy.deepcopy(self) - unpadded._griddata = unpadded._griddata[ypadding:-ypadding,xpadding:-xpadding] - unpadded._minC = unpadded._minC[ypadding:-ypadding,xpadding:-xpadding] + rows = slice(ypadding, -ypadding if ypadding else None) + cols = slice(xpadding, -xpadding if xpadding else None) + unpadded._griddata = unpadded._griddata[rows, cols] + unpadded._minC = unpadded._minC[rows, cols] unpadded._georef_info.xllcenter += xpadding*unpadded._georef_info.dx unpadded._georef_info.yllcenter += ypadding*unpadded._georef_info.dx unpadded._georef_info.nx -= 2*xpadding @@ -3223,7 +4324,7 @@ def save(self, filename): @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -3232,7 +4333,7 @@ def get_band(gdal_dataset, band_number): nodata_elements = np.where(grid == nodata) from numpy import uint8 if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -3257,7 +4358,7 @@ def get_band(gdal_dataset, band_number): @classmethod def mosaic(cls, tiles): - + """Reassemble tiles, carrying the curvature band along with the widths.""" xmin = np.nan xmax = np.nan ymin = np.nan @@ -3297,395 +4398,531 @@ def mosaic(cls, tiles): return return_object class DiscreteFlowAccumulation(BaseSpatialGrid): - + """Area drained by a single steepest-descent walk from each outlet. + + Unlike :class:`Area`, which accumulates over the whole D8 network, this + follows one path per outlet, stepping to the lowest unvisited neighbour + each time, and records the running area along it. It is used to trace + individual flow paths across surfaces (debris flows, lava) where the + network abstraction does not apply. + + :: + + dfa = DiscreteFlowAccumulation(elevation=dem, outlets=[(x, y)]) + + Keyword arguments + ----------------- + mask : BaseSpatialGrid + Confine the walk to cells where the mask is 1. + terminations_only : bool + Record the total area only at the end of each path. + display_output : bool + Report progress outlet by outlet. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation', 'outlets'), '_create_from_elevation_outlets'), (('elevation', ), '_create_from_elevation'),) - + def _create_from_elevation(self, *args, **kwargs): + """Start a walk from every valid cell.""" + kwargs = dict(kwargs) elevation = Elevation() elevation._copy_info_from_grid(kwargs['elevation']) if kwargs.get('mask') is not None: - i = np.where(kwargs['mask']._griddata != 1) - elevation._griddata[i] = np.NaN + outside = np.where(kwargs['mask']._griddata != 1) + elevation._griddata[outside] = np.nan kwargs['elevation'] = elevation - i = np.where(~np.isnan(elevation._griddata)) - rcs = zip(i[0].tolist(),i[1].tolist()) - kwargs['outlets'] = elevation._rowscols_to_xy(rcs) + rows, cols = np.where(~np.isnan(elevation._griddata)) + kwargs['outlets'] = elevation._rowscols_to_xy( + list(zip(rows.tolist(), cols.tolist()))) self._create_from_elevation_outlets(*args, **kwargs) - + def _create_from_elevation_outlets(self, *args, **kwargs): - + elevation = kwargs['elevation'] self._copy_info_from_grid(elevation, True) - adjust = [(-1, -1), (0, -1), (1,-1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)] - - counter = 1 - - for outlet in kwargs['outlets']: + adjust = [(-1, -1), (0, -1), (1, -1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)] + + per_pixel_area = self._area_per_pixel() + terminations_only = kwargs.get('terminations_only') is True + outlets = list(kwargs['outlets']) + + def neighbour_elevations(cell): + """Neighbour elevations, NaN off-grid, as a plain float array.""" + values = np.empty(8, dtype=float64) + for k, (di, dj) in enumerate(adjust): + value = elevation[cell[0] + di, cell[1] + dj] + values[k] = np.nan if value is None else value + return values + + for counter, outlet in enumerate(outlets, start=1): if kwargs.get('display_output') is True: - print('Evaluating outlet ' + str(counter) + ' / ' + str(len(kwargs['outlets']))) - counter += 1 + print('Evaluating outlet {0} / {1}'.format(counter, len(outlets))) + (ij, ) = elevation._xy_to_rowscols((outlet,)) - visited = (ij, ) - ij_a = [(ij[0] + x[0], ij[1] + x[1]) for x in adjust] - e_a = np.array([elevation[i[0], i[1]] if elevation[i[0], i[1]] is not None else np.NaN for i in ij_a]) - new = True - area = 0 - while None not in e_a and np.sum(np.isnan(e_a)) == 0 and new: - area += self._area_per_pixel()[ij[0], ij[1]] - if kwargs.get('terminations_only') is not True: + if ij[0] is None: + continue + + seen = {tuple(ij)} + e_a = neighbour_elevations(ij) + area = 0.0 + + # `None not in numpy_array` never matched, so the original loop + # only ever stopped on NaN. + while not np.any(np.isnan(e_a)): + area += per_pixel_area[ij[0], ij[1]] + if not terminations_only: self._griddata[ij[0], ij[1]] = area - sls = np.argsort(e_a) - new = False - for sl in sls: + moved = False + for sl in np.argsort(e_a): this_ij = (ij[0] + adjust[sl][0], ij[1] + adjust[sl][1]) - if this_ij not in visited: - visited += (this_ij, ) - ij = this_ij - new = True - break - if new: - ij_a = [(ij[0] + x[0], ij[1] + x[1]) for x in adjust] - e_a = np.array([elevation[i[0], i[1]] if elevation[i[0], i[1]] is not None else np.NaN for i in ij_a]) - if kwargs.get('terminations_only') is True: - self._griddata[ij[0], ij[1]] = area + if this_ij in seen: + continue + seen.add(this_ij) + ij = this_ij + moved = True + break + if not moved: + break + e_a = neighbour_elevations(ij) + + if terminations_only: + self._griddata[ij[0], ij[1]] = area def _area_per_pixel(self, *args, **kwargs): return self._georef_info.dx**2 * np.ones((self._georef_info.ny, self._georef_info.nx)) def _mean_pixel_dimension(self, *args, **kwargs): - return self._georef_info.dx * np.ones_like(kwargs['elevation']._griddata, self.dtype) + # Reading the shape from kwargs['elevation'] raised KeyError whenever + # this was called without one; the grid knows its own shape. + return self._georef_info.dx * np.ones( + (self._georef_info.ny, self._georef_info.nx), dtype=float64) + +class GeographicDiscreteFlowAccumulation(GeographicGridMixin, DiscreteFlowAccumulation): + """Discrete flow accumulation on a latitude/longitude grid.""" -class GeographicDiscreteFlowAccumulation(GeographicGridMixin, DiscreteFlowAccumulation): pass class FlowLength(BaseSpatialGrid): - + """Longest upstream flow distance reaching each cell. + + :: + + length = FlowLength(flow_direction=d8) + + Alongside the distances, the grid records which upstream neighbour + supplies each cell's longest path. That "main stem" network is what + :class:`Relief`, :class:`ScaledRelief` and :class:`Ksi` follow, and what + :meth:`locations_along_flow_path_from_outlet` walks. + + .. note:: + The main-stem codes use the same ArcGIS convention as + :class:`FlowDirectionD8`, pointing *from* a cell *at* its upstream + donor. Before version 1.0 they used a row-flipped convention that + was internally consistent but disagreed with every other + flow-direction grid in the library, so ``*_directions`` side-files + written by older versions must be regenerated. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction',), '_create_from_flow_direction_and_sorted_indexes')) - + def _create_from_flow_direction_and_sorted_indexes(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) self.__calculate_flow_length(*args, **kwargs) - + def __calculate_flow_length(self, *args, **kwargs): - - flow_dir = kwargs['flow_direction'] - if kwargs.get('sorted_indexes') is not None: - idcs = kwargs.get('sorted_indexes') - else: - idcs = flow_dir.sort() - - self.__flow_directions = np.zeros_like(flow_dir._griddata, np.uint8) - [ind_i, ind_j] = np.unravel_index(idcs, flow_dir._griddata.shape) - dx = self._mean_pixel_dimension(*args, **kwargs) * flow_dir.pixel_scale() + flow_dir = kwargs['flow_direction'] + kernels.warn_if_slow(flow_dir._griddata.size) + + step = flow_dir.step_lengths(self._mean_pixel_dimension(*args, **kwargs)) + length, main_stem = kernels.flow_length( + flow_dir.receivers, + flow_dir.topological_order, + np.ascontiguousarray(step, dtype=float64)) + + self._griddata = np.asarray(length, dtype=self.dtype) + self.__flow_directions = np.asarray(main_stem, dtype=np.uint8) + + @property + def main_stem_directions(self): + """Codes pointing from each cell at its longest-path donor.""" + return self.__flow_directions + + @staticmethod + def __flow_direction_for_length(from_index, to_index): + """Code stored at ``to_index`` that points back at ``from_index``.""" + d_i = from_index[0] - to_index[0] + d_j = from_index[1] - to_index[1] + for d in range(8): + if int(kernels.D8_DI[d]) == d_i and int(kernels.D8_DJ[d]) == d_j: + return int(kernels.D8_CODES[d]) + return 0 - for i, j in zip(ind_i, ind_j): # Loop through all the data in sorted order - - i_next, j_next, is_good = flow_dir.get_flow_to_cell(i,j) - - if is_good: - next_l = self._griddata[i,j] + dx[i,j] - if next_l > self._griddata[i_next, j_next]: - self._griddata[i_next, j_next] = next_l - self.__flow_directions[i_next, j_next] = self.__flow_direction_for_length((i,j), (i_next,j_next)) - - def __flow_direction_for_length(self, from_index, to_index): - (i_from, j_from) = from_index - (i_to, j_to) = to_index - - d_i = i_to - i_from - d_j = j_to - j_from - - if d_i == 0: - if d_j == 1: - return 16 - if d_j == 0: - return 0 - if d_j == -1: - return 1 - elif d_i == 1: - if d_j == 1: - return 8 - if d_j == 0: - return 4 - if d_j == -1: - return 2 - elif d_i == -1: - if d_j == 1: - return 32 - if d_j == 0: - return 64 - if d_j == -1: - return 128 - - return None - def points_with_length(self, length, fd): - + """Map coordinates where the flow path first exceeds ``length``. + + Used to pick a consistent set of basin outlets: every returned point + has ``length`` of channel upstream of it. + """ tolerance = np.nanmin(self._mean_pixel_dimension()) - max_length = length - min_length = length - tolerance*2.0 - + min_length = length - tolerance * 2.0 + indexes_of_locations = list() - ind = np.where(np.logical_and((self._griddata >= min_length), (self._griddata <= max_length))) - for (this_row,this_col) in zip(ind[0],ind[1]): + ind = np.where((self._griddata >= min_length) & (self._griddata <= length)) + for (this_row, this_col) in zip(ind[0], ind[1]): (next_row, next_col, is_good) = fd.get_flow_to_cell(this_row, this_col) - if next_row is not None and next_col is not None and is_good: - this_value = self[this_row, this_col] - next_value = self[next_row, next_col] - if this_value <= length and next_value >= length: - indexes_of_locations.append((this_row, this_col)) - + if not is_good: + continue + if self[this_row, this_col] <= length <= self[next_row, next_col]: + indexes_of_locations.append((this_row, this_col)) + return self._rowscols_to_xy(indexes_of_locations) - + def indexes_along_flow_path_from_outlet(self, outlet): + """``(row, col)`` along the longest flow path upstream of ``outlet``.""" ij = self._xy_to_rowscols((outlet,))[0] + if ij[0] is None: + return [] return self.__get_upstream_indexes((ij,)) - + def locations_along_flow_path_from_outlet(self, outlet): + """As above, in map coordinates.""" return self._rowscols_to_xy(self.indexes_along_flow_path_from_outlet(outlet)) - + def locations_along_flow_path_from_outlets(self, outlets): + """Concatenated main-stem paths for several outlets.""" points = tuple() for outlet in outlets: points += tuple(self.locations_along_flow_path_from_outlet(outlet)) return points - + + def __step_upstream(self, i, j): + """The single main-stem donor of ``(i, j)``, or ``None``.""" + code = self.__flow_directions[i, j] + if code == 0: + return None + for d in range(8): + if code == kernels.D8_CODES[d]: + up_i = i + int(kernels.D8_DI[d]) + up_j = j + int(kernels.D8_DJ[d]) + if 0 <= up_i < self._georef_info.ny and 0 <= up_j < self._georef_info.nx: + return (up_i, up_j) + return None + return None + def __get_upstream_indexes(self, index): - - (i,j) = index[0] - indexes = list(index) - - - flow_code = self.__flow_directions[i,j] - if flow_code == 1: - indexes += self.__get_upstream_indexes(((i,j+1),)) - elif flow_code == 2: - indexes += self.__get_upstream_indexes(((i-1,j+1),)) - elif flow_code == 4: - indexes += self.__get_upstream_indexes(((i-1,j),)) - elif flow_code == 8: - indexes += self.__get_upstream_indexes(((i-1,j-1),)) - elif flow_code == 16: - indexes += self.__get_upstream_indexes(((i,j-1),)) - elif flow_code == 32: - indexes += self.__get_upstream_indexes(((i+1,j-1),)) - elif flow_code == 64: - indexes += self.__get_upstream_indexes(((i+1,j),)) - elif flow_code == 128: - indexes += self.__get_upstream_indexes(((i+1, j+1),)) - + """Walk the main stem upstream. Iterative, so long rivers are fine.""" + (i, j) = index[0] + indexes = [(i, j)] + seen = {(i, j)} + while True: + nxt = self.__step_upstream(i, j) + if nxt is None or nxt in seen: + break + indexes.append(nxt) + seen.add(nxt) + (i, j) = nxt return indexes - + def __map_flow_from_cell(self, index, **kwargs): - - (i,j) = index[0] - - return_dict = dict() - flow_code = self.__flow_directions[i,j] - - return_dict['index'] = index[0] - return_dict['distance'] = self._griddata[i,j] - for arg in kwargs: - return_dict[arg] = kwargs[arg][i,j] - - return_dict['next'] = [] - return_dict['distance_scale'] = 1.0 - - if flow_code == 1: - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell(((i,j+1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 2: - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell(((i-1,j+1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 4: - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell(((i-1,j),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 8: - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell(((i-1,j-1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 16: - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell(((i,j-1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 32: - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell(((i+1,j-1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 64: - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell(((i+1,j),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 128: - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell(((i+1,j+1),), **kwargs) - return_dict['next'].append(child_dict) - - if len(return_dict.get('next', 0)) == 0: - return_dict.pop('next') - - return return_dict + """Nested dictionary along the main stem upstream of ``index``. + + Each node carries ``index``, ``distance`` (the flow length at that + cell), ``distance_scale`` (the step length from its parent, in cell + widths) and one entry per keyword grid. + """ + (i, j) = index[0] + + def make_node(cell, scale): + node = {'index': cell, + 'distance': self._griddata[cell[0], cell[1]], + 'distance_scale': scale} + for name, grid in kwargs.items(): + node[name] = grid[cell[0], cell[1]] + return node + + root = make_node((i, j), 1.0) + node = root + seen = {(i, j)} + while True: + nxt = self.__step_upstream(i, j) + if nxt is None or nxt in seen: + break + diagonal = (nxt[0] != i) and (nxt[1] != j) + child = make_node(nxt, SQRT2 if diagonal else 1.0) + node['next'] = [child] + node = child + seen.add(nxt) + (i, j) = nxt + + return root def is_along_flow_length(self, from_index, to_index): - + """True when ``from_index`` supplies ``to_index``'s longest path.""" (i_to, j_to) = to_index flow_code = self.__flow_directions[i_to, j_to] - + return flow_code == self.__flow_direction_for_length(from_index, to_index) - + def clip_to_extent(self, extent): - + """Clip both the lengths and the main-stem codes to ``extent``.""" return_grid = super(FlowLength, self).clip_to_extent(extent) - - import copy - return_grid.__flow_directions = copy.deepcopy(self.__flow_directions) - lower_left = (extent[0]+self._georef_info.dx, extent[2]+self._georef_info.dx) - upper_right = (extent[1]-self._georef_info.dx, extent[3]-self._georef_info.dx) - idx = self._xy_to_rowscols((lower_left, upper_right)) - return_grid.__flow_directions = return_grid._griddata[idx[1][0]+1:idx[0][0]+1,idx[0][1]:idx[1][1]] + + # Clip the direction grid the same way as the data, through a + # temporary grid -- the old version assigned the clipped *lengths* + # into the direction array. + helper = BaseSpatialGrid() + helper._georef_info = self._georef_info + helper._griddata = self.__flow_directions + return_grid.__flow_directions = helper.clip_to_extent(extent)._griddata.astype(np.uint8) return return_grid def map_values_to_recursive_list(self, outlet, **kwargs): - + """Nested main-stem dictionary upstream of ``outlet``.""" v = (outlet, ) (ij_outlet, ) = self._xy_to_rowscols(v) + if ij_outlet[0] is None: + raise Error.InputError('map_values_to_recursive_list', + 'outlet {0} lies outside the grid'.format(outlet)) return self.__map_flow_from_cell((ij_outlet,), **kwargs) - + def save(self, filename): - + """Write the lengths, plus the main-stem codes to ``filename_directions``.""" super(FlowLength, self).save(filename) flow_dir_name = filename + "_directions" - self._create_gdal_representation_from_array(self._georef_info, 'GTiff', self.__flow_directions, np.uint8, flow_dir_name, ['COMPRESS=LZW']) - + dataset = self._create_gdal_representation_from_array( + self._georef_info, 'GTiff', self.__flow_directions, np.uint8, + flow_dir_name, ['COMPRESS=LZW']) + dataset.FlushCache() + del dataset + @classmethod def load(cls, filename): - + """Read back a grid written by :meth:`save`.""" return_object_bsp = BaseSpatialGrid.load(filename) return_object = cls() return_object._georef_info = return_object_bsp._georef_info return_object._griddata = return_object_bsp._griddata - flow_dir_filename = filename + "_directions" + flow_dir_filename = filename + "_directions" gdal_dataset = gdal.Open(flow_dir_filename) - band = gdal_dataset.GetRasterBand(1) - return_object.__flow_directions = band.ReadAsArray().astype(np.uint8) - - gdal_file = None + try: + band = gdal_dataset.GetRasterBand(1) + return_object.__flow_directions = band.ReadAsArray().astype(np.uint8) + finally: + gdal_dataset = None return return_object - class GeographicFlowLength(GeographicGridMixin, FlowLength): + """Flow length on a latitude/longitude grid.""" + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), (('flow_direction',), '_create_from_flow_direction_and_sorted_indexes')) class MaxFlowLengthTrackingMixin(object): - + """Propagate a value downstream along longest-flow-length paths only. + + Subclasses set :attr:`_propagate_mode` to ``'carry'`` (the receiver takes + the donor's value) or ``'sum'`` (the receiver adds it) and may define + :meth:`_propagation_gate` to suppress the transfer for some cells. + """ + + #: ``'carry'`` or ``'sum'``. + _propagate_mode = 'carry' + + def _propagation_gate(self, *args, **kwargs): + """0/1 grid of cells allowed to pass their value on; ``None`` = all.""" + return None + def _calculate_by_tracking_down_max_flow_length(self, *args, **kwargs): flow_dir = kwargs['flow_direction'] flow_length = kwargs['flow_length'] - if kwargs.get('sorted_indexes') is not None: - idcs = kwargs.get('sorted_indexes') - else: - idcs = flow_dir.sort() - - [ind_i, ind_j] = np.unravel_index(idcs, flow_dir._griddata.shape) - - for i, j in zip(ind_i, ind_j): # Loop through all the data in sorted order - i_next, j_next, is_good = flow_dir.get_flow_to_cell(i,j) - if is_good and flow_length.is_along_flow_length((i,j), (i_next, j_next)): - self._calculate_grid_value((i,j), (i_next, j_next), *args, **kwargs) - + kernels.warn_if_slow(flow_dir._griddata.size) + + self._griddata = np.asarray(kernels.propagate_along_main_stem( + flow_dir.receivers, + flow_dir.topological_order, + np.ascontiguousarray(flow_length.main_stem_directions, dtype=np.uint8), + self._propagation_gate(*args, **kwargs), + np.ascontiguousarray(self._griddata, dtype=float64), + self._propagate_mode)) + + class Relief(BaseSpatialGrid, MaxFlowLengthTrackingMixin): - + """Height of the head of the longest flow path above each cell. + + :: + + relief = Relief(flow_direction=d8, elevation=dem, flow_length=length) + + Pass ``area`` and ``Ao`` together to stop the relief signal being carried + down from cells with less than ``Ao`` of drainage area, which keeps + hillslope noise out of the channel network. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), - (('flow_direction', 'elevation', 'flow_length'), '_create_from_flow_direction_sorted_indexes_and_elevation')) - + (('gdal_filename',), '_read_gdal'), + (('flow_direction', 'elevation', 'flow_length'), '_create_from_flow_direction_sorted_indexes_and_elevation')) + + _propagate_mode = 'carry' + + def _propagation_gate(self, *args, **kwargs): + area = kwargs.get('area') + Ao = kwargs.get('Ao') + if area is None or Ao is None: + return None + return (np.asarray(area._griddata) > Ao).astype(np.uint8) + def _create_from_flow_direction_sorted_indexes_and_elevation(self, *args, **kwargs): self._copy_info_from_grid(kwargs['elevation']) self._calculate_by_tracking_down_max_flow_length(*args, **kwargs) self._griddata = self._griddata - kwargs['elevation']._griddata - + def _calculate_grid_value(self, pos, next_pos, *args, **kwargs): - (i,j) = pos + """Per-cell transfer rule. + + Retained for subclasses that override it; the bulk path uses the + compiled kernel instead. + """ + (i, j) = pos (i_next, j_next) = next_pos if kwargs.get('area') is not None and kwargs.get('Ao') is not None: - if kwargs.get('area')[i,j] <= kwargs['Ao']: - self._griddata[i_next, j_next] = self._griddata[i_next, j_next] + if kwargs['area'][i, j] <= kwargs['Ao']: return - - self._griddata[i_next, j_next] = self._griddata[i,j] - + + self._griddata[i_next, j_next] = self._griddata[i, j] + + class ScaledRelief(Relief): - + """:class:`Relief` scaled by ``Ao**theta``. + + That scaling puts relief in the same units as :class:`Ksi`, so the ratio + of the two is a channel steepness. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction', 'flooded_dem', 'elevation', 'flow_length', 'Ao', 'theta'), '_create_scaled_from_flow_direction_flooded_dem_and_elevation'), (('flow_direction', 'elevation', 'flow_length', 'Ao', 'theta'), '_create_scaled_from_flow_direction_sorted_indexes_and_elevation')) - + def _create_scaled_from_flow_direction_flooded_dem_and_elevation(self, *args, **kwargs): - self._create_from_flow_direction_flooded_dem_and_elevation(*args, **kwargs) - self._griddata = self._griddata * (np.power(kwargs['Ao'],kwargs['theta'])) - + # The original called a `_create_from_flow_direction_flooded_dem_and_elevation` + # that was never defined, so supplying `flooded_dem` raised + # AttributeError. The flooded DEM plays no part in the calculation + # beyond being an elevation surface, so route it to the same code. + self._create_from_flow_direction_sorted_indexes_and_elevation(*args, **kwargs) + self._griddata = self._griddata * (np.power(kwargs['Ao'], kwargs['theta'])) + def _create_scaled_from_flow_direction_sorted_indexes_and_elevation(self, *args, **kwargs): self._create_from_flow_direction_sorted_indexes_and_elevation(*args, **kwargs) - self._griddata = self._griddata * (np.power(kwargs['Ao'],kwargs['theta'])) + self._griddata = self._griddata * (np.power(kwargs['Ao'], kwargs['theta'])) + class Ksi(BaseSpatialGrid, MaxFlowLengthTrackingMixin): - + r"""The chi coordinate integrated along longest-flow-length paths. + + .. math:: + \chi = \int \left(\frac{A_0}{A}\right)^{\theta} \mathrm{d}x + + :: + + ksi = Ksi(area=area, flow_direction=d8, flow_length=length, + theta=0.45, Ao=1e6) + + Cells with less than ``Ao`` of drainage area are left out of the + integral, so chi is defined only on the channel network. + + .. note:: + Before version 1.0 the integrand was ``(Ao / (A - Ao))**theta``, which + diverges as ``A`` approaches ``Ao`` and does not match the definition + of chi in the literature. Values from older runs are not comparable + with these. + + See also :class:`Chi`, which integrates upstream from chosen outlets over + the whole network rather than along main stems. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('area','flow_direction','theta', 'Ao', 'flow_length'), '_create_from_inputs')) - + + _propagate_mode = 'sum' + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) - area_grid = kwargs['area']._griddata - kwargs['Ao'] - area_grid[area_grid <= 0] = np.nan - self._griddata = np.zeros_like(area_grid) - i = np.where(area_grid > 0) - self._griddata[i] = ( (kwargs['Ao'] / area_grid[i]) ** kwargs['theta']) * self._mean_pixel_dimension(*args, **kwargs)[i] * kwargs['flow_direction'].pixel_scale()[i] + + area = np.asarray(kwargs['area']._griddata, dtype=float64) + Ao = kwargs['Ao'] + theta = kwargs['theta'] + + step = (self._mean_pixel_dimension(*args, **kwargs) + * kwargs['flow_direction'].pixel_scale(float64)) + + self._griddata = np.zeros(area.shape, dtype=float64) + on_network = area > Ao + self._griddata[on_network] = ( + (Ao / area[on_network]) ** theta) * step[on_network] + self._calculate_by_tracking_down_max_flow_length(*args, **kwargs) - + def _calculate_grid_value(self, pos, next_pos, *args, **kwargs): - (i,j) = pos + """Per-cell transfer rule; the bulk path uses the compiled kernel.""" + (i, j) = pos (i_next, j_next) = next_pos - self._griddata[i_next, j_next] += self._griddata[i,j] - + self._griddata[i_next, j_next] += self._griddata[i, j] class GeographicKsi(GeographicGridMixin, Ksi): + """Chi along longest-flow-length paths on a latitude/longitude grid.""" + pass class RestoredElevation(BaseSpatialGrid): - + r"""Topography implied by a uniform channel steepness. + + Starting from the given outlets, elevations are rebuilt upstream from + + .. math:: \frac{\mathrm{d}z}{\mathrm{d}x} = k_s A^{-\theta} + + and the divides are then allowed to migrate towards whichever side ends + up lower. Repeating that (``iterations`` times) converges on the + landscape a steady, spatially uniform ``ks`` would produce, which can be + compared against the real DEM to find where it is out of steady state. + + :: + + restored = RestoredElevation(flow_direction=d8, elevation=dem, + area=area, theta=0.45, ks=50, + outlets=[(x, y)], iterations=5) + + Keyword arguments + ----------------- + randomize : bool + Randomise elevations inside the basin first, then re-route, so the + result does not inherit the real network's planform. + fix_external_outlets : bool + Hold the outer boundary of the basin fixed while divides migrate. + verbose : bool + Report convergence each iteration (default ``True``). + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction','elevation','area','theta','ks','outlets', 'iterations'), '_fill_dem')) - + def _fill_dem(self, *args, **kwargs): import copy self._copy_info_from_grid(kwargs['elevation'], False) @@ -3696,583 +4933,558 @@ def _fill_dem(self, *args, **kwargs): iterations = kwargs['iterations'] ks = kwargs['ks'] theta = kwargs['theta'] - v = self._xy_to_rowscols(outlets) + verbose = kwargs.get('verbose', True) + v = [rc for rc in self._xy_to_rowscols(outlets) if rc[0] is not None] pixel_dimension = self._mean_pixel_dimension(*args, **kwargs) - - - mask = Mask(flow_direction = flow_direction, outlets = outlets) + + mask = Mask(flow_direction=flow_direction, outlets=outlets) external_divides = copy.deepcopy(mask) - - if kwargs.get('fix_external_outlets') == True: + + if kwargs.get('fix_external_outlets') is True: from scipy.ndimage import binary_erosion as erosion - external_divides._griddata = external_divides._griddata - erosion(external_divides._griddata, np.ones((3,3)), border_value = 0).astype(int) + external_divides._griddata = ( + external_divides._griddata + - erosion(external_divides._griddata, np.ones((3, 3)), + border_value=0).astype(int)) else: external_divides._griddata = np.zeros_like(external_divides._griddata, int) + if randomize: - filled = FilledElevation(elevation = self, mask = mask, randomize = randomize, outlets = outlets) + filled = FilledElevation(elevation=self, mask=mask, randomize=randomize, + outlets=outlets) flow_direction.update_flow_codes_in_mask(filled, mask) - i = np.where(mask._griddata == 1) - self._griddata[i] = filled._griddata[i] + inside = np.where(mask._griddata == 1) + self._griddata[inside] = filled._griddata[inside] area = self.__recalculate_area(area, flow_direction, pixel_dimension, outlets) - for i in range(iterations): + + self.convergence = [] + for iteration in range(iterations): last_grid = self._griddata.copy() - print('Iteration {0}'.format(i)) - print('Filling outlets.') divides = self.__fill_outlets(area, flow_direction, pixel_dimension, v, ks, theta) - print('Migrating divides') - flow_direction = self.__migrate_divides(flow_direction, divides, external_divides) + flow_direction = self.__migrate_divides(flow_direction, divides, external_divides, + verbose=verbose) area = self.__recalculate_area(area, flow_direction, pixel_dimension, outlets) - change_in_elevation = np.mean((self._griddata - last_grid)**2) - print('Change: {0}'.format(change_in_elevation)) - + change_in_elevation = float(np.nanmean((self._griddata - last_grid) ** 2)) + self.convergence.append(change_in_elevation) + if verbose: + print('Iteration {0}: mean squared change {1:g}'.format( + iteration, change_in_elevation)) + + self.flow_direction = flow_direction + self.area = area + def __is_unit_area(self, idx, flow_direction): - - (i,j) = idx - - if flow_direction[i,j+1] == 16 or flow_direction[i+1, j+1] == 32 or \ - flow_direction[i+1, j] == 64 or flow_direction[i+1, j-1] == 128 or \ - flow_direction[i, j-1] == 1 or flow_direction[i-1, j-1] == 2 or \ - flow_direction[i-1, j] == 4 or flow_direction[i-1, j+1] == 8: - return False - else: - return True - + """True when no neighbour drains into ``idx`` (it is a divide cell).""" + return not flow_direction.get_upstream_cell_indexes(idx[0], idx[1]) + def __migrate_divides(self, *args, **kwargs): - + """Redirect divide-adjacent cells towards whichever side is lower.""" flow_direction = args[0] divides = args[1] external_divides = args[2] + verbose = kwargs.get('verbose', True) migrated = 0 + ny, nx = self._georef_info.ny, self._georef_info.nx + for (i, j) in divides: - if external_divides[i,j] != 1: - try: - if self[i,j+1] > self[i,j] and external_divides[i,j+1] != 1: - flow_direction[i,j+1] = 16 - migrated += 1 - except: - pass - - try: - if self[i+1, j+1] > self[i,j] and external_divides[i+1,j+1] != 1: - flow_direction[i+1, j+1] = 32 - migrated += 1 - except: - pass - - try: - if self[i+1, j] > self[i,j] and external_divides[i+1,j] != 1: - flow_direction[i+1, j] = 64 - migrated += 1 - except: - pass - - try: - if self[i+1, j-1] > self[i,j] and external_divides[i+1,j-1] != 1: - flow_direction[i+1, j-1] = 128 - migrated += 1 - except: - pass - - try: - if self[i, j-1] > self[i,j] and external_divides[i,j-1] != 1: - flow_direction[i, j-1] = 1 - migrated += 1 - except: - pass - - try: - if self[i-1,j-1] > self[i,j] and external_divides[i-1,j-1] != 1: - flow_direction[i-1, j-1] = 2 - migrated += 1 - except: - pass - - try: - if self[i-1, j] > self[i,j] and external_divides[i-1,j] != 1: - flow_direction[i-1, j] = 4 - migrated += 1 - except: - pass - - try: - if self[i-1, j+1] > self[i,j] and external_divides[i-1,j+1] != 1: - flow_direction[i-1, j+1] = 8 - migrated += 1 - except: - pass - - print("Migrated " + str(migrated) + " divides.") + if external_divides[i, j] == 1: + continue + here = self[i, j] + if here is None: + continue + for d in range(8): + ni = i + int(kernels.D8_DI[d]) + nj = j + int(kernels.D8_DJ[d]) + # Explicit bounds checks instead of eight try/except blocks: + # the exception handlers also swallowed real errors, and + # negative indices wrapped instead of being rejected. + if not (0 <= ni < ny and 0 <= nj < nx): + continue + if external_divides[ni, nj] == 1: + continue + neighbour = self[ni, nj] + if neighbour is None or not neighbour > here: + continue + flow_direction[ni, nj] = int(kernels.D8_CODES[(d + 4) % 8]) + migrated += 1 + + flow_direction._invalidate_network() + if verbose: + print("Migrated " + str(migrated) + " divides.") return flow_direction - + def __fill_outlets(self, area, flow_direction, pixel_dimension, outlet_indexes, ks, theta): - divides = list() for ind in outlet_indexes: - divides += self.__fill_upstream_points(ind, ks, theta, area, flow_direction, pixel_dimension) + divides += self.__fill_upstream_points(ind, ks, theta, area, flow_direction, + pixel_dimension) return divides - + def __fill_upstream_points(self, ind, ks, theta, area, flow_direction, pixel_dimension): - - (i, j) = ind - elevation_for_cell = self[i,j] - is_divide = True + """Integrate dz/dx = ks * A**-theta upstream from ``ind``. + + Returns the divide cells reached. Iterative: the recursive original + needed a million-frame recursion limit and still overflowed on real + basins. + """ + ny, nx = self._georef_info.ny, self._georef_info.nx divides = list() - - try: - if flow_direction[i, j+1] == 16: - self._griddata[i,j+1] = elevation_for_cell + pixel_dimension[i,j]*1.00*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i,j+1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i+1, j+1] == 32: - self._griddata[i+1,j+1] = elevation_for_cell + pixel_dimension[i,j]*1.414*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i+1,j+1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i+1, j] == 64: - self._griddata[i+1,j] = elevation_for_cell + pixel_dimension[i,j]*1.00*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i+1,j), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i+1, j-1] == 128: - self._griddata[i+1,j-1] = elevation_for_cell + pixel_dimension[i,j]*1.414*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i+1,j-1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i, j-1] == 1: - self._griddata[i,j-1] = elevation_for_cell + pixel_dimension[i,j]*1.00*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i,j-1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i-1, j-1] == 2: - self._griddata[i-1,j-1] = elevation_for_cell + pixel_dimension[i,j]*1.414*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i-1,j-1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i-1, j] == 4: - self._griddata[i-1,j] = elevation_for_cell + pixel_dimension[i,j]*1.00*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i-1,j), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i-1, j+1] == 8: - self._griddata[i-1,j+1] = elevation_for_cell + pixel_dimension[i,j]*1.414*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i-1,j+1), ks, theta, area, flow_direction, pixel_dimension) + stack = [tuple(ind)] + seen = {tuple(ind)} + + while stack: + (i, j) = stack.pop() + elevation_for_cell = self._griddata[i, j] + a = area[i, j] + is_divide = True + + for d in range(8): + ui = i + int(kernels.D8_DI[d]) + uj = j + int(kernels.D8_DJ[d]) + if not (0 <= ui < ny and 0 <= uj < nx): + continue + if flow_direction[ui, uj] != kernels.D8_CODES[(d + 4) % 8]: + continue is_divide = False - except: - pass - - if is_divide: - return [(i,j)] - else: - return divides - + if (ui, uj) in seen: + continue + seen.add((ui, uj)) + step = pixel_dimension[i, j] * kernels.D8_DIST[d] + self._griddata[ui, uj] = elevation_for_cell + step * ks * a ** (-theta) + stack.append((ui, uj)) + + if is_divide: + divides.append((i, j)) + + return divides + def __recalculate_area(self, area, flow_direction, pixel_dimension, outlets): - outlet_indexes = self._xy_to_rowscols(outlets) - for outlet in outlet_indexes: - self.__recurse_area(outlet, area, flow_direction, pixel_dimension) + """Re-accumulate drainage area after the network has been rewired.""" + weights = np.asarray(pixel_dimension, dtype=float64) ** 2 + accumulated = np.asarray(kernels.accumulate( + flow_direction.receivers, + flow_direction.topological_order, + np.ascontiguousarray(weights))) + + inside = self.__basin_mask(flow_direction, outlets) + area._griddata = np.where(inside, accumulated, area._griddata) return area - - def __recurse_area(self, ind, area, flow_direction, pixel_dimension): - - (i, j) = ind - - this_area = pixel_dimension[i,j]**2 - - try: - if flow_direction[i, j+1] == 16: - ind = (i, j+1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i+1, j+1] == 32: - ind = (i+1,j+1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i+1, j] == 64: - ind = (i+1,j) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i+1, j-1] == 128: - ind = (i+1, j-1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i, j-1] == 1: - ind = (i,j-1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i-1, j-1] == 2: - ind = (i-1, j-1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i-1, j] == 4: - ind = (i-1, j) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i-1, j+1] == 8: - ind = (i-1, j+1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - area[i,j] = this_area - return this_area - + + @staticmethod + def __basin_mask(flow_direction, outlets): + return flow_direction.basin_mask(outlets) + class GeographicRestoredElevation(GeographicGridMixin, RestoredElevation): + """Restored topography on a latitude/longitude grid.""" + pass class ChiScaledRelief(BaseSpatialGrid): + """Height above the basin outlet, scaled by ``Ao**theta``. + + Plotted against :class:`Chi`, the slope of this quantity is the channel + steepness index. + + :: + + relief = ChiScaledRelief(elevation=dem, flow_direction=d8, + theta=0.45, Ao=1e6, outlets=[(x, y)]) + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation', 'flow_direction', 'theta', 'Ao', 'outlets'), '_create_from_inputs'), (('elevation', 'flow_direction', 'flow_length', 'theta', 'Ao', 'basin_length'), '_create_from_basin_length')) - + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['elevation'], True) - outlet_indexes = self._xy_to_rowscols(kwargs['outlets']) elevation = kwargs['elevation'] - scale = np.power(kwargs['Ao'],kwargs['theta']) - outlet_number = 1 - for outlet_index in outlet_indexes: - indexes = kwargs['flow_direction'].get_indexes_of_upstream_cells(outlet_index[0], outlet_index[1]) - elevation_of_outlet = kwargs['elevation'][outlet_index[0],outlet_index[1]] - for index in indexes: - self[index[0],index[1]] = (elevation[index[0], index[1]] - elevation_of_outlet) * scale + flow_direction = kwargs['flow_direction'] + scale = np.power(kwargs['Ao'], kwargs['theta']) + + outlet_indexes = [rc for rc in self._xy_to_rowscols(kwargs['outlets']) + if rc[0] is not None] + for outlet_number, outlet_index in enumerate(outlet_indexes, start=1): + # One vectorised basin extraction per outlet, rather than an + # element-by-element write through __setitem__. + basin = flow_direction.basin_mask( + self._rowscols_to_xy((outlet_index,))) + elevation_of_outlet = elevation[outlet_index[0], outlet_index[1]] + self._griddata[basin] = ( + np.asarray(elevation._griddata)[basin] - elevation_of_outlet) * scale if kwargs.get('output_flag', False): - print('Outlet ' + str(outlet_number) + '/' + str(len(outlet_indexes)) + ' completed.') - outlet_number = outlet_number + 1 - + print('Outlet {0}/{1} completed.'.format(outlet_number, len(outlet_indexes))) + def _create_from_basin_length(self, *args, **kwargs): - kwargs['outlets'] = kwargs['flow_length'].points_with_length(kwargs['basin_length'],kwargs['flow_direction']) + """Use every point with ``basin_length`` of channel upstream as an outlet.""" + kwargs = dict(kwargs) + kwargs['outlets'] = kwargs['flow_length'].points_with_length( + kwargs['basin_length'], kwargs['flow_direction']) kwargs['output_flag'] = True - return self._create_from_inputs(*args, **kwargs) + return self._create_from_inputs(*args, **kwargs) class Chi(BaseSpatialGrid): - + r"""The chi coordinate, integrated upstream from chosen outlets. + + .. math:: + \chi(x) = \int_{x_{\mathrm{outlet}}}^{x} + \left(\frac{A_0}{A(x')}\right)^{\theta}\,\mathrm{d}x' + + :: + + chi = Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e6, + outlets=[(x, y)]) + + Chi is 0 at each outlet and increases upstream. Cells outside the + outlets' drainage areas are left as 0. + + Keyword arguments + ----------------- + trapezoid : bool + Average the integrand across each step instead of taking it at the + upstream cell. This is the rule TopoToolbox's ``chitransform`` uses; + the default (``False``) reproduces the original TopoAnalysis result. + maximum_length : float + Stop integrating beyond this flow distance from the outlet. + mask : BaseSpatialGrid + Restrict the integration to cells where the mask is non-zero. + + .. note:: + Before version 1.0, chi at the outlet itself was one cell-step of the + integrand rather than 0. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('area','flow_direction', 'theta', 'Ao', 'outlets'), '_create_from_inputs'), (('area', 'flow_direction', 'flow_length', 'theta', 'Ao', 'basin_length'), '_create_from_basin_length')) - - + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) self.__calculate_chi(*args, **kwargs) - + def _create_from_basin_length(self, *args, **kwargs): - kwargs['outlets'] = kwargs['flow_length'].points_with_length(kwargs['basin_length'],kwargs['flow_direction']) + """Use every point with ``basin_length`` of channel upstream as an outlet.""" + kwargs = dict(kwargs) + kwargs['outlets'] = kwargs['flow_length'].points_with_length( + kwargs['basin_length'], kwargs['flow_direction']) kwargs['output_flag'] = True return self._create_from_inputs(*args, **kwargs) - + def __calculate_chi(self, *args, **kwargs): - pixel_dimension = self._mean_pixel_dimension(*args, **kwargs) area = kwargs['area'] flow_direction = kwargs['flow_direction'] - Ao = kwargs['Ao'] - theta = kwargs['theta'] - outlet_indexes = self._xy_to_rowscols(kwargs['outlets']) - self.__occupied = np.zeros_like(area._griddata, dtype=int) - outlet_number = 1 - lmax = kwargs.get('maximum_length') - - for outlet in outlet_indexes: - self.__recurse_chi(outlet, area, flow_direction, pixel_dimension, Ao, theta, 0, 1, kwargs.get('mask'), 0.0, lmax = lmax) - if kwargs.get('output_flag', False): - print('Outlet ' + str(outlet_number) + '/' + str(len(outlet_indexes)) + ' completed.') - outlet_number += 1 - self.__occupied = None - try: - self._griddata = self._griddata * kwargs['mask']._griddata - except: - pass - - def __recurse_chi(self, v, area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = None): - - (i,j) = v - - if mask is not None: - if mask[i,j] == 0: - if area[i,j] > 1E4: - return - - if self.__occupied[i,j] == 1: - return - - dl = pixel_dimension[i,j] * scale - l = l + dl - - if lmax is not None and l >= lmax: - return - - self._griddata[i,j] = chi + (Ao / area[i,j])**theta * dl - self.__occupied[i,j] = 1; - chi = self._griddata[i,j] - + kernels.warn_if_slow(flow_direction._griddata.size) + + step = flow_direction.step_lengths(self._mean_pixel_dimension(*args, **kwargs)) + + outlets = [] + for (row, col) in self._xy_to_rowscols(kwargs['outlets']): + if row is not None: + outlets.append(row * self._georef_info.nx + col) + + mask = kwargs.get('mask') + mask_array = None if mask is None else (np.asarray(mask._griddata) != 0).astype(np.uint8) + + chi, distance = kernels.chi( + flow_direction.receivers, + flow_direction.topological_order, + np.ascontiguousarray(area._griddata, dtype=float64), + np.ascontiguousarray(step, dtype=float64), + np.array(outlets, dtype=np.int64), + kwargs['Ao'], + kwargs['theta'], + bool(kwargs.get('trapezoid', False)), + float(kwargs.get('maximum_length') or 0.0), + mask_array) + + chi = np.asarray(chi) + self.distance_from_outlet = np.asarray(distance) + # Cells the integration never reached come back as NaN; the grid's + # documented empty value is 0. + self._griddata = np.where(np.isnan(chi), 0.0, chi) + + if kwargs.get('output_flag', False): + print('{0} outlets completed.'.format(len(outlets))) - try: - if flow_direction[i, j+1] == 16: - scale = 1.0 - self.__recurse_chi((i, j+1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i+1, j+1] == 32: - scale = 1.414 - self.__recurse_chi((i+1, j+1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i+1, j] == 64: - scale = 1.0 - self.__recurse_chi((i+1, j), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i+1, j-1] == 128: - scale = 1.414 - self.__recurse_chi((i+1, j-1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i, j-1] == 1: - scale = 1.0 - self.__recurse_chi((i, j-1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i-1, j-1] == 2: - scale = 1.414 - self.__recurse_chi((i-1, j-1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i-1, j] == 4: - scale = 1.0 - self.__recurse_chi((i-1, j), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i-1, j+1] == 8: - scale = 1.414 - self.__recurse_chi((i-1, j+1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass class GeographicChi(GeographicGridMixin, Chi): + """Chi on a latitude/longitude grid.""" + pass -class CrossDivideDChi(BaseSpatialGrid): - +class _CrossDivideMixin(object): + """Shared machinery for the across-divide chi contrast grids.""" + + def _divide_pairs(self, *args, **kwargs): + """``(from_rows, from_cols, to_rows, to_cols)`` for each divide pair.""" + paired_divides = self._paired_divides(*args, **kwargs) + if not paired_divides: + return None + + # zip() is a one-shot iterator in Python 3 and is not subscriptable, + # so the original `zip(*pairs)[0]` raised immediately. + from_point, to_point = zip(*paired_divides) + + from_idx = self._xy_to_rowscols(from_point) + to_idx = self._xy_to_rowscols(to_point) + + keep = [k for k in range(len(from_idx)) + if from_idx[k][0] is not None and to_idx[k][0] is not None] + if not keep: + return None + + from_rows = np.array([from_idx[k][0] for k in keep], dtype=int) + from_cols = np.array([from_idx[k][1] for k in keep], dtype=int) + to_rows = np.array([to_idx[k][0] for k in keep], dtype=int) + to_cols = np.array([to_idx[k][1] for k in keep], dtype=int) + return from_rows, from_cols, to_rows, to_cols + + +class CrossDivideDChi(_CrossDivideMixin, BaseSpatialGrid): + """Difference in chi across each drainage divide. + + A large value means the two sides of the divide are far from + steady state, so the divide is expected to migrate towards the + higher-chi side (Willett et al. 2014). + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('chi','flow_direction'), '_create_from_inputs')) - + + def _paired_divides(self, *args, **kwargs): + return kwargs['flow_direction'].paired_divides() + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) - paired_divides = kwargs['flow_direction'].paired_divides() - divide_columns = zip(*paired_divides) - from_point = divide_columns[0] - to_point = divide_columns[1] - - from_idx = self._xy_to_rowscols(from_point) - to_idx = self._xy_to_rowscols(to_point) - - from_idx_column = zip(*from_idx) - to_idx_column = zip(*to_idx) - - minchi = np.minimum(kwargs['chi']._griddata[to_idx_column[0],to_idx_column[1]], kwargs['chi']._griddata[from_idx_column[0],from_idx_column[1]]) - maxchi = np.maximum(kwargs['chi']._griddata[to_idx_column[0],to_idx_column[1]], kwargs['chi']._griddata[from_idx_column[0],from_idx_column[1]]) - - self._griddata[from_idx_column[0],from_idx_column[1]] = (maxchi - minchi) * (minchi != 0).astype(float) - -class NormalizedCrossDivideDChi(BaseSpatialGrid): - + pairs = self._divide_pairs(*args, **kwargs) + if pairs is None: + return + from_rows, from_cols, to_rows, to_cols = pairs + + chi = kwargs['chi']._griddata + a = chi[to_rows, to_cols] + b = chi[from_rows, from_cols] + minchi = np.minimum(a, b) + maxchi = np.maximum(a, b) + + self._griddata[from_rows, from_cols] = (maxchi - minchi) * (minchi != 0).astype(float) + + +class NormalizedCrossDivideDChi(_CrossDivideMixin, BaseSpatialGrid): + """Chi contrast across each divide, divided by the mean chi there. + + Normalising makes divides in high- and low-chi parts of a landscape + comparable. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('chi','flow_direction'), '_create_from_inputs')) - + + def _paired_divides(self, *args, **kwargs): + return kwargs['flow_direction'].paired_divides(mask=kwargs['chi']) + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) - paired_divides = kwargs['flow_direction'].paired_divides(mask = kwargs['chi']) - divide_columns = zip(*paired_divides) - from_point = divide_columns[0] - to_point = divide_columns[1] - - from_idx = self._xy_to_rowscols(from_point) - to_idx = self._xy_to_rowscols(to_point) - - from_idx_column = zip(*from_idx) - to_idx_column = zip(*to_idx) - - minchi = np.minimum(kwargs['chi']._griddata[to_idx_column[0],to_idx_column[1]], kwargs['chi']._griddata[from_idx_column[0],from_idx_column[1]]) - maxchi = np.maximum(kwargs['chi']._griddata[to_idx_column[0],to_idx_column[1]], kwargs['chi']._griddata[from_idx_column[0],from_idx_column[1]]) - + pairs = self._divide_pairs(*args, **kwargs) + if pairs is None: + return + from_rows, from_cols, to_rows, to_cols = pairs + + chi = kwargs['chi']._griddata + a = chi[to_rows, to_cols] + b = chi[from_rows, from_cols] + minchi = np.minimum(a, b) + maxchi = np.maximum(a, b) + rangechi = (maxchi - minchi) * (minchi != 0).astype(float) - meanchi = (maxchi + minchi) / 2 - - self._griddata[from_idx_column[0],from_idx_column[1]] = rangechi / meanchi + meanchi = (maxchi + minchi) / 2.0 + with np.errstate(divide='ignore', invalid='ignore'): + self._griddata[from_rows, from_cols] = np.where( + meanchi != 0, rangechi / meanchi, 0.0) - class Deflection(BaseSpatialGrid): - + r"""Flexural deflection of an elastic plate under a topographic load. + + Solves the thin-plate equation in the Fourier domain: + + .. math:: + w(k) = \frac{-\rho_c\,g\,h(k)} + {(\rho_m - \rho_c)\,g + D\,(2\pi k)^4} + + :: + + w = Deflection(elevation=dem, D=1e23, rho_m=3300, rho_c=2700, g=9.81) + + Pass ``restored_elevation`` as well to get the *change* in deflection + between two surfaces, which is the isostatic response to the erosion + between them. + + Because the solution is spectral, the load is implicitly periodic; pad + the DEM if edge effects matter. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation', 'D', 'rho_m', 'rho_c', 'g'), '_deflect')) - + def _deflect(self, *args, **kwargs): if kwargs.get('restored_elevation'): - self._griddata = self.__deflect_elevation(kwargs['elevation'], **kwargs) - self.__deflect_elevation(kwargs['restored_elevation'], **kwargs) + self._griddata = (self.__deflect_elevation(kwargs['elevation'], **kwargs) + - self.__deflect_elevation(kwargs['restored_elevation'], **kwargs)) else: self._griddata = self.__deflect_elevation(kwargs['elevation'], **kwargs) - + def __deflect_elevation(self, *args, **kwargs): elevation = args[0] self._copy_info_from_grid(elevation, False) + kwargs = dict(kwargs) kwargs['flow_direction'] = elevation - dx = np.mean(self._mean_pixel_dimension(*args, **kwargs)) - gr = -self._griddata * kwargs['rho_c'] * kwargs['g'] - grf = np.fft.fft2(gr) - Rx = np.arange(((self._georef_info.nx-1)*dx)/2,0,-dx) - Ry = np.arange(((self._georef_info.ny-1)*dx)/2,0,-dx) - lx = len(Rx) - ly = len(Ry) - wn_x = np.array((1 / dx) * np.arange(0,lx,1) / lx) - wn_y = np.array((1 / dx) * np.arange(0,ly,1) / ly) - wn_x = np.concatenate((wn_x,np.flipud(wn_x))) - wn_y = np.concatenate((wn_y,np.flipud(wn_y))) - [WN_x, WN_y] = np.meshgrid(wn_x, wn_y) - - R = 1 / ((kwargs['rho_m']-kwargs['rho_c'])*kwargs['g'] + (2*3.141)**4 * kwargs['D']*(WN_x**2 + WN_y**2)**2) - - grfR = grf * R - - return np.real(np.fft.ifft2(grfR)) - + dx = float(np.mean(self._mean_pixel_dimension(*args, **kwargs))) + + load = -np.nan_to_num(np.asarray(self._griddata, dtype=float64)) \ + * kwargs['rho_c'] * kwargs['g'] + load_hat = np.fft.fft2(load) + + # np.fft.fftfreq gives exactly one wavenumber per sample, in the + # order fft2 expects. Building the axes by hand from half-width + # ranges produced arrays whose length did not have to match the grid, + # so the multiply below could raise a broadcast error. + wn_x = np.fft.fftfreq(self._georef_info.nx, d=dx) + wn_y = np.fft.fftfreq(self._georef_info.ny, d=dx) + WN_x, WN_y = np.meshgrid(wn_x, wn_y) + + # (2*pi)**4, not (2*3.141)**4 -- the truncated constant was 0.2% low + # in the flexural term. + response = 1.0 / ((kwargs['rho_m'] - kwargs['rho_c']) * kwargs['g'] + + (2 * np.pi) ** 4 * kwargs['D'] + * (WN_x ** 2 + WN_y ** 2) ** 2) + + return np.real(np.fft.ifft2(load_hat * response)) + class GeographicDeflection(GeographicGridMixin, Deflection): + """Flexural deflection on a latitude/longitude grid.""" + pass - + class ChannelSlope(BaseSpatialGrid): - + """Downstream gradient of each cell along its D8 flow direction. + + :: + + slope = ChannelSlope(flow_direction=d8, elevation=dem) + + Positive values are downhill. Cells with no downstream neighbour stay 0. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction','elevation'), '_create_from_flow_direction_and_elevation')) - + def _create_from_flow_direction_and_elevation(self, *args, **kwargs): - + flow_dir = kwargs['flow_direction'] elevation = kwargs['elevation'] self._copy_info_from_grid(flow_dir, True) - if flow_dir.__class__ == FlowDirectionD8: - self.__calc_D8_slope(flow_dir, elevation) + # isinstance, not an exact class comparison, so GeographicFlowDirection + # and any other subclass is handled too. + if isinstance(flow_dir, FlowDirectionD8): + self.__calc_D8_slope(flow_dir, elevation, *args, **kwargs) - def __calc_D8_slope(self, fd, dem): + def __calc_D8_slope(self, fd, dem, *args, **kwargs): + + receivers = fd.receivers.reshape(-1) + has_receiver = receivers >= 0 + + z = np.asarray(dem._griddata, dtype=float64).reshape(-1) + step = fd.step_lengths(self._mean_pixel_dimension(*args, **kwargs)).reshape(-1) + + slope = np.zeros(z.shape, dtype=float64) + donors = np.flatnonzero(has_receiver) + targets = receivers[donors] + # Vectorised: the original walked every cell in Python, which cost + # minutes on a large DEM for what is a single subtraction per cell. + slope[donors] = (z[donors] - z[targets]) / step[donors] + + self._griddata = slope.reshape(self._georef_info.ny, self._georef_info.nx) - dx = self._georef_info.dx - idcs = fd.sort() # Get the sorted indices of the array in reverse order (e.g. largest first) - - for idx in idcs: # Loop through all the data in sorted order - [i, j] = np.unravel_index(idx, fd._griddata.shape) # Get the row/column indices - - i_next, j_next, is_good = fd.get_flow_to_cell(i,j) - if is_good: - dist = np.sqrt((i-i_next)**2 + (j-j_next)**2)*dx - self._griddata[i,j] = (dem[i,j] - dem[i_next, j_next]) / dist - def mosaicFolder(folderPath, fileSuffix, outfile): - #This runs the gdal utility gdal_merge on the command line, is mainly here so that I don't have to continue looking - #up how to actually accomplish this - #use os to get files, existing gdal functions to merge them - files = glob.glob1(folderPath, '*'+fileSuffix) # get all the files in the path - ## Need to use existing gdal_merge.py.... need to check on interpolation options. - # could also use gdal_warp i think. Should consider whether I care about the possibility of using the c - #utility... - argument = ['python', 'gdal_merge.py', '-o', outfile] - for file in files: - argument.append(os.path.join(folderPath, file)) - # sys.argv = argv - subprocess.call(argument) + """Merge every ``*fileSuffix`` raster in a folder into ``outfile``. + + Uses GDAL's Python API directly rather than shelling out to + ``gdal_merge.py``, which is not on the path of most installations. + """ + files = sorted(glob.glob1(folderPath, '*' + fileSuffix)) + if not files: + raise Error.InputError( + 'mosaicFolder', + 'no files matching *{0} in {1}'.format(fileSuffix, folderPath)) + + paths = [os.path.join(folderPath, name) for name in files] + vrt = gdal.BuildVRT('', paths) + try: + gdal.Translate(outfile, vrt, creationOptions=['COMPRESS=LZW']) + finally: + vrt = None + def plot(*args, **kwargs): - + """Scatter one grid's values against another's, cell by cell. + + :: + + plot(area, slope, xlabel='Drainage area', ylabel='Slope', + decimation_factor=50) + + Parameters + ---------- + xlabel, ylabel : str + Axis labels. + symbol : str + Matplotlib format string; ``'k.'`` by default. + indexes : tuple of ndarray + Restrict to these cells instead of every cell valid in both grids. + decimation_factor : int + Plot only every n-th point. + interactive : bool + Leave the figure open without blocking. + + Returns + ------- + matplotlib.axes.Axes + """ grid1 = args[0]._griddata grid2 = args[1]._griddata interactive = kwargs.pop('interactive', True) - if kwargs.get('xlabel') is None: - xlabel = 'Grid 1' - - if kwargs.get('ylabel') is None: - ylabel = 'Grid 2' - - if kwargs.get('symbol') is None: - symbol = 'k.' - - if kwargs.get('indexes') is None: - valid_indexes = np.where( ~np.isnan(grid1+grid2) ) - else: - valid_indexes = kwargs.get('indexes') - - if kwargs.get('decimation_factor') is not None: - valid_indexes = (valid_indexes[0][::kwargs.get('decimation_factor')],valid_indexes[1][::kwargs.get('decimation_factor')]) - - plt.plot(grid1[valid_indexes], grid2[valid_indexes], symbol) + # Reading these with .get() into a local only inside the `is None` + # branch meant that actually *passing* xlabel left the name unbound and + # raised NameError two lines later. + xlabel = kwargs.pop('xlabel', None) or 'Grid 1' + ylabel = kwargs.pop('ylabel', None) or 'Grid 2' + symbol = kwargs.pop('symbol', None) or 'k.' + valid_indexes = kwargs.pop('indexes', None) + decimation_factor = kwargs.pop('decimation_factor', None) + + if valid_indexes is None: + valid_indexes = np.where(~np.isnan(grid1 + grid2)) + + if decimation_factor is not None: + valid_indexes = tuple(axis[::decimation_factor] for axis in valid_indexes) + + plt.plot(grid1[valid_indexes], grid2[valid_indexes], symbol, **kwargs) plt.xlabel(xlabel) plt.ylabel(ylabel) if interactive: @@ -4282,8 +5494,4 @@ def plot(*args, **kwargs): plt.ioff() plt.show(block=True) - ax = plt.gca() - - return ax - - + return plt.gca() diff --git a/demMethods.py b/demMethods.py index 5432aea..cd6a327 100644 --- a/demMethods.py +++ b/demMethods.py @@ -1,73 +1,73 @@ -from dem import FilledElevation +"""Batch pipelines that chain the grid classes into a finished analysis.""" + +import numpy as np + +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d +except ImportError: # pragma: no cover + import dem as d + def processAll(prefix_name, Ao, theta, base_name = '.'): - from dem import Elevation, FlowDirectionD8, GeographicArea, Area, GeographicFlowLength, GeographicKsi, ScaledRelief - elevation_name = base_name + "/" + prefix_name + "_dem_15s" area_name = base_name + "/" + prefix_name + "_acc_15s" d8_name = base_name + "/" + prefix_name + "_dir_15s" - elevation = Elevation(gdal_filename = elevation_name) + elevation = d.Elevation(gdal_filename = elevation_name) elevation.save(prefix_name + "_elevation") - area = Area(gdal_filename = area_name) - d8 = FlowDirectionD8(gdal_filename = d8_name) + area = d.Area(gdal_filename = area_name) + d8 = d.FlowDirectionD8(gdal_filename = d8_name) d8.save(prefix_name + "_flow_direction") idx = area.sort(reverse = False) - area = GeographicArea(flow_direction = d8, sorted_indexes = idx) + area = d.GeographicArea(flow_direction = d8, sorted_indexes = idx) area.save(prefix_name + "_area") - flow_length = GeographicFlowLength(flow_direction = d8, sorted_indexes = idx) + flow_length = d.GeographicFlowLength(flow_direction = d8, sorted_indexes = idx) flow_length.save(prefix_name + "_flow_length") - ksi = GeographicKsi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length, sorted_indexes = idx) + ksi = d.GeographicKsi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length, sorted_indexes = idx) ksi.save(prefix_name + "_ksi_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) - relief = ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta, sorted_indexes = idx) + relief = d.ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta, sorted_indexes = idx) relief.save(prefix_name + "_relief_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) def processAllUTM(prefix_name, EPSGprojectionCode, Ao, theta, base_name = '.'): - from dem import Elevation, FlowDirectionD8, Area, FlowLength, Ksi, ScaledRelief - full_path_without_suffix = base_name + "/" + prefix_name elevation_unfilled_ascii_filename = full_path_without_suffix + ".txt" - elevation = Elevation(ai_ascii_filename = elevation_unfilled_ascii_filename, EPSGprojectionCode= EPSGprojectionCode) + elevation = d.Elevation(ai_ascii_filename = elevation_unfilled_ascii_filename, EPSGprojectionCode= EPSGprojectionCode) elevation.save(full_path_without_suffix + "_elevation") - filled = FilledElevation(elevation = elevation) + filled = d.FilledElevation(elevation = elevation) filled.save(full_path_without_suffix + "_filled") - d8 = FlowDirectionD8(flooded_dem = filled) + d8 = d.FlowDirectionD8(flooded_dem = filled) d8.save(full_path_without_suffix + "_flow_direction") - area = Area(flow_direction = d8) + area = d.Area(flow_direction = d8) area.save(full_path_without_suffix + "_area") - flow_length = FlowLength(flow_direction = d8) + flow_length = d.FlowLength(flow_direction = d8) flow_length.save(full_path_without_suffix + "_flow_length") - ksi = Ksi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length) + ksi = d.Ksi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length) ksi.save(full_path_without_suffix + "_ksi_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) - relief = ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta) + relief = d.ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta) relief.save(full_path_without_suffix + "_relief_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) def processForTheta(prefix_name, Ao, theta, base_name = '.'): - from dem import FlowDirectionD8, GeographicArea, GeographicFlowLength, GeographicKsi, ScaledRelief, Elevation - area_name = base_name + "/" + prefix_name + "_area" d8_name = base_name + "/" + prefix_name + "_flow_direction" flow_length_name = base_name + "/" + prefix_name + "_flow_length" elevation_name = base_name + "/" + prefix_name + "_elevation" - area = GeographicArea.load(area_name) - d8 = FlowDirectionD8.load(d8_name) - flow_length = GeographicFlowLength.load(flow_length_name) - elevation = Elevation.load(elevation_name) + area = d.GeographicArea.load(area_name) + d8 = d.FlowDirectionD8.load(d8_name) + flow_length = d.GeographicFlowLength.load(flow_length_name) + elevation = d.Elevation.load(elevation_name) idx = area.sort(reverse = False) - ksi = GeographicKsi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length, sorted_indexes = idx) + ksi = d.GeographicKsi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length, sorted_indexes = idx) ksi.save(prefix_name + "_ksi_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) - relief = ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta, sorted_indexes = idx, area = area) + relief = d.ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta, sorted_indexes = idx, area = area) relief.save(prefix_name + "_relief_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) def plotGrids(x_grid, y_grid, plot_string, **kwargs): - - import numpy as np - + """Scatter one grid against another, keeping only non-negative y.""" x_vec = np.ndarray.flatten(x_grid._griddata) y_vec = np.ndarray.flatten(y_grid._griddata) @@ -82,8 +82,7 @@ def plotGrids(x_grid, y_grid, plot_string, **kwargs): return x_vec, y_vec def extract_values_from_grid(x_grid, y_grid, ignore_zeros=False): - - import numpy as np + """Flatten two grids to matching 1-D vectors.""" x_vec = np.ndarray.flatten(x_grid._griddata) y_vec = np.ndarray.flatten(y_grid._griddata) if ignore_zeros: @@ -94,7 +93,7 @@ def extract_values_from_grid(x_grid, y_grid, ignore_zeros=False): return x_vec, y_vec def create_density(x, y, x_boundaries, y_boundaries): - import numpy as np + """2-D histogram of paired grid values.""" H, xedges, yedges = np.histogram2d(x, y, bins = (x_boundaries, y_boundaries)) return H, xedges, yedges diff --git a/demRecursionTools.py b/demRecursionTools.py index 744c872..e12b308 100644 --- a/demRecursionTools.py +++ b/demRecursionTools.py @@ -1,5 +1,16 @@ +"""Profile extraction and chi/steepness fitting over recursive basin maps. + +These functions consume the nested dictionaries produced by +:meth:`TopoAnalysis.dem.FlowDirectionD8.map_values_to_recursive_list`. +""" + import numpy as np +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d +except ImportError: # pragma: no cover + import dem as d + def extract_chi_elevation_values(ld_list, de, theta, chi_o, elevation, chi, base_elevation, A_mdx = None, xo = 500.0): @@ -113,7 +124,8 @@ def hi_list(ld_list): elevation, dA = extract_dA_elevation_values(ld_list) - if np.nan in elevation: + # `np.nan in array` is always False (NaN != NaN); test explicitly. + if np.isnan(elevation).any(): return 0.0 mean_elevation = np.mean(elevation) @@ -124,7 +136,6 @@ def hi_list(ld_list): def area_elevation_for_mainstem_and_tributaries(outlet, flow_direction, elevation, area, theta = 0.5, minimum_area = 1.0E7): - import dem as d mean_pixel_dimension = d.BaseSpatialGrid() mean_pixel_dimension._copy_info_from_grid(area, True) mean_pixel_dimension._griddata = area._mean_pixel_dimension() @@ -163,7 +174,8 @@ def get_elevations_and_areas(ld_list, area, elevation, de, tributary_ld, minimum for trb_ld in tributary_ld: this_area = [trb_ld['area']] this_elevation = [trb_ld['elevation']] - this_de = [trb_ld['de']+trb_ld['distance_scale']] + # '*' not '+': de is a length and distance_scale is a multiplier. + this_de = [trb_ld['de'] * trb_ld['distance_scale']] (this_area, this_elevation, this_de, next_tributary_ld) = get_elevations_and_areas(trb_ld, this_area, this_elevation, this_de, next_tributary_ld, minimum_area) area.append(this_area) elevation.append(this_elevation) @@ -203,7 +215,7 @@ def chi_for_profile(area, de, theta): def best_ks_with_wrss(chi, elevation): A = np.vstack([chi]).T - sol = np.linalg.lstsq(A, elevation) + sol = np.linalg.lstsq(A, elevation, rcond=None) m = sol[0] WRSS = sol[1] return (m, WRSS) @@ -291,7 +303,7 @@ def chi_for_profile(area, de, theta): def best_ks_with_wrss(chi, elevation): A = np.vstack([chi]).T - sol = np.linalg.lstsq(A, elevation) + sol = np.linalg.lstsq(A, elevation, rcond=None) m = sol[0] WRSS = sol[1] return (m, WRSS) @@ -356,7 +368,6 @@ def map_ld_link(current_chi, ld_list, downstream_sign, minimum_area): return_map[index] = (next_chi, next_elevation) map_ld_link(next_chi, next_list, downstream_sign, minimum_area) - import dem as d mean_pixel_dimension = d.BaseSpatialGrid() mean_pixel_dimension._copy_info_from_grid(area, True) mean_pixel_dimension._griddata = area._mean_pixel_dimension() diff --git a/docs/algorithms.md b/docs/algorithms.md new file mode 100644 index 0000000..0d7ec6a --- /dev/null +++ b/docs/algorithms.md @@ -0,0 +1,397 @@ +# Algorithms + +What each computation actually does, the conventions it assumes, and where +it will disagree with another package. + +--- + +## Depression filling — Priority-Flood + +A DEM straight off a sensor is full of closed depressions: real ones +(playas, craters, karst) and spurious ones (noise, bridges, vegetation +returns). Flow routing needs every cell to have a downhill path to the edge, +so the depressions are filled first. + +TopoAnalysis uses the Priority-Flood family of Barnes, Lehman and Mulla +(2014). The idea is simple: start from the cells water can leave through, +and flood inwards, always processing the lowest cell you have reached so far. + +``` +push every seed cell onto a min-heap, mark it closed +while the heap is not empty: + c = pop the lowest cell + for each of c's eight neighbours n that is not closed: + close n + raise n to at least c's elevation + push n +``` + +Because cells are processed in nondecreasing elevation order, the first time +you reach a cell you have reached it by the lowest possible path, so the +elevation you raise it to is the lowest that lets water out. The filled +surface is the pointwise-minimal surface with no depressions. + +### Seeds + +Water leaves the DEM at the grid perimeter and at no-data holes, so the seed +set is: + +- every cell on the outer ring, and +- every valid cell that touches a `NaN`. + +This is the same seed set TopoToolbox's `fillsinks` uses. A depression that +touches a no-data hole is therefore *not* filled — it drains into the hole. + +You can supply your own seeds with `outlets=[(x, y), ...]`, which floods +inwards from those points instead. + +### The three variants + +`TopoAnalysis.kernels.priority_flood` implements all three algorithms from +the paper: + +| Variant | Paper | When it is used | +|---|---|---| +| Priority-Flood | Algorithm 1 | `reference_algorithm=True`, as a test oracle | +| Improved Priority-Flood | Algorithm 2 | the default for a flat fill | +| Priority-Flood + ε | Algorithm 4 | whenever `aggradation_slope > 0` | + +Algorithm 2 adds a FIFO queue alongside the heap: a cell that needs no +filling sits at the current flood level and cannot be reordered, so it can +skip the heap entirely. That is what makes the practical cost O(n) instead +of O(n log n). The result is identical to Algorithm 1 — the test suite +asserts it on random surfaces. + +The FIFO shortcut is only sound while every cell it holds is at exactly the +current level. Two options break that, and both fall back to Algorithm 1: + +- **ε filling**, where each filled cell is raised by its own increment; and +- **`maximum_pit_depth`**, which re-queues an unfilled cell *below* the + current level. + +### Flat fill or ε fill? + +A flat fill produces exactly level plateaus, which have no downhill +direction — D8 routing has nothing to follow across them. TopoAnalysis +therefore defaults to raising each filled cell by + + parent_elevation + aggradation_slope × step_length + +with `aggradation_slope = 1e-12` per map unit. That is far below any +meaningful elevation difference but strictly monotonic, so every filled cell +has a defined downslope neighbour. + +Set `aggradation_slope=0` for the flat fill TopoToolbox produces. TopoToolbox +then handles the flats separately, by building an auxiliary "geodesic" +surface inside them so cells drain towards the nearest sill. The two +approaches give *different but equally valid* flow directions inside flats; +see [topotoolbox_parity.md](topotoolbox_parity.md). + +### Leaving real basins alone + +`maximum_pit_depth=20.0` leaves any depression deeper than 20 m unfilled. +The flood still traverses it — so the terrain upstream is routed — but the +cells keep their elevations, and the basin becomes a local base level. + +> The original implementation marked such cells closed without queueing +> them, which orphaned everything upstream of a deep pit. + +### Cost + +O(n) time and O(n) memory in practice. About 100 ms per million cells in +C++; 5 s in NumPy. + +--- + +## Flow direction — D8 + +Each cell drains to exactly one of its eight neighbours: the one reached by +the steepest *gradient*, + + slope = (z_centre − z_neighbour) / distance + +with `distance = dx` for the four cardinal neighbours and `√2 · dx` for the +four diagonals. Dividing by distance matters: a diagonal step covers 41% +more ground, so an equal drop over a diagonal is a shallower slope. Choosing +by raw drop biases the network towards diagonals and inflates flow lengths. + +Ties go to the first direction in the order E, SE, S, SW, W, NW, N, NE. Any +rule is arbitrary; what matters is that it is deterministic, so two runs +agree. + +Codes follow the ArcGIS convention, with row 0 at the north edge: + +``` +| 32 64 128 | | (i-1,j-1) (i-1,j) (i-1,j+1) | +| 16 X 1 | == | (i ,j-1) X (i ,j+1) | +| 8 4 2 | | (i+1,j-1) (i+1,j) (i+1,j+1) | +``` + +0 means the cell has no downstream neighbour: a pit, an outlet at the grid +edge, or no-data. On a filled DEM the only zeros are edge outlets. + +For a geographic (lat/lon) grid the distance uses the local cell size, which +varies with latitude. + +--- + +## The flow network + +Once directions are known, the grid is a forest: every cell has one +*receiver* and any number of *donors*, and following receivers always +terminates. + +```python +d8.receivers # flat index of each cell's receiver, or -1 +d8.topological_order # every donor appears before its receiver +``` + +The order comes from Kahn's algorithm on the donor counts, **not** from +sorting by elevation. That distinction matters: a flow-direction grid loaded +from a file carries no elevations, so an elevation sort is unavailable — +and the original code silently sorted such grids by their *direction codes*, +producing wrong accumulations. + +Kahn's algorithm also detects cycles, which a hand-edited or externally +produced flow-direction grid can contain. `d8.cycle_count` reports them and +a warning is raised; a valid network has none. + +Given the order, everything downstream is one linear pass: + +| Quantity | Pass | Recurrence | +|---|---|---| +| accumulation | forward | `A[recv] += A[k]` | +| flow length | forward | `L[recv] = max(L[recv], L[k] + step)` | +| distance to outlet | backward | `D[k] = D[recv] + step` | +| chi | backward | `χ[k] = χ[recv] + integrand · step` | +| basin labels | backward | `label[k] = label[recv]` | +| minima imposition | forward | `z[recv] = min(z[recv], z[k] − sl·step)` | +| Strahler order | forward | the order-increment rule | + +--- + +## Drainage area + +`Area` accumulates the per-cell area downstream, so each cell holds its own +area plus every cell upstream of it, in **square map units**. + +TopoToolbox's `flowacc` accumulates unit weights instead and reports the +**number of cells**. `TopoAnalysis.topotoolbox.flowacc` does the same; the +two differ by exactly `dx²` on a projected grid. + +For a geographic grid, `GeographicArea` uses the true spherical cell area + + A(lat) = R² · Δλ · (sin(lat + Δφ/2) − sin(lat − Δφ/2)) + +with `R = 6 371 000 m`. Cells shrink towards the poles: at 60° latitude they +are half the area of equatorial ones, and using a constant `dx²` would +overestimate drainage area by that factor. + +### Gating + +`evaluate_at` or `mask` restrict accumulation to a subset of cells. A gated- +out cell contributes nothing and passes nothing downstream — the first cell +*outside* the gate still receives what the gate delivers, and the flow +stops there. `ValleyArea` uses this to accumulate only over valley floors. + +--- + +## Flow length and the main stem + +`FlowLength` records, for each cell, the length of the **longest** flow path +reaching it — the standard "flow length" of a basin, and the distance used +to define its relief. + +Alongside the distances it records which upstream neighbour supplies each +cell's longest path. Chaining those donors from an outlet traces the main +stem. Ties are broken towards the donor with the lower flat index, so the +main stem does not depend on which valid topological order was used. + +`Relief`, `ScaledRelief` and `Ksi` all propagate along that main-stem +network, which is why they need a `FlowLength` as well as a flow direction. + +> Before version 1.0 these codes used a row-flipped convention, internally +> consistent but incompatible with `FlowDirectionD8`. `*_directions` files +> written by older versions must be regenerated. + +--- + +## Chi + +For a river in steady state under the stream-power law, elevation is linear +in the integral quantity + + χ(x) = ∫[outlet → x] (A₀ / A(x'))^θ dx' + +(Perron & Royden 2013). The slope of elevation against χ is the steepness +index k_sn, and A₀ makes χ dimensionally a length — usually 1 km² = 10⁶ m². + +Two implementations, because they answer different questions. + +**`Chi`** integrates upstream from outlets you name, over the whole network +above each one. Use it when the basins are the unit of analysis. + +**`Ksi`** integrates along longest-flow-length paths across the whole grid, +without outlets. Use it for a regional map. + +### The integration rule + +Per step from a cell to its receiver: + +- *left endpoint* (the default): `χ[k] = χ[recv] + (A₀/A[k])^θ · dl` +- *trapezoid* (`trapezoid=True`): averages the integrand across the step, + `χ[k] = χ[recv] + ½((A₀/A[k])^θ + (A₀/A[recv])^θ) · dl` + +TopoToolbox's `chitransform` uses the trapezoid rule. Area grows downstream +so the integrand shrinks downstream, which makes the left-endpoint rule the +larger of the two. Use `trapezoid=True` when comparing. + +χ is 0 at each outlet and grows upstream. + +> The `Ksi` integrand used to be `(A₀ / (A − A₀))^θ`, which diverges as area +> approaches the threshold. It is now `(A₀ / A)^θ`. + +--- + +## Terrain derivatives + +### Slope + +`MaxSlope` is the magnitude of the centred-difference gradient, +`√((dz/dx)² + (dz/dy)²)`, with the boundary padded by replicating the edge. + +`topotoolbox.gradient8` is the steepest *downward* gradient over the eight +neighbours, clamped at 0 — a different quantity, always non-negative, 0 in +pits and flats. + +`topotoolbox.arcslope` is the ArcGIS/Horn 3×3 estimator: a 1-2-1 weighted +stencil over an `8·dx` denominator, with no √2 anywhere. It is less +sensitive to single-cell noise than a plain centred difference. + +### Measuring over a length scale + +On a 1 m lidar DEM a 3×3 stencil measures the noise, not the landscape. +`calculate_gradient_over_length_scale(L)` and +`calculate_laplacian_over_length_scale(L)` use a stencil reaching +`N = ⌈L/dx⌉` cells either side, so the sampled cells are `2N·dx` apart. A +band `N` cells wide around the edge comes back as `NaN`. + +### Curvature + +`Laplacian` is `∇²z` from SciPy's 5-point stencil, scaled by `1/dx²`. +Positive in valleys, negative on ridges, and the basis of the +valley-floor masks that `ValleyArea` uses. + +`principal_curvatures()` returns the two principal curvatures of the surface +from the second fundamental form. `topotoolbox.curvature` offers the five +TopoToolbox flavours — profile, plan, tangential, mean and total — from the +Evans 3×3 quadratic fit. + +### Hillshade + +`Hillshade` follows the ESRI definition and returns 0–255: + + H = 255 · (cos(zenith)·cos(slope) + sin(zenith)·sin(slope)·cos(azimuth − aspect)) + +with `zenith = 90° − altitude`, `azimuth_math = 360° − azimuth + 90°`, and +`aspect = atan2(dz/dy, −dz/dx)`. Values below 0 — slopes facing away from +the light — are clamped to 0. + +> The `−dz/dx` is easy to miss and easy to get wrong; without it the +> illumination is mirrored east–west, which is what the original did. + +`topotoolbox.hillshade` returns the raw cosine in [−1, 1] from a +central-difference surface normal, which is what TopoToolbox does (it only +scales to 0–255 when plotting). + +--- + +## Valley extraction + +`ValleyArea` builds a valley-floor mask in three steps: + +1. **Convergence.** Cells whose Laplacian exceeds `valley_laplace_value`. +2. **Connectivity.** `PriorityFillGrid` floods outwards from cells with more + than `min_area_value` of drainage area through the convergent cells, so + only the parts that connect to a real channel survive. Isolated + convergent patches on hillslopes are dropped. +3. **Closing** (optional, `iterations=n`). Dilate then erode, to bridge + one-cell gaps without growing the network. + +Drainage area is then accumulated over that mask only. +`MainstemValleyArea` restricts it further, to the longest flow path. + +`MultiscaleCurvatureValleyWidth` takes a different route: it fits a local +quadratic at a range of window sizes and records, at each cell, the window +that minimises the smaller principal curvature. That scale is the valley +width, because a window matched to the valley resolves it best. + +--- + +## Restored topography + +`RestoredElevation` asks what the landscape would look like if channel +steepness were uniform. Starting from the outlets it integrates + + dz/dx = k_s · A^(−θ) + +upstream, then lets each divide migrate towards whichever side ended up +lower, recomputes drainage area, and repeats. Comparing the result with the +real DEM shows where the landscape is out of steady state. + +`randomize=True` randomises elevations inside the basin first and re-routes, +so the result does not inherit the real network's planform. + +--- + +## Flexural isostasy + +`Deflection` solves the thin elastic plate equation spectrally: + + w(k) = −ρ_c g h(k) / [(ρ_m − ρ_c) g + D (2π|k|)⁴] + +Long wavelengths approach the Airy limit `−h ρ_c/(ρ_m − ρ_c)`; short ones +are supported by plate strength. Because it is spectral the load is +implicitly periodic, so pad the DEM if edge effects matter. + +Pass `restored_elevation` as well to get the *change* in deflection between +two surfaces — the isostatic response to the erosion between them. + +--- + +## Numerical conventions + +| | | +|---|---| +| Row order | row 0 is the **north** edge | +| No-data | `NaN` in floating-point grids; 0 in flow-direction grids | +| Cell registration | coordinates refer to cell **centres** | +| `geoTransform` | GDAL's 6-tuple, origin at the **outer corner** of the NW cell | +| Diagonal distance | `√2 · dx`, never 1.41 | +| Precision | float64 throughout (TopoToolbox often works in float32) | +| Cells | assumed square; `dx` is used for both axes | + +--- + +## References + +Barnes, R., Lehman, C. and Mulla, D. (2014). Priority-flood: An optimal +depression-filling and watershed-labeling algorithm for digital elevation +models. *Computers & Geosciences* 62, 117–127. + +Horn, B.K.P. (1981). Hill shading and the reflectance map. *Proceedings of +the IEEE* 69, 14–47. + +O'Callaghan, J.F. and Mark, D.M. (1984). The extraction of drainage networks +from digital elevation data. *Computer Vision, Graphics and Image +Processing* 28, 323–344. + +Perron, J.T. and Royden, L. (2013). An integral approach to bedrock river +profile analysis. *Earth Surface Processes and Landforms* 38, 570–576. + +Schwanghart, W. and Scherler, D. (2014). TopoToolbox 2 — MATLAB-based +software for topographic analysis and modeling in Earth surface sciences. +*Earth Surface Dynamics* 2, 1–7. + +Willett, S.D., McCoy, S.W., Perron, J.T., Goren, L. and Chen, C.-Y. (2014). +Dynamic reorganization of river basins. *Science* 343, 1248765. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..4fd55ca --- /dev/null +++ b/docs/api.md @@ -0,0 +1,1993 @@ +# API reference + +Generated from the docstrings by `docs/generate_api.py`, so +`help(TopoAnalysis.Area)` in a session shows the same text. For how the +pieces fit together start with [guide.md](guide.md); for what the +computations actually do see [algorithms.md](algorithms.md). + +Attributes prefixed with a single underscore (`_griddata`, +`_georef_info`) are not private in practice -- they are how you reach +the data -- but they are not listed here; see +[guide.md](guide.md#the-grid-model). + +## `TopoAnalysis.dem` -- the grid classes + +Contents: + +- **Core grid types**: `BaseSpatialGrid`, `Georef_info`, `BaseSpatialShape`, `ValueGrid`, `Elevation`, `GeographicElevation` +- **Depression filling**: `PriorityQueueMixIn`, `FilledElevation`, `PriorityFillGrid` +- **Flow routing**: `FlowDirection`, `FlowDirectionD8`, `FlowLength`, `GeographicFlowLength`, `Mask`, `DiscreteFlowAccumulation`, `GeographicDiscreteFlowAccumulation` +- **Drainage area**: `Area`, `GeographicArea`, `LogArea`, `ValleyArea`, `GeographicValleyArea`, `MainstemValleyArea`, `GeographicMainstemValleyArea` +- **Chi and channel steepness**: `Chi`, `GeographicChi`, `Ksi`, `GeographicKsi`, `Relief`, `ScaledRelief`, `ChiScaledRelief`, `ChannelSlope`, `CrossDivideDChi`, `NormalizedCrossDivideDChi`, `RestoredElevation`, `GeographicRestoredElevation` +- **Along-flow smoothing**: `AlongFlowSmoothing`, `KsFromChiWithSmoothing`, `GeographicKsFromChiWithSmoothing`, `ThetaFromChiWithSmoothing`, `GeographicThetaFromChiWithSmoothing`, `ChannelSlopeWithSmoothing`, `ChannelDownSlopeWithSmoothing`, `ChannelUpSlopeWithSmoothing` +- **Terrain attributes**: `CalculationMixin`, `Hillshade`, `GeographicHillshade`, `MaxSlope`, `GeographicMaxSlope`, `Laplacian`, `GeographicLaplacian`, `Gradient`, `LocalRelief`, `MultiscaleCurvatureValleyWidth`, `ScarpWavelet` +- **Isostasy**: `Deflection`, `GeographicDeflection` +- **Mixins**: `GDALMixin`, `GeographicGridMixin`, `MaxFlowLengthTrackingMixin` + +### Core grid types + +#### `BaseSpatialGrid` + +*inherits* `GDALMixin` + +A georeferenced raster, and the base class of every grid here. + +Construct one by keyword; the combination you pass selects how it is +built (see :attr:`required_inputs_and_actions`):: + + BaseSpatialGrid(gdal_filename='grid.tif') + BaseSpatialGrid(dx=30.0, grid=array) + BaseSpatialGrid(nx=100, ny=80, dx=30.0) # random values + BaseSpatialGrid(nx=100, ny=80, projection=wkt, geo_transform=gt) + +The data live in ``_griddata``, a 2-D NumPy array with row 0 at the +**north** edge and ``NaN`` for no-data; the georeferencing lives in +``_georef_info`` (a :class:`Georef_info`). + +Indexing is by ``(row, col)`` and is forgiving: reading outside the grid +returns ``None`` and writing outside it is ignored, which is what the +neighbourhood traversals in this module rely on. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `nx`, `ny`, `dx` +- `dx`, `grid` + +*Stored as* `float64` + +| Member | Signature | Summary | +|---|---|---| +| `apply_moving_window` | `(moving_window)` | Apply a :class:`~TopoAnalysis.MovingWindow.MovingWindow` to the grid. | +| `average_over_distance` | `(distance, grid=None)` | Mean of the grid within ``distance`` of each cell. | +| `calculate_gradient_over_length_scale` | `(length_scale)` | Centred finite-difference gradient measured over ``length_scale``. | +| `calculate_laplacian_over_length_scale` | `(length_scale)` | Laplacian measured over ``length_scale`` with a centred stencil. | +| `clip_to_bounds` | `(bounds)` | Clip to ``((xmin, xmax), (ymin, ymax))``. | +| `clip_to_extent` | `(extent)` | Clip to ``(xmin, xmax, ymin, ymax)``, keeping every cell inside it. | +| `clip_to_mask_grid` | `(mask_grid)` | Reproject and resample this grid onto ``mask_grid``'s frame, in place. | +| `clip_to_shapefile` | `(shapefile)` | Cut the grid to a vector boundary, in place. | +| `coordinate_vectors` | `()` | ``(x, y)`` coordinates of the cell centres, west-to-east and | +| `export_arc_e00_grid` | `(filename)` | Write an ArcInfo E00 interchange grid. | +| `extent` | `()` | ``(xmin, xmax, ymin, ymax)`` of the grid's outer pixel edges. | +| `extent_of_data` | `()` | ``(xmin, xmax, ymin, ymax)`` of the smallest box holding all valid data. | +| `find_nearest_cell_with_greatest_value` | `(index, pixel_radius=5)` | Index of the highest-valued cell within ``pixel_radius``. | +| `find_nearest_cell_with_value` | `(index, value, pixel_radius)` | Index of the cell within ``pixel_radius`` whose value is closest to ``value``. | +| `find_nearest_cell_with_value_greater_than` | `(index, value, pixel_radius)` | Index of the cell within ``pixel_radius`` whose value is the smallest | +| `get_XY_matrices` | `()` | Meshgrid of cell-centre coordinates. | +| `get_XY_matricies` | `()` | Meshgrid of cell-centre coordinates. | +| `load` | `(filename)` | Read a grid previously written by :meth:`save`. *(classmethod)* | +| `location_in_grid` | `(xo)` | True when ``xo = (x, y)`` lands on a cell holding real data. | +| `mosaic` | `(tiles)` | Reassemble tiles produced by :meth:`tile` into a single grid. *(classmethod)* | +| `plot` | `(**kwargs)` | Display the grid with ``imshow``, georeferenced in map coordinates. | +| `principal_curvatures` | `()` | Maximum and minimum principal curvature of the surface. | +| `remove_padding` | `(xpadding=10, ypadding=10)` | Strip the overlap that :meth:`tile` added, ready for :meth:`mosaic`. | +| `resample` | `(de, interpolation='quintic')` | Resample onto a grid of spacing ``de``. | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | +| `set_value_at_rowscols` | `(value, rowscols)` | Set ``value`` at each ``(row, col)`` in ``rowscols``. | +| `snap_locations_to_closest_value` | `(v, value, pixel_radius=5)` | Move each ``(x, y)`` to the nearby cell whose value best matches. | +| `snap_locations_to_greatest_value` | `(v, pixel_radius=5)` | Move each ``(x, y)`` to the highest-valued cell within ``pixel_radius``. | +| `sort` | `(reverse=True, force=False, mask=None)` | Flat indices of the cells, ordered by value. | +| `tile` | `(tile_xdim=400, tile_ydim=400, tile_xpadding=10, tile_ypadding=10)` | Split the grid into overlapping tiles. | +| `vectorize` | `(filename)` | Polygonize the grid into an ESRI shapefile called ``filename.shp``. | +| `write_to_ai` | `(filename, nodata_value=-9999.0)` | Write the grid as an ArcInfo ASCII grid. | + +#### `Georef_info` + +Where a grid sits on the ground. + +Attributes +---------- +geoTransform : tuple + The GDAL 6-tuple ``(x_origin, dx, 0, y_origin, 0, -dy)``, whose + origin is the *outer corner* of the north-west cell. +projection : str or int + WKT, or the integer 0 when no projection is set. +dx : float + Cell size in map units. Cells are assumed square. +xllcenter, yllcenter : float + Centre of the south-west cell. +nx, ny : int + Columns and rows. + +#### `BaseSpatialShape` + +A vector layer, used to cut rasters to a boundary. + +:: + + shape = BaseSpatialShape(shapefile_name='basin.shp') + mask = shape.createMaskFromShape(dem._georef_info, + dem._georef_info.projection, + gdal.GDT_Byte) + +| Member | Signature | Summary | +|---|---|---| +| `createMaskFromShape` | `(geoRefInfo, projection, dtype, noDataValue=0)` | Rasterise the shape onto ``geoRefInfo``'s frame, 1 inside and 0 out. | + +#### `ValueGrid` + +*inherits* `BaseSpatialGrid` + +A plain raster of values, with a helper for scattered writes. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `dx`, `grid` + +| Member | Signature | Summary | +|---|---|---| +| `set_value_at_indexes` | `(indexes, value)` | Set ``value`` at each ``(row, col)`` pair in ``indexes``. | + +#### `Elevation` + +*inherits* `CalculationMixin`, `BaseSpatialGrid` + +A DEM. + +Adds the terrain-derivative helpers of :class:`CalculationMixin` to the +base raster, plus the edge detection that depression filling starts from. + +| Member | Signature | Summary | +|---|---|---| +| `findDEMedge` | `()` | Cells where water can leave the DEM. | +| `outlets_at_coastlines` | `(iterations=3)` | Map coordinates of the cells forming the land/sea boundary. | +| `track_flow_downhill` | `(starting_point, maximum_pit_depth=20)` | Walk downhill from ``starting_point``, hopping over shallow pits. | + +#### `GeographicElevation` + +*inherits* `GeographicGridMixin`, `Elevation` + +A DEM on a latitude/longitude grid. + +### Depression filling + +#### `PriorityQueueMixIn` + +Depression filling by Priority-Flood. + +Mix into a grid class to give it :meth:`_flood`, which removes closed +depressions so that every cell has a downhill path to an outlet. + +The algorithms are those of + + Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal + depression-filling and watershed-labeling algorithm for digital + elevation models." *Computers & Geosciences* 62, 117-127. + +and run in :mod:`TopoAnalysis.kernels`, in C++ where the extension is +built and in NumPy otherwise. + +Attributes +---------- +aggradation_slope : float + Gradient of the increment added while filling. The default of 1e-12 + makes filled surfaces very slightly convergent, which gives D8 routing + something to follow across what would otherwise be a dead flat + plateau. Set it to 0 for the flat fill that TopoToolbox's + ``fillsinks`` produces. + +| Member | Signature | Summary | +|---|---|---| +| `randomize_subbasins_with_mask` | `(*args, **kwargs)` | Replace elevations inside a mask with random values and re-flood. | + +#### `FilledElevation` + +*inherits* `PriorityQueueMixIn`, `Elevation` + +A DEM with its closed depressions removed. + +:: + + filled = FilledElevation(elevation=dem) + +Filling uses Priority-Flood (Barnes et al. 2014). By default the filled +surface carries a gradient of +:attr:`PriorityQueueMixIn.aggradation_slope` across former depressions so +that D8 routing has a direction to follow; pass +``aggradation_slope=0`` -- or set the class attribute -- for the flat fill +that TopoToolbox's ``fillsinks`` produces. + +Other keywords (``mask``, ``outlets``, ``maximum_pit_depth``, +``clip_to_fill``) are passed to :meth:`PriorityQueueMixIn._flood`. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation` + +#### `PriorityFillGrid` + +*inherits* `PriorityQueueMixIn`, `BaseSpatialGrid` + +Cells of a mask that are connected to a set of outlets. + +Floods outwards from ``outlets`` through the cells where ``mask`` is 1 +and returns a 0/1 grid of everything reached. Used to grow a valley +network from a curvature mask, keeping only the parts that connect to a +real channel. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `outlets`, `mask` + +*Stored as* `uint8` + +### Flow routing + +#### `FlowDirection` + +*inherits* `BaseSpatialGrid` + +Base class for flow-direction grids. + +#### `FlowDirectionD8` + +*inherits* `FlowDirection` + +Single-direction (D8) flow directions. + +Values are ArcGIS direction codes; 0 means the cell has no downstream +neighbour (a pit, an outlet at the grid edge, or no-data):: + + | 32 64 128 | + | 16 X 1 | + | 8 4 2 | + +Build one from a filled DEM:: + + d8 = FlowDirectionD8(flooded_dem=FilledElevation(elevation=dem)) + +Direction is chosen by steepest *gradient*, so the drop to a diagonal +neighbour is divided by ``sqrt(2) * dx`` before being compared with the +cardinal ones. Ties go to the first direction in the order E, SE, S, SW, +W, NW, N, NE. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flooded_dem` +- `elevation` + +*Stored as* `uint8` + +| Member | Signature | Summary | +|---|---|---| +| `basin_mask` | `(outlets)` | Boolean array marking every cell draining to any of ``outlets``. | +| `bounds_of_basin_for_outlet` | `(outlet)` | ``((xmin, xmax), (ymin, ymax))`` of the basin draining to ``outlet``. | +| `convert_rivertools_directions_to_arc` | `(no_data=0)` | Convert RiverTools flow codes in place to the ArcGIS convention. | +| `cycle_count` | `` | How many cells lie on circular flow paths (0 for a valid network). | +| `divides` | `()` | Grid marking drainage divides: cells with no upstream neighbour. | +| `divides_for_outlets` | `(outlet1, outlet2)` | Cells on each side of the divide shared by two basins. | +| `get_flow_to_cell` | `(i, j)` | Index of the cell that ``(i, j)`` drains into. | +| `get_indexes_of_upstream_cells` | `(i, j)` | Tuple of ``(row, col)`` for every cell upstream of ``(i, j)``. | +| `get_indexes_of_upstream_cells_for_location` | `(x, y)` | As :meth:`get_indexes_of_upstream_cells`, from a map coordinate. | +| `get_upstream_cell_indexes` | `(i, j)` | Indices of the immediate neighbours that drain into ``(i, j)``. | +| `invalidate_network` | `()` | Drop the cached receivers and ordering. | +| `locations_of_paired_hollows` | `(outlet1, outlet2, area, Ao=100000.0)` | Channel heads reached by flowing down from a shared divide. | +| `map_values_to_recursive_list` | `(outlet, **kwargs)` | Nested dictionary of the basin upstream of ``outlet``. | +| `paired_divides` | `(mask=None)` | Pair each divide cell with the cell directly across the divide. | +| `pixel_scale` | `(dtype=)` | Step-length multiplier for each cell: 1 cardinal, sqrt(2) diagonal. | +| `receivers` | `` | Flat index of the cell each cell drains to, or -1 for none. | +| `search_down_flow_direction` | `(start, search_length=inf)` | The cells along the flow path from ``start``, without distances. | +| `search_down_flow_direction_from_rowscols_location` | `(start, return_rowscols=False, search_length=inf)` | Follow flow downstream from a ``(row, col)`` start. | +| `search_down_flow_direction_from_xy_location` | `(start, return_rowscols=False, search_length=inf)` | Follow flow downstream from an ``(x, y)`` start. | +| `search_down_flow_direction_with_length` | `(start, search_length=inf)` | Follow flow downstream from ``start``, recording distance travelled. | +| `set_value_at_rowscols` | `(value, rowscols)` | Set ``value`` at each ``(row, col)`` in ``rowscols``. | +| `step_lengths` | `(pixel_dimension=None)` | Distance from each cell to its receiver, in map units. | +| `topological_order` | `` | Flat cell indices ordered so every donor precedes its receiver. | +| `update_flow_codes_in_mask` | `(*args, **kwargs)` | Re-derive flow codes inside a mask after the DEM has changed. | + +#### `FlowLength` + +*inherits* `BaseSpatialGrid` + +Longest upstream flow distance reaching each cell. + +:: + + length = FlowLength(flow_direction=d8) + +Alongside the distances, the grid records which upstream neighbour +supplies each cell's longest path. That "main stem" network is what +:class:`Relief`, :class:`ScaledRelief` and :class:`Ksi` follow, and what +:meth:`locations_along_flow_path_from_outlet` walks. + +.. note:: + The main-stem codes use the same ArcGIS convention as + :class:`FlowDirectionD8`, pointing *from* a cell *at* its upstream + donor. Before version 1.0 they used a row-flipped convention that + was internally consistent but disagreed with every other + flow-direction grid in the library, so ``*_directions`` side-files + written by older versions must be regenerated. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction` + +| Member | Signature | Summary | +|---|---|---| +| `clip_to_extent` | `(extent)` | Clip both the lengths and the main-stem codes to ``extent``. | +| `indexes_along_flow_path_from_outlet` | `(outlet)` | ``(row, col)`` along the longest flow path upstream of ``outlet``. | +| `is_along_flow_length` | `(from_index, to_index)` | True when ``from_index`` supplies ``to_index``'s longest path. | +| `load` | `(filename)` | Read back a grid written by :meth:`save`. *(classmethod)* | +| `locations_along_flow_path_from_outlet` | `(outlet)` | As above, in map coordinates. | +| `locations_along_flow_path_from_outlets` | `(outlets)` | Concatenated main-stem paths for several outlets. | +| `main_stem_directions` | `` | Codes pointing from each cell at its longest-path donor. | +| `map_values_to_recursive_list` | `(outlet, **kwargs)` | Nested main-stem dictionary upstream of ``outlet``. | +| `points_with_length` | `(length, fd)` | Map coordinates where the flow path first exceeds ``length``. | +| `save` | `(filename)` | Write the lengths, plus the main-stem codes to ``filename_directions``. | + +#### `GeographicFlowLength` + +*inherits* `GeographicGridMixin`, `FlowLength` + +Flow length on a latitude/longitude grid. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction` + +#### `Mask` + +*inherits* `BaseSpatialGrid` + +0/1 grid marking the cells that drain to a set of outlets. + +:: + + mask = Mask(flow_direction=d8, outlets=[(x, y), ...]) + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `outlets` + +*Stored as* `uint8` + +| Member | Signature | Summary | +|---|---|---| +| `perform_erosion` | `(structure=None, iterations=1)` | Morphological erosion: shrink the mask by ``iterations`` cells. | +| `perform_opening` | `(structure=None, iterations=1)` | Morphological opening: erode then dilate, removing specks. | + +#### `DiscreteFlowAccumulation` + +*inherits* `BaseSpatialGrid` + +Area drained by a single steepest-descent walk from each outlet. + +Unlike :class:`Area`, which accumulates over the whole D8 network, this +follows one path per outlet, stepping to the lowest unvisited neighbour +each time, and records the running area along it. It is used to trace +individual flow paths across surfaces (debris flows, lava) where the +network abstraction does not apply. + +:: + + dfa = DiscreteFlowAccumulation(elevation=dem, outlets=[(x, y)]) + +Keyword arguments +----------------- +mask : BaseSpatialGrid + Confine the walk to cells where the mask is 1. +terminations_only : bool + Record the total area only at the end of each path. +display_output : bool + Report progress outlet by outlet. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `outlets` +- `elevation` + +#### `GeographicDiscreteFlowAccumulation` + +*inherits* `GeographicGridMixin`, `DiscreteFlowAccumulation` + +Discrete flow accumulation on a latitude/longitude grid. + +### Drainage area + +#### `Area` + +*inherits* `BaseSpatialGrid` + +D8 contributing (drainage) area. + +:: + + area = Area(flow_direction=d8) + +Values are in squared map units: each cell contributes its own area +(``dx**2``, or the true spherical cell area for a +:class:`GeographicArea`) plus everything upstream of it. + +To count contributing *cells* instead -- which is what TopoToolbox's +``flowacc`` returns -- divide by the per-cell area, or pass +``weights=np.ones(...)``. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction` +- `dx`, `grid` + +| Member | Signature | Summary | +|---|---|---| +| `areas_between` | `(fd, min_area, max_area)` | Channel heads: cells in an area band whose donors are all below it. | +| `areas_greater_than` | `(min_area)` | Map coordinates of every cell whose area is at least ``min_area``. | + +#### `GeographicArea` + +*inherits* `GeographicGridMixin`, `Area` + +Drainage area on a latitude/longitude grid, using true cell areas. + +#### `LogArea` + +*inherits* `BaseSpatialGrid` + +Base-10 logarithm of drainage area. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `area` + +*Stored as* `float64` + +#### `ValleyArea` + +*inherits* `_ValleyMaskMixin`, `Area` + +Drainage area accumulated only over valley-floor cells. + +:: + + va = ValleyArea(flow_direction=d8, area=area, laplace=laplacian, + valley_laplace_value=0.01, min_area_value=1e6) + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `area`, `laplace`, `valley_laplace_value`, `min_area_value` + +#### `GeographicValleyArea` + +*inherits* `GeographicGridMixin`, `ValleyArea` + +Valley-floor area on a latitude/longitude grid. + +#### `MainstemValleyArea` + +*inherits* `_ValleyMaskMixin`, `Area` + +Valley-floor area accumulated along the main stem only. + +Where :class:`ValleyArea` sums every tributary, this follows only the +longest flow path into each cell, so the result is the valley area of the +trunk stream rather than of the whole basin. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `area`, `laplace`, `valley_laplace_value`, `min_area_value`, `flow_direction` + +#### `GeographicMainstemValleyArea` + +*inherits* `GeographicGridMixin`, `MainstemValleyArea` + +Main-stem valley area on a latitude/longitude grid. + +### Chi and channel steepness + +#### `Chi` + +*inherits* `BaseSpatialGrid` + +The chi coordinate, integrated upstream from chosen outlets. + +.. math:: + \chi(x) = \int_{x_{\mathrm{outlet}}}^{x} + \left(\frac{A_0}{A(x')}\right)^{\theta}\,\mathrm{d}x' + +:: + + chi = Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e6, + outlets=[(x, y)]) + +Chi is 0 at each outlet and increases upstream. Cells outside the +outlets' drainage areas are left as 0. + +Keyword arguments +----------------- +trapezoid : bool + Average the integrand across each step instead of taking it at the + upstream cell. This is the rule TopoToolbox's ``chitransform`` uses; + the default (``False``) reproduces the original TopoAnalysis result. +maximum_length : float + Stop integrating beyond this flow distance from the outlet. +mask : BaseSpatialGrid + Restrict the integration to cells where the mask is non-zero. + +.. note:: + Before version 1.0, chi at the outlet itself was one cell-step of the + integrand rather than 0. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `area`, `flow_direction`, `theta`, `Ao`, `outlets` +- `area`, `flow_direction`, `flow_length`, `theta`, `Ao`, `basin_length` + +#### `GeographicChi` + +*inherits* `GeographicGridMixin`, `Chi` + +Chi on a latitude/longitude grid. + +#### `Ksi` + +*inherits* `BaseSpatialGrid`, `MaxFlowLengthTrackingMixin` + +The chi coordinate integrated along longest-flow-length paths. + +.. math:: + \chi = \int \left(\frac{A_0}{A}\right)^{\theta} \mathrm{d}x + +:: + + ksi = Ksi(area=area, flow_direction=d8, flow_length=length, + theta=0.45, Ao=1e6) + +Cells with less than ``Ao`` of drainage area are left out of the +integral, so chi is defined only on the channel network. + +.. note:: + Before version 1.0 the integrand was ``(Ao / (A - Ao))**theta``, which + diverges as ``A`` approaches ``Ao`` and does not match the definition + of chi in the literature. Values from older runs are not comparable + with these. + +See also :class:`Chi`, which integrates upstream from chosen outlets over +the whole network rather than along main stems. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `area`, `flow_direction`, `theta`, `Ao`, `flow_length` + +#### `GeographicKsi` + +*inherits* `GeographicGridMixin`, `Ksi` + +Chi along longest-flow-length paths on a latitude/longitude grid. + +#### `Relief` + +*inherits* `BaseSpatialGrid`, `MaxFlowLengthTrackingMixin` + +Height of the head of the longest flow path above each cell. + +:: + + relief = Relief(flow_direction=d8, elevation=dem, flow_length=length) + +Pass ``area`` and ``Ao`` together to stop the relief signal being carried +down from cells with less than ``Ao`` of drainage area, which keeps +hillslope noise out of the channel network. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `elevation`, `flow_length` + +#### `ScaledRelief` + +*inherits* `Relief` + +:class:`Relief` scaled by ``Ao**theta``. + +That scaling puts relief in the same units as :class:`Ksi`, so the ratio +of the two is a channel steepness. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `flooded_dem`, `elevation`, `flow_length`, `Ao`, `theta` +- `flow_direction`, `elevation`, `flow_length`, `Ao`, `theta` + +#### `ChiScaledRelief` + +*inherits* `BaseSpatialGrid` + +Height above the basin outlet, scaled by ``Ao**theta``. + +Plotted against :class:`Chi`, the slope of this quantity is the channel +steepness index. + +:: + + relief = ChiScaledRelief(elevation=dem, flow_direction=d8, + theta=0.45, Ao=1e6, outlets=[(x, y)]) + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `flow_direction`, `theta`, `Ao`, `outlets` +- `elevation`, `flow_direction`, `flow_length`, `theta`, `Ao`, `basin_length` + +#### `ChannelSlope` + +*inherits* `BaseSpatialGrid` + +Downstream gradient of each cell along its D8 flow direction. + +:: + + slope = ChannelSlope(flow_direction=d8, elevation=dem) + +Positive values are downhill. Cells with no downstream neighbour stay 0. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `elevation` + +#### `CrossDivideDChi` + +*inherits* `_CrossDivideMixin`, `BaseSpatialGrid` + +Difference in chi across each drainage divide. + +A large value means the two sides of the divide are far from +steady state, so the divide is expected to migrate towards the +higher-chi side (Willett et al. 2014). + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `chi`, `flow_direction` + +#### `NormalizedCrossDivideDChi` + +*inherits* `_CrossDivideMixin`, `BaseSpatialGrid` + +Chi contrast across each divide, divided by the mean chi there. + +Normalising makes divides in high- and low-chi parts of a landscape +comparable. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `chi`, `flow_direction` + +#### `RestoredElevation` + +*inherits* `BaseSpatialGrid` + +Topography implied by a uniform channel steepness. + +Starting from the given outlets, elevations are rebuilt upstream from + +.. math:: \frac{\mathrm{d}z}{\mathrm{d}x} = k_s A^{-\theta} + +and the divides are then allowed to migrate towards whichever side ends +up lower. Repeating that (``iterations`` times) converges on the +landscape a steady, spatially uniform ``ks`` would produce, which can be +compared against the real DEM to find where it is out of steady state. + +:: + + restored = RestoredElevation(flow_direction=d8, elevation=dem, + area=area, theta=0.45, ks=50, + outlets=[(x, y)], iterations=5) + +Keyword arguments +----------------- +randomize : bool + Randomise elevations inside the basin first, then re-route, so the + result does not inherit the real network's planform. +fix_external_outlets : bool + Hold the outer boundary of the basin fixed while divides migrate. +verbose : bool + Report convergence each iteration (default ``True``). + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `elevation`, `area`, `theta`, `ks`, `outlets`, `iterations` + +#### `GeographicRestoredElevation` + +*inherits* `GeographicGridMixin`, `RestoredElevation` + +Restored topography on a latitude/longitude grid. + +### Along-flow smoothing + +#### `AlongFlowSmoothing` + +Collect the cells within a window along each flow path. + +Mixed into the ``*WithSmoothing`` classes, which fit a model over that +window at every cell. The window is set either by drop +(``vertical_interval``) or by distance (``horizontal_interval``), and +extends both upstream and downstream of the cell being evaluated. + +#### `KsFromChiWithSmoothing` + +*inherits* `BaseSpatialGrid`, `AlongFlowSmoothing` + +Channel steepness fitted to chi--elevation in a moving window. + +:: + + ks = KsFromChiWithSmoothing(elevation=dem, area=area, + flow_direction=d8, theta=0.45, + vertical_interval=50.0) + +At every channel cell, chi and elevation are extracted over a window +along the flow path and a line through the origin is fitted; its slope +is k_s. ``save()`` writes seven bands: ks, the number of profiles +crossing each cell, mean squared error, sum of squares, r-squared, +p-value, and the number of points in the regression. + +Needs ``statsmodels``. Pass ``verbose=False`` to silence the progress +ticker. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `theta`, `vertical_interval` +- `elevation`, `area`, `flow_direction`, `theta`, `horizontal_interval` + +| Member | Signature | Summary | +|---|---|---| +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | + +#### `GeographicKsFromChiWithSmoothing` + +*inherits* `GeographicGridMixin`, `KsFromChiWithSmoothing` + +Windowed steepness fit on a latitude/longitude grid. + +#### `ThetaFromChiWithSmoothing` + +*inherits* `BaseSpatialGrid`, `AlongFlowSmoothing` + +Concavity fitted to chi--elevation in a moving window. + +As :class:`KsFromChiWithSmoothing`, but ``theta`` is optimised at each +cell rather than held fixed: the value returned is the one that makes +the chi--elevation relation most nearly linear. The search is bounded +at +/-10. + +Needs ``statsmodels``. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `vertical_interval`, `min_area` +- `elevation`, `area`, `flow_direction`, `horizontal_interval`, `min_area` + +| Member | Signature | Summary | +|---|---|---| +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | + +#### `GeographicThetaFromChiWithSmoothing` + +*inherits* `GeographicGridMixin`, `ThetaFromChiWithSmoothing` + +Windowed concavity fit on a latitude/longitude grid. + +#### `ChannelSlopeWithSmoothing` + +*inherits* `BaseSpatialGrid`, `AlongFlowSmoothing` + +Channel gradient measured over a window along the flow path. + +Far less noisy than a cell-to-cell slope, which on a real DEM is mostly +vertical quantisation. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `vertical_interval` +- `elevation`, `area`, `flow_direction`, `horizontal_interval` + +| Member | Signature | Summary | +|---|---|---| +| `calc_channel_slope` | `(i, j, elevation, de, find_points_along_path)` | Gradient across the window centred on ``(i, j)``. | +| `points_along_path` | `(points, i, j)` | Which part of the window to measure over; the whole of it here. | + +#### `ChannelDownSlopeWithSmoothing` + +*inherits* `ChannelSlopeWithSmoothing` + +Channel gradient over the downstream half of the window. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `vertical_interval` +- `elevation`, `area`, `flow_direction`, `horizontal_interval` + +| Member | Signature | Summary | +|---|---|---| +| `points_along_path` | `(points, i, j)` | Only the downstream half of the window. | + +#### `ChannelUpSlopeWithSmoothing` + +*inherits* `ChannelSlopeWithSmoothing` + +Channel gradient over the upstream half of the window. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `vertical_interval` +- `elevation`, `area`, `flow_direction`, `horizontal_interval` + +| Member | Signature | Summary | +|---|---|---| +| `points_along_path` | `(points, i, j)` | Only the upstream half of the window. | + +### Terrain attributes + +#### `CalculationMixin` + +Finite-difference terrain derivatives. + +Supplies the slope, curvature and boundary-padding helpers that +:class:`Elevation`, :class:`Hillshade` and :class:`MaxSlope` share. + +| Member | Signature | Summary | +|---|---|---| +| `assignBCs` | `(grid, nx, ny)` | Pad a grid by one cell, replicating the edge values. | +| `calcAverageSlopeOfGridSubset` | `(gridSubset, dx)` | Least-squares plane through a block of cells; returns its slopes. | +| `calcContourCurvature` | `(grid, dx)` | Contour (plan) curvature: the curvature of the contour lines. | +| `calcFiniteCurv` | `(grid, dx)` | Laplacian from a 5-point centred stencil, same shape as the input. | + +#### `Hillshade` + +*inherits* `CalculationMixin`, `BaseSpatialGrid` + +Shaded relief, following the ESRI hillshade definition. + +:: + + hs = Hillshade(elevation=dem, azimuth=315, inclination=45) + +``azimuth`` is the compass bearing of the light source in degrees +(0 = north, 90 = east) and ``inclination`` its height above the horizon. +Values run 0-255. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `azimuth`, `inclination` + +*Stored as* `uint8` + +| Member | Signature | Summary | +|---|---|---| +| `calcHillshade` | `(az, elev, z_factor=1.0)` | Fill the grid with an ESRI-style hillshade. | + +#### `GeographicHillshade` + +*inherits* `GeographicGridMixin`, `Hillshade` + +Hillshade on a latitude/longitude grid. + +#### `MaxSlope` + +*inherits* `CalculationMixin`, `BaseSpatialGrid` + +Magnitude of the topographic gradient, from centred differences. + +:: + + slope = MaxSlope(elevation=dem) + +Values are dimensionless (rise over run); take ``arctan`` for degrees. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation` + +*Stored as* `float64` + +| Member | Signature | Summary | +|---|---|---| +| `calcSlope` | `()` | Replace the grid with the gradient magnitude of its own values. | + +#### `GeographicMaxSlope` + +*inherits* `GeographicGridMixin`, `MaxSlope` + +Gradient magnitude on a latitude/longitude grid. + +#### `Laplacian` + +*inherits* `CalculationMixin`, `BaseSpatialGrid` + +Laplacian of elevation -- positive in valleys, negative on ridges. + +:: + + curvature = Laplacian(elevation=dem) + +Uses SciPy's 5-point stencil, scaled by ``1 / dx**2``, so the units are +inverse length. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation` + +*Stored as* `float64` + +#### `GeographicLaplacian` + +*inherits* `GeographicGridMixin`, `Laplacian` + +Laplacian of elevation on a latitude/longitude grid. + +#### `Gradient` + +*inherits* `BaseSpatialGrid` + +The two components of the topographic gradient. + +:: + + gradient = Gradient(elevation=dem) + gradient._gx, gradient._gy + +Stored as two bands rather than one, so direction survives ``save()``. +:meth:`average_gradient` smooths both components over a length scale, +which is how a regional aspect is obtained. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation` + +| Member | Signature | Summary | +|---|---|---| +| `average_gradient` | `(distance)` | Both gradient components averaged over a disc of radius ``distance``. | +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `plot` | `(**kwargs)` | Display the grid with ``imshow``, georeferenced in map coordinates. | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | + +#### `LocalRelief` + +*inherits* `BaseSpatialGrid` + +Elevation range within a circular window. + +:: + + relief = LocalRelief(elevation=dem, pixel_radius=25) + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `pixel_radius` + +#### `MultiscaleCurvatureValleyWidth` + +*inherits* `BaseSpatialGrid` + +Valley width from the scale at which curvature is most negative. + +A local quadratic is fitted at a range of window sizes; the width +recorded at each cell is the window that minimises the smaller principal +curvature, i.e. the scale at which the valley is best resolved. The +curvature itself is kept in ``_minC``. + +:: + + width = MultiscaleCurvatureValleyWidth( + elevation=dem, area=area, area_cutoff=1e6, + min_width=30.0, max_width=600.0) + +Pass ``use_dask=True`` to evaluate the scales in parallel. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `area_cutoff`, `max_width`, `min_width` + +| Member | Signature | Summary | +|---|---|---| +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `mosaic` | `(tiles)` | Reassemble tiles, carrying the curvature band along with the widths. *(classmethod)* | +| `remove_padding` | `(xpadding=10, ypadding=10)` | Strip tile overlap from both the widths and the curvatures. | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | + +#### `ScarpWavelet` + +*inherits* `BaseSpatialGrid` + +Template-matched scarp amplitude, age, orientation and signal-to-noise. + +Holds the four-band output of a scarp template search. Load one with +:meth:`load`; the bands become ``_A``, ``_kt``, ``_orientation`` and +``_SNR``. :meth:`template_window` needs the separate ``scarplet`` +package. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` + +| Member | Signature | Summary | +|---|---|---| +| `calculate_valid_orientations` | `(window_size, age, num=46, discard_max_min=True)` | Range of template orientations that stay inside the valid data. | +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `load_elevation` | `(filename)` | Attach the DEM the templates were matched against. | +| `plot_orientations` | `(*args, **kwargs)` | Draw scarp orientations over a hillshade, faded by signal-to-noise. | +| `template_window` | `(window_size, age, orientation, use_pixels=False)` | The scarp template mask at one size, age and orientation. | +| `valid_data` | `()` | 1 where the attached DEM has usable, above-sea-level data; NaN elsewhere. | + +### Isostasy + +#### `Deflection` + +*inherits* `BaseSpatialGrid` + +Flexural deflection of an elastic plate under a topographic load. + +Solves the thin-plate equation in the Fourier domain: + +.. math:: + w(k) = \frac{-\rho_c\,g\,h(k)} + {(\rho_m - \rho_c)\,g + D\,(2\pi k)^4} + +:: + + w = Deflection(elevation=dem, D=1e23, rho_m=3300, rho_c=2700, g=9.81) + +Pass ``restored_elevation`` as well to get the *change* in deflection +between two surfaces, which is the isostatic response to the erosion +between them. + +Because the solution is spectral, the load is implicitly periodic; pad +the DEM if edge effects matter. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `D`, `rho_m`, `rho_c`, `g` + +#### `GeographicDeflection` + +*inherits* `GeographicGridMixin`, `Deflection` + +Flexural deflection on a latitude/longitude grid. + +### Mixins + +#### `GDALMixin` + +Reading and writing rasters through GDAL. + +Mixed into :class:`BaseSpatialGrid`; nothing here is meant to be called +directly. Every method reaches GDAL through the module-level lazy +proxies, so the mixin can be present without GDAL being installed. + +| Member | Signature | Summary | +|---|---|---| +| `getDEMcoords` | `(GdalData, dx)` | Cell-centre x and y coordinate lists for an open GDAL dataset. | + +#### `GeographicGridMixin` + +Spherical geometry for grids in degrees of latitude and longitude. + +Mix it in *before* the grid class to override the cell area and cell +dimension with their true, latitude-dependent values:: + + class GeographicArea(GeographicGridMixin, Area): + pass + +Every ``Geographic*`` class in this module is exactly that. + +#### `MaxFlowLengthTrackingMixin` + +Propagate a value downstream along longest-flow-length paths only. + +Subclasses set :attr:`_propagate_mode` to ``'carry'`` (the receiver takes +the donor's value) or ``'sum'`` (the receiver adds it) and may define +:meth:`_propagation_gate` to suppress the transfer for some cells. + +### Module-level functions in `TopoAnalysis.dem` + +| Function | Summary | +|---|---| +| `gdal_is_available()` | True when the GDAL Python bindings can be imported. | +| `mosaicFolder(folderPath, fileSuffix, outfile)` | Merge every ``*fileSuffix`` raster in a folder into ``outfile``. | +| `plot(*args, **kwargs)` | Scatter one grid's values against another's, cell by cell. | + +## `TopoAnalysis.topotoolbox` -- TopoToolbox-equivalent functions + +TopoToolbox-equivalent terrain analysis. + +Every function here reproduces the numerical result of the identically named +function in TopoToolbox 2 (Schwanghart & Scherler 2014), the MATLAB package, +so that a result computed in TopoAnalysis can be checked against one computed +in MATLAB. + +The rest of TopoAnalysis keeps its own idioms -- drainage area in square +metres, ESRI-style hillshade, chi integrated with a left-endpoint rule. This +module exists for the cases where you need the *same number*, not just the +same quantity. :doc:`docs/topotoolbox_parity` tabulates the correspondence +and lists the places where exact agreement is not achievable. + +See [topotoolbox_parity.md](topotoolbox_parity.md) for the full +correspondence table and the places where exact agreement is not +achievable. + +| Function | Summary | +|---|---| +| `arcslope(elevation, unit='tangent')` | ArcGIS/Horn 3x3 slope magnitude. | +| `aspect(elevation, classify=False)` | Down-slope direction in degrees clockwise from north. | +| `cellarea(grid, unit='m')` | Area of each cell of a geographic (lat/lon) grid. | +| `chitransform(flow_direction, area, mn=0.45, a0=1000000.0, outlets=None, correctcellsize=True, stream_mask=None)` | The chi integral transform. | +| `coord2sub(grid, x, y)` | Map coordinates to 0-based ``(row, col)`` subscripts. | +| `curvature(elevation, ctype='profc')` | Surface curvature from the Evans 3x3 quadratic fit. | +| `drainagebasins(flow_direction, outlets=None, valid=None)` | Label drainage basins. | +| `fillsinks(elevation, maxdepth=None)` | Fill topographic depressions to a flat surface. | +| `flowacc(flow_direction, weights=None)` | Flow accumulation in **number of cells**. | +| `flowdistance(flow_direction, direction='upstream')` | Along-flow distance in map units. | +| `flowobj(elevation, preprocess='fill')` | Build a D8 flow-direction grid the way ``FLOWobj`` does. | +| `getcoordinates(grid)` | ``(x, y)`` cell-centre coordinate vectors. | +| `getextent(grid)` | ``(xmin, xmax, ymin, ymax)`` of the outermost **cell centres**. | +| `gradient8(elevation, unit='tangent')` | Steepest downward gradient over the 8 neighbours. | +| `hillshade(elevation, azimuth=315.0, altitude=60.0, exaggerate=1.0)` | Shaded relief as the cosine of the illumination incidence angle. | +| `identifyflats(elevation, output='both')` | Locate flat areas, their sills and the closed basins. | +| `imposemin(flow_direction, elevation, sl=0.0)` | Carve elevations so every flow path descends at gradient >= ``sl``. | +| `ksn(flow_direction, elevation, area, theta=0.45, correctcellsize=True, min_gradient=1e-05)` | Normalized channel steepness index, :math:`k_{sn} = S A^{\theta}`. | +| `localtopography(elevation, radius=5000.0, type='range')` | Statistic of elevation within a circular window. | +| `mchi(flow_direction, elevation, chi)` | Gradient of elevation in chi space, :math:`M_{\chi}`. | +| `streamorder(flow_direction, stream_mask, order_type='strahler')` | Strahler or Shreve stream order. | +| `sub2coord(grid, row, col)` | 0-based ``(row, col)`` subscripts to cell-centre map coordinates. | + +### `fillsinks(elevation, maxdepth=None)` + +Fill topographic depressions to a flat surface. + +Equivalent to ``GRIDobj/fillsinks(DEM)`` and, with ``maxdepth``, to +``fillsinks(DEM, maxdepth)``. + +Unlike :class:`TopoAnalysis.dem.FilledElevation`, whose default adds a +tiny gradient across filled areas so that D8 routing has a direction to +follow, this fills to an exactly flat plateau -- which is what +TopoToolbox does, and what ``identifyflats`` then relies on. + +Parameters +---------- +elevation : BaseSpatialGrid + The DEM. ``NaN`` is no-data and, as in TopoToolbox, acts as an + outlet: water reaching a NaN cell leaves the grid. +maxdepth : float, optional + Leave depressions deeper than this unfilled. + +Returns +------- +Elevation + A new grid; the input is not modified. + +### `identifyflats(elevation, output='both')` + +Locate flat areas, their sills and the closed basins. + +Equivalent to ``GRIDobj/identifyflats``. + +* A **flat** is a cell with no strictly lower neighbour -- the MATLAB test + is ``imerode(dem, ones(3)) == dem``. An isolated pit therefore counts + as a flat. The grid's outer ring is cleared, as is the one-cell halo + around any no-data. +* A **sill** is a non-flat cell at the same elevation as an adjacent + flat: the point a flat drains through. +* A **closed basin** is a regional minimum that does not touch the grid + edge. + +Parameters +---------- +elevation : BaseSpatialGrid + Normally the output of :func:`fillsinks`. +output : {'both', 'flats', 'sills', 'closedbasins', 'all'} + ``'both'`` returns ``(flats, sills)``; ``'all'`` adds the closed + basins. + +Returns +------- +BaseSpatialGrid or tuple + Boolean grids. + +### `imposemin(flow_direction, elevation, sl=0.0)` + +Carve elevations so every flow path descends at gradient >= ``sl``. + +Equivalent to ``FLOWobj/imposemin(FD, DEM, sl)``. Cells are only ever +lowered, so channels are cut through obstructions instead of being +drowned by them. + +Returns +------- +Elevation + A new grid. + +### `flowobj(elevation, preprocess='fill')` + +Build a D8 flow-direction grid the way ``FLOWobj`` does. + +Parameters +---------- +elevation : BaseSpatialGrid +preprocess : {'fill', 'none'} + ``'fill'`` removes depressions first (TopoToolbox's + ``'preprocess','fill'``). ``'carve'`` is not implemented; see the + note below. + +Returns +------- +FlowDirectionD8 + +Notes +----- +TopoToolbox routes flow across the flat areas left by filling using an +auxiliary "geodesic" topography, so that flat cells drain towards the +nearest sill. TopoAnalysis instead adds a small gradient while filling +(:attr:`~TopoAnalysis.dem.PriorityQueueMixIn.aggradation_slope`). Both +produce a valid, connected drainage network with identical basin +boundaries wherever the terrain is not flat, but **inside flats the two +networks differ**. TopoToolbox's ``'carve'`` preprocessing lowers the +DEM instead of raising it and is a different surface again. + +### `flowacc(flow_direction, weights=None)` + +Flow accumulation in **number of cells**. + +Equivalent to ``FLOWobj/flowacc(FD)``: every cell counts itself, so the +minimum value is 1. + +Parameters +---------- +flow_direction : FlowDirectionD8 +weights : ndarray or BaseSpatialGrid, optional + Per-cell weight; defaults to 1. + +Returns +------- +BaseSpatialGrid + +See Also +-------- +TopoAnalysis.dem.Area : the same accumulation weighted by cell area. + +### `flowdistance(flow_direction, direction='upstream')` + +Along-flow distance in map units. + +Equivalent to ``FLOWobj/flowdistance(FD, direction)`` with no seed +points. + +Parameters +---------- +direction : {'upstream', 'downstream', 'maxdownstream'} + ``'upstream'`` gives the distance from each cell down to its outlet. + ``'downstream'`` (a synonym for ``'maxdownstream'``) gives the length + of the longest flow path reaching each cell -- the quantity + :class:`TopoAnalysis.dem.FlowLength` computes. + +### `drainagebasins(flow_direction, outlets=None, valid=None)` + +Label drainage basins. + +Equivalent to ``FLOWobj/drainagebasins``. Without ``outlets`` every +terminal cell that something drains into seeds its own basin; with them, +each cell is labelled by the 1-based position of the outlet it drains to. + +Parameters +---------- +outlets : sequence of (x, y), optional +valid : ndarray or BaseSpatialGrid, optional + Restrict labelling to these cells (e.g. ``~np.isnan(dem._griddata)``). + +Returns +------- +BaseSpatialGrid + Integer labels; 0 means "reaches none of the outlets". + +Notes +----- +TopoToolbox numbers basins in the order its edge list happens to meet +them, so the *partition* matches but the integer labels generally do +not. Compare partitions, not label values. + +### `gradient8(elevation, unit='tangent')` + +Steepest downward gradient over the 8 neighbours. + +Equivalent to ``GRIDobj/gradient8``. The cardinal and diagonal drops are +divided by ``cellsize`` and ``sqrt(2)*cellsize`` respectively, and the +result is clamped at 0, so pits and flats give exactly 0 rather than a +negative gradient. + +Parameters +---------- +unit : {'tangent', 'degree', 'radian', 'sine', 'percent'} + +### `arcslope(elevation, unit='tangent')` + +ArcGIS/Horn 3x3 slope magnitude. + +Equivalent to ``GRIDobj/arcslope``. Uses the 1-2-1 weighted 3x3 stencil +with an ``8 * cellsize`` denominator -- there is no ``sqrt(2)`` anywhere; +the diagonals enter through the weights. + +``NaN`` holes are filled from their nearest valid neighbour before +differencing, then restored, which is how TopoToolbox handles them here +(and is the opposite of what :func:`gradient8` does). + +### `hillshade(elevation, azimuth=315.0, altitude=60.0, exaggerate=1.0)` + +Shaded relief as the cosine of the illumination incidence angle. + +Equivalent to ``GRIDobj/hillshade`` with its default ``'surfnorm'`` +method, including its defaults of ``azimuth=315`` and ``altitude=60``. + +Returns +------- +BaseSpatialGrid + Values in ``[-1, 1]``. TopoToolbox only scales to 0-255 when it is + plotting, so neither does this. + +See Also +-------- +TopoAnalysis.dem.Hillshade : the ESRI formulation, returning 0-255. + +### `aspect(elevation, classify=False)` + +Down-slope direction in degrees clockwise from north. + +Equivalent to ``GRIDobj/aspect``. 0 = north, 90 = east, 180 = south, +270 = west. Perfectly flat cells come out as 90, exactly as they do in +TopoToolbox. + +Parameters +---------- +classify : bool + Return the 8-class Gomez-Plaza wetness ranking (1 = N, 3 = NE, + 5 = E, 7 = SE, 8 = S, 6 = SW, 4 = W, 2 = NW) instead of degrees. + +### `curvature(elevation, ctype='profc')` + +Surface curvature from the Evans 3x3 quadratic fit. + +Equivalent to ``GRIDobj/curvature``. + +Parameters +---------- +ctype : {'profc', 'planc', 'tangc', 'meanc', 'total'} + Profile, plan, tangential, mean, or total curvature. ``'profc'`` + (the default) is positive where a slope steepens downhill. + +Returns +------- +BaseSpatialGrid + Units are 1/length, except ``'total'`` which is 1/length^2. + +Notes +----- +Cells where the gradient vanishes give 0/0; TopoToolbox sets those to 0, +and so does this. Cells adjacent to no-data therefore come out as 0 +rather than ``NaN`` -- again matching TopoToolbox. + +### `cellarea(grid, unit='m')` + +Area of each cell of a geographic (lat/lon) grid. + +Equivalent to ``GRIDobj/cellarea``. Cells are treated as spherical +quadrangles on a sphere whose total area is +:data:`EARTH_SURFACE_AREA_KM2`. + +Parameters +---------- +unit : {'m', 'km'} + Square metres (the default) or square kilometres. + +Returns +------- +BaseSpatialGrid + Constant along each row, since area depends only on latitude. + +### `localtopography(elevation, radius=5000.0, type='range')` + +Statistic of elevation within a circular window. + +Equivalent to ``GRIDobj/localtopography``. The window radius is given in +map units and rounded up to a whole number of cells. + +Parameters +---------- +type : {'range', 'max', 'min', 'mean', 'median', 'std'} + +Notes +----- +TopoToolbox's default disk (``strel('disk', r)`` with ``N=8``) is a +periodic-line approximation whose exact shape is an internal MATLAB +detail. This uses an exact disc -- TopoToolbox's ``N=0`` -- so values +can differ slightly near the window scale. + +### `streamorder(flow_direction, stream_mask, order_type='strahler')` + +Strahler or Shreve stream order. + +Equivalent to ``FLOWobj/streamorder`` / ``STREAMobj/streamorder``. + +Parameters +---------- +stream_mask : ndarray or BaseSpatialGrid + Non-zero on the channel network, e.g. ``flowacc(fd)._griddata > 1000``. +order_type : {'strahler', 'shreve'} + +Returns +------- +BaseSpatialGrid + Integer orders; 0 off the network. + +### `chitransform(flow_direction, area, mn=0.45, a0=1000000.0, outlets=None, correctcellsize=True, stream_mask=None)` + +The chi integral transform. + +Equivalent to ``STREAMobj/chitransform``: + +.. math:: + \chi = \int_{\mathrm{outlet}}^{x} + \left(\frac{A_0}{A(x')}\right)^{m/n}\,\mathrm{d}x' + +integrated upstream with the trapezoidal rule, so chi is 0 at each outlet +and increases upstream. + +Parameters +---------- +area : BaseSpatialGrid + Flow accumulation. With ``correctcellsize=True`` (the default, as in + TopoToolbox) it is taken to be in **number of cells** and multiplied + by ``cellsize**2``; pass ``False`` if it is already in m^2, e.g. if + it came from :class:`TopoAnalysis.dem.Area`. +mn : float + The m/n concavity ratio. TopoToolbox's default is 0.45. +a0 : float + Reference area in m^2. TopoToolbox's default is 1e6 (1 km^2). +outlets : sequence of (x, y), optional + Integration starts here. Defaults to every terminal cell of the + network, which is what a STREAMobj built over the whole grid gives. +stream_mask : ndarray or BaseSpatialGrid, optional + Restrict the integration to the channel network. + +Returns +------- +BaseSpatialGrid + Chi in map units; ``NaN`` away from the integrated network. + +### `ksn(flow_direction, elevation, area, theta=0.45, correctcellsize=True, min_gradient=1e-05)` + +Normalized channel steepness index, :math:`k_{sn} = S A^{\theta}`. + +Equivalent to ``STREAMobj/ksn``, including its unconditional minima +imposition at a gradient of 1e-5 before the slope is measured. + +Parameters +---------- +area : BaseSpatialGrid + Flow accumulation in cells (see ``correctcellsize``). +theta : float + Reference concavity; TopoToolbox's default is 0.45. +min_gradient : float + Gradient imposed by the carving step. TopoToolbox hard-codes 1e-5. + +Returns +------- +BaseSpatialGrid + ``ksn`` with a reference area of 1 m^2 -- TopoToolbox's ``ksn`` has + no ``a0``. Multiply by ``a0**theta`` to compare against + ``chiplot``'s ``ks``. + +### `mchi(flow_direction, elevation, chi)` + +Gradient of elevation in chi space, :math:`M_{\chi}`. + +Equivalent to ``STREAMobj/mchi``: a forward difference assigned to the +upstream cell, with no carving and no smoothing. + +``M_chi`` scales with the ``a0`` baked into ``chi``: computing chi with +``a0=1`` puts it in the same units as :func:`ksn`, and +``mchi(a0) * a0**mn`` is invariant. + +It is *close to* but not equal to :func:`ksn`, despite what TopoToolbox's +own docstring says: ``ksn`` carves the DEM first and evaluates the +integrand at the upstream cell, while ``mchi`` uses the raw elevations +and inherits chi's trapezoidal averaging. + +### `getcoordinates(grid)` + +``(x, y)`` cell-centre coordinate vectors. + +Equivalent to ``GRIDobj/getcoordinates``. ``x`` runs west to east; +``y`` runs **north to south**, matching the row order of the grid. + +### `getextent(grid)` + +``(xmin, xmax, ymin, ymax)`` of the outermost **cell centres**. + +Equivalent to ``GRIDobj/getextent``. Note this is half a cell inside +:meth:`TopoAnalysis.dem.BaseSpatialGrid.extent`, which returns the pixel +edges that matplotlib wants. + +### `coord2sub(grid, x, y)` + +Map coordinates to 0-based ``(row, col)`` subscripts. + +Equivalent to ``GRIDobj/coord2sub`` apart from the index base. Points +outside the grid give ``NaN``. + +Notes +----- +Halves are rounded away from zero, as MATLAB does. Python's built-in +``round`` -- which +:meth:`TopoAnalysis.dem.BaseSpatialGrid._xy_to_rowscols` uses -- rounds +halves to even, so the two disagree for points landing exactly on a cell +boundary. + +### `sub2coord(grid, row, col)` + +0-based ``(row, col)`` subscripts to cell-centre map coordinates. + +Equivalent to ``GRIDobj/sub2coord``. Subscripts outside the grid are +extrapolated rather than rejected, as in TopoToolbox. + +## `TopoAnalysis.kernels` -- backend dispatch + +Backend dispatch for the TopoAnalysis grid kernels. + +Import the kernels from here rather than from :mod:`TopoAnalysis.fastops` or +the compiled extension directly:: + + from TopoAnalysis import kernels + kernels.priority_flood(z, mode="flat") + +If the C++ extension built successfully it is used; otherwise the pure-NumPy +implementations in :mod:`TopoAnalysis.fastops` take over. Both produce the +same numbers -- the test suite asserts it -- so the choice is purely about +speed. + +Set the environment variable ``TOPOANALYSIS_PURE_PYTHON=1`` to force the +NumPy path, which is useful when debugging or comparing the two. + +| Function | Summary | +|---|---| +| `accumulate(recv, order, weights, gate=None)` | Downstream accumulation; see :func:`TopoAnalysis.fastops.accumulate`. | +| `backend() -> 'str'` | ``'c++'`` or ``'python'``. | +| `chi(recv, order, area, step_length, outlets, A0, theta, trapezoid=False, max_length=0.0, mask=None)` | ``(chi, distance)``; see :func:`TopoAnalysis.fastops.chi`. | +| `default_seeds(elevations)` | Flat indices of the cells a flood should start from. | +| `downstream_distance(recv, order, step_length)` | See :func:`TopoAnalysis.fastops.downstream_distance`. | +| `drainage_basins(recv, order, outlets=None, valid=None)` | See :func:`TopoAnalysis.fastops.drainage_basins`. | +| `flow_directions(elevations, cellsize_grid=None, cellsize=1.0)` | Steepest-descent D8 codes; see :func:`TopoAnalysis.fastops.flow_directions`. | +| `flow_length(recv, order, step_length)` | ``(length, from_codes)``; see :func:`TopoAnalysis.fastops.flow_length`. | +| `have_extension() -> 'bool'` | True when the compiled kernels are in use. | +| `imposemin(recv, order, step_length, sl, elevations)` | Carve in place; see :func:`TopoAnalysis.fastops.imposemin`. | +| `priority_flood(elevations, closed=None, seeds=None, mode='flat', epsilon=0.0, cellsize=1.0, max_pit_depth=0.0, track_visited=False, reference_algorithm=False)` | Fill depressions in place; see :func:`TopoAnalysis.fastops.priority_flood`. | +| `propagate_along_main_stem(recv, order, main_stem_from_codes, gate, values, mode)` | See :func:`TopoAnalysis.fastops.propagate_along_main_stem`. | +| `receivers(codes)` | Receiver index per cell; see :func:`TopoAnalysis.fastops.receivers`. | +| `stream_order(recv, order, is_stream, kind='strahler')` | See :func:`TopoAnalysis.fastops.stream_order`. | +| `topological_order(recv)` | ``(order, n_cells_on_cycles)``; see :func:`TopoAnalysis.fastops.topological_order`. | +| `upstream_mask(recv, outlets)` | See :func:`TopoAnalysis.fastops.upstream_mask`. | + +## `TopoAnalysis.fastops` -- the reference NumPy kernels + +Pure-NumPy implementations of the TopoAnalysis grid kernels. + +Every function here mirrors, name for name and result for result, one of the +functions in the compiled ``TopoAnalysis._topoanalysis`` extension. They are +the reference semantics: the C++ code is validated against them in the test +suite, and they are what :mod:`TopoAnalysis.kernels` falls back to when the +extension is unavailable. + +The depression-filling routine implements the Priority-Flood algorithms of + + Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal + depression-filling and watershed-labeling algorithm for digital elevation + models." *Computers & Geosciences* 62, 117-127. + +These implementations are correct but roughly one to two orders of magnitude +slower than the compiled kernels on real DEMs. If you are processing +anything larger than a few hundred thousand cells, build the extension:: + + pip install -e . + +### `default_seeds(elevations)` + +Flat indices of the cells a flood should start from. + +These are the grid perimeter plus every valid cell touching a no-data +cell -- the same set that TopoToolbox's ``fillsinks`` keeps in its marker +image. + +Parameters +---------- +elevations : ndarray + 2-D elevation grid; ``NaN`` marks no-data. + +Returns +------- +ndarray of int64 + Flat (C-order) indices of the seed cells. + +### `priority_flood(elevations, closed=None, seeds=None, mode='flat', epsilon=0.0, cellsize=1.0, max_pit_depth=0.0, track_visited=False, reference_algorithm=False)` + +Fill depressions in ``elevations`` in place (Barnes et al. 2014). + +Parameters +---------- +elevations : ndarray + 2-D float64 grid, modified in place. ``NaN`` is no-data. +closed : ndarray of uint8, optional + Cells with a non-zero entry are never processed. Use this to restrict + the flood to a mask. +seeds : ndarray of int64, optional + Flat indices to start from. Defaults to :func:`default_seeds`. +mode : {'flat', 'epsilon'} + ``'flat'`` fills depressions to a level surface (Algorithm 2), which + is what TopoToolbox's ``fillsinks`` does. ``'epsilon'`` adds a small + upslope increment so the result has no flats (Algorithm 4). +epsilon : float + Gradient of the increment used by ``mode='epsilon'``. Each filled + cell is raised to ``parent + epsilon * cellsize * step``. Zero means + "use the smallest representable increment". +cellsize : float + Grid spacing, used only to scale ``epsilon``. +max_pit_depth : float + Depressions deeper than this are left unfilled. Zero disables the + check. Unlike the original TopoAnalysis implementation, the flood + still traverses such depressions, so terrain beyond them is reached. +track_visited : bool + Also return a mask of the cells the flood reached. +reference_algorithm : bool + Use Algorithm 1 (everything through the priority queue) instead of the + faster Algorithm 2. The filled surface is identical; this exists for + testing. + +Returns +------- +dict + ``{'visited': ndarray|None, 'cells_filled': int, 'max_fill_depth': float}`` + +### `flow_directions(elevations, cellsize_grid=None, cellsize=1.0)` + +Steepest-descent D8 flow directions as ArcGIS codes. + +The drop to each neighbour is divided by the centre-to-centre distance, so +a diagonal neighbour only wins if its *gradient* is steeper. Ties go to +the first direction in the order E, SE, S, SW, W, NW, N, NE. + +Cells with no lower neighbour -- pits, and no-data cells -- get code 0. + +### `receivers(codes)` + +Flat index of the cell each cell drains to, or -1 if there is none. + +### `topological_order(recv)` + +Order the cells donors-first. + +Returns ``(order, n_cells_on_cycles)``. Cells that lie on a cycle -- only +possible in a hand-edited or externally supplied flow-direction grid -- +cannot be ordered and are appended at the end in index order. + +### `accumulate(recv, order, weights, gate=None)` + +Accumulate ``weights`` downstream along the D8 network. + +``order`` is accepted for signature compatibility with the compiled +kernel; the result does not depend on which valid topological order is +used, so this implementation derives its own level structure -- which +lets it run vectorised rather than one cell at a time. + +### `flow_length(recv, order, step_length)` + +Longest upstream flow distance, and the main-stem donor direction. + +Returns ``(length, from_codes)``. ``from_codes[k]`` is the direction code +pointing from ``k`` back at the donor that supplies its longest path. +Ties are settled in favour of the donor with the lower flat index, so the +main stem does not depend on which topological order was used. + +### `propagate_along_main_stem(recv, order, main_stem_from_codes, gate, values, mode)` + +Carry or sum ``values`` downstream, but only along longest-flow paths. + +``mode='carry'`` overwrites the receiver with the donor's value (used by +:class:`~TopoAnalysis.dem.Relief`); ``mode='sum'`` adds it (used by +:class:`~TopoAnalysis.dem.Ksi`). + +### `upstream_mask(recv, outlets)` + +Mask of every cell draining to any of ``outlets`` (flat indices). + +### `chi(recv, order, area, step_length, outlets, A0, theta, trapezoid=False, max_length=0.0, mask=None)` + +Integrate the chi coordinate upstream from ``outlets``. + +``chi(cell) = chi(receiver) + (A0 / A)^theta * step_length`` + +With ``trapezoid=True`` the integrand is averaged between the cell and its +receiver, which is the rule TopoToolbox's ``chitransform`` uses. Cells +outside the outlets' drainage areas stay ``NaN``. + +Returns ``(chi, distance_from_outlet)``. + +### `drainage_basins(recv, order, outlets=None, valid=None)` + +Label drainage basins. + +With ``outlets`` given, each cell is labelled with the 1-based position of +the outlet it drains to (0 if it reaches none). Without, every terminal +cell -- pit or edge outlet -- seeds its own basin, matching TopoToolbox's +``drainagebasins(FD)``. + +## `TopoAnalysis.demRecursionTools` + +Profile extraction and chi/steepness fitting over recursive basin maps. + +These functions consume the nested dictionaries produced by +:meth:`TopoAnalysis.dem.FlowDirectionD8.map_values_to_recursive_list`. + +| Function | Summary | +|---|---| +| `area_elevation_for_mainstem_and_tributaries(outlet, flow_direction, elevation, area, theta=0.5, minimum_area=10000000.0)` | | +| `best_ks_and_theta_with_wrss(elevation, flow_direction_or_length, area, outlet, xo=500)` | | +| `best_ks_and_theta_with_wrss_list(ld_list, de, xo=500, maxiter=100, maxfun=200)` | | +| `best_ks_theta(outlet, flow_direction, elevation, area, minimum_area)` | | +| `best_ks_theta_wrss_for_outlet(outlet, flow_direction, elevation, area, minimum_area=10000000.0)` | | +| `best_ks_with_r2_list(ld_list, de, theta, xo=500)` | | +| `best_ks_with_wrss_list(ld_list, de, theta, xo=500)` | | +| `chi_elevation(ld_list, de, theta, xo=500.0)` | | +| `extract_chi_elevation_values(ld_list, de, theta, chi_o, elevation, chi, base_elevation, A_mdx=None, xo=500.0)` | | +| `extract_dA_elevation_values(ld_list)` | | +| `extract_profile_values(ld_list, xo=500.0, items=())` | | +| `hi(elevation, flow_direction, dA, outlet)` | | +| `hi_list(ld_list)` | | +| `map_chi_profiles(elevation, flow_direction, area, outlet, minimum_area=1000000.0, theta=0.5, start_at=0.0, downstream=True, Ao=1000000.0)` | | +| `uninformative_SS_list(ld_list, de, xo=500)` | | + +## `TopoAnalysis.plotting` + +Longitudinal- and chi-profile plotting helpers. + +| Function | Summary | +|---|---| +| `interactive_chi_profiles_and_map_view(prefix, code, plot_code, dem, fd, area, hillshade, minimum_area=10000000.0, theta=0.5, Ao=1000000.0)` | | +| `plot_chi_profiles(elevation, flow_direction, area, outlet, plot_code, minimum_area=1000000.0, figure=None, theta=0.5, start_at=0.0, downstream=True, Ao=1000000.0)` | | +| `plot_chi_profiles_with_outlet_code(prefix, code, plot_code, dem, fd, area, minimum_area=10000000.0, theta=0.5)` | | +| `plot_downstream_profile(elevation, flow_direction, outlet, plot_code, downstream=True, start_at=0.0, mean_pixel_dimension=None, figure=None)` | | +| `plot_profiles(elevation, flow_direction, area, outlet, plot_code, minimum_area=1000000.0, figure=None)` | | +| `plot_profiles_with_outlet_code(prefix, code, plot_code, dem, fd, area, minimum_area=10000000.0)` | | +| `plot_recursive_upstream_profiles(elevation, flow_direction, area, outlet, plot_code, downstream=False, start_at=0.0, figure=None, minimum_area=1000000.0)` | | + +## `TopoAnalysis.datasets` + +Utilities for generating synthetic data and loading examples + +| Function | Summary | +|---|---| +| `sinusoid_grid(ny, nx, width, amp=1, sig=0, slope_y=None)` | Returns a synthetic landscape with sinusoidal ridges and valleys | +| `triangle_grid(ny, nx, width, amp=1, sig=0, slope_y=None)` | Returns a synthetic landscape with triangular ridges and valleys | + +## `TopoAnalysis.utils` + +Convenience wrappers for fitting channel steepness at a single outlet. + +| Function | Summary | +|---|---| +| `calc_ks_for_outlet(outlet, theta, **kwargs)` | Fit channel steepness to the basin draining to ``outlet``. | + +## `TopoAnalysis.analysis` + +Quadrat sampling of a raster. + +### `Quadrats` + + + +| Method | Summary | +|---|---| +| `load_data(filename, band=1)` | Read a raster band with rasterio. | +| `make_quadrats(dx, dy=None)` | | +| `map_quadrats(func, **kwargs)` | | +| `plot(values, ax=None, **kwargs)` | | +| `quiver(u, v, ax=None, **kwargs)` | | + +## `TopoAnalysis.cli` + +Command-line entry point: ``topoanalysis-process``. + +Runs the standard pipeline over a DEM and writes the derived grids as +GeoTIFFs beside it:: + + topoanalysis-process srtm.tif --outdir results --theta 0.45 --a0 1e6 + +Use ``topoanalysis-process --info`` to report which compute backend is +active and whether GDAL is available. + +| Function | Summary | +|---|---| +| `main(argv=None)` | Run the CLI. Returns a process exit status. | + +## `TopoAnalysis.MovingWindow` + +Moving-window filters over a grid. + +A :class:`MovingWindow` applies a reducing function to the cells inside a +window centred on every cell. Subclasses choose the window shape; a +concrete class supplies the reduction as :attr:`~MovingWindow.function`:: + + class WindowMean(CircularMovingWindow): + function = staticmethod(np.mean) + + grid.apply_moving_window(WindowMean(window_radius=200.0)) + +Cells outside the grid, and no-data cells, are dropped from the window +rather than filled, so the function always sees real data. + +For the common reductions (min, max, mean, median, standard deviation) +:func:`TopoAnalysis.topotoolbox.localtopography` is far faster, because it +uses SciPy's separable filters instead of a Python loop. + +### `CircularMovingWindow` + +A disc-shaped window of radius ``round(window_radius / dx)`` cells. + +### `MovingWindow` + +Abstract base for moving-window filters. + +Parameters +---------- +window_radius : float + Radius of the window in map units. ``window_dimension`` is accepted + as a synonym for backward compatibility. + +| Method | Summary | +|---|---| +| `apply_moving_window(grid, dx, dtype)` | Run the window over every cell of ``grid``. | + +### `RectangularMovingWindow` + +A square window of side ``2 * round(window_radius / dx) + 1`` cells. + +## `TopoAnalysis.error` + +### `Error` + +Base class for exceptions in this module. + +### `InputError` + +Exception raised for errors in the input. + +Attributes: + expr -- input expression in which the error occurred + msg -- explanation of the error + +### `TransitionError` + +Raised when an operation attempts a state transition that's not +allowed. + +Attributes: + prev -- state at beginning of transition + next -- attempted new state + msg -- explanation of why the specific transition is not allowed + diff --git a/docs/generate_api.py b/docs/generate_api.py new file mode 100644 index 0000000..1f2badb --- /dev/null +++ b/docs/generate_api.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Regenerate ``docs/api.md`` from the live docstrings. + + python docs/generate_api.py + +Run it after adding or renaming anything public. The output is checked in +so the reference is readable on the repository page without building Sphinx. +""" + +from __future__ import annotations + +import inspect +import os +import pathlib +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) + +from TopoAnalysis import (MovingWindow, analysis, cli, datasets, dem, # noqa: E402 + demRecursionTools, error, fastops, kernels, + plotting, topotoolbox, utils) + +#: Grid classes, grouped the way the guide introduces them. +GROUPS = [ + ('Core grid types', ['BaseSpatialGrid', 'Georef_info', 'BaseSpatialShape', + 'ValueGrid', 'Elevation', 'GeographicElevation']), + ('Depression filling', ['PriorityQueueMixIn', 'FilledElevation', 'PriorityFillGrid']), + ('Flow routing', ['FlowDirection', 'FlowDirectionD8', 'FlowLength', + 'GeographicFlowLength', 'Mask', 'DiscreteFlowAccumulation', + 'GeographicDiscreteFlowAccumulation']), + ('Drainage area', ['Area', 'GeographicArea', 'LogArea', 'ValleyArea', + 'GeographicValleyArea', 'MainstemValleyArea', + 'GeographicMainstemValleyArea']), + ('Chi and channel steepness', ['Chi', 'GeographicChi', 'Ksi', 'GeographicKsi', + 'Relief', 'ScaledRelief', 'ChiScaledRelief', + 'ChannelSlope', 'CrossDivideDChi', + 'NormalizedCrossDivideDChi', 'RestoredElevation', + 'GeographicRestoredElevation']), + ('Along-flow smoothing', ['AlongFlowSmoothing', 'KsFromChiWithSmoothing', + 'GeographicKsFromChiWithSmoothing', + 'ThetaFromChiWithSmoothing', + 'GeographicThetaFromChiWithSmoothing', + 'ChannelSlopeWithSmoothing', + 'ChannelDownSlopeWithSmoothing', + 'ChannelUpSlopeWithSmoothing']), + ('Terrain attributes', ['CalculationMixin', 'Hillshade', 'GeographicHillshade', + 'MaxSlope', 'GeographicMaxSlope', 'Laplacian', + 'GeographicLaplacian', 'Gradient', 'LocalRelief', + 'MultiscaleCurvatureValleyWidth', 'ScarpWavelet']), + ('Isostasy', ['Deflection', 'GeographicDeflection']), + ('Mixins', ['GDALMixin', 'GeographicGridMixin', 'MaxFlowLengthTrackingMixin']), +] + + +def first_line(obj): + doc = inspect.getdoc(obj) or '' + return doc.strip().split('\n')[0] if doc else '' + + +def signature(obj): + """A call signature with the bound first argument removed.""" + try: + sig = str(inspect.signature(obj)) + except (TypeError, ValueError): + return '(...)' + return (sig.replace('self, ', '').replace('(self)', '()') + .replace('cls, ', '').replace('(cls)', '()')) + + +def esc(text): + """Escape the one character that breaks a Markdown table.""" + return text.replace('|', '\\|') + + +def function_table(module, names=None): + rows = [] + for name in sorted(names or getattr(module, '__all__', [])): + obj = getattr(module, name, None) + if not inspect.isfunction(obj): + continue + rows.append('| `{0}{1}` | {2} |'.format( + name, esc(signature(obj)), esc(first_line(obj)))) + return (['| Function | Summary |', '|---|---|'] + rows + ['']) if rows else [] + + +def class_section(out, cls, name): + w = out.append + bases = [b.__name__ for b in cls.__bases__ if b is not object] + w('#### `{0}`\n'.format(name)) + if bases: + w('*inherits* `{0}`\n'.format('`, `'.join(bases))) + doc = inspect.getdoc(cls) + if doc: + w(doc + '\n') + + combos = getattr(cls, 'required_inputs_and_actions', None) + if combos and 'required_inputs_and_actions' in vars(cls): + w('*Constructor keywords* (any one combination)\n') + for keys, _action in combos: + w('- `{0}`'.format('`, `'.join(keys))) + w('') + + dtype = vars(cls).get('dtype') + if dtype is not None: + w('*Stored as* `{0}`\n'.format(np.dtype(dtype).name)) + + rows = [] + for mname, member in sorted(vars(cls).items()): + if mname.startswith('_'): + continue + if isinstance(member, property): + rows.append((mname, '', first_line(member.fget))) + continue + target = member.__func__ if isinstance(member, (classmethod, staticmethod)) else member + if not inspect.isfunction(target): + continue # class constants such as dtype, reported above + kind = '' + if isinstance(member, classmethod): + kind = ' *(classmethod)*' + elif isinstance(member, staticmethod): + kind = ' *(staticmethod)*' + rows.append((mname, signature(target), first_line(target) + kind)) + + if rows: + w('| Member | Signature | Summary |') + w('|---|---|---|') + for mname, sig, summary in rows: + w('| `{0}` | `{1}` | {2} |'.format(mname, esc(sig), esc(summary))) + w('') + + +def build(): + out = [] + w = out.append + + w('# API reference\n') + w('Generated from the docstrings by `docs/generate_api.py`, so') + w('`help(TopoAnalysis.Area)` in a session shows the same text. For how the') + w('pieces fit together start with [guide.md](guide.md); for what the') + w('computations actually do see [algorithms.md](algorithms.md).\n') + w('Attributes prefixed with a single underscore (`_griddata`,') + w('`_georef_info`) are not private in practice -- they are how you reach') + w('the data -- but they are not listed here; see') + w('[guide.md](guide.md#the-grid-model).\n') + + w('## `TopoAnalysis.dem` -- the grid classes\n') + w('Contents:\n') + for title, names in GROUPS: + w('- **{0}**: {1}'.format(title, ', '.join('`{0}`'.format(n) for n in names))) + w('') + + for title, names in GROUPS: + w('### {0}\n'.format(title)) + for name in names: + cls = getattr(dem, name, None) + if cls is not None: + class_section(out, cls, name) + + w('### Module-level functions in `TopoAnalysis.dem`\n') + out.extend(function_table(dem, ['mosaicFolder', 'plot', 'gdal_is_available'])) + + w('## `TopoAnalysis.topotoolbox` -- TopoToolbox-equivalent functions\n') + w(inspect.getdoc(topotoolbox).split('Example')[0].strip() + '\n') + w('See [topotoolbox_parity.md](topotoolbox_parity.md) for the full') + w('correspondence table and the places where exact agreement is not') + w('achievable.\n') + out.extend(function_table(topotoolbox)) + for name in topotoolbox.__all__: + obj = getattr(topotoolbox, name) + if inspect.isfunction(obj): + w('### `{0}{1}`\n'.format(name, signature(obj))) + w((inspect.getdoc(obj) or '') + '\n') + + w('## `TopoAnalysis.kernels` -- backend dispatch\n') + w(inspect.getdoc(kernels) + '\n') + out.extend(function_table(kernels)) + + w('## `TopoAnalysis.fastops` -- the reference NumPy kernels\n') + w(inspect.getdoc(fastops).split('Conventions')[0].strip() + '\n') + for name in fastops.__all__: + obj = getattr(fastops, name) + if inspect.isfunction(obj): + w('### `{0}{1}`\n'.format(name, signature(obj))) + w((inspect.getdoc(obj) or '') + '\n') + + for mod, title in ((demRecursionTools, 'TopoAnalysis.demRecursionTools'), + (plotting, 'TopoAnalysis.plotting'), + (datasets, 'TopoAnalysis.datasets'), + (utils, 'TopoAnalysis.utils'), + (analysis, 'TopoAnalysis.analysis'), + (cli, 'TopoAnalysis.cli'), + (MovingWindow, 'TopoAnalysis.MovingWindow')): + w('## `{0}`\n'.format(title)) + doc = inspect.getdoc(mod) + if doc: + w(doc + '\n') + names = sorted(n for n, o in vars(mod).items() + if not n.startswith('_') and inspect.isfunction(o) + and o.__module__ == mod.__name__) + out.extend(function_table(mod, names)) + for cname in sorted(n for n, o in vars(mod).items() + if not n.startswith('_') and inspect.isclass(o) + and o.__module__ == mod.__name__): + cls = getattr(mod, cname) + w('### `{0}`\n'.format(cname)) + w((inspect.getdoc(cls) or '') + '\n') + rows = ['| `{0}{1}` | {2} |'.format(mname, esc(signature(member)), + esc(first_line(member))) + for mname, member in sorted(vars(cls).items()) + if not mname.startswith('_') and inspect.isfunction(member)] + if rows: + w('| Method | Summary |') + w('|---|---|') + out.extend(rows) + w('') + + w('## `TopoAnalysis.error`\n') + for cname in ('Error', 'InputError', 'TransitionError'): + cls = getattr(error, cname) + w('### `{0}`\n'.format(cname)) + w((inspect.getdoc(cls) or '') + '\n') + + return '\n'.join(out) + '\n' + + +if __name__ == '__main__': + target = pathlib.Path(__file__).parent / 'api.md' + target.write_text(build()) + print('wrote {0} ({1} lines)'.format(target, build().count('\n'))) diff --git a/docs/guide.md b/docs/guide.md new file mode 100644 index 0000000..fefdf18 --- /dev/null +++ b/docs/guide.md @@ -0,0 +1,282 @@ +# User guide + +## The grid model + +Everything in TopoAnalysis is a grid. A grid holds a 2-D NumPy array and the +georeferencing that places it on the ground: + +```python +grid._griddata # the array; row 0 is the NORTH edge +grid._georef_info.dx # cell size, in map units +grid._georef_info.nx, .ny # columns, rows +grid._georef_info.xllcenter, .yllcenter # centre of the SOUTH-WEST cell +grid._georef_info.geoTransform # the GDAL 6-tuple +grid._georef_info.projection # WKT, or 0 if unset +``` + +`NaN` is no-data in floating-point grids. + +Subclasses add meaning rather than structure. `Elevation` is a grid of +heights, `FlowDirectionD8` a grid of direction codes, `Area` a grid of +contributing areas — all of them are `BaseSpatialGrid` underneath and share +its clipping, tiling, resampling and plotting. + +## Constructing grids + +Grids are built by keyword, and the combination you pass selects the +constructor: + +```python +from TopoAnalysis import Elevation + +Elevation(gdal_filename='dem.tif') # from a file +Elevation(ai_ascii_filename='dem.asc', EPSGprojectionCode=32611) +Elevation(dx=30.0, grid=array) # from an array +Elevation(nx=100, ny=80, projection=wkt, geo_transform=gt) # empty +Elevation(nx=100, ny=80, dx=30.0) # random values +``` + +Derived grids name the grid they come from: + +```python +FilledElevation(elevation=dem) +FlowDirectionD8(flooded_dem=filled) +Area(flow_direction=d8) +Ksi(area=area, flow_direction=d8, flow_length=length, theta=0.45, Ao=1e6) +``` + +Each class lists what it accepts in `required_inputs_and_actions`, and an +unrecognised combination raises an error that prints the list. + +## The analysis chain + +```python +from TopoAnalysis import (Elevation, FilledElevation, FlowDirectionD8, + Area, FlowLength, Chi, Ksi, ScaledRelief) + +dem = Elevation(gdal_filename='srtm.tif') +filled = FilledElevation(elevation=dem) +d8 = FlowDirectionD8(flooded_dem=filled) +area = Area(flow_direction=d8) +length = FlowLength(flow_direction=d8) +``` + +Each step depends on the one before. Filling removes closed depressions so +every cell has a downhill path out; routing turns that surface into a +network; accumulation and length are properties of the network. + +### Filling + +```python +filled = FilledElevation(elevation=dem) # default +filled = FilledElevation(elevation=dem, aggradation_slope=0.0) # flat +filled = FilledElevation(elevation=dem, maximum_pit_depth=20.0) +filled = FilledElevation(elevation=dem, outlets=[(x, y)]) +print(filled.fill_report) # {'cells_filled': ..., 'max_fill_depth': ...} +``` + +By default the filled surface carries a very slight gradient +(`aggradation_slope = 1e-12` per map unit) across what were depressions, so +D8 routing has a direction to follow. `aggradation_slope=0` gives a flat +fill, which is what TopoToolbox's `fillsinks` returns — but then the flats +have no defined flow direction. + +`maximum_pit_depth` leaves genuinely closed basins — playas, craters — +unfilled, while still routing the terrain beyond them. + +### Flow direction + +D8 codes follow the ArcGIS convention, with row 0 at the north: + +``` +| 32 64 128 | +| 16 X 1 | +| 8 4 2 | +``` + +0 means the cell has no downstream neighbour: a pit, an outlet at the grid +edge, or no-data. + +The grid also exposes the network directly, which is what makes the +downstream computations linear-time: + +```python +d8.receivers # flat index of each cell's receiver, or -1 +d8.topological_order # donors before receivers +d8.cycle_count # 0 for a valid network +d8.step_lengths() # distance from each cell to its receiver +d8.basin_mask([(x, y)]) # boolean mask of everything draining to a point +``` + +### Drainage area + +```python +area = Area(flow_direction=d8) # square map units +``` + +Each cell contributes its own area plus everything upstream. For a +geographic (lat/lon) grid use `GeographicArea`, which uses the true spherical +cell area — cells shrink towards the poles. + +To count *cells* instead, which is what TopoToolbox reports: + +```python +from TopoAnalysis import topotoolbox as tt +cells = tt.flowacc(d8) +``` + +## Chi and channel steepness + +Chi is the integral + + chi(x) = integral from the outlet to x of (A0 / A)^theta dx' + +and a channel in steady state plots as a straight line in chi–elevation +space, with slope k_sn. There are two ways to compute it here. + +**`Chi`** integrates upstream from outlets you name, over the whole network +above each one: + +```python +outlets = area.areas_greater_than(1e8) +chi = Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e6, outlets=outlets) +``` + +**`Ksi`** integrates along longest-flow-length paths across the whole grid, +with no outlets needed: + +```python +ksi = Ksi(area=area, flow_direction=d8, flow_length=length, + theta=0.45, Ao=1e6) +``` + +Pair either with the matching relief grid and the slope of the resulting +plot is the steepness index: + +```python +relief = ScaledRelief(flow_direction=d8, elevation=dem, flow_length=length, + Ao=1e6, theta=0.45, area=area) + +from TopoAnalysis import plot +plot(ksi, relief, xlabel='chi (m)', ylabel='scaled relief (m)') +``` + +`Chi` takes `trapezoid=True` to average the integrand across each step +instead of taking it at the upstream cell. That is the rule TopoToolbox uses, +and the one to pick when comparing against it. + +### Fitting steepness cell by cell + +`KsFromChiWithSmoothing` fits chi–elevation over a moving window along each +flow path, giving a map of k_sn rather than one number per basin: + +```python +ks = KsFromChiWithSmoothing(elevation=dem, area=area, flow_direction=d8, + theta=0.45, vertical_interval=50.0, + area_threshold=1e6) +ks.save('ksn.tif') # 7 bands: ks, n, mse, ss, r2, p, n_regression +``` + +Use `horizontal_interval` instead of `vertical_interval` to set the window +by distance rather than by drop. `ThetaFromChiWithSmoothing` fits the +concavity instead of holding it fixed. Both need `statsmodels`. + +### Profiles + +For individual profiles rather than maps, extract the basin as a nested +dictionary and work on that: + +```python +from TopoAnalysis import demRecursionTools, plotting + +tree = d8.map_values_to_recursive_list(outlet, elevation=dem, area=area) +e, c = demRecursionTools.chi_elevation(tree, area._mean_pixel_dimension(), + [0.45], xo=1000.0) +ks, r2 = demRecursionTools.best_ks_with_r2_list(tree, de, [0.45]) + +plotting.plot_profiles(dem, d8, area, outlet, 'k-') +plotting.plot_chi_profiles(dem, d8, area, outlet, 'k-', theta=0.45) +``` + +Each node of the tree carries `index`, `distance_scale` (the step length +from its parent, 1 or sqrt(2)), `next` (its upstream children), and one entry +per grid you passed as a keyword. + +## Terrain attributes + +```python +from TopoAnalysis import Hillshade, MaxSlope, Laplacian, LocalRelief + +Hillshade(elevation=dem, azimuth=315, inclination=45) # ESRI, 0-255 +MaxSlope(elevation=dem) # gradient magnitude +Laplacian(elevation=dem) # + in valleys +LocalRelief(elevation=dem, pixel_radius=25) # range in a disc + +dem.principal_curvatures() # (k1, k2) +dem.calculate_gradient_over_length_scale(500.0) # (dz/dx, dz/dy) +dem.calculate_laplacian_over_length_scale(500.0) +``` + +Measuring over an explicit length scale is usually what you want on a noisy +DEM: a 3×3 stencil on 1 m lidar measures the noise, not the landscape. + +## Working with large DEMs + +The compiled kernels handle a few million cells comfortably. Beyond that, +process in tiles: + +```python +tiles = dem.tile(tile_xdim=2000, tile_ydim=2000, + tile_xpadding=100, tile_ypadding=100) + +results = [] +for tile in tiles: + filled = FilledElevation(elevation=tile) + d8 = FlowDirectionD8(flooded_dem=filled) + results.append(Area(flow_direction=d8).remove_padding(100, 100)) + +mosaic = Area.mosaic(results) +``` + +The padding gives each tile enough context that the interior is correct; it +is stripped before the tiles are reassembled. Pick padding larger than the +longest flow path you need to resolve within a tile — for drainage area, +which integrates over the whole basin, tiling is only valid when basins fit +inside a tile. + +## Choosing a backend + +```python +import TopoAnalysis +TopoAnalysis.backend() # 'c++' or 'python' +``` + +Set `TOPOANALYSIS_PURE_PYTHON=1` to force the NumPy kernels. The two produce +identical results — the test suite asserts it — so this is for debugging and +for comparing, not for changing answers. + +## Reading and writing + +```python +grid.save('out.tif') # LZW-compressed GeoTIFF, NaN as nodata +grid.write_to_ai('out.asc') # ArcInfo ASCII, -9999 as nodata +grid.vectorize('polygons') # -> polygons.shp +Elevation.load('out.tif') +``` + +`FlowLength.save` also writes a `*_directions` side-file holding its +main-stem direction codes; `FlowLength.load` expects to find it. + +## Cross-checking against TopoToolbox + +See [topotoolbox_parity.md](topotoolbox_parity.md). The short version: + +```python +from TopoAnalysis import topotoolbox as tt + +tt.fillsinks(dem) # flat fill +tt.flowobj(dem) # FLOWobj(DEM,'preprocess','fill') +tt.flowacc(fd) # in CELLS, not m^2 +tt.chitransform(fd, a) # trapezoidal, mn=0.45, a0=1e6 +tt.ksn(fd, dem, a) # carves at 1e-5 first, no a0 +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..4dcd0be --- /dev/null +++ b/docs/index.md @@ -0,0 +1,82 @@ +# TopoAnalysis documentation + +Digital elevation model analysis for tectonic geomorphology. + +```bash +git clone +cd TopoAnalysis +python3 install.py +``` + +That installs everything, GDAL included — both its C library and its Python +bindings, which pip cannot supply on its own. + +## Start here + +- **[Installation](installation.md)** — the one-command install, how GDAL is + obtained on each platform, and the offline and no-compiler cases. +- **[User guide](guide.md)** — the grid model, the analysis chain from DEM + to channel steepness, and worked examples. +- **[Algorithms](algorithms.md)** — what each computation actually does, + the conventions it assumes, and where it will disagree with another + package. +- **[API reference](api.md)** — every public class and function, generated + from the docstrings. +- **[TopoToolbox parity](topotoolbox_parity.md)** — the function-by-function + correspondence with the MATLAB package, and the places where exact + agreement is not achievable. +- **[Changelog](../CHANGELOG.md)** — the defects repaired in 1.0, and the + handful of results that change because they were wrong. + +## Thirty seconds + +```python +from TopoAnalysis import Elevation, FilledElevation, FlowDirectionD8, Area + +dem = Elevation(gdal_filename='srtm.tif') +filled = FilledElevation(elevation=dem) # Priority-Flood, Barnes et al. 2014 +d8 = FlowDirectionD8(flooded_dem=filled) # steepest-descent D8 +area = Area(flow_direction=d8) # contributing area, m^2 +area.save('area.tif') +``` + +## Two things to know before you start + +**Row 0 is the north edge.** Grids are stored the way a GeoTIFF is, so the +first row of `_griddata` is the top of the map and row index increases +southwards. Every convention in the library follows from that: the D8 code +for "south" is 4, `y` decreases as the row index grows, and +`geoTransform[5]` is negative. + +**Drainage area is in square metres, not cells.** `Area` accumulates +`dx**2` per cell. TopoToolbox's `flowacc` counts cells. When comparing the +two, or reading a paper that reports one, check which is meant — +`TopoAnalysis.topotoolbox.flowacc` gives the cell count. + +## Which backend am I on? + +```python +import TopoAnalysis +TopoAnalysis.backend() # 'c++' or 'python' +``` + +or from a shell: + +```bash +topoanalysis-process --info +``` + +Both backends produce identical numbers; the C++ one is about 50× faster. +`TOPOANALYSIS_PURE_PYTHON=1` forces the NumPy path. + +## Regenerating these pages + +`api.md` is generated from the docstrings: + +```bash +python docs/generate_api.py # or: make docs +``` + +The rest are written by hand. They are plain Markdown and are meant to be +read as they are, on the repository page or in an editor -- there is no +Sphinx build. The `[docs]` extra exists for anyone who wants to wire one up. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..42d2bb3 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,392 @@ +# Installation + +```bash +git clone +cd TopoAnalysis +python3 install.py +``` + +That is the whole procedure. `install.py` creates a virtual environment, +installs the dependencies, compiles the Priority-Flood kernels, installs GDAL +— the C library *and* the Python bindings — and then proves the result works +by writing a georeferenced GeoTIFF and reading it back. + +It uses nothing but the standard library, so it runs on a machine where +nothing has been set up yet. When it finishes it tells you how to activate +the environment. + +``` +======================================================================== +6. Checking the installation +======================================================================== + TopoAnalysis 1.0.0 + compute backend : c++ + in-memory pipeline : ok (max area 8100 m^2) + GDAL available : True + GDAL version : 3.13.3 + GeoTIFF round trip : ok (EPSG:32611, 5x5) +``` + +## What it installs, and what is optional + +| Component | Needed for | If it is missing | +|---|---|---| +| NumPy, SciPy, matplotlib | everything | hard requirement | +| a C++17 compiler | the fast kernels | falls back to NumPy, same numbers, ~50× slower | +| GDAL | reading and writing raster files | everything except file I/O still works | + +Neither the compiler nor GDAL is required, and `install.py` will finish +without them rather than failing. + +## Options + +```bash +python3 install.py --no-gdal # skip GDAL entirely +python3 install.py --venv ~/envs/topo # a virtualenv somewhere specific +python3 install.py --system # no virtualenv; use this interpreter +python3 install.py --gdal-strategy source # force a from-source GDAL build +python3 install.py --allow-root # let apt/dnf use sudo +python3 install.py --jobs 8 # parallel compile jobs +python3 install.py --dry-run # show the plan, change nothing +``` + +`--dry-run` is worth running first on an unfamiliar machine. It reports what +GDAL is present, what is wrong with it, and exactly which commands an install +would run. + +`--venv` forces a virtual environment even from inside a conda environment, +which is otherwise used as it stands. + +**Exit status.** `0` means everything asked for is installed and verified. +`1` means something is missing — including the case where TopoAnalysis +installed correctly but GDAL did not, which the output states explicitly. Use +`--no-gdal` if you do not want GDAL and do not want its absence to be an +error. + +**`--system` and PEP 668.** Debian, Ubuntu, Fedora and Homebrew mark their +Python installations as externally managed, and pip refuses to install into +them. `install.py --system` stops with an explanation rather than letting pip +fail obscurely. Inside a container, where there is no host environment to +protect, add `--break-system-packages`. + +### Why it makes a virtualenv by default + +Because installing into a system Python usually fails, and when it does not +fail it is a bad idea. Debian, Ubuntu, Fedora and Homebrew all mark their +Python installations "externally managed" (PEP 668), so `pip install` refuses +to touch them. Creating `.venv` sidesteps that and gives a from-source GDAL +build somewhere self-contained to live. If you are already inside a +virtualenv or a conda environment, that one is used and nothing new is +created. + +--- + +# GDAL + +GDAL is the one dependency pip cannot supply on its own, so it gets its own +tool. If you only want the summary: run `topoanalysis-gdal install` and it +will work out what to do. + +## Why it is difficult + +There is no GDAL wheel on PyPI. Not for your platform — for *any* platform, +at *any* version. Every release is an sdist whose `setup.py` shells out to +`gdal-config` and compiles the SWIG bindings against a GDAL C library that +must already be on the machine. So installing GDAL is two jobs that have to +agree with each other: + +1. the **C library**, `libgdal`, which pip knows nothing about, and +2. the **Python bindings**, `osgeo`, whose version must match it. + +`pip install GDAL` does only the second half, and fails confusingly when the +first half is absent or a different version. + +## The tool + +```bash +topoanalysis-gdal # what is installed, and what is wrong with it +topoanalysis-gdal plan # what an install would do, step by step +topoanalysis-gdal install # do it +``` + +`doctor` is the one to reach for when something is wrong: + +``` +python : 3.11.9 (/home/you/TopoAnalysis/.venv/bin/python) +numpy : 2.1.0 +GDAL bindings : 3.4.1 +GDAL C library : 3.8.4 +gdal-config : /usr/bin/gdal-config +numpy bridge : MISSING (ReadAsArray will fail) +projection lookup : works + +problems: + [version-mismatch] the bindings are 3.4.1 but the C library is 3.8.4; they + must agree to at least major.minor + [no-gdal-array] osgeo.gdal_array is missing, so Band.ReadAsArray() will + fail. The bindings were built without numpy present. +``` + +## The routes it can take + +`topoanalysis-gdal install` picks the cheapest one that stands a chance. +`--strategy` forces a particular choice. + +| Strategy | When it is chosen | Cost | +|---|---|---| +| `bindings` | the C library is already here; just build `osgeo` against it | seconds | +| `conda` | a conda environment is active | a minute or two | +| `wheel` | Windows, where prebuilt wheels bundling the C library are the only practical option | a minute | +| `system` | apt/dnf/pacman/zypper/apk/brew can supply the C library | a few minutes, needs root except under Homebrew | +| `source` | nothing else is available | about half an hour | + +### `source` — building GDAL without root + +This is the route that makes an unattended install possible on a machine with +no conda, no package manager and no administrator rights. It downloads, +verifies and builds: + +``` +sqlite3 only when the system copy is unusable. PROJ needs the library, the + header, and the sqlite3 command, which it runs to assemble proj.db. +PROJ mandatory for GDAL 3.x. Built without TIFF and curl, which are only + needed for the optional high-accuracy datum grids. +GDAL built against its own bundled libtiff, libgeotiff, libpng, libjpeg + and zlib, so the only external dependency left is PROJ. +``` + +Using GDAL's bundled libraries is what makes this realistic: there are no +system `-dev` packages to hunt for. The result reads GeoTIFF, ArcInfo ASCII, +ENVI, HFA and the other built-in drivers — everything TopoAnalysis uses. + +Everything installs into one prefix, by default the active virtualenv, so +nothing outside it is touched and deleting the environment removes GDAL with +it. The only prerequisites are a C/C++ compiler and CMake. + +Every tarball is pinned by SHA-256, checked against what the projects +themselves publish. A download that does not match the pin is not built. To +move a pin — a newer GDAL, or a local mirror — set all three variables +together, so the integrity check cannot be disabled by accident: + +```bash +export TOPOANALYSIS_GDAL_SOURCE_GDAL_VERSION=3.12.4 +export TOPOANALYSIS_GDAL_SOURCE_GDAL_URL=https://example.org/gdal-3.12.4.tar.gz +export TOPOANALYSIS_GDAL_SOURCE_GDAL_SHA256=<64 hex digits> +``` + +The pinned GDAL is deliberately recent. GDAL bundles its own copies of zlib +and libpng, and the copies shipped up to about 3.9 do not compile against a +current Apple SDK: both take their "Mac OS classic" branch, because that is +selected on `TARGET_OS_MAC`, which modern SDKs define on every Apple +platform. zlib then redefines `fdopen` to `NULL` and the system `_stdio.h` +fails to parse. + +## Platform notes + +**conda (any platform)** — the most reliable route, and the only one that is +equally good on Windows: + +```bash +conda env create -f environment.yml +conda activate topoanalysis +python -m pip install -e . +``` + +`environment.yml` puts `conda-forge` first, because mixing GDAL from +`defaults` with compilers from `conda-forge` produces a C++ ABI mismatch that +only appears at import time. + +**Ubuntu / Debian** — `sudo apt install gdal-bin libgdal-dev`, then +`topoanalysis-gdal install`. Note that 22.04 ships GDAL 3.4 and 24.04 ships +3.8; both are below the version where the sdist started declaring numpy as a +build dependency, so install the bindings with `topoanalysis-gdal` rather than +plain pip — see the trap below. + +**macOS** — Homebrew's GDAL is built for the Mac's native architecture. If +your Python is x86_64 running under Rosetta on an Apple Silicon machine, the +two cannot link. `topoanalysis-gdal doctor` detects this and will not try to +"fix" it by reinstalling the same thing; use conda, a matching Python, or +`--strategy source`, which builds for whichever architecture the interpreter +is. + +**Windows** — use conda, or let `topoanalysis-gdal install` fetch a prebuilt +wheel that bundles the C library. Building GDAL from source on Windows is not +attempted. + +**Docker** — the `osgeo/gdal` images already have both halves installed and +matched; `topoanalysis-gdal doctor` will report nothing to do and +`pip install -e .` is enough. + +## Two traps worth knowing about + +**The silent `gdal_array` trap.** GDAL sdists only grew a `pyproject.toml` in +version 3.9. Before that they declare no build dependencies at all, so under +pip's default build isolation numpy is absent when the bindings compile — and +the numpy bridge, `osgeo.gdal_array`, is quietly skipped. Everything imports +cleanly; `Band.ReadAsArray()` then fails at run time. Ubuntu 22.04 and 24.04 +both ship GDAL versions on the wrong side of that line, so this is the common +case, not an exotic one. `topoanalysis-gdal` installs the bindings with +`--no-build-isolation` against an interpreter that already has numpy, which +both avoids the trap and builds the bridge against the numpy you will +actually run. + +**The architecture trap.** Described under macOS above. The symptom is a +missing-symbol error at link or import time that says nothing about +architectures. + +--- + +## Checking the installation + +```bash +topoanalysis-process --info +topoanalysis-gdal doctor +``` + +``` +TopoAnalysis 1.0.0 + compute backend : c++ + GDAL available : True + fill algorithm : Barnes, R., Lehman, C., Mulla, D. (2014). ... +``` + +Then run the tests: + +```bash +python -m pytest -q # a few seconds +TOPOANALYSIS_PURE_PYTHON=1 python -m pytest -q # same tests, NumPy kernels +``` + +The second run is worth doing after any change to the C++. It re-runs every +test against the NumPy kernels, and the suite contains explicit +backend-comparison tests (`test_kernels.py::test_fill_backends_agree` and +`test_routing_backends_agree`) that call both implementations on the same +inputs and require identical output — so a divergence shows up as a failure +rather than as a quietly different answer. + +## When the compiler is missing + +The extension is declared `optional`, so a failed compile is a warning, not an +installation failure: + +``` +TopoAnalysis: pybind11 not found; building without the compiled kernels. +``` + +The library still installs and works. To get the speed back: + +| Platform | What to install | +|---|---| +| Ubuntu / Debian | `sudo apt install build-essential python3-dev` | +| RHEL / Fedora | `sudo dnf install gcc-c++ python3-devel` | +| macOS | `xcode-select --install` | +| Windows | [Build Tools for Visual Studio](https://visualstudio.microsoft.com/visual-cpp-build-tools/), "Desktop development with C++" | +| conda (any) | `conda install -c conda-forge compilers pybind11` | + +Then reinstall: + +```bash +python -m pip install --force-reinstall --no-deps -e . +python -c "import TopoAnalysis; print(TopoAnalysis.backend())" # -> c++ +``` + +Editing a header under `_cpp/` does trigger a rebuild — the extension declares +them in `depends` — but a stale `build/` directory can still confuse +setuptools. `make distclean && make build` clears it. + +## Installing without a network + +`pip install` needs `pybind11` at build time. If the machine is offline, fetch +the wheel somewhere with a network and carry it over: + +```bash +# on a connected machine +pip download pybind11 setuptools wheel -d wheelhouse + +# on the target +python -m pip install --no-index --find-links wheelhouse pybind11 setuptools wheel +python -m pip install --no-build-isolation --no-index -e . +``` + +`--no-build-isolation` is the important flag: without it pip builds in a fresh +environment and tries to download `pybind11` again. + +A from-source GDAL build also needs a network, but only to fetch its three +tarballs. They are cached under `~/.cache/TopoAnalysis/downloads`, so copying +that directory across is enough to make the build work offline. + +## Building from an sdist + +```bash +python -m build --sdist +python -m pip install dist/TopoAnalysis-1.0.0.tar.gz +``` + +The sdist carries `_cpp/*.hpp` and `_cpp/*.cpp`, so the kernels are rebuilt on +the target machine for its own Python and architecture. + +## Both import styles + +The repository directory *is* the package, so which style works depends on +what is on `sys.path`: + +```python +from TopoAnalysis import dem # parent directory on the path (installed) +import dem # the package directory itself on the path +``` + +Every module supports both. The first is preferred; the second is how the +original scripts in this repository loaded it, and is kept working +deliberately. + +## Optional extras + +```bash +pip install -e ".[stats]" # statsmodels, for the steepness regressions +pip install -e ".[parallel]" # dask, for multiscale curvature +pip install -e ".[quadrats]" # rasterio, for analysis.Quadrats +pip install -e ".[all]" # all of the above +pip install -e ".[docs]" # sphinx, if you want to build a site +``` + +`[all]` deliberately excludes GDAL. Listing it there would make +`pip install .[all]` fail on any machine without a GDAL C library already +present, taking the rest of the package down with it. + +Each is imported only when the code that needs it runs, and the error names +the missing package and how to get it. + +There is a `[gdal]` extra, but it will not work on its own: it installs the +bindings, which cannot build without the C library. Use `topoanalysis-gdal +install` instead, which installs both. + +## Troubleshooting + +**`ImportError: TopoAnalysis needs the GDAL Python bindings ...`** — you called +something that reads or writes a file. Run `topoanalysis-gdal install`, or +build grids in memory with `dx=..., grid=...`. + +**`AttributeError` or a numpy error from `ReadAsArray`** — the `gdal_array` +trap above. `topoanalysis-gdal doctor` confirms it; `topoanalysis-gdal +install` rebuilds the bindings correctly. + +**`symbol not found` or a segfault when importing `osgeo`** — either a C++ ABI +mismatch from mixing conda channels, or the architecture trap. Rebuild the +environment from `environment.yml` with `conda-forge` first, or run +`topoanalysis-gdal doctor`, which checks the architecture explicitly. + +**`ERROR 1: PROJ: proj_create_from_database: Cannot find proj.db`** — PROJ's +data files are missing or `PROJ_LIB` points somewhere wrong. +`topoanalysis-gdal doctor` reports this as `projection lookup : FAILS`. + +**`backend()` says `python` after installing a compiler** — the wheel was +cached. `pip install --force-reinstall --no-deps --no-cache-dir -e .` + +**`RuntimeWarning: TopoAnalysis is using its pure-Python kernels on a grid of N +cells`** — exactly what it says; see the compiler table above. + +**Tests fail with `ModuleNotFoundError: No module named 'TopoAnalysis'`** — run +them from the package directory (`cd TopoAnalysis && pytest`) or from its +parent (`pytest TopoAnalysis/tests`). Both work, and so does a checkout under +any other name (a GitHub zip unpacks to `TopoAnalysis-main`); +`tests/conftest.py` sorts out the import either way. diff --git a/docs/topotoolbox_parity.md b/docs/topotoolbox_parity.md new file mode 100644 index 0000000..d321dc4 --- /dev/null +++ b/docs/topotoolbox_parity.md @@ -0,0 +1,249 @@ +# Parity with TopoToolbox + +[TopoToolbox 2](https://topotoolbox.wordpress.com/) (Schwanghart & Scherler +2014) is the MATLAB package that covers much of the same ground as +TopoAnalysis. This page says, function by function, where the two agree, +where they differ, and what to do about it. + +`TopoAnalysis.topotoolbox` holds functions written to reproduce +TopoToolbox's numbers exactly. The rest of TopoAnalysis keeps its own +conventions — area in square metres, ESRI hillshade, left-endpoint chi — +because those are what the rest of the library is built on. + +```python +from TopoAnalysis import dem, topotoolbox as tt + +z = dem.Elevation(gdal_filename='srtm.tif') +fd = tt.flowobj(z) # == FLOWobj(DEM,'preprocess','fill') +a = tt.flowacc(fd) # == flowacc(FD) +c = tt.chitransform(fd, a) # == chitransform(S,A) +``` + +--- + +## Correspondence table + +**Agreement** is one of: + +- **exact** — the same arithmetic on the same inputs; +- **exact¹** — exact up to float64-vs-float32 rounding (TopoToolbox works in + single precision when the DEM is single); +- **equivalent** — the same quantity, related by a stated conversion; +- **differs** — genuinely different results; the reason is given. + +### Depressions and flats + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `fillsinks(DEM)` | `tt.fillsinks(dem)` | exact | Flat fill; seeds are the perimeter plus every cell touching NaN; 8-connected. | +| | `FilledElevation(elevation=dem)` | differs | Adds `aggradation_slope` (default 1e-12/m) across filled areas. Pass `aggradation_slope=0` for the flat fill. | +| `fillsinks(DEM, maxdepth)` | `tt.fillsinks(dem, maxdepth=...)` | exact | All-or-nothing per connected depression, iterated as MATLAB does — not a per-cell depth test. | +| `identifyflats(DEM)` | `tt.identifyflats(dem)` | exact | Returns `(flats, sills)`; `output='all'` adds the closed basins. Includes MATLAB's cleared rim and no-data halo, and its treatment of an isolated pit as a flat. | +| `imposemin(FD, DEM, sl)` | `tt.imposemin(fd, dem, sl)` | exact | Carves rather than fills: only lowers cells. | +| `elevateminima` | — | — | Not implemented. | + +### Flow + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `FLOWobj(DEM,'preprocess','fill')` | `tt.flowobj(dem)` | **differs inside flats** — see below | Identical elsewhere. | +| `FLOWobj(DEM,'preprocess','carve')` | — | — | Not implemented; `tt.flowobj` raises rather than guessing. | +| `FLOWobj2GRIDobj(FD)` | `d8._griddata` | exact | Same ArcGIS codes, same row-0-is-north orientation. | +| `flowacc(FD)` | `tt.flowacc(fd)` | exact | **Number of cells**, minimum 1. | +| | `Area(flow_direction=d8)` | equivalent | `Area = flowacc × dx²`. | +| `flowacc(FD, W0)` | `tt.flowacc(fd, weights)` | exact | | +| `flowdistance(FD,'upstream')` | `tt.flowdistance(fd,'upstream')` | exact¹ | Distance to the outlet. TopoToolbox accumulates in float32. | +| `flowdistance(FD,'downstream')` | `tt.flowdistance(fd,'downstream')` | exact¹ | Longest path reaching each cell. | +| | `FlowLength(flow_direction=d8)` | exact¹ | The same quantity as `'downstream'`. | +| `drainagebasins(FD)` | `tt.drainagebasins(fd)` | **partition** exact, labels differ | Both number basins in edge-list order; the orders differ. Compare partitions. | +| `drainagebasins(FD,IX)` | `tt.drainagebasins(fd, outlets=...)` | exact | Labels are positions in the outlet list. | +| `streamorder(S,'strahler')` | `tt.streamorder(fd, mask)` | exact | | +| `streamorder(S,'shreve')` | `tt.streamorder(fd, mask, 'shreve')` | exact | | + +### Terrain attributes + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `gradient8(DEM)` | `tt.gradient8(dem)` | exact | Steepest descent over 8 neighbours, clamped at 0. Diagonals use `√2·dx`. | +| `gradient8(DEM,unit)` | `tt.gradient8(dem, unit)` | exact | `tangent`, `degree`, `radian`, `sine`, `percent`. | +| `arcslope(DEM)` | `tt.arcslope(dem)` | exact² | Horn 3×3, `8·dx` denominator. ²NaN infill uses SciPy's distance transform; ties next to a NaN hole can break differently from MATLAB's `bwdist`. | +| | `MaxSlope(elevation=dem)` | differs | Centred-difference gradient magnitude — a different estimator. | +| `hillshade(DEM)` | `tt.hillshade(dem)` | exact | Defaults azimuth 315, altitude 60 (the *code's* defaults; the MATLAB docstring disagrees with itself). Returns a cosine in [−1, 1]. | +| | `Hillshade(elevation=dem, azimuth=, inclination=)` | differs | ESRI formulation: 0–255, and an aspect of `atan2(dz/dy, −dz/dx)` from centred differences. Strongly correlated with the TopoToolbox version but not equal. | +| `hillshade(...,'method','mdow')` | — | — | Not implemented. | +| `aspect(DEM)` | `tt.aspect(dem)` | exact | Degrees clockwise from north; flat cells report 90. | +| `aspect(DEM,true)` | `tt.aspect(dem, classify=True)` | exact | The 8-class Gómez-Plaza ranking. | +| `curvature(DEM,type)` | `tt.curvature(dem, ctype)` | exact | `profc`, `planc`, `tangc`, `meanc`, `total`. | +| | `Laplacian(elevation=dem)` | differs | `∇²z` from a 5-point stencil — a different quantity from any of the five. | +| `cellarea(DEM)` | `tt.cellarea(grid)` | exact³ | ³Assumes a sphere of total area 510 072 000 km². `GeographicArea` uses R = 6 371 000 m, which is 1.5 × 10⁻⁵ smaller. | +| `localtopography(DEM,r,'range')` | `tt.localtopography(dem, r, 'range')` | **≈** | TopoToolbox's default disc is a periodic-line approximation; this uses an exact disc (TopoToolbox's `N=0`). `'mean'` averages over that binary disc where MATLAB's `fspecial('disk')` weights the rim fractionally; `'std'` uses the N−1 sample deviation, as `stdfilt` does. | +| | `LocalRelief(elevation=dem, pixel_radius=)` | exact⁵ | Identical to `tt.localtopography(..., 'range')` with the matching radius, on grids with no no-data. ⁵`LocalRelief` does not mask `NaN`, so around a hole the two differ; use `tt.localtopography` there. | +| `roughness`, `evansslope` | — | — | Not implemented. | + +### Chi and steepness + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `chitransform(S,A)` | `tt.chitransform(fd, a)` | exact | `mn=0.45`, `a0=1e6`, trapezoidal, area in **cells**. | +| | `Chi(..., trapezoid=True)` | exact | Given the same outlets and area in m². | +| | `Chi(...)` (default) | differs | Left-endpoint rule instead of trapezoidal. | +| | `Ksi(...)` | differs | Integrates along longest-flow-length paths, not upstream from outlets. | +| `cumtrapz(S,g)` | — | equivalent | The integration inside `tt.chitransform`. | +| `ksn(S,DEM,A)` | `tt.ksn(fd, dem, a)` | exact | Carves at `sl=1e-5` first; no `a0` (reference area 1 m²). | +| `mchi(S,DEM,chi)` | `tt.mchi(fd, dem, chi)` | exact | `dz/dχ`, no carving. | +| `chiplot`, `slopearea` | — | — | Not implemented; `KsFromChiWithSmoothing` is the nearest equivalent. | + +### Referencing + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `getcoordinates(DEM)` | `tt.getcoordinates(grid)` | exact | `y` runs **north to south**, matching row order. | +| | `grid.coordinate_vectors()` | differs in order | `y` runs south to north. | +| `getextent(DEM)` | `tt.getextent(grid)` | exact | Outermost **cell centres**. | +| | `grid.extent()` | differs | Outer pixel **edges** — what `imshow` wants; half a cell larger each way. | +| `coord2sub`, `coord2ind` | `tt.coord2sub(grid, x, y)` | exact⁴ | ⁴0-based here, 1-based in MATLAB. | +| | `grid._xy_to_rowscols(v)` | **≈** | Rounds halves to even (Python) rather than away from zero (MATLAB); differs only for points landing exactly on a cell boundary. | +| `sub2coord`, `ind2coord` | `tt.sub2coord(grid, r, c)` | exact⁴ | | +| `GRIDobj2ascii` | `grid.write_to_ai(path)` | equivalent | Both centre-registered; TopoAnalysis writes `-9999` for no-data. | +| `GRIDobj2geotiff` | `grid.save(path)` | equivalent | TopoAnalysis adds LZW compression and writes NaN as the no-data value. | + +--- + +## Where exact agreement is not achievable + +### Flow directions inside flats + +This is the one difference that changes a drainage network. + +After filling, a depression becomes a plateau with no downhill direction. +The two packages resolve it differently: + +- **TopoToolbox** fills flat and then builds an *auxiliary topography* — a + grey-weighted geodesic distance transform inside each flat — so cells + drain towards the nearest sill. Flow directions inside a flat come from + the ranking that surface induces. +- **TopoAnalysis** adds a tiny gradient while filling + (`aggradation_slope`), so each cell's direction points back along the + path the flood arrived by. + +Both give a valid, connected network, and both drain each flat to the same +sill. **Which cells they route through inside the flat is different.** +Drainage area therefore differs inside flats and immediately downstream of +them. Everything outside flats is identical. + +There is no way to reconcile this short of implementing the geodesic +transform. If exact agreement inside flats matters, compute the flow +directions in MATLAB and load them: + +```python +d8 = FlowDirectionD8(gdal_filename='flowdir_from_matlab.tif') +``` + +The code convention is identical, so the grid transfers without conversion. + +TopoToolbox's default is actually `'carve'`, not `'fill'` — it *lowers* the +DEM through depressions rather than raising it. That produces a different +surface again, and is not implemented here. + +### Tie-breaking on equal gradients + +Where two neighbours are equally steep, TopoToolbox prefers the diagonal +(its test is `G1 <= G2`) and then the larger linear index; TopoAnalysis +takes the first in the order E, SE, S, SW, W, NW, N, NE. Exact ties are rare +on a real DEM with an ε-filled surface, but common on synthetic or +integer-valued ones. + +### Border cells of `hillshade` and `aspect` + +None — these agree exactly. MATLAB's `surfnorm` expands the grid with a +*quadratic* ghost cell (`3·z₁ − 3·z₂ + z₃`), and a centred difference across +that ghost is algebraically the second-order one-sided difference, which is +what `numpy.gradient(..., edge_order=2)` computes. +`test_surfnorm_matches_matlab_including_the_border` checks it against a +literal port of `surfnorm.m`. + +### Precision + +TopoToolbox works in whatever class the DEM is, usually float32 from a +GeoTIFF; TopoAnalysis works in float64 throughout. Accumulated quantities — +flow length especially, which sums thousands of terms — drift apart in the +last few digits. Compare with a relative tolerance of about 1e-6, not +exactly. + +### Basin labels + +Both packages number basins in the order their edge list happens to meet +them, and the two orders differ. The *partition* is identical. Compare +partitions: + +```python +import numpy as np + +def same_partition(a, b): + """True when two labellings group the cells identically.""" + pairs = np.unique(np.stack([a.ravel(), b.ravel()]), axis=1) + return (len(np.unique(pairs[0])) == pairs.shape[1] + and len(np.unique(pairs[1])) == pairs.shape[1]) +``` + +### Earth radius + +`tt.cellarea` uses TopoToolbox's sphere (total area 510 072 000 km², so +R = 6 371 047 m); `GeographicArea` uses R = 6 371 000 m. Areas differ by +1.5 × 10⁻⁵ — negligible for most work, but it will show up in an exact +comparison. Setting the radius brings them to within 3 × 10⁻¹²: + +```python +grid.earth_radius = (tt.EARTH_SURFACE_AREA_KM2 / (4 * np.pi)) ** 0.5 * 1000.0 +grid._GeographicGridMixin__app = None # the area grid is cached +``` + +The residue is because `GeographicArea` evaluates the longitude difference +per cell, so its areas vary in the last few bits along a row, whereas +`cellarea` computes one value per row and broadcasts it. + +--- + +## Cross-checking in practice + +A recipe for comparing a result against MATLAB: + +```python +import numpy as np +from TopoAnalysis import dem, topotoolbox as tt + +z = dem.Elevation(gdal_filename='srtm.tif') +fd = tt.flowobj(z) +a = tt.flowacc(fd) # cells, like flowacc(FD) +c = tt.chitransform(fd, a, mn=0.45, a0=1e6) + +matlab = dem.BaseSpatialGrid(gdal_filename='chi_from_matlab.tif') +difference = c._griddata - matlab._griddata +print(np.nanmax(np.abs(difference)), + np.nanpercentile(np.abs(difference), 99)) +``` + +Expect agreement to ~1e-6 relative away from flats, and visible differences +inside them. If the difference is a constant factor, check the units: area +in cells versus m², or `a0` versus `a0^mn`. + +## How this was verified + +Without MATLAB available, parity was established three ways: + +1. **From the source.** Every function above was transcribed from the + TopoToolbox MATLAB source in this repository — the exact stencils, + constants, default values, rounding rules and NaN handling — and each + transcription was independently re-checked against the source. +2. **Against closed forms.** Planes, quadratics and spherical cells have + answers that can be written down: a 1:1 east-facing plane has gradient + exactly 1, aspect exactly 90°, curvature exactly 0. `test_topotoolbox.py` + checks those. +3. **Against each other.** Where TopoAnalysis and the parity module compute + the same quantity by different routes, the tests assert the documented + conversion holds — `flowacc × dx² == Area`, `chitransform == + Chi(trapezoid=True)`, `localtopography('range') == LocalRelief`. + +What this does **not** establish is bit-for-bit agreement with a specific +MATLAB release. If you have MATLAB, running the same DEM through both and +comparing is worth doing, and the differences above are what to expect. diff --git a/environment.yml b/environment.yml index 093c386..67ce615 100644 --- a/environment.yml +++ b/environment.yml @@ -1,9 +1,24 @@ -name: TopoAnalysis +# Conda environment for TopoAnalysis. +# +# conda env create -f environment.yml +# conda activate topoanalysis +# pip install -e . +# +# conda-forge is listed first because it is the only channel that reliably +# provides a working GDAL together with a matching libstdc++. +name: topoanalysis channels: - - defaults - conda-forge + - defaults dependencies: - - numpy - - matplotlib - - gdal - - scipy + - python>=3.8 + - numpy>=1.20 + - scipy>=1.7 + - matplotlib-base>=3.3 + - gdal>=3.0 # file I/O only; the rest of the library runs without it + - pybind11>=2.6 # builds the Priority-Flood kernels + - compilers # a C++17 toolchain on every platform + - statsmodels>=0.12 # channel-steepness regressions + - dask>=2021.0 # optional parallel multiscale curvature + - pytest>=7.0 + - pip diff --git a/fastops.py b/fastops.py new file mode 100644 index 0000000..4f262b5 --- /dev/null +++ b/fastops.py @@ -0,0 +1,811 @@ +"""Pure-NumPy implementations of the TopoAnalysis grid kernels. + +Every function here mirrors, name for name and result for result, one of the +functions in the compiled ``TopoAnalysis._topoanalysis`` extension. They are +the reference semantics: the C++ code is validated against them in the test +suite, and they are what :mod:`TopoAnalysis.kernels` falls back to when the +extension is unavailable. + +The depression-filling routine implements the Priority-Flood algorithms of + + Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal + depression-filling and watershed-labeling algorithm for digital elevation + models." *Computers & Geosciences* 62, 117-127. + +These implementations are correct but roughly one to two orders of magnitude +slower than the compiled kernels on real DEMs. If you are processing +anything larger than a few hundred thousand cells, build the extension:: + + pip install -e . + +Conventions +----------- +Grids are 2-D, row-major, with row 0 at the north edge. ``NaN`` is no-data. +Flow-direction codes follow the ArcGIS convention:: + + | 32 64 128 | + | 16 X 1 | + | 8 4 2 | +""" + +from __future__ import annotations + +import heapq +import math + +import numpy as np + +__all__ = [ + "D8_DI", + "D8_DJ", + "D8_DIST", + "D8_CODES", + "default_seeds", + "priority_flood", + "flow_directions", + "receivers", + "topological_order", + "accumulate", + "flow_length", + "propagate_along_main_stem", + "upstream_mask", + "chi", + "drainage_basins", +] + +# Neighbour order E, SE, S, SW, W, NW, N, NE -- so that the ArcGIS code for +# direction ``d`` is simply ``1 << d``. +D8_DI = np.array([0, 1, 1, 1, 0, -1, -1, -1], dtype=np.int64) +D8_DJ = np.array([1, 1, 0, -1, -1, -1, 0, 1], dtype=np.int64) +D8_DIST = np.array([1.0, math.sqrt(2.0)] * 4, dtype=np.float64) +D8_CODES = np.array([1, 2, 4, 8, 16, 32, 64, 128], dtype=np.uint8) + +_SQRT2 = math.sqrt(2.0) + + +def _as_2d_float(grid, name): + arr = np.asarray(grid, dtype=np.float64) + if arr.ndim != 2: + raise ValueError("{0} must be a 2-D array".format(name)) + return arr + + +def _shifted(grid, di, dj, fill): + """``grid`` translated so that element (i, j) holds grid[i+di, j+dj].""" + out = np.full_like(grid, fill) + ny, nx = grid.shape + src_i = slice(max(di, 0), ny + min(di, 0)) + src_j = slice(max(dj, 0), nx + min(dj, 0)) + dst_i = slice(max(-di, 0), ny + min(-di, 0)) + dst_j = slice(max(-dj, 0), nx + min(-dj, 0)) + out[dst_i, dst_j] = grid[src_i, src_j] + return out + + +# --------------------------------------------------------------------------- +# Depression filling +# --------------------------------------------------------------------------- + + +def default_seeds(elevations): + """Flat indices of the cells a flood should start from. + + These are the grid perimeter plus every valid cell touching a no-data + cell -- the same set that TopoToolbox's ``fillsinks`` keeps in its marker + image. + + Parameters + ---------- + elevations : ndarray + 2-D elevation grid; ``NaN`` marks no-data. + + Returns + ------- + ndarray of int64 + Flat (C-order) indices of the seed cells. + """ + z = _as_2d_float(elevations, "elevations") + ny, nx = z.shape + if ny == 0 or nx == 0: + return np.empty(0, dtype=np.int64) + nodata = np.isnan(z) + + seed = np.zeros((ny, nx), dtype=bool) + seed[0, :] = True + seed[-1, :] = True + seed[:, 0] = True + seed[:, -1] = True + for di, dj in zip(D8_DI, D8_DJ): + seed |= _shifted(nodata.astype(np.float64), int(di), int(dj), 0.0) > 0.5 + seed &= ~nodata + return np.flatnonzero(seed).astype(np.int64) + + +def priority_flood( + elevations, + closed=None, + seeds=None, + mode="flat", + epsilon=0.0, + cellsize=1.0, + max_pit_depth=0.0, + track_visited=False, + reference_algorithm=False, +): + """Fill depressions in ``elevations`` in place (Barnes et al. 2014). + + Parameters + ---------- + elevations : ndarray + 2-D float64 grid, modified in place. ``NaN`` is no-data. + closed : ndarray of uint8, optional + Cells with a non-zero entry are never processed. Use this to restrict + the flood to a mask. + seeds : ndarray of int64, optional + Flat indices to start from. Defaults to :func:`default_seeds`. + mode : {'flat', 'epsilon'} + ``'flat'`` fills depressions to a level surface (Algorithm 2), which + is what TopoToolbox's ``fillsinks`` does. ``'epsilon'`` adds a small + upslope increment so the result has no flats (Algorithm 4). + epsilon : float + Gradient of the increment used by ``mode='epsilon'``. Each filled + cell is raised to ``parent + epsilon * cellsize * step``. Zero means + "use the smallest representable increment". + cellsize : float + Grid spacing, used only to scale ``epsilon``. + max_pit_depth : float + Depressions deeper than this are left unfilled. Zero disables the + check. Unlike the original TopoAnalysis implementation, the flood + still traverses such depressions, so terrain beyond them is reached. + track_visited : bool + Also return a mask of the cells the flood reached. + reference_algorithm : bool + Use Algorithm 1 (everything through the priority queue) instead of the + faster Algorithm 2. The filled surface is identical; this exists for + testing. + + Returns + ------- + dict + ``{'visited': ndarray|None, 'cells_filled': int, 'max_fill_depth': float}`` + """ + if mode not in ("flat", "epsilon"): + raise ValueError("mode must be 'flat' or 'epsilon'") + + z = elevations + if not isinstance(z, np.ndarray) or z.dtype != np.float64 or z.ndim != 2: + raise TypeError("elevations must be a 2-D float64 ndarray (modified in place)") + if not z.flags['C_CONTIGUOUS']: + # reshape(-1) would return a *copy* of a non-contiguous array, so the + # fill would land in the copy and the caller's grid would come back + # untouched while the report claimed success. The compiled kernel + # rejects the same input, so both backends agree. + raise TypeError( + "elevations must be C-contiguous to be filled in place; " + "pass np.ascontiguousarray(elevations)") + ny, nx = z.shape + n = ny * nx + flat = z.reshape(-1) + + if closed is None: + closed_flat = np.zeros(n, dtype=bool) + else: + closed_flat = np.asarray(closed, dtype=bool).reshape(-1).copy() + if closed_flat.size != n: + raise ValueError("closed must have the same shape as elevations") + closed_flat |= np.isnan(flat) + + if seeds is None: + seed_idx = default_seeds(z) + keep = seed_idx[~closed_flat[seed_idx]] + if closed is not None and keep.size == 0: + keep = _mask_boundary_seeds(closed_flat.reshape(ny, nx)) + seed_idx = keep + else: + seed_idx = np.asarray(seeds, dtype=np.int64).reshape(-1) + + use_epsilon = mode == "epsilon" + limit_depth = max_pit_depth > 0.0 + # The FIFO shortcut is only sound when every cell it holds sits at exactly + # the current flood level. Two options break that invariant: an epsilon + # fill raises each cell by its own increment, and a depth limit re-queues + # unfilled cells *below* the current level. In either case everything goes + # through the priority queue, i.e. the run degrades to Algorithm 1. + use_pit = not use_epsilon and not limit_depth and not reference_algorithm + + visited = np.zeros(n, dtype=np.uint8) if track_visited else None + unfilled = np.zeros(n, dtype=np.uint8) if limit_depth else None + cells_filled = 0 + max_fill_depth = 0.0 + + # ``order`` makes the heap ordering total so ties resolve first-in + # first-out, exactly as the compiled kernel does. + order = 0 + open_heap = [] + for k in seed_idx: + k = int(k) + if k < 0 or k >= n or closed_flat[k]: + continue + closed_flat[k] = True + heapq.heappush(open_heap, (flat[k], order, k)) + order += 1 + + from collections import deque + + pit = deque() + + while open_heap or pit: + if pit and open_heap and open_heap[0][0] <= flat[pit[0]]: + cur_z, _, cur = heapq.heappop(open_heap) + elif pit: + cur = pit.popleft() + cur_z = flat[cur] + else: + cur_z, _, cur = heapq.heappop(open_heap) + + if visited is not None: + visited[cur] = 1 + + i, j = divmod(cur, nx) + for d in range(8): + ni = i + int(D8_DI[d]) + nj = j + int(D8_DJ[d]) + if ni < 0 or nj < 0 or ni >= ny or nj >= nx: + continue + nk = ni * nx + nj + if closed_flat[nk]: + continue + closed_flat[nk] = True + if np.isnan(flat[nk]): + continue + + if use_epsilon: + if epsilon > 0.0: + target = cur_z + epsilon * cellsize * D8_DIST[d] + if target <= cur_z: + target = np.nextafter(cur_z, np.inf) + else: + target = np.nextafter(cur_z, np.inf) + else: + target = cur_z + + if flat[nk] < target: + depth = float(target) - float(flat[nk]) + if limit_depth and depth > max_pit_depth: + unfilled[nk] = 1 + heapq.heappush(open_heap, (flat[nk], order, nk)) + order += 1 + continue + flat[nk] = target + cells_filled += 1 + if depth > max_fill_depth: + max_fill_depth = depth + if use_pit: + pit.append(nk) + else: + heapq.heappush(open_heap, (flat[nk], order, nk)) + order += 1 + else: + heapq.heappush(open_heap, (flat[nk], order, nk)) + order += 1 + + return { + "visited": None if visited is None else visited.reshape(ny, nx), + "unfilled": None if unfilled is None else unfilled.reshape(ny, nx), + "cells_filled": cells_filled, + "max_fill_depth": max_fill_depth, + } + + +def _mask_boundary_seeds(closed_2d): + """Seeds on the edge of the open region when the mask excludes the rim.""" + open_region = ~closed_2d + ny, nx = closed_2d.shape + boundary = np.zeros((ny, nx), dtype=bool) + boundary[0, :] = True + boundary[-1, :] = True + boundary[:, 0] = True + boundary[:, -1] = True + for di, dj in zip(D8_DI, D8_DJ): + boundary |= _shifted(closed_2d.astype(np.float64), int(di), int(dj), 0.0) > 0.5 + return np.flatnonzero((boundary & open_region).reshape(-1)).astype(np.int64) + + +# --------------------------------------------------------------------------- +# Flow directions +# --------------------------------------------------------------------------- + + +def flow_directions(elevations, cellsize_grid=None, cellsize=1.0): + """Steepest-descent D8 flow directions as ArcGIS codes. + + The drop to each neighbour is divided by the centre-to-centre distance, so + a diagonal neighbour only wins if its *gradient* is steeper. Ties go to + the first direction in the order E, SE, S, SW, W, NW, N, NE. + + Cells with no lower neighbour -- pits, and no-data cells -- get code 0. + """ + z = _as_2d_float(elevations, "elevations") + if cellsize_grid is None: + de = np.full(z.shape, float(cellsize)) + else: + de = _as_2d_float(cellsize_grid, "cellsize_grid") + if de.shape != z.shape: + raise ValueError("cellsize_grid must match the elevation grid") + + best_slope = np.zeros(z.shape) + codes = np.zeros(z.shape, dtype=np.uint8) + + for d in range(8): + neighbour = _shifted(z, int(D8_DI[d]), int(D8_DJ[d]), np.nan) + slope = (z - neighbour) / (de * D8_DIST[d]) + better = slope > best_slope + # NaN comparisons are False, so no-data neighbours never win. + best_slope = np.where(better, slope, best_slope) + codes = np.where(better, D8_CODES[d], codes).astype(np.uint8) + + codes[np.isnan(z)] = 0 + return codes + + +def receivers(codes): + """Flat index of the cell each cell drains to, or -1 if there is none.""" + codes = np.asarray(codes, dtype=np.uint8) + if codes.ndim != 2: + raise ValueError("codes must be a 2-D array") + ny, nx = codes.shape + ii, jj = np.mgrid[0:ny, 0:nx] + + out = np.full(codes.shape, -1, dtype=np.int64) + for d in range(8): + sel = codes == D8_CODES[d] + if not sel.any(): + continue + ni = ii + int(D8_DI[d]) + nj = jj + int(D8_DJ[d]) + inside = sel & (ni >= 0) & (nj >= 0) & (ni < ny) & (nj < nx) + out[inside] = (ni * nx + nj)[inside] + return out + + +# --------------------------------------------------------------------------- +# Topological ordering and accumulation +# --------------------------------------------------------------------------- + + +def _levels(recv_flat, n): + """Kahn's algorithm, one frontier at a time. + + Returns ``(chunks, n_on_cycles)`` where each chunk is an array of cell + indices that can be processed simultaneously: no cell in a chunk drains + into another cell of the same chunk. + """ + has_recv = (recv_flat >= 0) & (recv_flat < n) & (recv_flat != np.arange(n)) + indegree = np.bincount(recv_flat[has_recv], minlength=n) + + frontier = np.flatnonzero(indegree == 0) + chunks = [] + seen = 0 + while frontier.size: + chunks.append(frontier) + seen += frontier.size + moving = frontier[has_recv[frontier]] + if moving.size == 0: + break + targets = recv_flat[moving] + indegree -= np.bincount(targets, minlength=n) + candidates = np.unique(targets) + frontier = candidates[indegree[candidates] == 0] + + return chunks, n - seen + + +def topological_order(recv): + """Order the cells donors-first. + + Returns ``(order, n_cells_on_cycles)``. Cells that lie on a cycle -- only + possible in a hand-edited or externally supplied flow-direction grid -- + cannot be ordered and are appended at the end in index order. + """ + recv_flat = np.asarray(recv, dtype=np.int64).reshape(-1) + n = recv_flat.size + chunks, n_cycles = _levels(recv_flat, n) + if chunks: + order = np.concatenate(chunks) + else: + order = np.empty(0, dtype=np.int64) + if n_cycles: + emitted = np.zeros(n, dtype=bool) + emitted[order] = True + order = np.concatenate([order, np.flatnonzero(~emitted)]) + return order.astype(np.int64), int(n_cycles) + + +def accumulate(recv, order, weights, gate=None): + """Accumulate ``weights`` downstream along the D8 network. + + ``order`` is accepted for signature compatibility with the compiled + kernel; the result does not depend on which valid topological order is + used, so this implementation derives its own level structure -- which + lets it run vectorised rather than one cell at a time. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + shape = np.asarray(weights).shape + recv_flat = recv_arr.reshape(-1) + n = recv_flat.size + + acc = np.asarray(weights, dtype=np.float64).reshape(-1).copy() + if gate is not None: + gate_flat = np.asarray(gate).reshape(-1).astype(bool) + acc[~gate_flat] = 0.0 + else: + gate_flat = None + + chunks, _ = _levels(recv_flat, n) + for chunk in chunks: + if gate_flat is not None: + chunk = chunk[gate_flat[chunk]] + if chunk.size == 0: + continue + r = recv_flat[chunk] + valid = (r >= 0) & (r < n) & (r != chunk) + if not valid.any(): + continue + np.add.at(acc, r[valid], acc[chunk[valid]]) + + return acc.reshape(shape) + + +def flow_length(recv, order, step_length): + """Longest upstream flow distance, and the main-stem donor direction. + + Returns ``(length, from_codes)``. ``from_codes[k]`` is the direction code + pointing from ``k`` back at the donor that supplies its longest path. + Ties are settled in favour of the donor with the lower flat index, so the + main stem does not depend on which topological order was used. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + step = np.asarray(step_length, dtype=np.float64).reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + + length = np.zeros(n, dtype=np.float64) + from_index = np.full(n, -1, dtype=np.int64) + + for k in order: + k = int(k) + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + candidate = length[k] + step[k] + incumbent = int(from_index[r]) + if candidate > length[r] or ( + candidate == length[r] and (incumbent < 0 or k < incumbent) + ): + length[r] = candidate + from_index[r] = k + + code_for_offset = { + (int(D8_DI[d]), int(D8_DJ[d])): int(D8_CODES[d]) for d in range(8) + } + from_codes = np.zeros(n, dtype=np.uint8) + contributing = np.flatnonzero(from_index >= 0) + for r in contributing: + r = int(r) + k = int(from_index[r]) + offset = (k // nx - r // nx, k % nx - r % nx) + from_codes[r] = code_for_offset.get(offset, 0) + + return length.reshape(ny, nx), from_codes.reshape(ny, nx) + + +def propagate_along_main_stem(recv, order, main_stem_from_codes, gate, values, mode): + """Carry or sum ``values`` downstream, but only along longest-flow paths. + + ``mode='carry'`` overwrites the receiver with the donor's value (used by + :class:`~TopoAnalysis.dem.Relief`); ``mode='sum'`` adds it (used by + :class:`~TopoAnalysis.dem.Ksi`). + """ + if mode not in ("carry", "sum"): + raise ValueError("mode must be 'carry' or 'sum'") + + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + codes_flat = np.asarray(main_stem_from_codes, dtype=np.uint8).reshape(-1) + out = np.asarray(values, dtype=np.float64).reshape(-1).copy() + gate_flat = None if gate is None else np.asarray(gate).reshape(-1).astype(bool) + order = np.asarray(order, dtype=np.int64).reshape(-1) + + code_for_offset = { + (int(D8_DI[d]), int(D8_DJ[d])): int(D8_CODES[d]) for d in range(8) + } + + for k in order: + k = int(k) + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + offset = (k // nx - r // nx, k % nx - r % nx) + code = code_for_offset.get(offset, 0) + if code == 0 or codes_flat[r] != code: + continue + if gate_flat is not None and not gate_flat[k]: + continue + if mode == "sum": + out[r] += out[k] + else: + out[r] = out[k] + + return out.reshape(ny, nx) + + +# --------------------------------------------------------------------------- +# Traversal +# --------------------------------------------------------------------------- + + +def upstream_mask(recv, outlets): + """Mask of every cell draining to any of ``outlets`` (flat indices).""" + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + + valid = (recv_flat >= 0) & (recv_flat < n) & (recv_flat != np.arange(n)) + donor_of = np.flatnonzero(valid) + target_of = recv_flat[valid] + sort = np.argsort(target_of, kind="stable") + donors_sorted = donor_of[sort] + starts = np.searchsorted(target_of[sort], np.arange(n), side="left") + ends = np.searchsorted(target_of[sort], np.arange(n), side="right") + + mask = np.zeros(n, dtype=np.uint8) + stack = [] + for k in np.asarray(outlets, dtype=np.int64).reshape(-1): + k = int(k) + if k < 0 or k >= n or mask[k]: + continue + mask[k] = 1 + stack.append(k) + while stack: + k = stack.pop() + for donor in donors_sorted[starts[k]:ends[k]]: + donor = int(donor) + if mask[donor]: + continue + mask[donor] = 1 + stack.append(donor) + return mask.reshape(ny, nx) + + +def chi( + recv, + order, + area, + step_length, + outlets, + A0, + theta, + trapezoid=False, + max_length=0.0, + mask=None, +): + """Integrate the chi coordinate upstream from ``outlets``. + + ``chi(cell) = chi(receiver) + (A0 / A)^theta * step_length`` + + With ``trapezoid=True`` the integrand is averaged between the cell and its + receiver, which is the rule TopoToolbox's ``chitransform`` uses. Cells + outside the outlets' drainage areas stay ``NaN``. + + Returns ``(chi, distance_from_outlet)``. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + area_flat = np.asarray(area, dtype=np.float64).reshape(-1) + step = np.asarray(step_length, dtype=np.float64).reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + mask_flat = None if mask is None else np.asarray(mask).reshape(-1).astype(bool) + + chi_flat = np.full(n, np.nan) + dist = np.full(n, np.nan) + for k in np.asarray(outlets, dtype=np.int64).reshape(-1): + k = int(k) + if k < 0 or k >= n: + continue + if mask_flat is not None and not mask_flat[k]: + continue + chi_flat[k] = 0.0 + dist[k] = 0.0 + + # Receivers before donors: walk the topological order backwards. + for k in order[::-1]: + k = int(k) + if not np.isnan(chi_flat[k]): + continue + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k or np.isnan(chi_flat[r]): + continue + if mask_flat is not None and not mask_flat[k]: + continue + a = area_flat[k] + if not a > 0.0: + continue + dl = step[k] + if max_length > 0.0 and dist[r] + dl > max_length: + continue + integrand = (A0 / a) ** theta + if trapezoid: + ar = area_flat[r] + integrand = 0.5 * (integrand + ((A0 / ar) ** theta if ar > 0.0 else integrand)) + chi_flat[k] = chi_flat[r] + integrand * dl + dist[k] = dist[r] + dl + + return chi_flat.reshape(ny, nx), dist.reshape(ny, nx) + + +def downstream_distance(recv, order, step_length): + """Along-flow distance from each cell to the end of its flow path. + + Zero at outlets and at cells with no receiver, increasing upstream -- + the same quantity as TopoToolbox's ``flowdistance(FD, 'upstream')``. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + shape = recv_arr.shape + n = recv_arr.size + recv_flat = recv_arr.reshape(-1) + step = np.asarray(step_length, dtype=np.float64).reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + + distance = np.zeros(n, dtype=np.float64) + for k in order[::-1]: + k = int(k) + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + distance[k] = distance[r] + step[k] + return distance.reshape(shape) + + +def imposemin(recv, order, step_length, sl, elevations): + """Carve ``elevations`` in place so flow paths descend at gradient >= ``sl``. + + Only lowers cells, never raises them -- the complement of depression + filling, and what TopoToolbox's ``imposemin`` does. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + n = recv_arr.size + recv_flat = recv_arr.reshape(-1) + step = np.asarray(step_length, dtype=np.float64).reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + if not isinstance(elevations, np.ndarray) or not elevations.flags['C_CONTIGUOUS']: + # As in priority_flood: reshape(-1) copies a non-contiguous array, so + # the carving would be thrown away. + raise TypeError( + "elevations must be a C-contiguous ndarray to be carved in place; " + "pass np.ascontiguousarray(elevations)") + z = elevations.reshape(-1) + + for k in order: + k = int(k) + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + candidate = z[k] - sl * step[k] + if candidate < z[r]: + z[r] = candidate + + +def stream_order(recv, order, is_stream, kind="strahler"): + """Strahler or Shreve stream order over the cells marked in ``is_stream``.""" + recv_arr = np.asarray(recv, dtype=np.int64) + shape = recv_arr.shape + n = recv_arr.size + recv_flat = recv_arr.reshape(-1) + stream = np.asarray(is_stream).reshape(-1).astype(bool) + order = np.asarray(order, dtype=np.int64).reshape(-1) + + result = np.zeros(n, dtype=np.int32) + + if kind == "shreve": + has_donor = np.zeros(n, dtype=bool) + for k in range(n): + if not stream[k]: + continue + r = int(recv_flat[k]) + if 0 <= r < n and r != k and stream[r]: + has_donor[r] = True + result[stream & ~has_donor] = 1 + for k in order: + k = int(k) + if not stream[k]: + continue + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k or not stream[r]: + continue + result[r] += result[k] + return result.reshape(shape) + + # Each junction is resolved as a whole -- the largest incoming order and + # how many donors carry it -- rather than donor by donor. The + # incremental form depends on the sequence the donors arrive in, which + # would make the answer depend on which valid topological order was used. + best = np.zeros(n, dtype=np.int32) + ties = np.zeros(n, dtype=np.int32) + + for k in order: + k = int(k) + if not stream[k]: + continue + result[k] = best[k] + 1 if ties[k] >= 2 else max(int(best[k]), 1) + + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k or not stream[r]: + continue + if result[k] > best[r]: + best[r] = result[k] + ties[r] = 1 + elif result[k] == best[r]: + ties[r] += 1 + + return result.reshape(shape) + + +def drainage_basins(recv, order, outlets=None, valid=None): + """Label drainage basins. + + With ``outlets`` given, each cell is labelled with the 1-based position of + the outlet it drains to (0 if it reaches none). Without, every terminal + cell -- pit or edge outlet -- seeds its own basin, matching TopoToolbox's + ``drainagebasins(FD)``. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + valid_flat = None if valid is None else np.asarray(valid).reshape(-1).astype(bool) + + labels = np.zeros(n, dtype=np.int32) + if outlets is None: + # Only a terminal cell that something actually drains into seeds a + # basin; an isolated cell stays 0, as in TopoToolbox. + has_donor = np.zeros(n, dtype=bool) + for k in range(n): + if valid_flat is not None and not valid_flat[k]: + continue + r = int(recv_flat[k]) + if 0 <= r < n and r != k: + has_donor[r] = True + next_label = 0 + for k in range(n): + if valid_flat is not None and not valid_flat[k]: + continue + r = recv_flat[k] + if (r < 0 or r >= n or r == k) and has_donor[k]: + next_label += 1 + labels[k] = next_label + else: + for position, k in enumerate(np.asarray(outlets, dtype=np.int64).reshape(-1), start=1): + k = int(k) + if 0 <= k < n and (valid_flat is None or valid_flat[k]): + labels[k] = position + + for k in order[::-1]: + k = int(k) + if labels[k] != 0: + continue + if valid_flat is not None and not valid_flat[k]: + continue + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + labels[k] = labels[r] + + return labels.reshape(ny, nx) diff --git a/gdal_setup.py b/gdal_setup.py new file mode 100644 index 0000000..5f403c2 --- /dev/null +++ b/gdal_setup.py @@ -0,0 +1,1561 @@ +"""Find, diagnose and install GDAL -- both the C library and the Python bindings. + +Why this module exists +---------------------- +GDAL is the one dependency ``pip install`` cannot supply by itself. There is +no GDAL wheel on PyPI: at *every* version, for *every* platform, the only +artifact published is an sdist whose ``setup.py`` shells out to +``gdal-config`` and compiles the SWIG bindings against a GDAL C library that +has to be on the machine already. So "install GDAL" is really two jobs that +must agree with each other: + + 1. the C library, ``libgdal``, which pip knows nothing about, and + 2. the Python bindings, ``osgeo``, whose version must match it. + +This module does both. It probes what is present, works out the cheapest +route to a working install, and takes it. In rough order of preference: + + ``bindings`` the C library is already here -- just build ``osgeo`` + against it. Seconds. + ``conda`` a conda environment is active; conda-forge ships both + halves, already matched. A minute or two. + ``wheel`` Windows only, where prebuilt wheels that bundle the C + library are the only practical option. + ``system`` apt/dnf/pacman/zypper/apk/brew supply the C library, then + fall through to ``bindings``. Needs administrator rights + except under Homebrew. + ``source`` download and build PROJ and GDAL from pinned, checksummed + tarballs into a prefix of our own, then fall through to + ``bindings``. Half an hour, but needs nothing but a + compiler and CMake -- no root, no package manager. + +Two traps this module exists to avoid +------------------------------------- +**The silent ``gdal_array`` trap.** GDAL sdists only grew a +``pyproject.toml`` in 3.9. Before that they declare no build dependencies at +all, so under pip's default build isolation numpy is absent when the bindings +compile -- and the numpy bridge, ``osgeo.gdal_array``, is quietly skipped. +Everything imports; ``Band.ReadAsArray()`` then fails at run time with an +unhelpful error. Ubuntu 22.04 ships GDAL 3.4 and 24.04 ships 3.8, so this is +the common case, not the exotic one. We install the bindings with +``--no-build-isolation`` against an interpreter that already has numpy, which +both fixes the trap and builds the bridge against the numpy the user will +actually run. + +**The architecture trap.** A Homebrew GDAL on Apple Silicon is arm64; a +Python installed under Rosetta is x86_64. The bindings cannot link the two, +and the error names a missing symbol rather than the real cause. The probe +checks for this before anything is built. + +Command line +------------ +:: + + topoanalysis-gdal # what is installed, and what is wrong + topoanalysis-gdal plan # what an install would do, step by step + topoanalysis-gdal install # do it +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import shutil +import subprocess +import sys + +try: # pragma: no cover - exercised by whichever import style is in use + from . import _gdal_source +except ImportError: # pragma: no cover + import _gdal_source + +__all__ = [ + "Plan", + "Probe", + "Step", + "activate_environment", + "apply_plan", + "ensure", + "main", + "make_plan", + "probe", +] + + +#: The oldest GDAL whose C API TopoAnalysis is known to work against. +MIN_LIBRARY_VERSION = (3, 0) + +#: GDAL sdists from this version on declare numpy in ``[build-system] +#: requires``. Below it, build isolation leaves numpy out and the +#: ``gdal_array`` bridge is silently dropped. See the module docstring. +NUMPY_BUILD_REQUIRES_FROM = (3, 9) + +#: Fallback when the GitHub API cannot be reached; see :func:`_windows_find_links`. +CGOHLKE_FALLBACK_TAG = "v2026.8.20" + + +# --------------------------------------------------------------------------- +# Probing +# --------------------------------------------------------------------------- + +# Run in a subprocess, never in-process. A bindings/library version mismatch +# or an architecture mismatch can abort the interpreter outright rather than +# raising ImportError, and a diagnostic tool that dies while diagnosing is +# worse than useless. +_PROBE_SCRIPT = r""" +import json, os, platform, sys +out = {"executable": sys.executable, "python": sys.version.split()[0]} +# The *target's* architecture, not ours. An x86_64 Python under Rosetta on +# an arm64 machine is the whole reason the architecture check exists, and +# asking the wrong process gives exactly the wrong answer. +out["machine"] = platform.machine() +# Asked here rather than inferred from the parent process: --python can point +# at a different interpreter, and it is *its* environment we install into. +out["sys_prefix"] = sys.prefix +out["base_prefix"] = getattr(sys, "base_prefix", sys.prefix) + +# The same data-file fixup TopoAnalysis applies at import time, repeated here +# rather than imported: the interpreter being probed very often does not have +# TopoAnalysis installed yet. Without it the probe would report a broken +# GDAL that works perfectly well in use. +def _activate(): + if os.environ.get("TOPOANALYSIS_NO_GDAL_ENV"): + return + root = os.environ.get("XDG_CACHE_HOME") or os.path.join( + os.path.expanduser("~"), ".cache") + for path in (os.path.join(sys.prefix, "share", "topoanalysis", "gdal-prefix.json"), + os.path.join(root, "TopoAnalysis", "gdal-prefix.json")): + try: + with open(path) as handle: + manifest = json.load(handle) + except Exception: + continue + recorded = manifest.get("python_prefix") + if recorded is not None and recorded != sys.prefix: + return + proj_lib = manifest.get("proj_lib") + if proj_lib and os.path.isfile(os.path.join(proj_lib, "proj.db")): + os.environ["PROJ_LIB"] = proj_lib + os.environ["PROJ_DATA"] = proj_lib + gdal_data = manifest.get("gdal_data") + if gdal_data and os.path.isdir(gdal_data): + os.environ["GDAL_DATA"] = gdal_data + return + +try: + _activate() +except Exception: + pass +try: + import numpy + out["numpy"] = numpy.__version__ +except Exception: + out["numpy"] = None +try: + from osgeo import gdal + out["bindings_version"] = gdal.__version__ + out["osgeo_file"] = getattr(gdal, "__file__", None) +except Exception as exc: + out["bindings_error"] = "%s: %s" % (type(exc).__name__, exc) +try: + from osgeo import gdal_array # noqa: F401 + out["has_gdal_array"] = True +except Exception as exc: + out["has_gdal_array"] = False + out["gdal_array_error"] = "%s: %s" % (type(exc).__name__, exc) +out["proj_env"] = { + name: os.environ.get(name) + for name in ("PROJ_LIB", "PROJ_DATA", "GDAL_DATA") + if os.environ.get(name) +} +try: + from osgeo import osr + try: + out["proj_search_paths"] = list(osr.GetPROJSearchPaths() or []) + except Exception: + pass + # Without this a failed lookup returns an error code rather than raising, + # and the diagnosis loses the one message that explains it -- PROJ names + # the offending proj.db and why it was rejected. + osr.UseExceptions() + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + out["projection_works"] = bool(srs.ExportToWkt()) + if not out["projection_works"]: + out["projection_error"] = "EPSG:4326 produced an empty definition" +except Exception as exc: + out["projection_works"] = False + out["projection_error"] = "%s: %s" % (type(exc).__name__, exc) +sys.stdout.write(json.dumps(out)) +""" + + +class Probe(object): + """A snapshot of the GDAL situation on one interpreter.""" + + def __init__(self, **fields): + self.python = fields.get("python_executable") or sys.executable + self.python_version = fields.get("python") + self.numpy_version = fields.get("numpy") + self.bindings_version = fields.get("bindings_version") + self.bindings_error = fields.get("bindings_error") + self.osgeo_file = fields.get("osgeo_file") + self.has_gdal_array = bool(fields.get("has_gdal_array")) + self.gdal_array_error = fields.get("gdal_array_error") + self.projection_works = bool(fields.get("projection_works")) + self.projection_error = fields.get("projection_error") + self.library_version = fields.get("library_version") + self.gdal_config = fields.get("gdal_config") + self.prefix = fields.get("prefix") + self.crashed = bool(fields.get("crashed")) + self.problems = tuple(fields.get("problems", ())) + self.sys_prefix = fields.get("sys_prefix") + self.base_prefix = fields.get("base_prefix") + self.machine = fields.get("machine") or platform.machine() + self.proj_env = dict(fields.get("proj_env") or {}) + self.proj_search_paths = tuple(fields.get("proj_search_paths") or ()) + + # -- derived ----------------------------------------------------------- + + @property + def is_virtualenv(self): + return bool(self.sys_prefix) and self.sys_prefix != self.base_prefix + + @property + def is_conda(self): + return bool(self.sys_prefix) and os.path.isdir( + os.path.join(self.sys_prefix, "conda-meta") + ) + + @property + def target_prefix(self): + """The environment an install should go into. + + A virtualenv or conda environment is self-contained, so a C library + built into it is removed with it and its ``bin`` is already on PATH + when the environment is active -- which is exactly what building the + bindings needs. Anything else falls back to a per-user directory + rather than writing into a system prefix. + """ + if self.is_virtualenv or self.is_conda: + return self.sys_prefix + return default_source_prefix() + + @property + def ok(self): + """True when nothing needs doing.""" + return not self.problems + + @property + def codes(self): + return tuple(code for code, _ in self.problems) + + def has(self, code): + return code in self.codes + + def describe(self): + """A human-readable report, as a list of lines.""" + lines = [ + "python : %s (%s)" % (self.python_version or "?", self.python), + "numpy : %s" % (self.numpy_version or "not installed"), + "GDAL bindings : %s" % (self.bindings_version or "not installed"), + "GDAL C library : %s" % (self.library_version or "not found"), + ] + if self.gdal_config: + lines.append("gdal-config : %s" % self.gdal_config) + lines.append( + "numpy bridge : %s" + % ("present" if self.has_gdal_array else "MISSING (ReadAsArray will fail)") + ) + lines.append( + "projection lookup : %s" % ("works" if self.projection_works else "FAILS") + ) + if self.problems: + lines.append("") + lines.append("problems:") + for code, message in self.problems: + lines.append(" [%s] %s" % (code, message)) + else: + lines.append("") + lines.append("GDAL is installed and working.") + return lines + + +def probe(python=None): + """Inspect ``python`` (default: this interpreter) for a working GDAL.""" + python = python or sys.executable + fields = {"python_executable": python} + + result = subprocess.run( + [python, "-c", _PROBE_SCRIPT], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + fields.update(json.loads(result.stdout.decode("utf8", "replace"))) + except ValueError: + # The probe did not even get to print: importing osgeo took the + # interpreter down with it. That is itself the finding. + fields["crashed"] = True + fields["bindings_error"] = ( + "the interpreter died while importing osgeo (exit %s). %s" + % (result.returncode, result.stderr.decode("utf8", "replace").strip()[:400]) + ) + + config = _gdal_config_info(sys_prefix=fields.get("sys_prefix")) + if config: + fields["gdal_config"] = config["path"] + fields["library_version"] = config["version"] + fields["prefix"] = config["prefix"] + + found = Probe(**fields) + found.problems = tuple(_diagnose(found, config)) + return found + + +def _diagnose(found, config): + """Turn a raw probe into a list of ``(code, message)`` problems.""" + problems = [] + + if found.bindings_version is None: + problems.append( + ( + "no-bindings", + "the osgeo Python bindings are not installed (%s)" + % (found.bindings_error or "import failed"), + ) + ) + + if config is None: + # Not a problem in itself when the bindings work -- a wheel or conda + # package can carry the library without shipping gdal-config -- but + # it is what blocks building bindings from an sdist. + if found.bindings_version is None: + problems.append( + ( + "no-library", + "no GDAL C library found: gdal-config is not on PATH", + ) + ) + else: + library = _parse_version(config["version"]) + if library and library[:2] < MIN_LIBRARY_VERSION: + problems.append( + ( + "old-library", + "GDAL %s is older than the %d.%d TopoAnalysis needs" + % (config["version"], MIN_LIBRARY_VERSION[0], MIN_LIBRARY_VERSION[1]), + ) + ) + + bindings = _parse_version(found.bindings_version) + if bindings and library and bindings[:2] != library[:2]: + problems.append( + ( + "version-mismatch", + "the bindings are %s but the C library is %s; they must " + "agree to at least major.minor" + % (found.bindings_version, config["version"]), + ) + ) + + mismatch = _architecture_mismatch(config["prefix"], found.machine) + if mismatch: + problems.append(("arch-mismatch", mismatch)) + + if found.bindings_version is not None and not found.has_gdal_array: + problems.append( + ( + "no-gdal-array", + "osgeo.gdal_array is missing, so Band.ReadAsArray() will fail " + "(%s). The bindings were built without numpy present." + % (found.gdal_array_error or "import failed"), + ) + ) + + if found.bindings_version is not None and not found.projection_works: + message = "looking up EPSG:4326 failed (%s)" % ( + found.projection_error or "unknown error" + ) + if found.proj_env: + message += ". These are set in the environment and override the " + message += "paths compiled into the library: " + ", ".join( + "%s=%s" % item for item in sorted(found.proj_env.items()) + ) + if found.proj_search_paths: + message += ". PROJ searched: " + ", ".join(found.proj_search_paths) + problems.append(("no-proj-data", message)) + + return problems + + +def _manifest_for(python_prefix): + """The GDAL manifest recorded inside ``python_prefix``, or None.""" + try: + with open(manifest_path(python_prefix)) as handle: + return json.load(handle) + except (OSError, ValueError): + return None + + +def _config_search_order(sys_prefix=None): + """Where to look for ``gdal-config``, most specific first. + + PATH is the *last* resort, not the first. ``--python`` can name an + interpreter in a different environment, and what matters is the library + that interpreter will load -- not whichever ``gdal-config`` happens to be + on the PATH of the shell this command was typed into. Getting that wrong + produces a confident and completely wrong version-mismatch diagnosis. + """ + candidates = [] + explicit = os.environ.get("GDAL_CONFIG") + if explicit: + candidates.append(explicit) + if sys_prefix: + manifest = _manifest_for(sys_prefix) + if manifest and manifest.get("prefix"): + candidates.append( + os.path.join(manifest["prefix"], "bin", "gdal-config") + ) + candidates.append(os.path.join(sys_prefix, "bin", "gdal-config")) + candidates.append("gdal-config") + return candidates + + +def _gdal_config_info(gdal_config=None, sys_prefix=None): + """Ask ``gdal-config`` about the installed C library, or return None.""" + if gdal_config: + candidates = [gdal_config] + else: + candidates = _config_search_order(sys_prefix) + + for program in candidates: + path = program if os.sep in program else shutil.which(program) + if not path or not os.path.exists(path): + continue + info = _interrogate_gdal_config(path) + if info is not None: + return info + return None + + +def _interrogate_gdal_config(path): + info = {"path": path} + for option in ("version", "prefix", "cflags", "libs"): + try: + result = subprocess.run( + [path, "--" + option], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + except OSError: + return None + if result.returncode != 0: + return None + info[option] = result.stdout.decode("utf8", "replace").strip() + return info if info.get("version") else None + + +def _parse_version(text): + """``'3.9.3'`` -> ``(3, 9, 3)``; None when it does not look like one.""" + if not text: + return None + match = re.match(r"(\d+)\.(\d+)(?:\.(\d+))?", str(text)) + if not match: + return None + return tuple(int(part) for part in match.groups(default="0")) + + +def _architecture_mismatch(prefix, machine=None): + """Describe an architecture clash between libgdal and the target Python. + + ``machine`` is the target interpreter's ``platform.machine()``, which is + not necessarily ours. Returns None when there is no clash, or when we + cannot tell -- an unknown answer must not block an install that would + have worked. + """ + if sys.platform not in ("darwin", "linux"): + return None + library = _find_library(prefix) + if library is None: + return None + + tool = shutil.which("file") + if tool is None: + return None + try: + result = subprocess.run( + [tool, "-L", library], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + except OSError: + return None + description = result.stdout.decode("utf8", "replace").lower() + + machine = (machine or platform.machine()).lower() + aliases = { + "x86_64": ("x86_64", "x86-64", "amd64"), + "amd64": ("x86_64", "x86-64", "amd64"), + "arm64": ("arm64", "aarch64"), + "aarch64": ("arm64", "aarch64"), + }.get(machine) + if aliases is None: + return None + if any(alias in description for alias in aliases): + return None + + # The library is a real binary of some architecture, just not ours. + for name, names in (("x86_64", ("x86_64", "x86-64")), ("arm64", ("arm64", "aarch64"))): + if any(alias in description for alias in names): + return ( + "the GDAL C library at %s is %s but this Python is %s; the " + "bindings cannot link against it. Install a GDAL built for " + "%s, or use a Python that matches the library." + % (library, name, machine, machine) + ) + return None + + +def _find_library(prefix): + if not prefix: + return None + suffixes = (".dylib",) if sys.platform == "darwin" else (".so",) + for libdir in ("lib", "lib64"): + directory = os.path.join(prefix, libdir) + if not os.path.isdir(directory): + continue + for entry in sorted(os.listdir(directory)): + if entry.startswith("libgdal") and any( + suffix in entry for suffix in suffixes + ): + return os.path.join(directory, entry) + return None + + +# --------------------------------------------------------------------------- +# Environments we might install into +# --------------------------------------------------------------------------- + + +def _conda_prefix(): + """The active conda environment's prefix, or None. + + ``conda-meta`` is the reliable marker: ``CONDA_PREFIX`` can be left over + from a shell that has since switched interpreters. + """ + if os.path.isdir(os.path.join(sys.prefix, "conda-meta")): + return sys.prefix + return None + + +def _conda_executable(): + """A conda-family program that can install into the current environment. + + ``CONDA_EXE`` comes first because ``conda`` itself is frequently only a + shell function, invisible to ``shutil.which``. mamba is preferred when + present purely because it is very much faster. + """ + for name in ("mamba", "micromamba"): + found = shutil.which(name) + if found: + return found + from_env = os.environ.get("CONDA_EXE") + if from_env and os.path.exists(from_env): + return from_env + return shutil.which("conda") + + +def default_source_prefix(): + """Per-user fallback prefix, used when there is no environment to use. + + :attr:`Probe.target_prefix` prefers the interpreter's own virtualenv or + conda environment; this is where it lands when there is neither. + """ + root = os.environ.get("XDG_DATA_HOME") or os.path.join( + os.path.expanduser("~"), ".local", "share" + ) + return os.path.join(root, "TopoAnalysis", "gdal") + + +#: System package managers, in the order we look for them. Each entry is +#: (program, install arguments, whether it needs administrator rights). +_SYSTEM_MANAGERS = ( + ("apt-get", ["install", "-y", "gdal-bin", "libgdal-dev"], True), + ("dnf", ["install", "-y", "gdal", "gdal-devel"], True), + ("yum", ["install", "-y", "gdal", "gdal-devel"], True), + ("zypper", ["install", "-y", "gdal", "libgdal-devel"], True), + ("pacman", ["-S", "--noconfirm", "gdal"], True), + ("apk", ["add", "--no-cache", "gdal", "gdal-dev"], True), + # Homebrew refuses to run as root, and does not need to. + ("brew", ["install", "gdal"], False), +) + + +def _system_manager(): + for program, arguments, needs_root in _SYSTEM_MANAGERS: + found = shutil.which(program) + if found: + return found, arguments, needs_root + return None + + +def _is_root(): + return hasattr(os, "geteuid") and os.geteuid() == 0 + + +# --------------------------------------------------------------------------- +# Plans +# --------------------------------------------------------------------------- + + +class Step(object): + """One thing an install does, describable without doing it.""" + + def __init__( + self, + summary, + command=None, + action=None, + env=None, + fallbacks=(), + needs_root=False, + minutes=1, + ): + self.summary = summary + self.command = command + self.action = action + self.env = env or {} + self.fallbacks = tuple(fallbacks) + self.needs_root = needs_root + self.minutes = minutes + + def __repr__(self): # pragma: no cover - debugging aid + return "" % self.summary + + +class Plan(object): + """An ordered list of :class:`Step` plus why it was chosen. + + ``blockers`` are reasons the plan cannot run at all. They are kept + separate from the steps so that ``plan`` can still show what *would* + happen -- which is the most useful thing to print when the answer is + "install these two tools first". + """ + + def __init__(self, strategy, steps=(), notes=(), blockers=()): + self.strategy = strategy + self.steps = tuple(steps) + self.notes = tuple(notes) + self.blockers = tuple(blockers) + + @property + def minutes(self): + return sum(step.minutes for step in self.steps) + + @property + def needs_root(self): + return any(step.needs_root for step in self.steps) + + def describe(self): + lines = ["strategy: %s" % self.strategy] + for note in self.notes: + lines.append(" note: %s" % note) + for blocker in self.blockers: + lines.append(" CANNOT RUN: %s" % blocker) + if not self.steps: + lines.append(" (nothing to do)") + return lines + lines.append("") + for number, step in enumerate(self.steps, 1): + lines.append(" %d. %s" % (number, step.summary)) + if step.command: + lines.append(" $ %s" % " ".join(step.command)) + lines.append("") + lines.append(" estimated time: about %d minute(s)" % max(1, self.minutes)) + return lines + + +def make_plan( + found=None, + strategy="auto", + python=None, + prefix=None, + allow_root=False, + jobs=None, +): + """Decide how to get a working GDAL, without doing anything yet. + + Parameters + ---------- + found : Probe, optional + A previous probe; taken fresh when omitted. + strategy : str + ``'auto'``, or one of ``'bindings'``, ``'conda'``, ``'wheel'``, + ``'system'``, ``'source'`` to force a route. + python : str, optional + Interpreter to install for; defaults to the current one. + prefix : str, optional + Where a source build should install; see + :func:`default_source_prefix`. + allow_root : bool + Permit steps that need administrator rights. Without it they are + still planned, so the user can see the command, but refused at + execution time. + jobs : int, optional + Parallel compile jobs for a source build. + """ + python = python or sys.executable + found = found if found is not None else probe(python) + notes = [] + + if strategy == "auto": + strategy, auto_notes = _choose_strategy(found, allow_root=allow_root) + notes.extend(auto_notes) + + if strategy == "none": + return Plan("none", (), ("GDAL is already installed and working.",)) + + if strategy == "proj-data": + return Plan( + "proj-data", + (), + ( + "GDAL and its bindings are fine; PROJ cannot read its " + "database. Nothing to install -- this is an environment " + "problem.", + ), + _proj_data_advice(found), + ) + + builder = { + "bindings": _plan_bindings, + "conda": _plan_conda, + "wheel": _plan_wheel, + "system": _plan_system, + "source": _plan_source, + }.get(strategy) + if builder is None: + raise ValueError("unknown strategy %r" % strategy) + + produced = builder( + found, python=python, prefix=prefix, allow_root=allow_root, jobs=jobs + ) + # Builders return (steps, notes), or (steps, notes, blockers) when they + # can tell in advance that this route cannot work on this machine. + steps, more_notes = produced[0], produced[1] + blockers = produced[2] if len(produced) > 2 else () + return Plan(strategy, steps, notes + list(more_notes), blockers) + + +def _proj_data_advice(found): + """What to actually do about a PROJ database that will not load.""" + advice = [] + if found is not None and found.proj_env: + advice.append( + "%s set in the environment, overriding the paths compiled into " + "the library. Activating a conda environment sets these, and they " + "then follow you into every other GDAL on the machine. Unset them " + "and try again: unset %s" + % ( + " and ".join(sorted(found.proj_env)), + " ".join(sorted(found.proj_env)), + ) + ) + else: + advice.append( + "PROJ's data files are missing from the installation itself. " + "Reinstall PROJ -- `conda install -c conda-forge proj` in a conda " + "environment, the system's proj-data package otherwise -- or " + "build GDAL with `topoanalysis-gdal install --strategy source`, " + "which installs a matching PROJ alongside it." + ) + return advice + + +def _choose_strategy(found, allow_root=False): + """Pick the cheapest route that stands a chance of working. + + ``allow_root`` matters here and not only at execution time: asking for it + is what makes the system package manager a usable route at all, and + without it a machine with apt would silently fall through to a half-hour + source build. + """ + notes = [] + + if found.ok: + return "none", notes + + # A pure numpy-bridge or PROJ-data failure is repaired by rebuilding the + # bindings against the library that is already here; no need to touch the + # C library at all. + # A PROJ data problem on its own is not fixed by installing anything: + # the library and bindings are fine, and something in the environment is + # pointing them at the wrong database. Reinstalling GDAL would leave + # that variable exactly where it was. + if set(found.codes) == {"no-proj-data"}: + return "proj-data", notes + + # ``no-proj-data`` *alongside* other problems is different: it usually + # just follows from bindings that are broken or mismatched, and goes away + # when they are rebuilt. + repairable = {"no-bindings", "no-gdal-array", "version-mismatch", "no-proj-data"} + wrong_architecture = found.has("arch-mismatch") + if wrong_architecture: + # Nothing installed on top of this library will fix it; say so plainly. + notes.append( + "the C library's architecture does not match this interpreter, so " + "the existing library cannot be reused" + ) + elif found.gdal_config and not found.has("old-library"): + if set(found.codes) <= repairable: + return "bindings", notes + + if found.is_conda and _conda_executable(): + notes.append("a conda environment is active, which is the most reliable route") + return "conda", notes + + if sys.platform == "win32": + notes.append( + "on Windows the only practical source of a GDAL C library is a " + "prebuilt wheel or conda" + ) + return "wheel", notes + + manager = _system_manager() + if manager is not None and found.has("old-library"): + # The too-old library almost certainly came from this very package + # manager -- a distro pinned to GDAL 2.x -- so asking it again just + # reinstalls the same version, and the bindings step would then + # faithfully build bindings for a library we have already rejected. + notes.append( + "%s is available but is where the too-old library came from; " + "a newer GDAL has to come from somewhere else" + % os.path.basename(manager[0]) + ) + elif manager is not None and wrong_architecture: + # The package manager is very often what produced the mismatch in the + # first place -- a Homebrew arm64 GDAL under a Rosetta x86_64 Python -- + # so asking it again would install the same wrong thing and loop. + notes.append( + "%s is available but would supply the same architecture again; " + "building from source is what actually resolves this" + % os.path.basename(manager[0]) + ) + elif manager is not None: + program, _, needs_root = manager + if not needs_root or _is_root() or allow_root: + return "system", notes + notes.append( + "%s could install the C library but needs administrator rights; " + "pass --allow-root to use it, or fall back to a source build" + % os.path.basename(program) + ) + + notes.append( + "no package manager can be used without help, so GDAL will be built " + "from source -- this needs only a compiler and CMake" + ) + return "source", notes + + +def _pip(python, *arguments): + return [python, "-m", "pip", "install"] + list(arguments) + + +def _bindings_requirements(library_version): + """Version specifiers to try for the bindings, best first. + + The bindings and the C library have to agree. An exact match is ideal; + failing that any patch release of the same major.minor will do, because + GDAL does not break its C API within a minor version. + """ + parsed = _parse_version(library_version) + if not parsed: + return ["gdal"] + major, minor, patch = parsed + return [ + "gdal==%d.%d.%d" % (major, minor, patch), + "gdal==%d.%d.*" % (major, minor), + ] + + +def _plan_bindings( + found, python, prefix=None, allow_root=False, jobs=None, config=None, force=None +): + """Build ``osgeo`` against a C library that is already installed. + + ``force`` re-installs even when the requested version is already + present. It has to default to on whenever we are *repairing* bindings, + because the commonest repair -- a missing ``gdal_array`` -- leaves an + ``osgeo`` of exactly the right version in place. Plain ``pip install + gdal==X.Y.Z`` would report "already satisfied" and change nothing, and + even with a reinstall pip would happily serve the broken wheel back out + of its own cache, since that wheel's name and version are correct. Only + ``--force-reinstall --no-cache-dir`` actually rebuilds. + """ + config = config or _gdal_config_info( + sys_prefix=found.sys_prefix if found is not None else None + ) + steps = [] + notes = [] + if config is None: + raise ValueError( + "the 'bindings' strategy needs a GDAL C library, but gdal-config " + "is not on PATH" + ) + + version = config["version"] + env = {} + # Make sure the gdal-config we chose is the one the sdist's setup.py + # finds, whatever else is on PATH. + env["PATH"] = os.path.dirname(config["path"]) + os.pathsep + os.environ.get("PATH", "") + libdir = _library_directory(config) + if libdir: + # Bake the library location into the extension so importing osgeo + # works without the user setting LD_LIBRARY_PATH first. + env["LDFLAGS"] = ( + "-L%s -Wl,-rpath,%s %s" % (libdir, libdir, os.environ.get("LDFLAGS", "")) + ).strip() + + steps.append( + Step( + "make sure the build tools and numpy are present", + command=_pip(python, "--upgrade", "setuptools", "wheel", "numpy"), + minutes=1, + ) + ) + + parsed = _parse_version(version) + if parsed and parsed[:2] < NUMPY_BUILD_REQUIRES_FROM: + notes.append( + "GDAL %s predates the sdist declaring numpy as a build dependency, " + "so the bindings are installed without build isolation -- otherwise " + "osgeo.gdal_array would be silently left out and ReadAsArray() " + "would fail at run time" % version + ) + + if force is None: + force = found is not None and found.bindings_version is not None + extra = ["--no-build-isolation"] + if force: + # --no-deps because step 1 has already put numpy in place, and + # --force-reinstall would otherwise tear down and rebuild the user's + # numpy as a side effect of repairing GDAL. + extra += ["--force-reinstall", "--no-cache-dir", "--no-deps"] + notes.append( + "the bindings are being rebuilt rather than merely requested: " + "this version pin can be satisfied by an install already present, " + "or by a wheel in pip's cache, that was built without numpy or " + "against a different C library" + ) + + requirements = _bindings_requirements(version) + steps.append( + Step( + "build the GDAL %s Python bindings against the installed C library" + % version, + command=_pip(python, *(extra + [requirements[0]])), + fallbacks=[ + _pip(python, *(extra + [requirement])) + for requirement in requirements[1:] + ], + env=env, + minutes=3, + ) + ) + return steps, notes + + +def _library_directory(config): + """Where libgdal actually lives, according to gdal-config. + + Taken from ``--libs`` rather than assumed to be ``/lib``: on + Fedora, RHEL and openSUSE a 64-bit library lands in ``lib64``, and an + RPATH pointing at an empty ``lib`` would leave the bindings unable to + find the library they were just built against. + """ + for token in (config.get("libs") or "").split(): + if token.startswith("-L"): + candidate = token[2:] + if candidate and os.path.isdir(candidate): + return candidate + # No usable -L: find the library itself rather than guess between lib + # and lib64, both of which commonly exist. + library = _find_library(config.get("prefix")) + return os.path.dirname(library) if library else None + + +def _plan_conda(found, python, prefix=None, allow_root=False, jobs=None): + executable = _conda_executable() + if executable is None: + raise ValueError("no conda, mamba or micromamba found") + target = (found.sys_prefix if found is not None else None) or _conda_prefix() or sys.prefix + command = [ + executable, + "install", + "-y", + "-p", + target, + "-c", + "conda-forge", + "gdal>=%d.%d" % MIN_LIBRARY_VERSION, + ] + notes = [ + "conda-forge ships the C library and the bindings as one package, " + "already matched, which removes both of the traps this module exists " + "to avoid" + ] + return [ + Step( + "install GDAL from conda-forge into %s" % target, + command=command, + minutes=3, + ) + ], notes + + +def _plan_wheel(found, python, prefix=None, allow_root=False, jobs=None): + """Windows: install a wheel that bundles the C library.""" + find_links = _windows_find_links() + notes = [ + "these wheels bundle the GDAL C library, so nothing else has to be " + "installed; they are built by Christoph Gohlke, who has maintained " + "the de-facto Windows builds of the scientific stack for years" + ] + return [ + Step( + "install prebuilt GDAL wheels for Windows", + command=_pip( + python, "--only-binary", ":all:", "--find-links", find_links, "gdal" + ), + minutes=2, + ) + ], notes + + +def _windows_find_links(): + """The newest cgohlke/geospatial-wheels release page, or a pinned one.""" + try: + import urllib.request + + with urllib.request.urlopen( + "https://api.github.com/repos/cgohlke/geospatial-wheels/releases/latest", + timeout=15, + ) as response: + tag = json.load(response)["tag_name"] + except Exception: + tag = CGOHLKE_FALLBACK_TAG + return ( + "https://github.com/cgohlke/geospatial-wheels/releases/expanded_assets/%s" % tag + ) + + +def _plan_system(found, python, prefix=None, allow_root=False, jobs=None): + manager = _system_manager() + if manager is None: + raise ValueError("no supported system package manager found") + program, arguments, needs_root = manager + + elevate = needs_root and not _is_root() + + def _command(arguments): + return (["sudo"] if elevate else []) + [program] + arguments + + steps = [] + notes = [] + if os.path.basename(program) == "apt-get": + # On a container image whose package lists were stripped -- which is + # every official Debian and Ubuntu image -- `apt-get install` fails + # outright with "Unable to locate package" until the lists are + # refreshed. + steps.append( + Step( + "refresh the apt package lists", + command=_command(["update"]), + needs_root=elevate, + minutes=1, + ) + ) + + steps.append( + Step( + "install the GDAL C library with %s" % os.path.basename(program), + command=_command(arguments), + needs_root=elevate, + minutes=3, + ) + ) + + # The bindings step cannot be planned in detail yet -- gdal-config does + # not exist until the step above has run -- so it is deferred and built + # when we get there. + steps.append( + Step( + "build the Python bindings against the newly installed C library", + action=_DeferredBindings(python), + minutes=3, + ) + ) + return steps, notes + + +def _plan_source(found, python, prefix=None, allow_root=False, jobs=None): + prefix = prefix or ( + found.target_prefix if found is not None else default_source_prefix() + ) + # So the plan quotes the versions the build will actually fetch. + _gdal_source.apply_environment_overrides() + missing = _gdal_source.missing_build_tools() + notes = [ + "GDAL will be built from pinned, checksummed tarballs and installed " + "into %s; nothing outside that prefix is modified" % prefix + ] + blockers = [] + if missing: + blockers.append( + "a source build needs %s, which %s not installed. %s" + % ( + " and ".join(missing), + "are" if len(missing) > 1 else "is", + _build_tool_hint(), + ) + ) + + steps = [] + for component in _gdal_source.components_needed(): + spec = _gdal_source.SOURCES[component] + steps.append( + Step( + "download and build %s %s" % (component, spec["version"]), + action=_SourceBuild(component, prefix, jobs), + minutes=spec["minutes"], + ) + ) + if "sqlite" not in _gdal_source.components_needed(): + notes.append("the system SQLite is usable, so it will not be rebuilt") + + steps.append( + Step( + "record the prefix so TopoAnalysis can find GDAL's data files", + action=_RecordPrefix( + prefix, found.sys_prefix if found is not None else None + ), + minutes=1, + ) + ) + steps.append( + Step( + "build the Python bindings against the GDAL we just built", + action=_DeferredBindings(python, prefix=prefix), + minutes=3, + ) + ) + return steps, notes, blockers + + +def _build_tool_hint(): + """How to get a compiler and CMake on this platform.""" + if sys.platform == "darwin": + return "Install them with: xcode-select --install && brew install cmake" + if sys.platform == "win32": + return ( + "Install the Visual Studio C++ build tools and CMake, or use " + "--strategy conda instead." + ) + manager = _system_manager() + if manager and os.path.basename(manager[0]) == "apt-get": + return "Install them with: sudo apt-get install build-essential cmake" + if manager and os.path.basename(manager[0]) in ("dnf", "yum"): + return "Install them with: sudo dnf install gcc-c++ make cmake" + return "Install a C/C++ compiler and CMake, or use --strategy conda." + + +class _SourceBuild(object): + """Deferred call into :mod:`TopoAnalysis._gdal_source`.""" + + def __init__(self, component, prefix, jobs): + self.component = component + self.prefix = prefix + self.jobs = jobs + + def __call__(self, log): + _gdal_source.build_one(self.component, self.prefix, jobs=self.jobs, log=log) + + +class _RecordPrefix(object): + """Note which environment's ``osgeo`` belongs to this GDAL prefix.""" + + def __init__(self, prefix, python_prefix=None): + self.prefix = prefix + self.python_prefix = python_prefix + + def __call__(self, log): + path = write_manifest(self.prefix, self.python_prefix) + log(" recorded %s" % path) + + +class _DeferredBindings(object): + """Plan and run the bindings step once the C library actually exists. + + ``system`` and ``source`` both install a C library as an earlier step, so + ``gdal-config`` cannot be interrogated when the plan is drawn up. + """ + + def __init__(self, python, prefix=None): + self.python = python + self.prefix = prefix + + def __call__(self, log): + search = None + if self.prefix: + search = os.path.join(self.prefix, "bin", "gdal-config") + if not os.path.exists(search): + raise RuntimeError( + "expected gdal-config at %s after the build, but it is not " + "there" % search + ) + config = _gdal_config_info(search) + if config is None: + raise RuntimeError( + "the C library was installed but gdal-config is still not on " + "PATH, so the bindings cannot be built" + ) + # The C library changed underneath whatever bindings may already be + # installed, so they must be rebuilt even if their version still + # nominally matches. + steps, notes = _plan_bindings(None, self.python, config=config, force=True) + for note in notes: + log(" note: %s" % note) + for step in steps: + log(" %s" % step.summary) + _run_step(step, log) + + +# --------------------------------------------------------------------------- +# Execution +# --------------------------------------------------------------------------- + + +def apply_plan(plan, allow_root=False, log=None, dry_run=False): + """Carry out ``plan``. Returns True when every step succeeded.""" + log = log or (lambda line: print(line, flush=True)) + + if dry_run: + for line in plan.describe(): + log(line) + return True + + if plan.blockers: + # Stopping here turns "cmake: command not found" three minutes into a + # build into a sentence that says what to install. + log("") + log("This route cannot run on this machine:") + for blocker in plan.blockers: + log(" %s" % blocker) + return False + + if plan.needs_root and not allow_root and not _is_root(): + log("") + log("This route needs administrator rights. Either run:") + for step in plan.steps: + if step.needs_root and step.command: + log(" %s" % " ".join(step.command)) + log("") + log("and then re-run this command, or pass --allow-root to let it use") + log("sudo directly, or choose --strategy source to build GDAL without") + log("any administrator rights at all.") + return False + + for number, step in enumerate(plan.steps, 1): + log("[%d/%d] %s" % (number, len(plan.steps), step.summary)) + _run_step(step, log) + return True + + +def _run_step(step, log): + if step.action is not None: + step.action(log) + return + + attempts = [step.command] + list(step.fallbacks) + environment = dict(os.environ) + environment.update(step.env) + + last = None + for index, command in enumerate(attempts): + log(" $ %s" % " ".join(command)) + result = subprocess.run(command, env=environment) + if result.returncode == 0: + return + last = result.returncode + if index + 1 < len(attempts): + log(" that failed (exit %d); trying %s" % (last, attempts[index + 1][-1])) + raise RuntimeError("%s failed (exit %s)" % (" ".join(attempts[0]), last)) + + +# --------------------------------------------------------------------------- +# Remembering a source-built prefix +# --------------------------------------------------------------------------- + + +MANIFEST_NAME = "gdal-prefix.json" + + +def manifest_path(python_prefix=None): + """Where the record of a TopoAnalysis-built GDAL lives. + + Preferably inside the environment the bindings were installed into, so + that two virtualenvs can each have their own build. The per-user cache + is the fallback for the case where that environment is not writable. + """ + if python_prefix: + return os.path.join(python_prefix, "share", "topoanalysis", MANIFEST_NAME) + root = os.environ.get("XDG_CACHE_HOME") or os.path.join( + os.path.expanduser("~"), ".cache" + ) + return os.path.join(root, "TopoAnalysis", MANIFEST_NAME) + + +def write_manifest(prefix, python_prefix=None): + """Record a GDAL prefix that TopoAnalysis built, and return the path. + + ``python_prefix`` is the environment whose ``osgeo`` was built against + ``prefix``. It is what makes :func:`activate_environment` safe: the data + files are only forced on an interpreter that is actually running the + library they belong to. + """ + python_prefix = python_prefix or sys.prefix + payload = { + "prefix": prefix, + "gdal_data": os.path.join(prefix, "share", "gdal"), + "proj_lib": os.path.join(prefix, "share", "proj"), + "gdal_version": _gdal_source.SOURCES["gdal"]["version"], + "python_prefix": python_prefix, + } + + # Inside the environment if we can, and only otherwise in the per-user + # cache. Writing both would leave a cache copy describing an environment + # it does not belong to, which is exactly the confusion the + # ``python_prefix`` field exists to prevent. + for candidate in (manifest_path(python_prefix), manifest_path()): + try: + os.makedirs(os.path.dirname(candidate), exist_ok=True) + with open(candidate, "w") as handle: + json.dump(payload, handle, indent=2) + except OSError: + continue + return candidate + raise RuntimeError( # pragma: no cover - both locations unwritable + "could not record the GDAL prefix in %s or %s" + % (manifest_path(python_prefix), manifest_path()) + ) + + +def read_manifest(): + """The manifest for this interpreter, or None.""" + for candidate in (manifest_path(sys.prefix), manifest_path()): + try: + with open(candidate) as handle: + return json.load(handle) + except (OSError, ValueError): + continue + return None + + +def activate_environment(): + """Point GDAL and PROJ at the data files belonging to the library in use. + + This exists because of a failure that is easy to hit and very hard to + read. Activating a conda environment exports ``GDAL_DATA`` and + ``PROJ_LIB``, and those variables outrank the paths compiled into any + *other* GDAL on the machine. A PROJ 9 library handed PROJ 8's + ``proj.db`` does not fall back gracefully; it refuses every coordinate + lookup with a message about ``DATABASE.LAYOUT.VERSION`` metadata that + names neither the environment variable nor the installation that set it. + + So these variables are *overridden*, not merely filled in when unset -- + a stale value is precisely the problem, and an unset one is comparatively + harmless. Two things keep that from being reckless: + + * it only happens when the manifest was written for this very + interpreter, which means its ``osgeo`` was built against the prefix + being pointed at, so these really are the matching data files; and + * ``TOPOANALYSIS_NO_GDAL_ENV=1`` turns it off entirely. + + Any failure here is swallowed: this must never be the reason an import + breaks. + """ + try: + if os.environ.get("TOPOANALYSIS_NO_GDAL_ENV"): + return + manifest = read_manifest() + if not manifest: + return + # A manifest from a different environment describes a library this + # interpreter is not running; its data files would be as wrong as + # the ones we are trying to displace. + recorded = manifest.get("python_prefix") + if recorded is not None and recorded != sys.prefix: + return + + proj_lib = manifest.get("proj_lib") + if proj_lib and os.path.isfile(os.path.join(proj_lib, "proj.db")): + # PROJ 9.1 renamed PROJ_LIB to PROJ_DATA and still honours both. + os.environ["PROJ_LIB"] = proj_lib + os.environ["PROJ_DATA"] = proj_lib + + gdal_data = manifest.get("gdal_data") + if gdal_data and os.path.isdir(gdal_data): + os.environ["GDAL_DATA"] = gdal_data + except Exception: # pragma: no cover - defensive by design + pass + + +# --------------------------------------------------------------------------- +# Convenience entry point +# --------------------------------------------------------------------------- + + +def ensure( + strategy="auto", + python=None, + prefix=None, + allow_root=False, + jobs=None, + dry_run=False, + log=None, +): + """Make GDAL work, and return the resulting :class:`Probe`. + + This is what :file:`install.py` calls. It is a no-op when GDAL already + works, so it is safe to call on every install. + """ + log = log or (lambda line: print(line, flush=True)) + python = python or sys.executable + + before = probe(python) + if before.ok: + log("GDAL %s is already installed and working." % before.bindings_version) + return before + + plan = make_plan( + before, + strategy=strategy, + python=python, + prefix=prefix, + allow_root=allow_root, + jobs=jobs, + ) + # apply_plan prints the plan itself on a dry run, so describing it here + # too would say everything twice. + if not dry_run: + for line in plan.describe(): + log(line) + log("") + + if not apply_plan(plan, allow_root=allow_root, log=log, dry_run=dry_run): + return before + if dry_run: + return before + + after = probe(python) + if after.ok: + log("") + log("GDAL %s is installed and working." % after.bindings_version) + return after + + +# --------------------------------------------------------------------------- +# Command line +# --------------------------------------------------------------------------- + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog="topoanalysis-gdal", + description=( + "Find, diagnose and install GDAL -- both the C library and the " + "Python bindings." + ), + ) + parser.add_argument( + "command", + nargs="?", + default="doctor", + choices=("doctor", "plan", "install"), + help=( + "doctor: report what is installed and what is wrong (default). " + "plan: show what an install would do. install: do it." + ), + ) + parser.add_argument( + "--strategy", + default="auto", + choices=("auto", "bindings", "conda", "wheel", "system", "source"), + help="force a particular route instead of choosing automatically", + ) + parser.add_argument( + "--prefix", + help="where a source build installs (default: the active virtualenv)", + ) + parser.add_argument( + "--python", + default=sys.executable, + help="install for this interpreter instead of the current one", + ) + parser.add_argument( + "--allow-root", + action="store_true", + help="permit steps that need sudo, instead of only printing them", + ) + parser.add_argument( + "--jobs", type=int, help="parallel compile jobs for a source build" + ) + arguments = parser.parse_args(argv) + + found = probe(arguments.python) + + if arguments.command == "doctor": + for line in found.describe(): + print(line) + if not found.ok: + print("") + print("Run 'topoanalysis-gdal plan' to see how this would be fixed.") + return 0 if found.ok else 1 + + try: + plan = make_plan( + found, + strategy=arguments.strategy, + python=arguments.python, + prefix=arguments.prefix, + allow_root=arguments.allow_root, + jobs=arguments.jobs, + ) + except ValueError as exc: + print("cannot plan an install: %s" % exc, file=sys.stderr) + return 2 + + if arguments.command == "plan": + for line in plan.describe(): + print(line) + return 0 + + try: + ok = apply_plan(plan, allow_root=arguments.allow_root) + except Exception as exc: + print("") + print("install failed: %s" % exc, file=sys.stderr) + return 1 + if not ok: + return 1 + + after = probe(arguments.python) + print("") + for line in after.describe(): + print(line) + return 0 if after.ok else 1 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/gui_tools.py b/gui_tools.py index b684d48..83a5721 100644 --- a/gui_tools.py +++ b/gui_tools.py @@ -1,16 +1,20 @@ -import matplotlib -matplotlib.use('TKAgg') +"""Interactive outlet picking. + +Needs an interactive matplotlib backend; if figures do not appear, select +one before importing, e.g. ``matplotlib.use('TkAgg')``. +""" + +import pickle as p -import TopoAnalysis.dem as d import matplotlib.pylab as plt import numpy as np -import pickle as p -import matplotlib.cm as cm -import sys -if sys.version_info[0] >= 3: - raw_input = input - +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d +except ImportError: # pragma: no cover + import dem as d + + def plot_dem(dem, hs): plt.close('all') @@ -33,8 +37,9 @@ def select_outlets(dem, fd, prefix, hs = None, outlet_filename = 'outlets.p', co print('\nClick on point upstream of outlet.') xy = plt.ginput(1)[0] xy_path = fd.search_down_flow_direction_from_xy_location(xy) - plt.plot(xy) - plt.plot(xy_path) + if not xy_path: + print('No flow path from that point; try again.') + continue xy_path_plot = list(zip(*xy_path)) path = plt.plot(xy_path_plot[0],xy_path_plot[1], color+'-') print('\nClick on the outlet location.') @@ -52,7 +57,7 @@ def select_outlets(dem, fd, prefix, hs = None, outlet_filename = 'outlets.p', co plt.plot(outlet_loc[0], outlet_loc[1], color+'o') path.pop(0).remove() - outlet_prefix = raw_input('Type a name for this outlet (leaving this blank will prevent outlet from being saved and will complete the selection process: ') + outlet_prefix = input('Type a name for this outlet (leaving this blank will prevent outlet from being saved and will complete the selection process: ') if outlet_prefix != '': import os.path diff --git a/install.py b/install.py new file mode 100644 index 0000000..9062a72 --- /dev/null +++ b/install.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""Install TopoAnalysis and everything it needs, from a fresh clone. + + git clone + cd TopoAnalysis + python3 install.py + +That is the whole procedure. The script creates an environment, installs the +dependencies, compiles the Priority-Flood kernels, installs GDAL -- the C +library *and* the Python bindings -- and then checks that the result actually +works by reading and writing a GeoTIFF. + +It uses nothing but the standard library, so it runs on a machine where +nothing has been set up yet. + +Why it makes a virtualenv by default +------------------------------------ +Because installing into a system Python usually fails, and when it does not +fail it is a bad idea. Debian, Ubuntu, Fedora and Homebrew all now mark their +Python installations "externally managed" (PEP 668), so ``pip install`` +refuses to touch them. Creating ``.venv`` sidesteps that, keeps this project +from disturbing anything else, and gives the from-source GDAL build somewhere +self-contained to live. If you are already inside a virtualenv or a conda +environment, that one is used instead and nothing new is created. + +Useful options +-------------- +``--no-gdal`` skip GDAL entirely; everything except file I/O works +``--gdal-strategy`` force a route: conda, system, source, bindings, wheel +``--allow-root`` let the system package manager use sudo +``--system`` install into the current interpreter, no virtualenv +``--jobs N`` parallel compile jobs +``--dry-run`` print what would happen and stop +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import sysconfig +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +MIN_PYTHON = (3, 8) + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + +_STEP = [0] + + +def heading(text): + _STEP[0] += 1 + print("") + print("=" * 72) + print("%d. %s" % (_STEP[0], text)) + print("=" * 72) + sys.stdout.flush() + + +def say(text=""): + print(text) + sys.stdout.flush() + + +def die(text): + print("") + print("ERROR: %s" % text, file=sys.stderr) + sys.exit(1) + + +def run(command, env=None, check=True): + say(" $ %s" % " ".join(command)) + result = subprocess.run(command, env=env) + if check and result.returncode != 0: + die("command failed (exit %d):\n %s" % (result.returncode, " ".join(command))) + return result.returncode + + +# --------------------------------------------------------------------------- +# Environment +# --------------------------------------------------------------------------- + + +def in_virtualenv(): + return sys.prefix != getattr(sys, "base_prefix", sys.prefix) + + +def in_conda(): + return os.path.isdir(os.path.join(sys.prefix, "conda-meta")) + + +def venv_python(directory): + """The interpreter inside a virtualenv directory.""" + if sys.platform == "win32": + return os.path.join(directory, "Scripts", "python.exe") + return os.path.join(directory, "bin", "python") + + +def venv_is_usable(python): + """True when a virtualenv's interpreter exists and has a working pip. + + Checking pip matters because a *partial* virtualenv is a real and common + thing. ``venv`` creates the interpreter before it bootstraps pip, and on + Debian and Ubuntu -- where pip's bootstrap lives in a separate + ``python3-venv`` package -- it fails in between, leaving a directory that + has ``bin/python`` and no way to install anything into it. + """ + if not os.path.exists(python): + return False + result = subprocess.run( + [python, "-c", "import pip"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + return result.returncode == 0 + + +def create_venv(directory): + """Make a virtualenv at ``directory``, reusing a usable one already there.""" + python = venv_python(directory) + if venv_is_usable(python): + say(" reusing the existing environment at %s" % directory) + return python + + existed = os.path.exists(directory) + if existed and os.path.exists(python): + die( + "there is a virtual environment in %s but it has no working pip,\n" + "so nothing can be installed into it. Remove it and re-run:\n" + " rm -rf %s\n" + "On Debian and Ubuntu the usual cause is a missing python3-venv:\n" + " sudo apt install python3-venv" % (directory, directory) + ) + + say(" creating a virtual environment in %s" % directory) + result = subprocess.run([sys.executable, "-m", "venv", directory]) + if result.returncode != 0 or not venv_is_usable(python): + # We made this directory a moment ago and it is not usable, so it is + # ours to clean up. Leaving the half-built shell behind would make + # the next run take the "reuse" path into the same dead end. + if not existed: + shutil.rmtree(directory, ignore_errors=True) + die( + "could not create a working virtual environment in %s.\n" + "On Debian and Ubuntu this is almost always a missing package:\n" + " sudo apt install python3-venv\n" + "Alternatively re-run with --system to install into this\n" + "interpreter instead." % directory + ) + return python + + +def externally_managed(): + """The PEP 668 marker path if this interpreter forbids pip installs. + + Debian, Ubuntu, Fedora and Homebrew all ship one. pip refuses to install + anything into such an interpreter, with an error that does not mention + this script at all -- so it is worth saying plainly up front. + """ + for key in ("stdlib", "platstdlib"): + directory = sysconfig.get_path(key) + if not directory: + continue + marker = os.path.join(directory, "EXTERNALLY-MANAGED") + if os.path.exists(marker): + return marker + return None + + +def activation_hint(directory): + if sys.platform == "win32": + return "%s\\Scripts\\activate" % directory + return "source %s/bin/activate" % directory + + +# --------------------------------------------------------------------------- +# The steps +# --------------------------------------------------------------------------- + + +def check_python(): + if sys.version_info[:2] < MIN_PYTHON: + die( + "TopoAnalysis needs Python %d.%d or newer; this is %s" + % (MIN_PYTHON[0], MIN_PYTHON[1], sys.version.split()[0]) + ) + say(" Python %s at %s" % (sys.version.split()[0], sys.executable)) + + +#: pip needs to understand PEP 660 to do an editable install of this project. +MIN_PIP = (21, 3) + + +def install_build_tools(python): + """setuptools, wheel, pybind11 and numpy, before anything else. + + pybind11 has to be present for the C++ kernels to compile, and numpy has + to be present before the GDAL bindings are built -- otherwise the numpy + bridge is silently skipped. Both are build-time requirements of things + installed later, so they go in first. + + pip itself is upgraded only when it is too old to do the job. This + script may be pointed at an environment the user uses for other work, and + replacing their pip as a side effect of installing one package is not + ours to do. + """ + version = pip_version(python) + if version is None or version < MIN_PIP: + say( + " pip %s is older than %d.%d; upgrading it" + % (".".join(map(str, version)) if version else "(unknown)", *MIN_PIP) + ) + run([python, "-m", "pip", "install", "--upgrade", "pip"]) + else: + say(" pip %s is recent enough; leaving it alone" % ".".join(map(str, version))) + + run([python, "-m", "pip", "install", "--upgrade", "setuptools", "wheel"]) + run([python, "-m", "pip", "install", "pybind11>=2.6", "numpy>=1.20"]) + + +def pip_version(python): + """``(major, minor)`` of the target interpreter's pip, or None.""" + result = subprocess.run( + [python, "-c", "import pip, sys; sys.stdout.write(pip.__version__)"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + return None + parts = result.stdout.decode("utf8", "replace").strip().split(".") + try: + return int(parts[0]), int(parts[1]) + except (IndexError, ValueError): + return None + + +def install_package(python, extras, editable): + target = "." if not extras else ".[%s]" % extras + command = [python, "-m", "pip", "install"] + if editable: + command.append("-e") + command.append(target) + run(command) + + +def install_gdal(python, strategy, prefix, allow_root, jobs, dry_run): + """Hand off to the GDAL bootstrapper, which lives with the package.""" + sys.path.insert(0, HERE) + try: + import gdal_setup + except ImportError as exc: # pragma: no cover - only a broken checkout + die("could not load the GDAL bootstrapper from %s: %s" % (HERE, exc)) + + try: + return gdal_setup.ensure( + strategy=strategy, + python=python, + prefix=prefix, + allow_root=allow_root, + jobs=jobs, + dry_run=dry_run, + log=say, + ) + except Exception as exc: + # A failed GDAL install is a partial failure, not a crash: everything + # except file I/O still works, and a traceback here would obscure the + # fact that the rest of the installation succeeded. + say("") + say("GDAL could not be installed: %s" % exc) + return None + + +_VERIFY = r""" +import sys, tempfile, os +import numpy as np +import TopoAnalysis +from TopoAnalysis import Elevation, FilledElevation, FlowDirectionD8, Area + +print(" TopoAnalysis %s" % TopoAnalysis.__version__) +print(" compute backend : %s" % TopoAnalysis.backend()) + +# A small DEM with a pit, so the fill has something to do. +z = np.array([[5.0, 4.0, 3.0, 4.0, 5.0], + [4.0, 3.0, 2.0, 3.0, 4.0], + [3.0, 2.0, 0.5, 2.0, 3.0], + [2.0, 1.0, 1.5, 1.0, 2.0], + [1.0, 0.0, 1.0, 0.0, 1.0]]) +dem = Elevation(dx=30.0, grid=z) +filled = FilledElevation(elevation=dem) +d8 = FlowDirectionD8(flooded_dem=filled) +area = Area(flow_direction=d8) +assert np.isfinite(area._griddata).any(), "flow accumulation produced nothing" +print(" in-memory pipeline : ok (max area %.0f m^2)" % np.nanmax(area._griddata)) + +available = TopoAnalysis.gdal_is_available() +print(" GDAL available : %s" % available) +if not available: + print(" (file I/O is unavailable; everything else works)") + sys.exit(0) + +from osgeo import gdal, osr +print(" GDAL version : %s" % gdal.__version__) + +# The real test: write a georeferenced GeoTIFF and read it back through +# TopoAnalysis. This exercises the C library, the numpy bridge and PROJ. +tmp = tempfile.mkdtemp() +path = os.path.join(tmp, "probe.tif") +try: + driver = gdal.GetDriverByName("GTiff") + ds = driver.Create(path, 5, 5, 1, gdal.GDT_Float32) + ds.SetGeoTransform((500000.0, 30.0, 0.0, 4000000.0, 0.0, -30.0)) + srs = osr.SpatialReference() + srs.ImportFromEPSG(32611) + ds.SetProjection(srs.ExportToWkt()) + ds.GetRasterBand(1).WriteArray(z.astype("float32")) + ds = None + + back = Elevation(gdal_filename=path) + assert back._georef_info.nx == 5 and back._georef_info.ny == 5, "wrong shape" + assert np.allclose(back._griddata, z), "values did not survive the round trip" + print(" GeoTIFF round trip : ok (EPSG:32611, %dx%d)" + % (back._georef_info.nx, back._georef_info.ny)) +finally: + import shutil as _sh + _sh.rmtree(tmp, ignore_errors=True) +""" + + +def verify(python): + """Prove the install works rather than assuming it does. + + Run from outside the source tree: importing from the checkout directory + would succeed even if the install had not, because the package directory + *is* the repository. Running elsewhere makes the check mean something. + """ + result = subprocess.run([python, "-c", _VERIFY], cwd=tempfile.gettempdir()) + return result.returncode == 0 + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Install TopoAnalysis and all of its dependencies, GDAL included.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--venv", + default=None, + help="virtualenv directory to create or reuse (default: .venv). " + "Giving this forces a virtualenv even from inside a conda " + "environment, which is otherwise used as-is.", + ) + parser.add_argument( + "--system", + action="store_true", + help="install into the current interpreter instead of a virtualenv", + ) + parser.add_argument( + "--break-system-packages", + action="store_true", + help="with --system, install into an externally managed (PEP 668) " + "Python anyway", + ) + parser.add_argument( + "--extras", + default="test", + help="optional dependency groups to install (default: test; 'all' " + "adds statsmodels, dask and rasterio; '' for none). GDAL is not an " + "extra -- this script installs it separately, C library and all.", + ) + parser.add_argument( + "--no-editable", + dest="editable", + action="store_false", + help="install a copy instead of linking to this checkout", + ) + parser.add_argument("--no-gdal", action="store_true", help="do not install GDAL") + parser.add_argument( + "--gdal-strategy", + default="auto", + choices=("auto", "bindings", "conda", "wheel", "system", "source"), + help="force a particular route for GDAL (default: auto)", + ) + parser.add_argument( + "--gdal-prefix", help="where a from-source GDAL build should install" + ) + parser.add_argument( + "--allow-root", + action="store_true", + help="permit system package manager steps that need sudo", + ) + parser.add_argument("--jobs", type=int, help="parallel compile jobs") + parser.add_argument( + "--dry-run", action="store_true", help="show what would happen, then stop" + ) + arguments = parser.parse_args(argv) + + say("TopoAnalysis installer") + say("running in %s" % HERE) + + if arguments.break_system_packages: + # Set in the environment rather than passed as a flag so that it also + # reaches the pip calls made later by the GDAL bootstrapper. + os.environ["PIP_BREAK_SYSTEM_PACKAGES"] = "1" + + # -- 1 ---------------------------------------------------------------- + heading("Checking the interpreter") + check_python() + + # -- 2 ---------------------------------------------------------------- + heading("Choosing an environment") + if arguments.system: + python = sys.executable + marker = externally_managed() + if marker and not arguments.break_system_packages: + die( + "this Python is marked externally managed (PEP 668), so pip\n" + "will refuse to install into it:\n" + " %s\n" + "It is managed by the operating system's package manager, and\n" + "installing over it can break system tools. Drop --system to\n" + "use a virtual environment instead, or pass\n" + "--break-system-packages if you really mean it." % marker + ) + say(" --system given: installing into %s" % python) + activate = None + elif not arguments.venv and (in_virtualenv() or in_conda()): + # An environment the user already chose and activated. Respect it -- + # but only when they did not name one explicitly, because --venv is + # how you ask for a separate environment from inside conda. + python = sys.executable + say( + " already inside %s; installing into it" + % ("a conda environment" if in_conda() else "a virtualenv") + ) + say(" %s" % sys.prefix) + activate = None + else: + directory = arguments.venv or os.path.join(HERE, ".venv") + if arguments.dry_run: + # A real run would reuse a usable environment that is already + # there, so a dry run has to report on that one -- describing the + # outer interpreter instead would be describing a different + # machine state than the one an install would act on. + existing = venv_python(directory) + if venv_is_usable(existing): + say(" would reuse the existing environment at %s" % directory) + python = existing + else: + say(" would create a virtualenv in %s" % directory) + python = sys.executable + else: + python = create_venv(directory) + activate = directory + + if arguments.dry_run: + say("") + say("--dry-run: stopping before anything is installed.") + if activate and python == sys.executable: + say("Note: that environment does not exist yet, so the GDAL report") + say("below describes %s instead. A fresh" % sys.executable) + say("virtualenv starts with no GDAL bindings at all, whatever it") + say("says here.") + say("") + say("The GDAL plan for this machine would be:") + sys.path.insert(0, HERE) + import gdal_setup + + found = gdal_setup.probe(python) + for line in found.describe(): + say(" " + line) + if not found.ok: + say("") + plan = gdal_setup.make_plan( + found, + strategy=arguments.gdal_strategy, + python=python, + prefix=arguments.gdal_prefix, + allow_root=arguments.allow_root, + jobs=arguments.jobs, + ) + for line in plan.describe(): + say(" " + line) + return 0 + + # -- 3 ---------------------------------------------------------------- + heading("Installing build tools") + install_build_tools(python) + + # -- 4 ---------------------------------------------------------------- + heading("Installing TopoAnalysis and its dependencies") + os.chdir(HERE) + install_package(python, arguments.extras, arguments.editable) + + # -- 5 ---------------------------------------------------------------- + gdal_ok = True + if arguments.no_gdal: + heading("Skipping GDAL (--no-gdal)") + say(" everything except reading and writing raster files will work") + else: + heading("Installing GDAL (C library and Python bindings)") + found = install_gdal( + python, + arguments.gdal_strategy, + arguments.gdal_prefix, + arguments.allow_root, + arguments.jobs, + dry_run=False, + ) + gdal_ok = found is not None and found.ok + if not gdal_ok: + say("") + if found is not None: + say("GDAL is still not working:") + for line in found.describe(): + say(" " + line) + say("") + say("TopoAnalysis itself is installed and everything except file") + say("I/O will work. To retry just the GDAL part:") + say(" %s -m TopoAnalysis.gdal_setup install --strategy source" % python) + + # -- 6 ---------------------------------------------------------------- + heading("Checking the installation") + ok = verify(python) + + say("") + say("=" * 72) + if ok and gdal_ok: + say("TopoAnalysis is installed.") + elif ok: + say("TopoAnalysis is installed, but GDAL is not -- see above.") + else: + say("TopoAnalysis is installed, but the check did not pass -- see above.") + say("=" * 72) + if activate: + say("") + say("Activate the environment with:") + say(" %s" % activation_hint(activate)) + say("") + say("Then try:") + say(" topoanalysis-process --info") + say(" topoanalysis-gdal doctor") + + # A GDAL that was asked for and did not arrive is a partial failure, and + # a script that called this needs to be able to tell. The message above + # makes clear that the rest of the installation is fine; --no-gdal is how + # you say you did not want it in the first place. + return 0 if (ok and gdal_ok) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/kernels.py b/kernels.py new file mode 100644 index 0000000..0edb6bf --- /dev/null +++ b/kernels.py @@ -0,0 +1,264 @@ +"""Backend dispatch for the TopoAnalysis grid kernels. + +Import the kernels from here rather than from :mod:`TopoAnalysis.fastops` or +the compiled extension directly:: + + from TopoAnalysis import kernels + kernels.priority_flood(z, mode="flat") + +If the C++ extension built successfully it is used; otherwise the pure-NumPy +implementations in :mod:`TopoAnalysis.fastops` take over. Both produce the +same numbers -- the test suite asserts it -- so the choice is purely about +speed. + +Set the environment variable ``TOPOANALYSIS_PURE_PYTHON=1`` to force the +NumPy path, which is useful when debugging or comparing the two. +""" + +from __future__ import annotations + +import importlib.machinery +import importlib.util +import os +import warnings + +try: # pragma: no cover - exercised by whichever import style is in use + from . import fastops +except ImportError: # pragma: no cover + import fastops + +__all__ = [ + "backend", + "have_extension", + "priority_flood", + "flow_directions", + "receivers", + "topological_order", + "accumulate", + "flow_length", + "propagate_along_main_stem", + "upstream_mask", + "chi", + "drainage_basins", + "downstream_distance", + "imposemin", + "stream_order", + "default_seeds", + "D8_DI", + "D8_DJ", + "D8_DIST", + "D8_CODES", + "PRIORITY_FLOOD_REFERENCE", +] + +PRIORITY_FLOOD_REFERENCE = ( + "Barnes, R., Lehman, C., Mulla, D. (2014). Priority-flood: An optimal " + "depression-filling and watershed-labeling algorithm for digital elevation " + "models. Computers & Geosciences 62, 117-127. " + "doi:10.1016/j.cageo.2013.04.024" +) + +# Geometry constants are always taken from the NumPy module: both backends use +# the same neighbour ordering, and having one definition avoids them drifting. +D8_DI = fastops.D8_DI +D8_DJ = fastops.D8_DJ +D8_DIST = fastops.D8_DIST +D8_CODES = fastops.D8_CODES + +_FORCE_PYTHON = os.environ.get("TOPOANALYSIS_PURE_PYTHON", "").strip() not in ("", "0", "false", "False") + +def _load_extension(): + """Import the compiled kernels, or return None if they were not built.""" + try: + from . import _topoanalysis # type: ignore[attr-defined] + return _topoanalysis + except ImportError: + pass + + # Flat import (`import kernels` with this directory on sys.path): load the + # extension from *this file's* directory by path. A bare + # `import _topoanalysis` would search all of sys.path and could pick up an + # unrelated -- or stale, or ABI-incompatible -- module of the same name + # left lying around somewhere else. + here = os.path.dirname(os.path.abspath(__file__)) + for suffix in importlib.machinery.EXTENSION_SUFFIXES: + candidate = os.path.join(here, "_topoanalysis" + suffix) + if not os.path.exists(candidate): + continue + spec = importlib.util.spec_from_file_location("_topoanalysis", candidate) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except ImportError: # pragma: no cover - a broken build + return None + return module + return None + + +_ext = None if _FORCE_PYTHON else _load_extension() + + +def have_extension() -> bool: + """True when the compiled kernels are in use.""" + return _ext is not None + + +def backend() -> str: + """``'c++'`` or ``'python'``.""" + return "c++" if _ext is not None else "python" + + +def warn_if_slow(n_cells: int, threshold: int = 250_000) -> None: + """Warn once when a large grid is about to be processed in pure Python.""" + if _ext is None and n_cells >= threshold: + warnings.warn( + "TopoAnalysis is using its pure-Python kernels on a grid of " + "{0:,} cells; this will be slow. Build the C++ extension with " + "`pip install -e .` from the TopoAnalysis directory for a large " + "speed-up.".format(n_cells), + RuntimeWarning, + stacklevel=3, + ) + + +default_seeds = fastops.default_seeds + + +def priority_flood( + elevations, + closed=None, + seeds=None, + mode="flat", + epsilon=0.0, + cellsize=1.0, + max_pit_depth=0.0, + track_visited=False, + reference_algorithm=False, +): + """Fill depressions in place; see :func:`TopoAnalysis.fastops.priority_flood`.""" + if _ext is None: + return fastops.priority_flood( + elevations, + closed=closed, + seeds=seeds, + mode=mode, + epsilon=epsilon, + cellsize=cellsize, + max_pit_depth=max_pit_depth, + track_visited=track_visited, + reference_algorithm=reference_algorithm, + ) + return _ext.priority_flood( + elevations, + closed, + seeds, + mode, + epsilon, + cellsize, + max_pit_depth, + track_visited, + reference_algorithm, + ) + + +def flow_directions(elevations, cellsize_grid=None, cellsize=1.0): + """Steepest-descent D8 codes; see :func:`TopoAnalysis.fastops.flow_directions`.""" + if _ext is None: + return fastops.flow_directions(elevations, cellsize_grid, cellsize) + return _ext.flow_directions(elevations, cellsize_grid, cellsize) + + +def receivers(codes): + """Receiver index per cell; see :func:`TopoAnalysis.fastops.receivers`.""" + if _ext is None: + return fastops.receivers(codes) + return _ext.receivers(codes) + + +def topological_order(recv): + """``(order, n_cells_on_cycles)``; see :func:`TopoAnalysis.fastops.topological_order`.""" + if _ext is None: + return fastops.topological_order(recv) + return _ext.topological_order(recv) + + +def accumulate(recv, order, weights, gate=None): + """Downstream accumulation; see :func:`TopoAnalysis.fastops.accumulate`.""" + if _ext is None: + return fastops.accumulate(recv, order, weights, gate) + return _ext.accumulate(recv, order, weights, gate) + + +def flow_length(recv, order, step_length): + """``(length, from_codes)``; see :func:`TopoAnalysis.fastops.flow_length`.""" + if _ext is None: + return fastops.flow_length(recv, order, step_length) + return _ext.flow_length(recv, order, step_length) + + +def propagate_along_main_stem(recv, order, main_stem_from_codes, gate, values, mode): + """See :func:`TopoAnalysis.fastops.propagate_along_main_stem`.""" + if _ext is None: + return fastops.propagate_along_main_stem( + recv, order, main_stem_from_codes, gate, values, mode + ) + return _ext.propagate_along_main_stem( + recv, order, main_stem_from_codes, gate, values, mode + ) + + +def upstream_mask(recv, outlets): + """See :func:`TopoAnalysis.fastops.upstream_mask`.""" + if _ext is None: + return fastops.upstream_mask(recv, outlets) + return _ext.upstream_mask(recv, outlets) + + +def chi( + recv, + order, + area, + step_length, + outlets, + A0, + theta, + trapezoid=False, + max_length=0.0, + mask=None, +): + """``(chi, distance)``; see :func:`TopoAnalysis.fastops.chi`.""" + if _ext is None: + return fastops.chi( + recv, order, area, step_length, outlets, A0, theta, trapezoid, max_length, mask + ) + return _ext.chi( + recv, order, area, step_length, outlets, A0, theta, trapezoid, max_length, mask + ) + + +def drainage_basins(recv, order, outlets=None, valid=None): + """See :func:`TopoAnalysis.fastops.drainage_basins`.""" + if _ext is None: + return fastops.drainage_basins(recv, order, outlets, valid) + return _ext.drainage_basins(recv, order, outlets, valid) + + +def downstream_distance(recv, order, step_length): + """See :func:`TopoAnalysis.fastops.downstream_distance`.""" + if _ext is None: + return fastops.downstream_distance(recv, order, step_length) + return _ext.downstream_distance(recv, order, step_length) + + +def imposemin(recv, order, step_length, sl, elevations): + """Carve in place; see :func:`TopoAnalysis.fastops.imposemin`.""" + if _ext is None: + return fastops.imposemin(recv, order, step_length, sl, elevations) + return _ext.imposemin(recv, order, step_length, sl, elevations) + + +def stream_order(recv, order, is_stream, kind="strahler"): + """See :func:`TopoAnalysis.fastops.stream_order`.""" + if _ext is None: + return fastops.stream_order(recv, order, is_stream, kind) + return _ext.stream_order(recv, order, is_stream, kind) diff --git a/plotting.py b/plotting.py index bf6cbd2..83d64fa 100644 --- a/plotting.py +++ b/plotting.py @@ -1,13 +1,21 @@ -from . import dem as d +"""Longitudinal- and chi-profile plotting helpers.""" + import matplotlib.pylab as plt import numpy as np +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d + from . import demRecursionTools +except ImportError: # pragma: no cover + import dem as d + import demRecursionTools + def plot_downstream_profile(elevation, flow_direction, outlet, plot_code, downstream = True, start_at = 0.0, mean_pixel_dimension = None, figure = None): rc = flow_direction.search_down_flow_direction(outlet) - rows = np.array(list(zip(*rc))[0], dtype = np.int) - cols = np.array(list(zip(*rc))[1], dtype = np.int) + rows = np.array(list(zip(*rc))[0], dtype=int) + cols = np.array(list(zip(*rc))[1], dtype=int) if downstream: direction = 1.0 @@ -40,10 +48,7 @@ def plot_downstream_profile(elevation, flow_direction, outlet, plot_code, downst def plot_recursive_upstream_profiles(elevation, flow_direction, area, outlet, plot_code, downstream = False, start_at = 0.0, figure = None, minimum_area = 1.0E6): def plot_ld_link(current_length, ld_list, plot_code, downstream_sign, minimum_area): - if len(ld_list['index']) == 1: - (current_row, current_column) = ld_list['index'][0] - else: - (current_row, current_column) = ld_list['index'] + (current_row, current_column) = ld_list['index'] if ld_list.get('next') is None: @@ -149,9 +154,7 @@ def interactive_chi_profiles_and_map_view(prefix, code, plot_code, dem, fd, area outlets = pickle.load(open('outlets.p', 'rb')) outlet = outlets[prefix][code] - from demRecursionTools import map_chi_profiles - - chi_map = map_chi_profiles(dem, fd, area, outlet, minimum_area = minimum_area, theta = theta, Ao = Ao) + chi_map = demRecursionTools.map_chi_profiles(dem, fd, area, outlet, minimum_area = minimum_area, theta = theta, Ao = Ao) indexes = chi_map.keys() import operator (chi, _) = list(zip(*operator.itemgetter(*indexes)(chi_map))) diff --git a/process_dem.py b/process_dem.py index f8f90ef..ac5e0d7 100644 --- a/process_dem.py +++ b/process_dem.py @@ -1,4 +1,7 @@ -import dem as d +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d +except ImportError: # pragma: no cover + import dem as d def process_dem(dem_name, EPSGcode, folder_name = '.'): @@ -39,8 +42,16 @@ def calc_ks_and_associated_grids(dem_name, Ao, theta, v, folder_name = '.', use_ else: mask = None - from denudationRateAnalysis import calculate_ks_for_sample - return calculate_ks_for_sample(v, d8, ksi, relief, area, Ao = Ao, mask = mask) + # denudationRateAnalysis is a separate, unpublished module; import it + # lazily so that the rest of process_dem works without it. + try: + from denudationRateAnalysis import calculate_ks_for_sample + except ImportError as exc: # pragma: no cover - external dependency + raise ImportError( + "calc_ks_and_associated_grids needs the separate " + "denudationRateAnalysis module, which is not part of TopoAnalysis." + ) from exc + return calculate_ks_for_sample(v, d8, ksi, relief, area, Ao=Ao, mask=mask) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f31cfdb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,119 @@ +[build-system] +requires = ["setuptools>=64", "wheel", "pybind11>=2.6"] +build-backend = "setuptools.build_meta" + +[project] +name = "TopoAnalysis" +version = "1.0.0" +description = "Digital elevation model analysis for tectonic geomorphology" +readme = "README.md" +requires-python = ">=3.8" +authors = [ + { name = "Sam Johnstone" }, + { name = "George Hilley" }, +] +keywords = [ + "geomorphology", + "digital elevation model", + "flow routing", + "priority flood", + "chi analysis", + "channel steepness", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: C++", + "Topic :: Scientific/Engineering :: GIS", +] + +dependencies = [ + "numpy>=1.20", + "scipy>=1.7", + "matplotlib>=3.3", +] + +[project.optional-dependencies] +# GDAL is deliberately not a hard dependency, and this extra will not work on +# its own. There is no GDAL wheel on PyPI at any version for any platform: +# every release is an sdist that compiles the bindings against a GDAL C +# library which must already exist on the machine. Declaring it here would +# therefore turn a missing C library into a failed install of TopoAnalysis +# itself, rather than the missing-file-I/O it actually is. +# +# Use one of these instead, both of which install the C library too: +# python install.py (from a clone -- does everything) +# topoanalysis-gdal install (once TopoAnalysis is installed) +# +# The extra remains for the case where the C library is already present and +# only the bindings are wanted. +gdal = ["GDAL>=3.0"] +stats = ["statsmodels>=0.12"] +parallel = ["dask[delayed]>=2021.0"] +quadrats = ["rasterio>=1.2"] +test = ["pytest>=7.0", "pytest-cov"] +# The docs are Markdown and are read as-is on the repository page; +# these are only needed to build them as a Sphinx site. +docs = ["sphinx>=5.0", "sphinx-rtd-theme", "myst-parser"] +# Deliberately without GDAL. Including it would make `pip install .[all]` +# fail on any machine without a GDAL C library already installed -- which is +# most of them -- and take the rest of the package down with it. GDAL is +# installed by `python install.py` or `topoanalysis-gdal install`, both of +# which supply the C library as well. +all = [ + "statsmodels>=0.12", + "dask[delayed]>=2021.0", + "rasterio>=1.2", +] + +[project.urls] +Homepage = "https://github.com/stanford-geomorph/TopoAnalysis" + +[project.scripts] +topoanalysis-process = "TopoAnalysis.cli:main" +topoanalysis-gdal = "TopoAnalysis.gdal_setup:main" + +# --------------------------------------------------------------------------- +# The repository root *is* the package directory, so the package is mapped +# explicitly rather than discovered. This keeps ``import TopoAnalysis.dem`` +# working for anyone who already has the parent directory on their path, while +# still producing a normal installable distribution. +# --------------------------------------------------------------------------- +[tool.setuptools] +include-package-data = true +packages = ["TopoAnalysis"] + +[tool.setuptools.package-dir] +TopoAnalysis = "." + +[tool.setuptools.package-data] +# The test suite ships with the package so an installed copy can be +# checked: `pytest --pyargs` needs a package, so point pytest at the +# directory instead -- `python -m pytest $(python -c "import +# TopoAnalysis, os; print(os.path.dirname(TopoAnalysis.__file__))")/tests`. +TopoAnalysis = ["_cpp/*.hpp", "_cpp/*.cpp", "tests/*.py", "conftest.py"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +# Import test modules by path rather than by dotted name. The checkout +# directory is itself a package, and under the default "prepend" mode pytest +# derives a module name from the directory's *name* -- which is not a valid +# identifier when the checkout is called TopoAnalysis-main (a GitHub zip) or +# anything else with a hyphen in it. +addopts = "--import-mode=importlib" +# install.py puts .venv inside this directory, because the directory is the +# package. Nothing good comes of pytest walking into it. +norecursedirs = [".venv", "venv", "build", "dist", "*.egg-info", ".git"] +filterwarnings = [ + "error::DeprecationWarning:TopoAnalysis.*", +] +markers = [ + "gdal: requires the GDAL Python bindings", + "slow: takes more than a second", + "parity: checks numerical agreement with TopoToolbox", +] + +[tool.coverage.run] +source = ["TopoAnalysis"] +omit = ["*/tests/*", "*/gui_tools.py"] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..42e45c2 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,16 @@ +-r requirements.txt + +# Building the C++ kernels. +pybind11>=2.6 +setuptools>=64 +wheel + +# Optional runtime extras that the test suite exercises. +statsmodels>=0.12 + +# Testing and docs. +pytest>=7.0 +pytest-cov +sphinx>=5.0 +sphinx-rtd-theme +myst-parser diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..67b5ac4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +# Runtime dependencies. +# +# GDAL is deliberately absent: it cannot be installed reliably from PyPI on +# every platform, and TopoAnalysis only needs it for file I/O. Install it +# from conda-forge (see environment.yml) or your system package manager. +numpy>=1.20 +scipy>=1.7 +matplotlib>=3.3 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b8b71d5 --- /dev/null +++ b/setup.py @@ -0,0 +1,71 @@ +"""Build script for TopoAnalysis. + +Almost all configuration lives in ``pyproject.toml``. This file exists only +to describe the optional C++ extension, which cannot be expressed +declaratively. + +The extension is marked ``optional``: if no C++ compiler is available, or the +build fails for any other reason, installation still succeeds and TopoAnalysis +falls back to the pure-NumPy kernels in ``TopoAnalysis.fastops``. Results are +identical either way; only the speed differs. +""" + +from __future__ import annotations + +import os +import sys + +from setuptools import Extension, setup + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _pybind11_include_dirs(): + """Locate pybind11's headers, tolerating its absence.""" + try: + import pybind11 + except ImportError: # pragma: no cover - only hit on a broken build env + return None + return [pybind11.get_include()] + + +def _extensions(): + include_dirs = _pybind11_include_dirs() + if include_dirs is None: + sys.stderr.write( + "TopoAnalysis: pybind11 not found; building without the compiled " + "kernels. Install pybind11 and reinstall for a large speed-up.\n" + ) + return [] + + if sys.platform == "win32": + extra_compile_args = ["/O2", "/std:c++17", "/EHsc"] + extra_link_args = [] + else: + extra_compile_args = ["-O3", "-std=c++17", "-funroll-loops"] + extra_link_args = [] + if sys.platform == "darwin": + # Structured bindings and std::optional need a modern libc++. + extra_compile_args.append("-mmacosx-version-min=10.14") + extra_link_args.append("-mmacosx-version-min=10.14") + + return [ + Extension( + "TopoAnalysis._topoanalysis", + sources=[os.path.join("_cpp", "module.cpp")], + # Without this a header-only edit would not trigger a rebuild. + depends=[ + os.path.join("_cpp", "priority_flood.hpp"), + os.path.join("_cpp", "flow_routing.hpp"), + ], + include_dirs=include_dirs + [os.path.join(HERE, "_cpp")], + language="c++", + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + # Never fail the install because the compiler is unhappy. + optional=True, + ) + ] + + +setup(ext_modules=_extensions()) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..bfed24c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,169 @@ +"""Shared fixtures: small synthetic landscapes with known answers. + +This file also makes ``import TopoAnalysis`` resolve when the suite is run +from a checkout rather than from an installed copy. It lives in ``tests/`` +rather than beside ``__init__.py`` deliberately: a ``conftest.py`` next to a +package's ``__init__.py`` makes pytest import that package by the checkout +directory's *name*, which fails whenever the directory is called anything +other than ``TopoAnalysis`` -- a GitHub zip unpacks to ``TopoAnalysis-main``. +""" + +from __future__ import annotations + +import importlib.util +import os +import sys + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_PACKAGE_DIR = os.path.dirname(_TESTS_DIR) +_PARENT = os.path.dirname(_PACKAGE_DIR) + + +def _register_package(): + """Make ``import TopoAnalysis`` resolve to this checkout.""" + try: + import TopoAnalysis # noqa: F401 + return + except ImportError: + pass + + if os.path.basename(_PACKAGE_DIR) == "TopoAnalysis": + if _PARENT not in sys.path: + sys.path.insert(0, _PARENT) + return + + # The checkout has some other name; load it under the one the package + # expects rather than asking the user to rename their directory. + spec = importlib.util.spec_from_file_location( + "TopoAnalysis", os.path.join(_PACKAGE_DIR, "__init__.py"), + submodule_search_locations=[_PACKAGE_DIR]) + module = importlib.util.module_from_spec(spec) + sys.modules["TopoAnalysis"] = module + spec.loader.exec_module(module) + + +_register_package() + +import numpy as np # noqa: E402 +import pytest # noqa: E402 + +from TopoAnalysis import dem, kernels # noqa: E402 + + +# --------------------------------------------------------------------------- +# Surfaces whose correct answers can be written down +# --------------------------------------------------------------------------- + + +@pytest.fixture +def east_plane(): + """A 5x5 plane dropping 1 m per 1 m eastward. + + Everything flows east, gradient is exactly 1, aspect is exactly 90. + """ + return dem.Elevation(dx=1.0, grid=np.tile(np.arange(5, 0, -1, dtype=float), (5, 1))) + + +@pytest.fixture +def south_plane(): + """A 5x5 plane dropping 1 m per 1 m southward (row-increasing).""" + return dem.Elevation(dx=1.0, grid=np.tile( + np.arange(5, 0, -1, dtype=float).reshape(-1, 1), (1, 5))) + + +@pytest.fixture +def single_pit(): + """A 5x5 bowl: a rim at 10, a ring at 4-5 and a 1 m pit in the middle. + + Filling raises every interior cell to the 10 m rim. + """ + return dem.Elevation(dx=1.0, grid=np.array([ + [10, 10, 10, 10, 10], + [10, 5, 4, 5, 10], + [10, 4, 1, 4, 10], + [10, 5, 4, 5, 10], + [10, 10, 10, 10, 10]], dtype=float)) + + +@pytest.fixture +def plateau(): + """A 5x5 grid with a flat plateau at 3 and one outlet cell at 2.""" + return dem.Elevation(dx=1.0, grid=np.array([ + [5, 5, 5, 5, 5], + [5, 3, 3, 3, 5], + [5, 3, 3, 3, 5], + [5, 3, 3, 2, 5], + [5, 5, 5, 5, 5]], dtype=float)) + + +@pytest.fixture +def v_valley(): + """A 7-column V-shaped valley draining south along the middle column. + + The channel is column 3; every row drains inward then down. + """ + ny, nx = 9, 7 + col = np.abs(np.arange(nx) - 3.0) + z = np.tile(col, (ny, 1)) * 2.0 + z += np.arange(ny, dtype=float).reshape(-1, 1)[::-1] * 0.5 + return dem.Elevation(dx=10.0, grid=z) + + +@pytest.fixture +def rough_dome(): + """A 40x40 noisy dome with real depressions. + + The noise amplitude is chosen so that the surface genuinely has closed + depressions -- a smoother dome makes every filling test vacuous, because + there is nothing to fill. ``test_dome_fixture_has_depressions`` pins + that down. + """ + rng = np.random.default_rng(20240921) + n = 40 + y, x = np.mgrid[0:n, 0:n] / n + z = (300.0 * (1.0 - ((x - 0.5) ** 2 + (y - 0.5) ** 2) / 0.5) + + 12.0 * np.sin(5 * np.pi * x) * np.sin(5 * np.pi * y) + + rng.normal(0.0, 6.0, (n, n))) + # A handful of deliberate pits, so the depressions are not an accident of + # the random seed. + for (i, j) in ((9, 12), (22, 8), (30, 27), (15, 31)): + z[i, j] -= 40.0 + return dem.Elevation(dx=30.0, grid=z) + + +@pytest.fixture +def dome_network(rough_dome): + """``(elevation, filled, flow_direction, area, flow_length)`` for the dome.""" + filled = dem.FilledElevation(elevation=rough_dome) + flow_direction = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=flow_direction) + flow_length = dem.FlowLength(flow_direction=flow_direction) + return rough_dome, filled, flow_direction, area, flow_length + + +@pytest.fixture +def geographic_grid(): + """A 10x10 lat/lon grid near 40 degrees north, 0.01 degree cells.""" + grid = dem.GeographicElevation(dx=0.01, grid=np.zeros((10, 10))) + grid._georef_info.geoTransform = (-120.0, 0.01, 0.0, 40.1, 0.0, -0.01) + grid._georef_info.xllcenter = -119.995 + grid._georef_info.yllcenter = 40.005 + return grid + + +# --------------------------------------------------------------------------- +# Environment markers +# --------------------------------------------------------------------------- + + +def pytest_runtest_setup(item): + """Skip GDAL-dependent tests when the bindings are missing.""" + if any(mark.name == 'gdal' for mark in item.iter_markers()): + if not dem.gdal_is_available(): + pytest.skip('the GDAL Python bindings are not installed') + + +@pytest.fixture +def backend_name(): + """Which kernel backend the test run is exercising.""" + return kernels.backend() diff --git a/tests/test_flow.py b/tests/test_flow.py new file mode 100644 index 0000000..40cc497 --- /dev/null +++ b/tests/test_flow.py @@ -0,0 +1,572 @@ +"""Tests for the flow-routing grid classes.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from TopoAnalysis import dem + + +# --------------------------------------------------------------------------- +# Filling +# --------------------------------------------------------------------------- + + +def test_filled_elevation_removes_the_pit(single_pit): + filled = dem.FilledElevation(elevation=single_pit) + assert np.all(filled._griddata >= single_pit._griddata - 1e-12) + # With the default aggradation slope the fill is just above the rim. + assert filled._griddata[2, 2] > 10.0 + assert filled._griddata[2, 2] == pytest.approx(10.0, abs=1e-6) + assert filled.fill_report['cells_filled'] == 9 + + +def test_flat_fill_option_matches_the_rim_exactly(single_pit): + filled = dem.FilledElevation(elevation=single_pit, aggradation_slope=0.0) + assert np.all(filled._griddata == 10.0) + + +def test_fill_restricted_to_a_mask(single_pit): + """Only masked cells take part; the rest keep their exact values.""" + mask = dem.Mask(dx=1.0, grid=np.zeros((5, 5), dtype=np.uint8)) + mask._griddata[1:4, 1:4] = 1 + masked = dem.FilledElevation(elevation=single_pit, mask=mask) + unmasked = dem.FilledElevation(elevation=single_pit) + + outside = mask._griddata == 0 + assert np.array_equal(masked._griddata[outside], single_pit._griddata[outside]) + # And the mask must actually change the answer. Confined to the 3x3 + # interior, the flood spills over that block's own rim of 4 rather than + # over the grid's 10 m rim. + assert not np.allclose(masked._griddata, unmasked._griddata) + assert masked._griddata[2, 2] == pytest.approx(4.0, abs=1e-6) + assert unmasked._griddata[2, 2] == pytest.approx(10.0, abs=1e-6) + + +def test_fill_from_explicit_outlets(single_pit): + outlet = single_pit._rowscols_to_xy(((2, 0),)) + filled = dem.FilledElevation(elevation=single_pit, outlets=outlet) + assert np.all(filled._griddata >= single_pit._griddata - 1e-12) + + +def test_clip_to_fill_marks_unreached_cells(): + """Cells the flood cannot reach become no-data.""" + # Two basins separated by a wall of NaN: flooding from the left outlet + # cannot reach the right half. + z = np.array([ + [5., 4., 3., np.nan, 3., 4., 5.], + [5., 4., 3., np.nan, 3., 4., 5.], + [5., 4., 3., np.nan, 3., 4., 5.]]) + grid = dem.Elevation(dx=1.0, grid=z) + outlet = grid._rowscols_to_xy(((1, 2),)) + + filled = dem.FilledElevation(elevation=grid, outlets=outlet, clip_to_fill=True) + assert np.isfinite(filled._griddata[:, :3]).all(), 'the left basin is reached' + assert np.isnan(filled._griddata[:, 4:]).all(), 'the right basin is not' + + +def test_clip_to_fill_excises_depressions_left_by_a_depth_limit(): + """A depression too deep to fill is marked no-data, as it always was.""" + ny, nx = 11, 21 + z = np.tile(np.linspace(100.0, 80.0, nx), (ny, 1)) + z[4:7, 9:12] = 50.0 # a 40 m pit + grid = dem.Elevation(dx=10.0, grid=z) + + clipped = dem.FilledElevation(elevation=grid, maximum_pit_depth=5.0, + clip_to_fill=True) + assert int(np.isnan(clipped._griddata).sum()) == 9 + assert np.isnan(clipped._griddata[4:7, 9:12]).all() + + # Without the limit the pit is filled and nothing is clipped. + filled = dem.FilledElevation(elevation=grid, clip_to_fill=True) + assert np.isfinite(filled._griddata).all() + assert filled._griddata[5, 10] > 80.0 + + +def test_findDEMedge_matches_the_flood_seeds(rough_dome): + rows, cols = rough_dome.findDEMedge() + n = rough_dome._georef_info.nx + assert set((rows * n + cols).tolist()) == set( + dem.kernels.default_seeds(rough_dome._griddata).tolist()) + + +def test_randomize_subbasins_with_mask_runs(single_pit): + """The method used to call a name that Python had mangled away.""" + filled = dem.FilledElevation(elevation=single_pit) + mask = dem.Mask(dx=1.0, grid=np.ones((5, 5), dtype=np.uint8)) + outlets = single_pit._rowscols_to_xy(((0, 0),)) + filled.randomize_subbasins_with_mask(mask, outlets) + assert filled._griddata.shape == (5, 5) + + +# --------------------------------------------------------------------------- +# Flow direction +# --------------------------------------------------------------------------- + + +def test_flow_direction_on_a_plane(east_plane): + filled = dem.FilledElevation(elevation=east_plane) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + # Everything flows east except the last column, which leaves the grid. + assert np.all(d8._griddata[:, :-1] == 1) + assert np.all(d8._griddata[:, -1] == 0) + + +def test_flow_direction_covers_the_last_row_and_column(rough_dome): + """The old windowed implementation left the last row and column unrouted. + + On a dome every cell drains outwards, so the only cells without a + direction are those on the perimeter whose steepest neighbour is off the + grid -- never a whole row or column. + """ + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + assert (d8._griddata[1:-1, 1:-1] == 0).sum() == 0 + # The last row and the last column are what the old code skipped + # entirely; each must now be routed except where flow leaves the grid. + for label, line in (('last row', d8._griddata[-1, :]), + ('last column', d8._griddata[:, -1]), + ('first row', d8._griddata[0, :]), + ('first column', d8._griddata[:, 0])): + assert (line != 0).any(), '{0} is entirely unrouted'.format(label) + + +def test_flow_direction_from_an_unfilled_elevation(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + assert np.all(d8._griddata[:, :-1] == 1) + + +def test_get_flow_to_cell_round_trips(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + receivers = d8.receivers + nx = d8._georef_info.nx + + for i in (5, 17, 30): + for j in (4, 22, 38): + row, col, good = d8.get_flow_to_cell(i, j) + if good: + assert receivers[i, j] == row * nx + col + else: + assert receivers[i, j] == -1 + + +def test_upstream_cells_are_the_inverse_of_downstream(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + for (i, j) in ((10, 10), (20, 25), (33, 7)): + for (ui, uj) in d8.get_upstream_cell_indexes(i, j): + assert d8.get_flow_to_cell(ui, uj)[:2] == (i, j) + + +def test_upstream_cell_set_is_closed(rough_dome): + """Everything upstream of a cell must itself drain to that cell.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + upstream = set(d8.get_indexes_of_upstream_cells(20, 20)) + assert (20, 20) in upstream + for (i, j) in upstream: + if (i, j) == (20, 20): + continue + row, col, good = d8.get_flow_to_cell(i, j) + assert good and (row, col) in upstream + + +def test_get_indexes_of_upstream_cells_can_be_iterated_twice(rough_dome): + """The result used to be a single-use zip object.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + indexes = d8.get_indexes_of_upstream_cells(20, 20) + assert len(list(indexes)) == len(list(indexes)) > 0 + + +def test_deep_basins_do_not_need_recursion(): + """A 4000-cell snake would have overflowed the recursive traversal.""" + n = 4000 + z = np.arange(n, 0, -1, dtype=float).reshape(1, n) + grid = dem.Elevation(dx=1.0, grid=z) + d8 = dem.FlowDirectionD8(elevation=grid) + assert len(d8.get_indexes_of_upstream_cells(0, n - 1)) == n + + +def test_pixel_scale_uses_the_true_diagonal(): + codes = np.array([[1, 2], [4, 32]], dtype=np.uint8) + d8 = dem.FlowDirectionD8(dx=1.0, grid=codes) + scale = d8.pixel_scale(np.float64) + assert scale[0, 0] == 1.0 and scale[1, 0] == 1.0 + assert scale[0, 1] == pytest.approx(np.sqrt(2.0)) + assert scale[1, 1] == pytest.approx(np.sqrt(2.0)) + + +def test_rivertools_conversion_rotates_the_codes(): + # RiverTools starts its numbering one position anticlockwise of ArcGIS. + codes = np.array([[1, 2, 4, 8], [16, 32, 64, 128]], dtype=np.uint8) + d8 = dem.FlowDirectionD8(dx=1.0, grid=codes.copy()) + d8.convert_rivertools_directions_to_arc() + assert d8._griddata.tolist() == [[128, 1, 2, 4], [8, 16, 32, 64]] + + +def test_search_down_flow_direction_follows_the_plane(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + start = east_plane._rowscols_to_xy(((2, 0),))[0] + cells, lengths = d8.search_down_flow_direction_with_length(start) + assert [c[1] for c in cells] == [0, 1, 2, 3, 4] + assert list(lengths) == pytest.approx([0.0, 1.0, 2.0, 3.0, 4.0]) + + +def test_divides_have_no_upstream_neighbours(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + divides = d8.divides() + + rows, cols = np.where(divides._griddata == 1) + for (i, j) in list(zip(rows.tolist(), cols.tolist()))[:50]: + assert d8.get_upstream_cell_indexes(i, j) == [] + + +def test_bounds_of_basin_for_outlet(rough_dome): + """Comparing a float with None used to raise on the first iteration.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + ((x, y),) = d8._rowscols_to_xy(((20, 20),)) + + (xr, yr) = d8.bounds_of_basin_for_outlet((y, x)) + assert xr[0] <= x <= xr[1] + assert yr[0] <= y <= yr[1] + + +def test_divides_for_outlets_returns_reusable_sequences(v_valley): + """Two adjoining basins must share a non-empty divide, listed twice over.""" + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + # The valley's outlet, and a point part-way up it: the second basin is + # nested inside the first, so the two adjoin along its whole rim. + a = d8._rowscols_to_xy(((8, 3),))[0] + b = d8._rowscols_to_xy(((0, 3),))[0] + + first, second = d8.divides_for_outlets(a, b) + assert len(first) > 0 and len(second) > 0, 'the basins must actually meet' + # Both must survive a second pass; they used to be single-use zips. + assert len(list(first)) == len(list(first)) == len(first) + assert len(list(second)) == len(list(second)) == len(second) + # Every reported cell belongs to the basin it was reported for. + in_a = d8.basin_mask((a,)) + in_b = d8.basin_mask((b,)) + assert all(in_a[i, j] for (i, j) in first) + assert all(in_b[i, j] for (i, j) in second) + + +# --------------------------------------------------------------------------- +# Accumulation and length +# --------------------------------------------------------------------------- + + +def test_area_conserves_the_total(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + + cell_area = rough_dome._georef_info.dx ** 2 + assert area._griddata.min() == pytest.approx(cell_area) + + outlets = d8.receivers < 0 + assert area._griddata[outlets].sum() == pytest.approx( + cell_area * rough_dome._griddata.size) + + +def test_area_on_a_plane_grows_downstream(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + area = dem.Area(flow_direction=d8) + # Each row is an independent flow line: 1, 2, 3, 4, 5 cells. + assert area._griddata[2].tolist() == [1.0, 2.0, 3.0, 4.0, 5.0] + + +def test_area_ignores_a_loaded_grids_missing_sort_order(rough_dome, tmp_path): + """Accumulation must not depend on an elevation sort being cached.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + expected = dem.Area(flow_direction=d8)._griddata + + # A flow-direction grid built straight from codes has no sort cache, so + # sort() would have ordered cells by their direction *code*. + fresh = dem.FlowDirectionD8(dx=rough_dome._georef_info.dx, + grid=d8._griddata.copy()) + assert np.allclose(dem.Area(flow_direction=fresh)._griddata, expected) + + +def test_area_gate_restricts_accumulation(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + gate = dem.Mask(dx=rough_dome._georef_info.dx, + grid=np.zeros(rough_dome._griddata.shape, dtype=np.uint8)) + gate._griddata[10:30, 10:30] = 1 + + gated = dem.Area(flow_direction=d8, evaluate_at=gate) + plain = dem.Area(flow_direction=d8) + + # A mask that used to be ignored entirely must now reduce the totals ... + assert gated._griddata.max() < plain._griddata.max() + # ... and no cell can accumulate more than the gate could deliver. + cell = rough_dome._georef_info.dx ** 2 + assert gated._griddata.max() <= cell * int(gate._griddata.sum()) + # Cells outside the gate contribute nothing of their own, so any value + # they hold came from inside it. + assert np.all(gated._griddata <= plain._griddata + 1e-9) + + +def test_areas_between_finds_channel_heads(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + + cell = rough_dome._georef_info.dx ** 2 + heads = area.areas_between(d8, 5 * cell, 20 * cell) + assert len(heads) > 0 + for (x, y) in heads: + ((i, j),) = area._xy_to_rowscols(((x, y),)) + assert 5 * cell <= area._griddata[i, j] <= 20 * cell + + +def test_flow_length_on_a_plane(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + length = dem.FlowLength(flow_direction=d8) + assert length._griddata[2].tolist() == [0.0, 1.0, 2.0, 3.0, 4.0] + + +def test_flow_length_main_stem_uses_the_shared_code_convention(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + length = dem.FlowLength(flow_direction=d8) + # Cell (2,2)'s longest path comes from the west, so the code stored + # there must be 16 (W) in the same convention FlowDirectionD8 uses. + assert length.main_stem_directions[2, 2] == 16 + + +def test_flow_length_path_walks_upstream(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + length = dem.FlowLength(flow_direction=d8) + outlet = east_plane._rowscols_to_xy(((2, 4),))[0] + path = length.indexes_along_flow_path_from_outlet(outlet) + assert [c[1] for c in path] == [4, 3, 2, 1, 0] + + +def test_is_along_flow_length_agrees_with_the_stored_codes(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + length = dem.FlowLength(flow_direction=d8) + + hits = 0 + for i in range(5, 35, 7): + for j in range(5, 35, 7): + row, col, good = d8.get_flow_to_cell(i, j) + if good and length.is_along_flow_length((i, j), (row, col)): + hits += 1 + assert hits > 0 + + +@pytest.mark.gdal +def test_flow_length_round_trip(rough_dome, tmp_path): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + length = dem.FlowLength(flow_direction=d8) + + path = str(tmp_path / 'length.tif') + length.save(path) + back = dem.FlowLength.load(path) + + assert np.allclose(back._griddata, length._griddata) + assert np.array_equal(back.main_stem_directions, length.main_stem_directions) + + +# --------------------------------------------------------------------------- +# Chi, relief and steepness +# --------------------------------------------------------------------------- + + +def test_chi_is_zero_at_the_outlet(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,)) + chi = dem.Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e4, outlets=outlet) + + assert chi._griddata[outlet_rc] == 0.0 + assert chi._griddata.max() > 0.0 + + +def test_chi_trapezoid_option(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,)) + + left = dem.Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e4, outlets=outlet) + trap = dem.Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e4, outlets=outlet, + trapezoid=True) + assert not np.allclose(left._griddata, trap._griddata) + assert trap._griddata.max() <= left._griddata.max() + 1e-9 + + +def test_ksi_uses_the_ratio_of_areas(v_valley): + """chi must integrate (Ao/A)^theta, not (Ao/(A-Ao))^theta.""" + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + length = dem.FlowLength(flow_direction=d8) + + Ao, theta = 200.0, 0.45 + ksi = dem.Ksi(area=area, flow_direction=d8, flow_length=length, + theta=theta, Ao=Ao) + + # Pin the integrand exactly. At a cell where the channel network + # *starts* -- nothing above it on the main stem is on the network -- chi + # is one step of (Ao/A)^theta * step_length and nothing more. The old + # form, (Ao/(A - Ao))^theta, is a different number and diverges as A + # approaches Ao. + step = d8.step_lengths() + a = area._griddata + on_network = a > Ao + + sources = on_network.copy() + codes = length.main_stem_directions + ny, nx = a.shape + for i in range(ny): + for j in range(nx): + code = codes[i, j] + if not code: + continue + for d in range(8): + if code == dem.kernels.D8_CODES[d]: + ui = i + int(dem.kernels.D8_DI[d]) + uj = j + int(dem.kernels.D8_DJ[d]) + if 0 <= ui < ny and 0 <= uj < nx and on_network[ui, uj]: + sources[i, j] = False + break + assert sources.any(), 'the test needs at least one network source' + + expected = (Ao / a[sources]) ** theta * step[sources] + assert np.allclose(ksi._griddata[sources], expected) + + buggy = (Ao / (a[sources] - Ao)) ** theta * step[sources] + assert not np.allclose(ksi._griddata[sources], buggy) + assert np.isfinite(ksi._griddata[on_network]).all() + + +def test_relief_measures_the_drop_to_the_path_head(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + length = dem.FlowLength(flow_direction=d8) + + relief = dem.Relief(flow_direction=d8, elevation=v_valley, flow_length=length) + assert relief._griddata.min() >= -1e-9 + # The outlet of the longest path sees the largest relief. + assert relief._griddata.max() > 0.0 + + +def test_scaled_relief_accepts_a_flooded_dem(v_valley): + """The flooded_dem branch called a method that did not exist.""" + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + length = dem.FlowLength(flow_direction=d8) + + scaled = dem.ScaledRelief(flow_direction=d8, flooded_dem=filled, + elevation=v_valley, flow_length=length, + Ao=1e4, theta=0.45) + assert np.isfinite(scaled._griddata).all() + + +def test_chi_scaled_relief(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,)) + + relief = dem.ChiScaledRelief(elevation=v_valley, flow_direction=d8, + theta=0.45, Ao=1e4, outlets=outlet) + assert relief._griddata[outlet_rc] == pytest.approx(0.0) + assert relief._griddata.max() > 0.0 + + +def test_cross_divide_dchi_runs(rough_dome): + """zip() objects are not subscriptable; this used to raise immediately.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlets = area.areas_greater_than(0.5 * area._griddata.max()) + chi = dem.Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e4, + outlets=outlets[:1]) + + result = dem.CrossDivideDChi(chi=chi, flow_direction=d8) + assert result._griddata.shape == rough_dome._griddata.shape + normalized = dem.NormalizedCrossDivideDChi(chi=chi, flow_direction=d8) + assert np.isfinite(normalized._griddata).all() + + +# --------------------------------------------------------------------------- +# Masks and basins +# --------------------------------------------------------------------------- + + +def test_mask_matches_the_upstream_set(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + outlet = d8._rowscols_to_xy(((25, 25),)) + + mask = dem.Mask(flow_direction=d8, outlets=outlet) + expected = set(d8.get_indexes_of_upstream_cells(25, 25)) + assert int(mask._griddata.sum()) == len(expected) + for (i, j) in list(expected)[:100]: + assert mask._griddata[i, j] == 1 + + +def test_channel_slope_is_positive_downhill(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + slope = dem.ChannelSlope(flow_direction=d8, elevation=east_plane) + assert np.all(slope._griddata[:, :-1] == pytest.approx(1.0)) + assert np.all(slope._griddata[:, -1] == 0.0) + + +def test_discrete_flow_accumulation_walks_downhill(east_plane): + # The walk needs a full 8-neighbourhood at every step, so it must start + # away from the grid edge. + outlet = east_plane._rowscols_to_xy(((2, 1),)) + dfa = dem.DiscreteFlowAccumulation(elevation=east_plane, outlets=outlet) + # The walk descends east, ties going to the first neighbour in the + # search order, so the path is diagonal; all that matters is that it + # advanced and that area accumulated along it. + visited = dfa._griddata > 0 + assert visited.sum() > 1 + assert dfa._griddata.max() > dfa._griddata[2, 1] + + +def test_restored_elevation_converges(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,)) + + restored = dem.RestoredElevation(flow_direction=d8, elevation=v_valley, + area=area, theta=0.45, ks=20.0, + outlets=outlet, iterations=2, verbose=False) + assert len(restored.convergence) == 2 + assert np.isfinite(restored._griddata).all() + + +def test_deflection_of_a_uniform_load(): + """A flat load must deflect uniformly, at the Airy value.""" + load = dem.Elevation(dx=1000.0, grid=np.full((32, 32), 1000.0)) + rho_c, rho_m, g = 2700.0, 3300.0, 9.81 + + w = dem.Deflection(elevation=load, D=1e23, rho_m=rho_m, rho_c=rho_c, g=g) + airy = -1000.0 * rho_c / (rho_m - rho_c) + assert np.allclose(w._griddata, airy, rtol=1e-6) diff --git a/tests/test_gdal_setup.py b/tests/test_gdal_setup.py new file mode 100644 index 0000000..cf745d1 --- /dev/null +++ b/tests/test_gdal_setup.py @@ -0,0 +1,688 @@ +"""Tests for the GDAL bootstrapper. + +Everything here is offline and installs nothing. The parts that would touch +the network or a package manager are exercised through :func:`make_plan`, +which decides what to do without doing any of it -- that separation is the +main reason the module is shaped the way it is. +""" + +from __future__ import annotations + +import io +import json +import os +import subprocess +import sys +import tarfile + +import pytest + +from TopoAnalysis import _gdal_source, gdal_setup + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _isolate_gdal_environment(): + """Undo every environment change these tests can cause. + + ``activate_environment`` assigns to ``os.environ`` directly, so + ``monkeypatch`` only restores the variables it was itself asked to touch. + ``PROJ_DATA`` in particular was being left behind pointing at a fixture's + deliberately-invalid ``proj.db``, which then broke unrelated tests -- + invisibly on GDAL 3.4, which ignores ``PROJ_DATA``, and visibly on 3.13, + which does not. + """ + names = ("PROJ_LIB", "PROJ_DATA", "GDAL_DATA", "TOPOANALYSIS_NO_GDAL_ENV") + saved = {name: os.environ.get(name) for name in names} + yield + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def make_probe(**fields): + """A Probe with the given raw fields and its problems worked out.""" + config = fields.pop("_config", None) + found = gdal_setup.Probe(**fields) + found.problems = tuple(gdal_setup._diagnose(found, config)) + return found + + +WORKING = { + "bindings_version": "3.9.3", + "has_gdal_array": True, + "projection_works": True, + "gdal_config": "/opt/env/bin/gdal-config", + "_config": {"version": "3.9.3", "prefix": "/nonexistent-prefix", "path": "/opt/env/bin/gdal-config"}, +} + + +# --------------------------------------------------------------------------- +# Version handling +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "text, expected", + [ + ("3.9.3", (3, 9, 3)), + ("3.4.1", (3, 4, 1)), + ("3.10", (3, 10, 0)), + ("3.13.3dev", (3, 13, 3)), + ("", None), + (None, None), + ("not a version", None), + ], +) +def test_parse_version(text, expected): + assert gdal_setup._parse_version(text) == expected + + +def test_bindings_requirements_prefer_an_exact_match(): + """The bindings must match the C library; exact first, then the series.""" + assert gdal_setup._bindings_requirements("3.4.1") == ["gdal==3.4.1", "gdal==3.4.*"] + + +def test_bindings_requirements_without_a_version(): + assert gdal_setup._bindings_requirements(None) == ["gdal"] + + +# --------------------------------------------------------------------------- +# Diagnosis +# --------------------------------------------------------------------------- + + +def test_a_working_install_has_no_problems(): + found = make_probe(**WORKING) + assert found.ok + assert found.problems == () + assert "GDAL is installed and working." in found.describe() + + +def test_missing_bindings_and_library_are_both_reported(): + found = make_probe(bindings_error="ModuleNotFoundError: osgeo", _config=None) + assert set(found.codes) == {"no-bindings", "no-library"} + + +def test_missing_numpy_bridge_is_caught(): + """The trap this module exists for: osgeo imports but ReadAsArray fails.""" + fields = dict(WORKING, has_gdal_array=False, gdal_array_error="ImportError: numpy") + found = make_probe(**fields) + assert found.has("no-gdal-array") + assert any("ReadAsArray" in message for _, message in found.problems) + + +def test_version_mismatch_between_bindings_and_library(): + fields = dict(WORKING, bindings_version="3.4.1") + found = make_probe(**fields) + assert found.has("version-mismatch") + + +def test_patch_level_differences_are_not_a_mismatch(): + """GDAL does not break its C API within a minor version.""" + fields = dict(WORKING, bindings_version="3.9.1") + found = make_probe(**fields) + assert not found.has("version-mismatch") + + +def test_broken_projection_lookup_is_reported(): + fields = dict(WORKING, projection_works=False, projection_error="proj.db missing") + found = make_probe(**fields) + assert found.has("no-proj-data") + + +def test_library_older_than_supported(): + fields = dict( + WORKING, + bindings_version="2.4.4", + _config={"version": "2.4.4", "prefix": "/nonexistent-prefix", "path": "/x/gdal-config"}, + ) + found = make_probe(**fields) + assert found.has("old-library") + + +def test_describe_survives_a_completely_empty_probe(): + found = make_probe() + assert any("not installed" in line for line in found.describe()) + + +# --------------------------------------------------------------------------- +# Choosing a strategy +# --------------------------------------------------------------------------- + + +def test_a_working_install_plans_nothing(): + plan = gdal_setup.make_plan(make_probe(**WORKING)) + assert plan.strategy == "none" + assert plan.steps == () + + +def test_an_existing_library_is_reused_rather_than_reinstalled(monkeypatch): + """Only the bindings are missing, so only the bindings get built.""" + monkeypatch.setattr( + gdal_setup, + "_gdal_config_info", + lambda *a, **k: { + "version": "3.9.3", + "prefix": "/opt/env", + "path": "/opt/env/bin/gdal-config", + }, + ) + found = make_probe( + bindings_error="ModuleNotFoundError: osgeo", + gdal_config="/opt/env/bin/gdal-config", + _config={"version": "3.9.3", "prefix": "/nonexistent-prefix", "path": "/opt/env/bin/gdal-config"}, + ) + plan = gdal_setup.make_plan(found) + assert plan.strategy == "bindings" + joined = " ".join(" ".join(step.command or []) for step in plan.steps) + assert "gdal==3.9.3" in joined + # The whole point: build against the runtime numpy, not an isolated one. + assert "--no-build-isolation" in joined + + +def test_an_architecture_mismatch_does_not_go_back_to_the_package_manager(monkeypatch): + """Homebrew supplied the wrong architecture; asking it again would loop.""" + # Windows short-circuits to the prebuilt-wheel route before any of this, + # so pin the platform rather than let the assertion depend on the runner. + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: None) + monkeypatch.setattr( + gdal_setup, "_system_manager", lambda: ("/opt/homebrew/bin/brew", ["install", "gdal"], False) + ) + found = make_probe(**WORKING) + found.problems = (("arch-mismatch", "library is arm64 but Python is x86_64"),) + strategy, notes = gdal_setup._choose_strategy(found) + assert strategy == "source" + assert any("same architecture" in note for note in notes) + + +def test_a_conda_environment_is_preferred_over_a_source_build(tmp_path, monkeypatch): + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: "/opt/conda/bin/mamba") + # conda-meta is what marks a prefix as a conda environment. + (tmp_path / "conda-meta").mkdir() + found = make_probe(bindings_error="no osgeo", sys_prefix=str(tmp_path)) + assert found.is_conda + strategy, _ = gdal_setup._choose_strategy(found) + assert strategy == "conda" + + +def test_root_is_not_assumed_for_system_packages(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: None) + monkeypatch.setattr( + gdal_setup, "_system_manager", lambda: ("/usr/bin/apt-get", ["install", "-y", "libgdal-dev"], True) + ) + monkeypatch.setattr(gdal_setup, "_is_root", lambda: False) + found = make_probe(bindings_error="no osgeo") + strategy, notes = gdal_setup._choose_strategy(found) + assert strategy == "source" + assert any("administrator rights" in note for note in notes) + + +def test_allow_root_makes_the_package_manager_usable(monkeypatch): + """Otherwise --allow-root would still fall through to a half-hour build.""" + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: None) + monkeypatch.setattr( + gdal_setup, + "_system_manager", + lambda: ("/usr/bin/apt-get", ["install", "-y", "libgdal-dev"], True), + ) + monkeypatch.setattr(gdal_setup, "_is_root", lambda: False) + found = make_probe(bindings_error="no osgeo") + strategy, _ = gdal_setup._choose_strategy(found, allow_root=True) + assert strategy == "system" + + +def test_windows_uses_a_prebuilt_wheel(monkeypatch): + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: None) + found = make_probe(bindings_error="no osgeo") + strategy, _ = gdal_setup._choose_strategy(found) + assert strategy == "wheel" + + +def test_apt_refreshes_its_package_lists_first(monkeypatch): + """A stock Debian image has no package lists; install alone would fail.""" + monkeypatch.setattr( + gdal_setup, + "_system_manager", + lambda: ("/usr/bin/apt-get", ["install", "-y", "libgdal-dev"], True), + ) + monkeypatch.setattr(gdal_setup, "_is_root", lambda: True) + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="system") + assert plan.steps[0].command == ["/usr/bin/apt-get", "update"] + assert "install" in plan.steps[1].command + + +def test_repairing_broken_bindings_forces_a_rebuild(monkeypatch): + """pip would otherwise say 'already satisfied' and change nothing. + + The commonest repair is a missing gdal_array, which leaves osgeo at + exactly the right version -- and pip's wheel cache holds the bad build. + """ + monkeypatch.setattr( + gdal_setup, + "_gdal_config_info", + lambda *a, **k: { + "version": "3.9.3", + "prefix": "/opt/env", + "path": "/opt/env/bin/gdal-config", + }, + ) + fields = dict(WORKING, has_gdal_array=False, gdal_array_error="ImportError: numpy") + found = make_probe(**fields) + plan = gdal_setup.make_plan(found, strategy="bindings") + command = plan.steps[-1].command + assert "--force-reinstall" in command + assert "--no-cache-dir" in command + + +def test_a_first_install_of_the_bindings_does_not_force(monkeypatch): + """Nothing to displace, so leave pip's cache alone.""" + monkeypatch.setattr( + gdal_setup, + "_gdal_config_info", + lambda *a, **k: { + "version": "3.9.3", + "prefix": "/opt/env", + "path": "/opt/env/bin/gdal-config", + }, + ) + found = make_probe( + bindings_error="ModuleNotFoundError: osgeo", + gdal_config="/opt/env/bin/gdal-config", + _config={ + "version": "3.9.3", + "prefix": "/nonexistent-prefix", + "path": "/opt/env/bin/gdal-config", + }, + ) + plan = gdal_setup.make_plan(found, strategy="bindings") + assert "--force-reinstall" not in plan.steps[-1].command + + +# --------------------------------------------------------------------------- +# Plans +# --------------------------------------------------------------------------- + + +def test_a_source_plan_builds_the_library_then_the_bindings(): + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="source", prefix="/tmp/example-prefix") + summaries = [step.summary for step in plan.steps] + assert any("proj" in text for text in summaries) + assert any("gdal" in text for text in summaries) + # The bindings are always last: they need the library to exist first. + assert "bindings" in summaries[-1] + assert "/tmp/example-prefix" in " ".join(plan.notes) + + +def test_a_source_plan_can_be_described_without_running_anything(): + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="source", prefix="/tmp/example-prefix") + text = "\n".join(plan.describe()) + assert "strategy: source" in text + assert "estimated time" in text + + +def test_a_source_plan_refuses_to_start_without_a_compiler(monkeypatch, capsys): + """Better a sentence naming the missing tool than cmake failing later.""" + monkeypatch.setattr( + _gdal_source, "missing_build_tools", lambda: ["cmake", "a C/C++ compiler"] + ) + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="source", prefix="/tmp/example-prefix") + assert plan.blockers + assert "cmake" in plan.blockers[0] + # The steps are still listed, so `plan` can show what would happen. + assert plan.steps + assert gdal_setup.apply_plan(plan) is False + assert "cannot run on this machine" in capsys.readouterr().out + + +def test_a_source_plan_has_no_blockers_when_the_tools_are_present(monkeypatch): + monkeypatch.setattr(_gdal_source, "missing_build_tools", lambda: []) + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="source", prefix="/tmp/example-prefix") + assert plan.blockers == () + + +def test_forcing_the_bindings_strategy_without_a_library_is_an_error(monkeypatch): + monkeypatch.setattr(gdal_setup, "_gdal_config_info", lambda *a, **k: None) + found = make_probe(bindings_error="no osgeo") + with pytest.raises(ValueError, match="gdal-config"): + gdal_setup.make_plan(found, strategy="bindings") + + +def test_a_system_plan_marks_the_step_that_needs_root(monkeypatch): + monkeypatch.setattr( + gdal_setup, "_system_manager", lambda: ("/usr/bin/apt-get", ["install", "-y", "libgdal-dev"], True) + ) + monkeypatch.setattr(gdal_setup, "_is_root", lambda: False) + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="system") + assert plan.needs_root + assert plan.steps[0].command[0] == "sudo" + + +def test_a_plan_needing_root_refuses_to_run_without_permission(capsys): + plan = gdal_setup.Plan( + "system", + [gdal_setup.Step("install things", command=["sudo", "apt-get", "install"], needs_root=True)], + ) + assert gdal_setup.apply_plan(plan, allow_root=False) is False + printed = capsys.readouterr().out + assert "sudo apt-get install" in printed + assert "--strategy source" in printed + + +def test_dry_run_executes_nothing(capsys): + def explode(log): + raise AssertionError("a dry run must not run anything") + + plan = gdal_setup.Plan("source", [gdal_setup.Step("build the world", action=explode)]) + assert gdal_setup.apply_plan(plan, dry_run=True) is True + assert "build the world" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- +# The prefix manifest +# --------------------------------------------------------------------------- + + +@pytest.fixture +def built_gdal(tmp_path, monkeypatch): + """A recorded source-built GDAL belonging to a fake environment.""" + prefix = tmp_path / "gdal" + (prefix / "share" / "proj").mkdir(parents=True) + (prefix / "share" / "proj" / "proj.db").write_bytes(b"not really a database") + (prefix / "share" / "gdal").mkdir(parents=True) + + environment = tmp_path / "env" + environment.mkdir() + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.delenv("TOPOANALYSIS_NO_GDAL_ENV", raising=False) + # activate_environment only acts for the interpreter the bindings were + # built for, so the test has to be that interpreter. + monkeypatch.setattr(sys, "prefix", str(environment)) + gdal_setup.write_manifest(str(prefix), str(environment)) + return prefix, environment + + +def test_manifest_round_trip(built_gdal): + prefix, environment = built_gdal + manifest = gdal_setup.read_manifest() + assert manifest["prefix"] == str(prefix) + assert manifest["python_prefix"] == str(environment) + assert manifest["proj_lib"].endswith(os.path.join("share", "proj")) + + +def test_the_manifest_is_written_inside_the_environment(built_gdal): + """So two virtualenvs can each have their own build.""" + _, environment = built_gdal + assert (environment / "share" / "topoanalysis" / "gdal-prefix.json").exists() + + +def test_activate_environment_sets_data_paths(built_gdal, monkeypatch): + prefix, _ = built_gdal + monkeypatch.delenv("PROJ_LIB", raising=False) + monkeypatch.delenv("GDAL_DATA", raising=False) + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == str(prefix / "share" / "proj") + assert os.environ["PROJ_DATA"] == str(prefix / "share" / "proj") + assert os.environ["GDAL_DATA"] == str(prefix / "share" / "gdal") + + +def test_activate_environment_displaces_a_stale_setting(built_gdal, monkeypatch): + """The case this exists for: conda exported another GDAL's data paths. + + A PROJ 9 library handed PROJ 8's proj.db refuses every lookup, so leaving + the inherited value in place is not the conservative choice -- it is the + broken one. + """ + prefix, _ = built_gdal + monkeypatch.setenv("PROJ_LIB", "/opt/conda/envs/other/share/proj") + monkeypatch.setenv("GDAL_DATA", "/opt/conda/envs/other/share/gdal") + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == str(prefix / "share" / "proj") + assert os.environ["GDAL_DATA"] == str(prefix / "share" / "gdal") + + +def test_activate_environment_can_be_switched_off(built_gdal, monkeypatch): + monkeypatch.setenv("TOPOANALYSIS_NO_GDAL_ENV", "1") + monkeypatch.setenv("PROJ_LIB", "/somewhere/the/user/chose") + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == "/somewhere/the/user/chose" + + +def test_a_manifest_from_another_environment_is_ignored(built_gdal, monkeypatch): + """Those data files belong to a library this interpreter is not running.""" + monkeypatch.setattr(sys, "prefix", "/some/entirely/different/env") + monkeypatch.setenv("PROJ_LIB", "/opt/conda/envs/other/share/proj") + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == "/opt/conda/envs/other/share/proj" + + +def test_activate_environment_ignores_a_prefix_without_a_database( + tmp_path, monkeypatch +): + """An empty share/proj is not a usable replacement, so leave things be.""" + prefix = tmp_path / "gdal" + (prefix / "share" / "proj").mkdir(parents=True) # no proj.db + environment = tmp_path / "env" + environment.mkdir() + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr(sys, "prefix", str(environment)) + gdal_setup.write_manifest(str(prefix), str(environment)) + monkeypatch.setenv("PROJ_LIB", "/opt/conda/envs/other/share/proj") + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == "/opt/conda/envs/other/share/proj" + + +def test_activate_environment_is_silent_without_a_manifest(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + monkeypatch.setattr(sys, "prefix", str(tmp_path / "nothing-here")) + gdal_setup.activate_environment() # must not raise + + +# --------------------------------------------------------------------------- +# Source pins +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("component", sorted(_gdal_source.SOURCES)) +def test_every_source_is_pinned_by_checksum_over_https(component): + spec = _gdal_source.SOURCES[component] + assert spec["url"].startswith("https://") + assert len(spec["sha256"]) == 64 + assert set(spec["sha256"]) <= set("0123456789abcdef") + + +@pytest.mark.parametrize("component", sorted(_gdal_source.SOURCES)) +def test_the_pinned_url_matches_the_pinned_version(component): + """A version bumped without its URL would download the old tarball.""" + spec = _gdal_source.SOURCES[component] + major, minor, patch = gdal_setup._parse_version(spec["version"]) + plain = "%d.%d.%d" % (major, minor, patch) + # SQLite encodes 3.53.4 in its filenames as 3530400. + encoded = "%d%02d%02d00" % (major, minor, patch) + assert plain in spec["url"] or encoded in spec["url"], spec["url"] + + +def test_library_directory_prefers_what_gdal_config_reports(tmp_path): + """On Fedora and RHEL a 64-bit libgdal lives in lib64, not lib.""" + lib64 = tmp_path / "lib64" + lib64.mkdir() + (tmp_path / "lib").mkdir() + suffix = ".dylib" if sys.platform == "darwin" else ".so" + (lib64 / ("libgdal" + suffix)).write_bytes(b"") + + assert gdal_setup._library_directory( + {"libs": "-L%s -lgdal" % lib64, "prefix": str(tmp_path)} + ) == str(lib64) + + +def test_library_directory_finds_the_library_when_there_is_no_minus_l(tmp_path): + """Guessing 'lib' would point the RPATH at an empty directory.""" + lib64 = tmp_path / "lib64" + lib64.mkdir() + (tmp_path / "lib").mkdir() # exists but holds nothing + suffix = ".dylib" if sys.platform == "darwin" else ".so" + (lib64 / ("libgdal" + suffix)).write_bytes(b"") + + assert gdal_setup._library_directory( + {"libs": "-lgdal", "prefix": str(tmp_path)} + ) == str(lib64) + + +def test_a_too_old_cmake_is_caught_before_the_build_starts(monkeypatch): + """PROJ 9 and GDAL 3.13 both declare a 3.16 floor.""" + monkeypatch.setattr(_gdal_source, "_cmake_version", lambda cmake: (3, 10)) + monkeypatch.setattr(_gdal_source, "_c_compiler", lambda: "/usr/bin/cc") + monkeypatch.setattr(_gdal_source, "_cxx_compiler", lambda: "/usr/bin/c++") + monkeypatch.setattr(_gdal_source.shutil, "which", lambda name: "/usr/bin/" + name) + missing = _gdal_source.missing_build_tools() + assert any("3.16" in item and "3.10" in item for item in missing) + + +def test_a_recent_enough_cmake_is_accepted(monkeypatch): + monkeypatch.setattr(_gdal_source, "_cmake_version", lambda cmake: (3, 16)) + monkeypatch.setattr(_gdal_source, "_c_compiler", lambda: "/usr/bin/cc") + monkeypatch.setattr(_gdal_source, "_cxx_compiler", lambda: "/usr/bin/c++") + monkeypatch.setattr(_gdal_source.shutil, "which", lambda name: "/usr/bin/" + name) + assert _gdal_source.missing_build_tools() == [] + + +def test_an_unreadable_cmake_version_does_not_block_the_build(monkeypatch): + """An unknown answer must not veto a build that would have worked.""" + monkeypatch.setattr(_gdal_source, "_cmake_version", lambda cmake: None) + monkeypatch.setattr(_gdal_source, "_c_compiler", lambda: "/usr/bin/cc") + monkeypatch.setattr(_gdal_source, "_cxx_compiler", lambda: "/usr/bin/c++") + monkeypatch.setattr(_gdal_source.shutil, "which", lambda name: "/usr/bin/" + name) + assert _gdal_source.missing_build_tools() == [] + + +def test_a_source_build_needs_a_cxx_compiler_not_just_a_c_one(monkeypatch): + """PROJ and GDAL are C++17; a bare gcc without g++ is a real machine.""" + monkeypatch.setattr(_gdal_source, "_cxx_compiler", lambda: None) + monkeypatch.setattr(_gdal_source, "_c_compiler", lambda: "/usr/bin/cc") + monkeypatch.setattr(_gdal_source.shutil, "which", lambda name: "/usr/bin/" + name) + assert "a C++ compiler" in _gdal_source.missing_build_tools() + + +def test_a_stale_proj_variable_is_diagnosed_not_reinstalled(): + """Reinstalling GDAL would leave the offending variable exactly where it is.""" + found = make_probe( + bindings_version="3.9.3", + has_gdal_array=True, + projection_works=False, + projection_error="RuntimeError: proj.db lacks DATABASE.LAYOUT.VERSION", + gdal_config="/opt/env/bin/gdal-config", + proj_env={"PROJ_LIB": "/opt/conda/envs/other/share/proj"}, + _config={ + "version": "3.9.3", + "prefix": "/nonexistent-prefix", + "path": "/opt/env/bin/gdal-config", + }, + ) + assert found.codes == ("no-proj-data",) + plan = gdal_setup.make_plan(found) + assert plan.strategy == "proj-data" + assert plan.steps == () + assert any("unset PROJ_LIB" in blocker for blocker in plan.blockers) + # The offending value is named, because that is the whole diagnosis. + assert any("/opt/conda/envs/other/share/proj" in m for _, m in found.problems) + + +def test_build_order_covers_every_source(): + assert set(_gdal_source.BUILD_ORDER) == set(_gdal_source.SOURCES) + + +def test_overriding_a_pin_requires_its_checksum(monkeypatch): + monkeypatch.setenv("TOPOANALYSIS_GDAL_SOURCE_GDAL_URL", "https://example.org/g.tar.gz") + monkeypatch.delenv("TOPOANALYSIS_GDAL_SOURCE_GDAL_SHA256", raising=False) + with pytest.raises(_gdal_source.BuildError, match="must be set together"): + _gdal_source.apply_environment_overrides() + + +def test_components_needed_is_a_subset_of_the_build_order(): + needed = _gdal_source.components_needed() + assert set(needed) <= set(_gdal_source.BUILD_ORDER) + # PROJ and GDAL are never optional. + assert "proj" in needed and "gdal" in needed + + +def test_extracting_a_hostile_archive_is_refused(tmp_path): + """A tarball that writes outside its directory must not be unpacked.""" + archive = tmp_path / "evil.tar.gz" + payload = io.BytesIO(b"pwned") + with tarfile.open(archive, "w:gz") as tar: + info = tarfile.TarInfo("../escaped.txt") + info.size = len(payload.getvalue()) + tar.addfile(info, payload) + + with pytest.raises((_gdal_source.BuildError, tarfile.TarError)): + _gdal_source._extract(str(archive), str(tmp_path / "into")) + assert not (tmp_path / "escaped.txt").exists() + + +def test_a_corrupt_download_is_rejected(tmp_path): + """A cached file with the wrong checksum is discarded, not built.""" + cache = tmp_path / "cache" + cache.mkdir() + (cache / "thing.tar.gz").write_bytes(b"not what was pinned") + with pytest.raises(_gdal_source.BuildError): + # The URL is unreachable, so the only way this could succeed is by + # trusting the bad cached copy. + _gdal_source._download( + "https://localhost:1/thing.tar.gz", "0" * 64, str(cache), lambda line: None + ) + + +# --------------------------------------------------------------------------- +# The command line +# --------------------------------------------------------------------------- + + +def test_doctor_reports_on_the_running_interpreter(capsys): + code = gdal_setup.main(["doctor"]) + printed = capsys.readouterr().out + assert "GDAL bindings" in printed + assert code in (0, 1) + + +def test_plan_prints_a_plan_without_installing(capsys): + code = gdal_setup.main(["plan", "--strategy", "source", "--prefix", "/tmp/x"]) + printed = capsys.readouterr().out + assert code == 0 + assert "strategy: source" in printed + + +def test_installer_dry_run_is_harmless(tmp_path): + """``install.py --dry-run`` must not create or modify anything.""" + import TopoAnalysis + + root = os.path.dirname(os.path.abspath(TopoAnalysis.__file__)) + script = os.path.join(root, "install.py") + if not os.path.exists(script): # an installed copy may not ship it + pytest.skip("install.py is not part of this installation") + + before = set(os.listdir(root)) + result = subprocess.run( + [sys.executable, script, "--dry-run"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=root, + ) + assert result.returncode == 0, result.stdout.decode("utf8", "replace") + assert "stopping before anything is installed" in result.stdout.decode("utf8", "replace") + assert set(os.listdir(root)) == before diff --git a/tests/test_grid.py b/tests/test_grid.py new file mode 100644 index 0000000..5949e1c --- /dev/null +++ b/tests/test_grid.py @@ -0,0 +1,402 @@ +"""Tests for :class:`TopoAnalysis.dem.BaseSpatialGrid` and its geometry.""" + +from __future__ import annotations + +import os + +import numpy as np +import pytest + +from TopoAnalysis import dem, error + + +# --------------------------------------------------------------------------- +# Construction and dispatch +# --------------------------------------------------------------------------- + + +def test_grid_from_array_sets_a_usable_geotransform(): + data = np.arange(12, dtype=float).reshape(3, 4) + grid = dem.BaseSpatialGrid(dx=2.0, grid=data) + + assert grid._georef_info.nx == 4 + assert grid._georef_info.ny == 3 + assert grid._georef_info.dx == 2.0 + # A grid built in memory must still be georeferenced well enough to save. + gt = grid._georef_info.geoTransform + assert gt[1] == 2.0 and gt[5] == -2.0 + assert gt[0] == pytest.approx(grid._georef_info.xllcenter - 1.0) + assert gt[3] == pytest.approx(grid._georef_info.yllcenter + 2.5 * 2.0) + + +def test_unsatisfiable_keywords_raise_a_helpful_error(): + with pytest.raises(error.InputError) as excinfo: + dem.BaseSpatialGrid(not_a_real_keyword=1) + # The message must list what the class does accept. + assert 'gdal_filename' in excinfo.value.msg + + +def test_random_grid_is_bounded_and_seedable(): + """Values lie in [0, 1); seeding NumPy makes the grid reproducible.""" + grid = dem.BaseSpatialGrid(nx=6, ny=4, dx=1.0) + assert grid._griddata.shape == (4, 6) + assert grid._griddata.min() >= 0.0 and grid._griddata.max() < 1.0 + # Two grids drawn in a row differ -- it is a random grid, after all ... + assert not np.array_equal(grid._griddata, + dem.BaseSpatialGrid(nx=6, ny=4, dx=1.0)._griddata) + # ... but reseeding NumPy's global generator reproduces one exactly. + np.random.seed(1234) + first = dem.BaseSpatialGrid(nx=6, ny=4, dx=1.0)._griddata.copy() + np.random.seed(1234) + second = dem.BaseSpatialGrid(nx=6, ny=4, dx=1.0)._griddata + assert np.array_equal(first, second) + + +# --------------------------------------------------------------------------- +# Coordinates +# --------------------------------------------------------------------------- + + +def test_xy_and_rowcol_round_trip(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((7, 5))) + for row in range(7): + for col in range(5): + ((x, y),) = grid._rowscols_to_xy(((row, col),)) + assert grid._xy_to_rowscols(((x, y),)) == ((row, col),) + + +def test_row_zero_is_the_north_edge(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + ((_, y_top),) = grid._rowscols_to_xy(((0, 0),)) + ((_, y_bottom),) = grid._rowscols_to_xy(((2, 0),)) + assert y_top > y_bottom + + +def test_points_outside_the_grid_are_rejected(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + info = grid._georef_info + # One whole cell beyond the eastern edge -- this used to come back as a + # valid subscript of nx, which then raised IndexError downstream. + outside = (info.xllcenter + info.nx * info.dx, info.yllcenter) + assert grid._xy_to_rowscols((outside,)) == ((None, None),) + + +def test_getitem_returns_none_outside_the_grid(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.ones((3, 3))) + assert grid[1, 1] == 1.0 + assert grid[-1, 0] is None + assert grid[3, 0] is None + assert grid[0, 3] is None + + +def test_setitem_ignores_out_of_range_writes(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + grid[3, 0] = 5.0 # one past the last row + grid[0, 3] = 5.0 # one past the last column + grid[-1, 0] = 5.0 + assert not grid._griddata.any() + + +def test_coordinate_vectors_have_one_entry_per_cell(): + grid = dem.BaseSpatialGrid(dx=2.0, grid=np.zeros((4, 6))) + x, y = grid.coordinate_vectors() + # np.arange(start, start + (n-1)*dx, dx) yields n-1 values, which is what + # get_XY_matricies used to return. + assert x.size == 6 and y.size == 4 + X, Y = grid.get_XY_matricies() + assert X.shape == (4, 6) and Y.shape == (4, 6) + assert grid.get_XY_matrices()[0].shape == (4, 6) + + +def test_extent_covers_the_pixel_edges(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((3, 4))) + xmin, xmax, ymin, ymax = grid.extent() + assert xmax - xmin == pytest.approx(4 * 10.0) + assert ymax - ymin == pytest.approx(3 * 10.0) + + +# --------------------------------------------------------------------------- +# Clipping, tiling, resampling +# --------------------------------------------------------------------------- + + +def test_clip_to_extent_keeps_the_requested_cells(): + data = np.arange(100, dtype=float).reshape(10, 10) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + ((x0, y0),) = grid._rowscols_to_xy(((7, 2),)) # south-west corner + ((x1, y1),) = grid._rowscols_to_xy(((3, 6),)) # north-east corner + clipped = grid.clip_to_extent((x0, x1, y0, y1)) + + assert clipped._griddata.shape == (5, 5) + assert np.array_equal(clipped._griddata, data[3:8, 2:7]) + # The clip must be georeferenced where it was cut from. + assert clipped._rowscols_to_xy(((0, 0),))[0] == pytest.approx((x0, y1)) + + +def test_clip_rejects_an_extent_that_misses_the_grid(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((5, 5))) + with pytest.raises(error.InputError): + grid.clip_to_extent((1000.0, 1010.0, 1000.0, 1010.0)) + + +def test_extent_of_data_ignores_nodata_margins(): + data = np.full((8, 10), np.nan) + data[2:5, 3:7] = 1.0 + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + xmin, xmax, ymin, ymax = grid.extent_of_data() + ((x_left, y_bottom),) = grid._rowscols_to_xy(((4, 3),)) + ((x_right, y_top),) = grid._rowscols_to_xy(((2, 6),)) + assert (xmin, xmax, ymin, ymax) == pytest.approx( + (x_left, x_right, y_bottom, y_top)) + + +def test_tile_then_mosaic_reproduces_the_grid(): + rng = np.random.default_rng(4) + data = rng.random((37, 41)) + grid = dem.BaseSpatialGrid(dx=5.0, grid=data) + + padding = 4 + tiles = grid.tile(tile_xdim=16, tile_ydim=16, + tile_xpadding=padding, tile_ypadding=padding) + unpadded = [t.remove_padding(xpadding=padding, ypadding=padding) for t in tiles] + rebuilt = dem.BaseSpatialGrid.mosaic(unpadded) + + assert rebuilt._griddata.shape == data.shape + assert np.allclose(rebuilt._griddata, data) + assert rebuilt._georef_info.xllcenter == pytest.approx(grid._georef_info.xllcenter) + assert rebuilt._georef_info.yllcenter == pytest.approx(grid._georef_info.yllcenter) + + +def test_resample_preserves_a_linear_ramp(): + """A plane must resample to the same plane at any spacing.""" + nx = ny = 21 + x = np.arange(nx) * 4.0 + data = np.tile(x, (ny, 1)) + grid = dem.BaseSpatialGrid(dx=4.0, grid=data) + + coarse = grid.resample(8.0, interpolation='linear') + assert coarse._georef_info.dx == 8.0 + assert coarse._griddata.shape[1] == pytest.approx(nx // 2, abs=1) + + # The value at each new cell centre must equal the plane's value there. + xs, _ = coarse.coordinate_vectors() + expected = np.tile(xs - grid._georef_info.xllcenter, (coarse._georef_info.ny, 1)) + assert np.allclose(coarse._griddata, expected, atol=1e-8) + + +def test_resample_rejects_an_unknown_interpolation(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((6, 6))) + with pytest.raises(error.InputError): + grid.resample(2.0, interpolation='bilinear-ish') + + +# --------------------------------------------------------------------------- +# Sorting and value lookup +# --------------------------------------------------------------------------- + + +def test_sort_orders_cells_by_value(): + data = np.array([[3.0, 1.0], [4.0, 2.0]]) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + ascending = grid.sort(reverse=False) + assert data.reshape(-1)[ascending].tolist() == [1.0, 2.0, 3.0, 4.0] + descending = grid.sort(reverse=True) + assert data.reshape(-1)[descending].tolist() == [4.0, 3.0, 2.0, 1.0] + + +def test_sort_with_a_mask_keeps_only_masked_cells(): + data = np.array([[3.0, 1.0], [4.0, 2.0]]) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + mask = dem.BaseSpatialGrid(dx=1.0, grid=np.array([[1.0, 0.0], [1.0, 0.0]])) + + indices = grid.sort(reverse=False, force=True, mask=mask) + # Only the left column (values 3 and 4) survives, in ascending order. + assert data.reshape(-1)[indices].tolist() == [3.0, 4.0] + + +def test_find_nearest_cell_searches_the_full_radius(): + data = np.zeros((9, 9)) + data[4, 8] = 10.0 # four cells east of the centre + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + # range(i-r, i+r) stopped one short of the radius, so the far cell was + # invisible at exactly r = 4. + assert grid.find_nearest_cell_with_greatest_value((4, 4), pixel_radius=4) == (4, 8) + + +def test_find_nearest_cell_does_not_wrap_around_the_grid(): + data = np.zeros((5, 5)) + data[0, 0] = 1.0 + data[4, 4] = 99.0 # would be reached by a negative index + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + assert grid.find_nearest_cell_with_greatest_value((0, 0), pixel_radius=2) == (0, 0) + + +def test_find_nearest_cell_returns_none_when_there_is_no_data(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.full((5, 5), np.nan)) + assert grid.find_nearest_cell_with_greatest_value((2, 2), 2) == (None, None) + + +def test_set_value_at_rowscols_handles_an_empty_list(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + grid.set_value_at_rowscols(1.0, []) + assert not grid._griddata.any() + grid.set_value_at_rowscols(1.0, [(0, 0), (2, 2)]) + assert grid._griddata[0, 0] == 1.0 and grid._griddata[2, 2] == 1.0 + + +def test_value_grid_set_value_at_indexes(): + grid = dem.ValueGrid(dx=1.0, grid=np.zeros((3, 3))) + # zip() is a lazy iterator in Python 3; using one as a subscript raised. + grid.set_value_at_indexes([(0, 1), (2, 2)], 7.0) + assert grid._griddata[0, 1] == 7.0 and grid._griddata[2, 2] == 7.0 + + +# --------------------------------------------------------------------------- +# Smoothing and derivatives +# --------------------------------------------------------------------------- + + +def test_average_over_distance_preserves_a_constant_field(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.full((21, 21), 5.0)) + averaged = grid.average_over_distance(3.0) + assert averaged.shape == (21, 21) + assert np.allclose(averaged, 5.0) + + +@pytest.mark.parametrize('shape', [(21, 21), (20, 20), (20, 25)]) +def test_average_over_distance_is_centred(shape): + """A point source must smear symmetrically about its own cell.""" + data = np.zeros(shape) + centre = (shape[0] // 2, shape[1] // 2) + data[centre] = 1.0 + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + averaged = grid.average_over_distance(2.0) + + # The answer is the normalised disc, centred on the source cell. An + # off-by-half-a-cell kernel would place it one cell away. + ny, nx = shape + di = np.abs(np.arange(ny) - centre[0]) + dj = np.abs(np.arange(nx) - centre[1]) + di = np.minimum(di, ny - di) + dj = np.minimum(dj, nx - dj) + inside = np.hypot(di[:, None], dj[None, :]) <= 2.0 + expected = inside / inside.sum() + + assert np.allclose(averaged, expected, atol=1e-12) + assert averaged.sum() == pytest.approx(1.0) + + +def test_average_over_distance_rejects_a_radius_that_selects_nothing(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((5, 5))) + with pytest.raises(error.InputError): + grid.average_over_distance(-1.0) + + +def test_gradient_over_length_scale_on_a_plane(): + """A plane of slope 0.5 must give 0.5 whatever the measuring length.""" + nx = ny = 21 + data = np.tile(np.arange(nx) * 0.5, (ny, 1)) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + for length in (1.0, 3.0, 5.0): + sx, sy = grid.calculate_gradient_over_length_scale(length) + n = int(np.ceil(length)) + assert np.allclose(sx[n:-n, n:-n], 0.5) + assert np.allclose(sy[n:-n, n:-n], 0.0, atol=1e-12) + assert np.isnan(sx[0, 0]) + + +def test_laplacian_over_length_scale_on_a_quadratic(): + """d2/dx2 of x^2 is 2, at any measuring length.""" + nx = ny = 25 + x = np.arange(nx, dtype=float) + data = np.tile(x ** 2, (ny, 1)) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + for length in (1.0, 2.0, 4.0): + curvature = grid.calculate_laplacian_over_length_scale(length) + n = int(np.ceil(length)) + assert np.allclose(curvature[n:-n, n:-n], 2.0) + + +def test_length_scale_larger_than_the_grid_is_rejected(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((5, 5))) + with pytest.raises(error.InputError): + grid.calculate_gradient_over_length_scale(10.0) + + +def test_principal_curvatures_of_a_sphere_cap(): + """Both principal curvatures of a paraboloid of revolution are equal.""" + n = 41 + y, x = np.mgrid[-20:21, -20:21].astype(float) + data = -0.001 * (x ** 2 + y ** 2) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + k1, k2 = grid.principal_curvatures() + centre = (slice(15, 26), slice(15, 26)) + assert np.allclose(k1._griddata[centre], k2._griddata[centre], atol=2e-4) + assert np.all(np.isfinite(k1._griddata[centre])) + + +def test_assign_bcs_replicates_every_corner(): + grid = dem.Elevation(dx=1.0, grid=np.arange(9, dtype=float).reshape(3, 3)) + padded = grid.assignBCs(grid._griddata, 3, 3) + assert padded[0, 0] == grid._griddata[0, 0] + assert padded[0, -1] == grid._griddata[0, -1] + assert padded[-1, 0] == grid._griddata[-1, 0] + # The south-east corner used to copy the south-west one. + assert padded[-1, -1] == grid._griddata[-1, -1] + + +# --------------------------------------------------------------------------- +# File round-trips +# --------------------------------------------------------------------------- + + +@pytest.mark.gdal +def test_geotiff_round_trip(tmp_path): + rng = np.random.default_rng(9) + data = rng.random((11, 13)) * 100.0 + data[3, 4] = np.nan + grid = dem.Elevation(dx=25.0, grid=data) + + path = str(tmp_path / 'grid.tif') + grid.save(path) + back = dem.Elevation.load(path) + + assert np.array_equal(back._griddata, data, equal_nan=True) + assert back._georef_info.nx == 13 and back._georef_info.ny == 11 + assert back._georef_info.dx == pytest.approx(25.0) + assert back._georef_info.xllcenter == pytest.approx(grid._georef_info.xllcenter) + assert back._georef_info.yllcenter == pytest.approx(grid._georef_info.yllcenter) + + +@pytest.mark.gdal +def test_ascii_grid_round_trip_maps_nodata(tmp_path): + data = np.array([[1.0, 2.0], [np.nan, 4.0]]) + grid = dem.Elevation(dx=1.0, grid=data) + + path = str(tmp_path / 'grid.asc') + grid.write_to_ai(path) + text = open(path).read() + assert 'NODATA_value -9999' in text + assert 'nan' not in text.lower() + + back = dem.Elevation(ai_ascii_filename=path, EPSGprojectionCode=32610) + # The no-data flag must come back as NaN, not as an elevation of -9999. + assert np.isnan(back._griddata[1, 0]) + assert back._griddata[0, 0] == pytest.approx(1.0) + + +@pytest.mark.gdal +def test_save_flushes_to_disk(tmp_path): + grid = dem.Elevation(dx=1.0, grid=np.ones((4, 4))) + path = str(tmp_path / 'flushed.tif') + grid.save(path) + assert os.path.getsize(path) > 0 diff --git a/tests/test_kernels.py b/tests/test_kernels.py new file mode 100644 index 0000000..3ea262f --- /dev/null +++ b/tests/test_kernels.py @@ -0,0 +1,365 @@ +"""Tests for the compute kernels. + +Two things are checked here: + +1. that the compiled C++ kernels and the pure-NumPy ones agree exactly, so + the two backends are interchangeable; and +2. that the Priority-Flood implementation has the properties Barnes et al. + (2014) prove it has -- the filled surface is minimal, monotone, and + independent of which of the paper's algorithms produced it. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from TopoAnalysis import fastops, kernels + + +FILL_CASES = [ + ('flat', 0.0, 0.0), + ('epsilon', 0.0, 0.0), + ('epsilon', 1e-3, 0.0), + ('flat', 0.0, 3.0), + ('epsilon', 1e-4, 5.0), +] + + +def random_surface(seed, ny=17, nx=23, nodata_fraction=0.0): + rng = np.random.default_rng(seed) + z = rng.random((ny, nx)) * 20.0 + if nodata_fraction: + z[rng.random(z.shape) < nodata_fraction] = np.nan + return z + + +def build_network(z, cellsize=1.0): + """``(filled, codes, receivers, order)`` for a surface.""" + filled = np.ascontiguousarray(z, dtype=float) + kernels.priority_flood(filled, mode='epsilon', epsilon=1e-9, cellsize=cellsize) + codes = kernels.flow_directions(filled, cellsize=cellsize) + receivers = kernels.receivers(codes) + order, cycles = kernels.topological_order(receivers) + assert cycles == 0 + return filled, codes, np.asarray(receivers), np.asarray(order) + + +# --------------------------------------------------------------------------- +# Backend agreement +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('seed', [0, 1, 2, 3]) +@pytest.mark.parametrize('mode,epsilon,max_depth', FILL_CASES) +def test_fill_backends_agree(seed, mode, epsilon, max_depth): + z = random_surface(seed, nodata_fraction=0.08 if seed % 2 else 0.0) + + a, b = z.copy(), z.copy() + ra = kernels.priority_flood(a, mode=mode, epsilon=epsilon, + max_pit_depth=max_depth, track_visited=True) + rb = fastops.priority_flood(b, mode=mode, epsilon=epsilon, + max_pit_depth=max_depth, track_visited=True) + + assert np.array_equal(a, b, equal_nan=True) + assert np.array_equal(np.asarray(ra['visited']), np.asarray(rb['visited'])) + assert ra['cells_filled'] == rb['cells_filled'] + assert ra['max_fill_depth'] == pytest.approx(rb['max_fill_depth']) + + +@pytest.mark.parametrize('seed', [4, 5]) +def test_routing_backends_agree(seed): + z = random_surface(seed) + filled, codes, receivers, order = build_network(z) + + assert np.array_equal(codes, fastops.flow_directions(filled, cellsize=1.0)) + assert np.array_equal(receivers, fastops.receivers(codes)) + + weights = np.ones_like(filled) + gate = (filled > np.median(filled)).astype(np.uint8) + py_order, _ = fastops.topological_order(receivers) + + assert np.allclose(kernels.accumulate(receivers, order, weights), + fastops.accumulate(receivers, py_order, weights)) + assert np.allclose(kernels.accumulate(receivers, order, weights, gate), + fastops.accumulate(receivers, py_order, weights, gate)) + + length, codes_c = kernels.flow_length(receivers, order, weights) + length_py, codes_py = fastops.flow_length(receivers, py_order, weights) + assert np.allclose(length, length_py) + assert np.array_equal(codes_c, codes_py) + + outlets = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + assert np.array_equal(kernels.upstream_mask(receivers, outlets), + fastops.upstream_mask(receivers, outlets)) + assert np.array_equal(kernels.drainage_basins(receivers, order), + fastops.drainage_basins(receivers, py_order)) + + accumulated = np.asarray(kernels.accumulate(receivers, order, weights)) + for trapezoid in (False, True): + chi_c, dist_c = kernels.chi(receivers, order, accumulated, weights, + outlets, 1.0, 0.45, trapezoid, 0.0, None) + chi_py, dist_py = fastops.chi(receivers, py_order, accumulated, weights, + outlets, 1.0, 0.45, trapezoid, 0.0, None) + assert np.allclose(chi_c, chi_py, equal_nan=True) + assert np.allclose(dist_c, dist_py, equal_nan=True) + + assert np.allclose(kernels.downstream_distance(receivers, order, weights), + fastops.downstream_distance(receivers, py_order, weights)) + + carved_c, carved_py = filled.copy(), filled.copy() + kernels.imposemin(receivers, order, weights, 1e-3, carved_c) + fastops.imposemin(receivers, py_order, weights, 1e-3, carved_py) + assert np.allclose(carved_c, carved_py) + + streams = (accumulated > 5).astype(np.uint8) + for kind in ('strahler', 'shreve'): + assert np.array_equal(kernels.stream_order(receivers, order, streams, kind), + fastops.stream_order(receivers, py_order, streams, kind)) + + +# --------------------------------------------------------------------------- +# Priority-Flood properties (Barnes et al. 2014) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('seed', [10, 11, 12]) +@pytest.mark.parametrize('mode,epsilon', [('flat', 0.0), ('epsilon', 1e-6)]) +def test_improved_algorithm_matches_the_reference(seed, mode, epsilon): + """Algorithm 2 must return exactly what Algorithm 1 returns.""" + z = random_surface(seed) + improved, reference = z.copy(), z.copy() + kernels.priority_flood(improved, mode=mode, epsilon=epsilon) + kernels.priority_flood(reference, mode=mode, epsilon=epsilon, + reference_algorithm=True) + assert np.array_equal(improved, reference) + + +@pytest.mark.parametrize('seed', [20, 21]) +def test_filling_never_lowers_a_cell(seed): + z = random_surface(seed) + filled = z.copy() + kernels.priority_flood(filled, mode='flat') + assert np.all(filled >= z - 1e-12) + + +def test_flat_fill_produces_the_spill_elevation(single_pit): + """Every interior cell of the bowl ends up at the 10 m rim.""" + filled = np.ascontiguousarray(single_pit._griddata, dtype=float) + report = kernels.priority_flood(filled, mode='flat') + assert np.all(filled == 10.0) + assert report['cells_filled'] == 9 + assert report['max_fill_depth'] == pytest.approx(9.0) + + +def test_epsilon_fill_leaves_no_flats(single_pit): + """Priority-Flood+epsilon must leave a strictly descending path.""" + filled = np.ascontiguousarray(single_pit._griddata, dtype=float) + kernels.priority_flood(filled, mode='epsilon', epsilon=1e-3, cellsize=1.0) + interior = filled[1:-1, 1:-1] + assert np.all(interior > 10.0) + # The deepest cell must end up highest: it is furthest from the spill. + assert filled[2, 2] == interior.max() + + +def test_max_pit_depth_leaves_deep_depressions_alone(single_pit): + filled = np.ascontiguousarray(single_pit._griddata, dtype=float) + kernels.priority_flood(filled, mode='flat', max_pit_depth=2.0) + # The 9 m deep centre stays put ... + assert filled[2, 2] == pytest.approx(1.0) + # ... but the flood still reached the far side of it. + assert np.all(np.isfinite(filled)) + + +def test_nodata_acts_as_an_outlet(): + """A depression touching no-data drains into it and is not filled.""" + z = np.array([ + [10, 10, 10, 10, 10], + [10, 5, 5, 5, 10], + [np.nan, 5, 1, 5, 10], + [10, 5, 5, 5, 10], + [10, 10, 10, 10, 10]], dtype=float) + filled = z.copy() + kernels.priority_flood(filled, mode='flat') + # (2,1) sits next to the NaN, so it keeps its own elevation ... + assert filled[2, 1] == pytest.approx(5.0) + # ... and the pit only rises to the level it must cross to get there. + assert filled[2, 2] == pytest.approx(5.0) + assert np.isnan(filled[2, 0]) + + +def test_default_seeds_are_the_perimeter_and_nodata_fringe(): + z = np.arange(25, dtype=float).reshape(5, 5) + z[2, 2] = np.nan + seeds = set(kernels.default_seeds(z).tolist()) + + perimeter = {i * 5 + j for i in range(5) for j in range(5) + if i in (0, 4) or j in (0, 4)} + nan_fringe = {(2 + di) * 5 + (2 + dj) + for di in (-1, 0, 1) for dj in (-1, 0, 1)} - {2 * 5 + 2} + + assert seeds == perimeter | nan_fringe + + +# --------------------------------------------------------------------------- +# Flow routing +# --------------------------------------------------------------------------- + + +def test_d8_codes_follow_the_arcgis_convention(): + """Each of the eight codes must point at the neighbour it names.""" + # 32 64 128 + # 16 X 1 + # 8 4 2 + expected = {(0, 1): 1, (1, 1): 2, (1, 0): 4, (1, -1): 8, + (0, -1): 16, (-1, -1): 32, (-1, 0): 64, (-1, 1): 128} + for (di, dj), code in expected.items(): + z = np.ones((3, 3)) + z[1 + di, 1 + dj] = 0.0 + codes = kernels.flow_directions(z, cellsize=1.0) + assert codes[1, 1] == code, 'offset {0} should give code {1}'.format((di, dj), code) + + +def test_diagonals_compete_on_gradient_not_on_drop(): + """A diagonal neighbour only wins if its *gradient* is steeper.""" + z = np.ones((3, 3)) * 10.0 + z[1, 2] = 9.0 # east: drop 1 over 1 -> gradient 1.00 + z[2, 2] = 8.6 # south-east: drop 1.4 over sqrt(2) -> gradient 0.99 + assert kernels.flow_directions(z, cellsize=1.0)[1, 1] == 1 + + z[2, 2] = 8.5 # gradient 1.06, now steeper than east + assert kernels.flow_directions(z, cellsize=1.0)[1, 1] == 2 + + +def test_accumulation_conserves_weight(rough_dome): + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=30.0) + weights = np.ones_like(z) + accumulated = np.asarray(kernels.accumulate(receivers, order, weights)) + + outlets = receivers.reshape(-1) < 0 + # Everything that enters the grid must leave through some outlet. + assert accumulated.reshape(-1)[outlets].sum() == pytest.approx(weights.sum()) + assert accumulated.min() >= 1.0 + + +def test_accumulation_ignores_the_order_it_is_given(rough_dome): + """Any valid topological order must give the same accumulation.""" + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=30.0) + other_order, _ = fastops.topological_order(receivers) + weights = np.ones_like(z) + assert np.allclose(kernels.accumulate(receivers, order, weights), + kernels.accumulate(receivers, other_order, weights)) + + +def test_flow_length_tie_break_is_deterministic(): + """Equal-length paths must resolve the same way whatever the order.""" + z = np.array([ + [2.0, 3.0, 2.0], + [3.0, 1.0, 3.0], + [2.0, 3.0, 2.0]]) + _, _, receivers, order = build_network(z) + other_order, _ = fastops.topological_order(receivers) + step = np.ones_like(z) + _, codes_a = kernels.flow_length(receivers, order, step) + _, codes_b = kernels.flow_length(receivers, other_order, step) + assert np.array_equal(codes_a, codes_b) + + +def test_upstream_mask_covers_the_whole_basin(rough_dome): + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=30.0) + outlets = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + mask = np.asarray(kernels.upstream_mask(receivers, outlets)) + # Every cell drains to some outlet, so the union covers the grid. + assert mask.all() + + +def test_chi_is_zero_at_the_outlet_and_grows_upstream(v_valley): + z = np.ascontiguousarray(v_valley._griddata, dtype=float) + filled, _, receivers, order = build_network(z, cellsize=10.0) + step = np.full(z.shape, 10.0) + area = np.asarray(kernels.accumulate(receivers, order, np.full(z.shape, 100.0))) + + outlets = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + chi, distance = kernels.chi(receivers, order, area, step, outlets, + 1e4, 0.45, True, 0.0, None) + chi = np.asarray(chi) + + assert np.nanmin(chi) == 0.0 + # chi must increase monotonically upstream: every cell exceeds its receiver. + flat_chi = chi.reshape(-1) + for k, r in enumerate(receivers.reshape(-1)): + if r >= 0 and np.isfinite(flat_chi[k]) and np.isfinite(flat_chi[r]): + assert flat_chi[k] >= flat_chi[r] - 1e-12 + + +def test_trapezoid_and_left_endpoint_chi_differ_predictably(v_valley): + """The trapezoid rule must sit between the two endpoint rules.""" + z = np.ascontiguousarray(v_valley._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=10.0) + step = np.full(z.shape, 10.0) + area = np.asarray(kernels.accumulate(receivers, order, np.full(z.shape, 100.0))) + outlets = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + + left, _ = kernels.chi(receivers, order, area, step, outlets, 1e4, 0.45, False, 0.0, None) + trap, _ = kernels.chi(receivers, order, area, step, outlets, 1e4, 0.45, True, 0.0, None) + valid = np.isfinite(left) & np.isfinite(trap) + # Area grows downstream, so the integrand shrinks downstream and the + # left-endpoint (upstream) value is the larger of the two. + assert np.all(np.asarray(trap)[valid] <= np.asarray(left)[valid] + 1e-9) + + +def test_imposemin_only_lowers_and_enforces_the_gradient(rough_dome): + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + filled, _, receivers, order = build_network(z, cellsize=30.0) + step = np.full(z.shape, 30.0) + + carved = filled.copy() + kernels.imposemin(receivers, order, step, 1e-3, carved) + + assert np.all(carved <= filled + 1e-12) + flat = carved.reshape(-1) + for k, r in enumerate(receivers.reshape(-1)): + if r >= 0: + assert flat[k] - flat[r] >= 1e-3 * 30.0 - 1e-9 + + +def test_strahler_order_of_a_simple_confluence(): + """Two order-1 streams meeting make an order-2 stream.""" + z = np.array([[3.0, 9.0, 3.0], + [9.0, 2.0, 9.0], + [9.0, 1.0, 9.0]]) + _, _, receivers, order = build_network(z) + streams = np.array([[1, 0, 1], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) + + strahler = np.asarray(kernels.stream_order(receivers, order, streams, 'strahler')) + assert strahler[0, 0] == 1 and strahler[0, 2] == 1 + assert strahler[1, 1] == 2 and strahler[2, 1] == 2 + + shreve = np.asarray(kernels.stream_order(receivers, order, streams, 'shreve')) + assert shreve[1, 1] == 2 and shreve[2, 1] == 2 + + +def test_drainage_basins_partition_the_grid(rough_dome): + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=30.0) + labels = np.asarray(kernels.drainage_basins(receivers, order)) + + assert labels.min() >= 1, 'every cell drains somewhere on this surface' + # Each cell must carry the same label as the cell it drains into. + flat = labels.reshape(-1) + for k, r in enumerate(receivers.reshape(-1)): + if r >= 0: + assert flat[k] == flat[r] + + +def test_topological_order_reports_cycles(): + """A hand-made loop must be reported, not silently mis-ordered.""" + # Two cells pointing at each other: 1 -> east, 16 -> west. + codes = np.array([[1, 16]], dtype=np.uint8) + receivers = kernels.receivers(codes) + order, cycles = kernels.topological_order(receivers) + assert cycles == 2 + assert sorted(np.asarray(order).tolist()) == [0, 1] diff --git a/tests/test_modules.py b/tests/test_modules.py new file mode 100644 index 0000000..595667b --- /dev/null +++ b/tests/test_modules.py @@ -0,0 +1,319 @@ +"""Tests for the supporting modules: windows, datasets, profiles, the CLI.""" + +from __future__ import annotations + +import os + +import numpy as np +import pytest + +import matplotlib +matplotlib.use('Agg') # no display in CI + +from TopoAnalysis import (MovingWindow, analysis, datasets, dem, + demRecursionTools, error, plotting, utils) +from TopoAnalysis import cli + + +# --------------------------------------------------------------------------- +# MovingWindow +# --------------------------------------------------------------------------- + + +class WindowMean(MovingWindow.CircularMovingWindow): + function = staticmethod(np.mean) + + +class WindowRange(MovingWindow.RectangularMovingWindow): + function = staticmethod(lambda values: values.max() - values.min()) + + +def test_moving_window_needs_a_radius(): + with pytest.raises(error.InputError): + WindowMean() + + +def test_abstract_windows_cannot_be_instantiated(): + with pytest.raises(error.Error): + MovingWindow.MovingWindow(window_radius=1.0) + with pytest.raises(error.Error): + MovingWindow.RectangularMovingWindow(window_radius=1.0) + with pytest.raises(error.Error): + MovingWindow.CircularMovingWindow(window_radius=1.0) + + +def test_circular_window_is_a_disc_not_a_band(): + """The kernel used to square the row offset twice, giving a column.""" + window = WindowMean(window_radius=2.0) + rows, cols = window._build_search_kernel(1.0) + offsets = set(zip(rows.tolist(), cols.tolist())) + assert (0, 2) in offsets and (2, 0) in offsets + assert (2, 2) not in offsets, 'the corner is outside a radius-2 disc' + + +def test_moving_window_mean_of_a_constant_field(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.full((9, 9), 4.0)) + # The method used to call a mangled name that did not exist, and to pass + # the grid where the cell size was expected. + result = grid.apply_moving_window(WindowMean(window_radius=2.0)) + assert np.allclose(result._griddata, 4.0) + + +def test_moving_window_ignores_nodata(): + data = np.ones((7, 7)) + data[3, 3] = np.nan + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + result = grid.apply_moving_window(WindowMean(window_radius=1.0)) + # The NaN is dropped from the window rather than poisoning it. + assert result._griddata[3, 3] == pytest.approx(1.0) + + +def test_rectangular_window_range(): + data = np.tile(np.arange(9, dtype=float), (9, 1)) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + result = grid.apply_moving_window(WindowRange(window_radius=1.0)) + assert result._griddata[4, 4] == pytest.approx(2.0) + + +def test_window_dimension_is_still_accepted(): + """The original keyword name must keep working.""" + window = WindowMean(window_dimension=3.0) + assert window.window_radius == 3.0 + + +# --------------------------------------------------------------------------- +# datasets +# --------------------------------------------------------------------------- + + +def test_triangle_grid_has_the_requested_shape_and_relief(): + grid = datasets.triangle_grid(30, 40, width=10, amp=5.0) + assert grid._griddata.shape == (30, 40) + assert grid._griddata.min() == pytest.approx(0.0) + assert grid._griddata.max() == pytest.approx(10.0, rel=0.1) + + +def test_sinusoid_grid_spans_the_requested_amplitude(): + grid = datasets.sinusoid_grid(20, 40, width=10, amp=3.0) + row = grid._griddata[0] + assert np.allclose(grid._griddata, row, atol=1e-12), 'every row is the same' + assert row.min() == pytest.approx(0.0) + assert row.max() == pytest.approx(6.0, rel=0.05) + # Four ridges across 40 columns at a spacing of 10. + peaks = np.flatnonzero((row[1:-1] > row[:-2]) & (row[1:-1] > row[2:])) + 1 + assert len(peaks) == 4 + + +def test_slope_adds_a_ramp_rather_than_scaling_the_relief(): + """`triangle *= tilt` zeroed row 0 and amplified the rest.""" + flat = datasets.triangle_grid(20, 20, width=5, amp=1.0) + sloped = datasets.triangle_grid(20, 20, width=5, amp=1.0, slope_y=2.0) + assert not np.allclose(sloped._griddata[0], 0.0) + # The added ramp is the only difference between the two. + difference = sloped._griddata - flat._griddata + assert np.allclose(difference, difference[:, [0]]) + assert difference[1, 0] - difference[0, 0] == pytest.approx( + 2.0 * 20.0 / 19.0, rel=1e-6) + + +def test_synthetic_grids_are_georeferenced_enough_to_route(): + grid = datasets.triangle_grid(24, 24, width=6, amp=4.0, sig=0.05) + filled = dem.FilledElevation(elevation=grid) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + assert dem.Area(flow_direction=d8)._griddata.max() > 1.0 + # A grid built in memory used to have no geotransform, so save() failed. + assert grid._georef_info.geoTransform != 0 + + +# --------------------------------------------------------------------------- +# Recursive profile tools +# --------------------------------------------------------------------------- + + +@pytest.fixture +def profile_network(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,))[0] + return v_valley, d8, area, outlet + + +def test_map_values_to_recursive_list_structure(profile_network): + elevation, d8, area, outlet = profile_network + tree = d8.map_values_to_recursive_list(outlet, elevation=elevation, area=area) + + assert 'index' in tree and 'distance_scale' in tree + assert tree['distance_scale'] == 1.0, 'the root has no parent step' + assert tree['area'] == pytest.approx(area._griddata.max()) + + +def test_distance_scale_describes_the_step_from_the_parent(profile_network): + """It used to be overwritten on the parent by each child in turn.""" + elevation, d8, area, outlet = profile_network + tree = d8.map_values_to_recursive_list(outlet, elevation=elevation, area=area) + + stack = [tree] + checked = 0 + while stack: + node = stack.pop() + parent_index = node['index'] + for child in node.get('next', []): + di = child['index'][0] - parent_index[0] + dj = child['index'][1] - parent_index[1] + expected = np.sqrt(2.0) if (di and dj) else 1.0 + assert child['distance_scale'] == pytest.approx(expected) + checked += 1 + stack.append(child) + assert checked > 0 + + +def test_recursive_map_covers_the_whole_basin(profile_network): + elevation, d8, area, outlet = profile_network + tree = d8.map_values_to_recursive_list(outlet, elevation=elevation, area=area) + + seen = set() + stack = [tree] + while stack: + node = stack.pop() + seen.add(tuple(node['index'])) + stack.extend(node.get('next', [])) + + ((row, col),) = d8._xy_to_rowscols((outlet,)) + assert seen == set(d8.get_indexes_of_upstream_cells(row, col)) + + +def test_chi_elevation_and_ks_fit(profile_network): + elevation, d8, area, outlet = profile_network + de = area._mean_pixel_dimension() + tree = d8.map_values_to_recursive_list(outlet, elevation=elevation, area=area) + + e, c = demRecursionTools.chi_elevation(tree, de, np.array([0.45]), xo=10.0) + assert e.size == c.size > 0 + assert c.min() == pytest.approx(0.0) + + ks, r2 = demRecursionTools.best_ks_with_r2_list(tree, de, np.array([0.45]), xo=10.0) + assert np.isfinite(ks).all() + + +def test_hypsometric_integral_is_between_zero_and_one(profile_network): + elevation, d8, area, outlet = profile_network + dA = dem.BaseSpatialGrid(dx=area._georef_info.dx, + grid=area._area_per_pixel()) + hi = demRecursionTools.hi(elevation, d8, dA, outlet) + assert 0.0 <= hi <= 1.0 + + +def test_calc_ks_for_outlet_without_an_xo(profile_network): + """kwargs.pop('xo') used to raise KeyError when xo was left at default.""" + elevation, d8, area, outlet = profile_network + ks, r2 = utils.calc_ks_for_outlet(outlet, 0.45, xo=10.0, flow_direction=d8, + elevation=elevation, area=area) + assert np.isfinite(ks).all() + + +def test_map_chi_profiles(profile_network): + elevation, d8, area, outlet = profile_network + chi_map = demRecursionTools.map_chi_profiles( + elevation, d8, area, outlet, minimum_area=100.0, theta=0.45) + assert len(chi_map) > 0 + for (chi, relief) in chi_map.values(): + assert np.isfinite(chi) + + +# --------------------------------------------------------------------------- +# Plotting (smoke tests -- Agg backend, no display) +# --------------------------------------------------------------------------- + + +def test_plot_downstream_profile(profile_network): + elevation, d8, area, outlet = profile_network + rows, cols, length, profile = plotting.plot_downstream_profile( + elevation, d8, outlet, 'k-') + assert rows.size == cols.size == length.size == profile.size > 0 + + +def test_plot_two_grids(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + slope = dem.MaxSlope(elevation=rough_dome) + + # Passing xlabel used to leave the local name unbound and raise. + axes = dem.plot(area, slope, xlabel='area', ylabel='slope', + interactive=False, decimation_factor=7) + assert axes.get_xlabel() == 'area' + + +def test_grid_plot_accepts_an_axes(rough_dome): + import matplotlib.pyplot as plt + + fig, ax = plt.subplots() + returned = rough_dome.plot(ax=ax, interactive=False, colorbar=False) + assert returned is fig + plt.close(fig) + + +# --------------------------------------------------------------------------- +# Quadrats +# --------------------------------------------------------------------------- + + +def test_quadrats_partition_the_data(): + data = np.arange(100, dtype=float).reshape(10, 10) + quadrats = analysis.Quadrats(data=data, dx=5, dy=5) + assert len(quadrats.quadrats) == 4 + assert all(q.shape == (5, 5) for q in quadrats.quadrats) + assert quadrats.map_quadrats(np.sum) == [ + float(data[r:r + 5, c:c + 5].sum()) for r in (0, 5) for c in (0, 5)] + + +def test_quadrats_without_rasterio_message(): + quadrats = analysis.Quadrats(data=np.zeros((4, 4))) + try: + import rasterio # noqa: F401 + except ImportError: + with pytest.raises(ImportError, match='rasterio'): + quadrats.load_data('nowhere.tif') + + +# --------------------------------------------------------------------------- +# Command line +# --------------------------------------------------------------------------- + + +def test_cli_info(capsys): + assert cli.main(['--info']) == 0 + out = capsys.readouterr().out + assert 'compute backend' in out + assert 'Barnes' in out + + +def test_cli_rejects_an_unknown_product(): + with pytest.raises(SystemExit): + cli.main(['dem.tif', '--products', 'not-a-product']) + + +def test_cli_requires_a_dem(): + with pytest.raises(SystemExit): + cli.main([]) + + +@pytest.mark.gdal +def test_cli_end_to_end(tmp_path, rough_dome): + source = str(tmp_path / 'dem.tif') + rough_dome.save(source) + + outdir = str(tmp_path / 'out') + assert cli.main([source, '-o', outdir, '--products', + 'filled,flowdir,area,length,slope,hillshade', + '--min-area', '9000']) == 0 + + produced = sorted(os.listdir(outdir)) + for name in ('dem_filled.tif', 'dem_flowdir.tif', 'dem_area.tif', + 'dem_length.tif', 'dem_slope.tif', 'dem_hillshade.tif'): + assert name in produced + + back = dem.Area.load(os.path.join(outdir, 'dem_area.tif')) + assert back._griddata.max() > 0.0 diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..543ad43 --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,642 @@ +"""One test per repaired defect, so none of them can quietly return. + +Each test names the specific failure it guards against. Defects already +covered by the behavioural suites (filling, routing, terrain attributes) are +not repeated here. +""" + +from __future__ import annotations + +import importlib +import os +import subprocess +import sys + +import numpy as np +import pytest + +from TopoAnalysis import dem, error, fastops, kernels + +#: The package directory, and the directory that must be on sys.path for +#: ``import TopoAnalysis`` to resolve. +PACKAGE_DIR = os.path.dirname(os.path.abspath(dem.__file__)) +IMPORT_ROOT = os.path.dirname(PACKAGE_DIR) + + +def run_in_subprocess(script, **extra_env): + """Run ``script`` in a fresh interpreter that can import TopoAnalysis.""" + env = dict(os.environ, PYTHONPATH=IMPORT_ROOT, **extra_env) + return subprocess.run([sys.executable, '-c', script], capture_output=True, + text=True, cwd=IMPORT_ROOT, env=env) + + +# --------------------------------------------------------------------------- +# Module-level hygiene +# --------------------------------------------------------------------------- + + +def test_no_removed_numpy_or_scipy_aliases_remain(): + """np.NAN, np.float and scipy.ndimage.morphology were all removed upstream.""" + package = PACKAGE_DIR + banned = ('np.NAN', 'np.NaN', 'np.float)', 'np.int)', 'np.bool)', + 'scipy.ndimage.morphology', 'scipy.ndimage.filters', + 'interp2d', 'plt.hold') + + offenders = [] + for name in sorted(os.listdir(package)): + if not name.endswith('.py'): + continue + text = open(os.path.join(package, name)).read() + for token in banned: + # The tests themselves may name these tokens when describing them. + if token in text and not name.startswith('test_'): + offenders.append('{0}: {1}'.format(name, token)) + assert offenders == [] + + +def test_import_does_not_require_gdal(): + """GDAL must be imported lazily, so the package loads without it.""" + script = ( + 'import sys, types;' + 'blocker = types.ModuleType("osgeo");' + 'blocker.__getattr__ = lambda name: (_ for _ in ()).throw(ImportError());' + 'sys.modules["osgeo"] = blocker;' + 'import TopoAnalysis;' + 'print(TopoAnalysis.gdal_is_available())' + ) + result = run_in_subprocess(script) + assert result.returncode == 0, result.stderr + + +def test_recursion_limit_is_left_alone(): + """The module used to raise it to a million, which segfaults before it trips.""" + assert sys.getrecursionlimit() < 100000 + + +def test_constructor_dispatch_does_not_use_eval(): + """Constructors are selected with getattr, not by compiling source text.""" + import ast + + tree = ast.parse(open(dem.__file__).read()) + called = {node.func.id for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} + assert 'eval' not in called + assert 'exec' not in called + + +def test_both_import_styles_work(): + """`import dem` (flat) and `from TopoAnalysis import dem` must both work.""" + script = ('import sys; sys.path.insert(0, {0!r}); ' + 'import dem, kernels; print(kernels.backend())').format(PACKAGE_DIR) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() in ('c++', 'python') + + +# --------------------------------------------------------------------------- +# Geometry +# --------------------------------------------------------------------------- + + +def test_extent_of_data_handles_a_non_square_grid(): + """The column scan used to index rows, and ran off the end when nx != ny.""" + data = np.full((4, 12), np.nan) + data[1:3, 5:9] = 1.0 + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + xmin, xmax, ymin, ymax = grid.extent_of_data() + assert xmax > xmin and ymax > ymin + + +def test_tile_geotransform_is_self_consistent(): + """The tile transform put the north edge one cell too far north.""" + grid = dem.BaseSpatialGrid(dx=3.0, grid=np.arange(400, dtype=float).reshape(20, 20)) + for tile in grid.tile(tile_xdim=8, tile_ydim=8, tile_xpadding=2, tile_ypadding=2): + info = tile._georef_info + # yllcenter must invert out of the transform the same way the reader + # derives it. + assert info.geoTransform[3] - info.dx * (info.ny - 0.5) == pytest.approx( + info.yllcenter) + + +def test_flow_length_clip_keeps_the_direction_codes(): + """clip_to_extent used to copy the *lengths* into the direction grid.""" + n = 12 + z = np.tile(np.arange(n, 0, -1, dtype=float), (n, 1)) + grid = dem.Elevation(dx=1.0, grid=z) + d8 = dem.FlowDirectionD8(elevation=grid) + length = dem.FlowLength(flow_direction=d8) + + ((x0, y0),) = length._rowscols_to_xy(((9, 2),)) + ((x1, y1),) = length._rowscols_to_xy(((2, 9),)) + clipped = length.clip_to_extent((x0, x1, y0, y1)) + + codes = clipped.main_stem_directions + assert codes.shape == clipped._griddata.shape + assert codes.dtype == np.uint8 + # Direction codes are powers of two (or 0), never flow lengths. + assert set(np.unique(codes).tolist()) <= {0, 1, 2, 4, 8, 16, 32, 64, 128} + + +# --------------------------------------------------------------------------- +# Flow routing +# --------------------------------------------------------------------------- + + +def test_flow_code_for_position_prefers_the_steeper_gradient(): + """The per-cell re-router divided cardinals, not diagonals, by sqrt(2).""" + z = np.ones((3, 3)) * 10.0 + z[1, 2] = 9.0 # east, gradient 1.00 + z[2, 2] = 8.6 # south-east, gradient 0.99 + grid = dem.Elevation(dx=1.0, grid=z) + + d8 = dem.FlowDirectionD8(dx=1.0, grid=np.zeros((3, 3), dtype=np.uint8)) + mask = dem.Mask(dx=1.0, grid=np.zeros((3, 3), dtype=np.uint8)) + mask._griddata[1, 1] = 1 + d8.update_flow_codes_in_mask(grid, mask) + assert d8._griddata[1, 1] == 1, 'the steeper cardinal must win' + + z[2, 2] = 8.5 # south-east, gradient 1.06 + grid = dem.Elevation(dx=1.0, grid=z) + d8.update_flow_codes_in_mask(grid, mask) + assert d8._griddata[1, 1] == 2, 'the steeper diagonal must win' + + +def test_update_flow_codes_invalidates_the_cached_network(): + z = np.tile(np.arange(5, 0, -1, dtype=float), (5, 1)) + grid = dem.Elevation(dx=1.0, grid=z) + d8 = dem.FlowDirectionD8(elevation=grid) + first = d8.receivers.copy() + + mask = dem.Mask(dx=1.0, grid=np.ones((5, 5), dtype=np.uint8)) + flipped = dem.Elevation(dx=1.0, grid=np.fliplr(z).copy()) + d8.update_flow_codes_in_mask(flipped, mask) + + assert not np.array_equal(d8.receivers, first) + + +def test_mainstem_valley_area_differs_from_valley_area(): + """The override was name-mangled out of reach, so the two were identical.""" + n = 24 + y, x = np.mgrid[0:n, 0:n].astype(float) + z = 100.0 - y * 2.0 + np.abs(x - n / 2) * 1.5 + elevation = dem.Elevation(dx=10.0, grid=z) + + filled = dem.FilledElevation(elevation=elevation) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + laplace = dem.Laplacian(elevation=elevation) + + common = dict(flow_direction=d8, area=area, laplace=laplace, + valley_laplace_value=-1e9, min_area_value=1e3) + whole = dem.ValleyArea(**common) + mainstem = dem.MainstemValleyArea(**common) + + assert not np.allclose(whole._griddata, mainstem._griddata) + # Only the trunk contributes, so the mainstem totals cannot exceed the + # full accumulation anywhere. + assert np.all(mainstem._griddata <= whole._griddata + 1e-9) + + +def test_area_gate_actually_gates(): + """`mask[i, j] is not None` was always true, so the mask did nothing.""" + n = 10 + z = np.tile(np.arange(n, 0, -1, dtype=float), (n, 1)) + d8 = dem.FlowDirectionD8(elevation=dem.Elevation(dx=1.0, grid=z)) + + gate = dem.Mask(dx=1.0, grid=np.zeros((n, n), dtype=np.uint8)) + gate._griddata[:, :5] = 1 + + gated = dem.Area(flow_direction=d8, evaluate_at=gate) + plain = dem.Area(flow_direction=d8) + + # Inside the gate, accumulation is unaffected ... + assert gated._griddata[0, 4] == pytest.approx(plain._griddata[0, 4]) + # ... the first cell outside it still receives what the gate delivers ... + assert gated._griddata[0, 5] == pytest.approx(plain._griddata[0, 4]) + # ... but passes nothing further on, and adds nothing of its own. + assert gated._griddata[0, 6] == 0.0 + assert gated._griddata[0, 9] == 0.0 + assert plain._griddata[0, 9] == pytest.approx(10.0) + + +# --------------------------------------------------------------------------- +# Regression grids that fit a model +# --------------------------------------------------------------------------- + + +@pytest.fixture +def smoothing_inputs(): + n = 30 + y, x = np.mgrid[0:n, 0:n].astype(float) + z = 200.0 - y * 3.0 + np.abs(x - n / 2) * 2.0 + elevation = dem.Elevation(dx=20.0, grid=z) + filled = dem.FilledElevation(elevation=elevation) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + return elevation, d8, area + + +def test_ks_from_chi_with_smoothing_counts_each_crossing(smoothing_inputs): + """`self._n[rows, cols] += 1` only counted a repeated cell once.""" + pytest.importorskip('statsmodels') + elevation, d8, area = smoothing_inputs + + ks = dem.KsFromChiWithSmoothing( + elevation=elevation, area=area, flow_direction=d8, theta=0.45, + vertical_interval=15.0, area_threshold=2000.0, verbose=False) + + assert ks._n.sum() > 0 + assert ks._n.max() > 1, 'busy cells must be counted once per profile' + assert np.isfinite(ks._griddata[~np.isnan(ks._griddata)]).all() + + +def test_channel_slope_with_smoothing_accepts_a_horizontal_interval(smoothing_inputs): + """The horizontal_interval entry pointed at a method that did not exist.""" + elevation, d8, area = smoothing_inputs + slope = dem.ChannelSlopeWithSmoothing( + elevation=elevation, area=area, flow_direction=d8, + horizontal_interval=100.0, min_area=2000.0, verbose=False) + assert np.isfinite(slope._griddata[~np.isnan(slope._griddata)]).all() + + +def test_channel_up_and_down_slope_variants(smoothing_inputs): + elevation, d8, area = smoothing_inputs + common = dict(elevation=elevation, area=area, flow_direction=d8, + vertical_interval=15.0, min_area=2000.0, verbose=False) + down = dem.ChannelDownSlopeWithSmoothing(**common) + up = dem.ChannelUpSlopeWithSmoothing(**common) + assert down._griddata.shape == up._griddata.shape + assert not np.array_equal(np.nan_to_num(down._griddata), + np.nan_to_num(up._griddata)) + + +def test_theta_from_chi_with_smoothing(smoothing_inputs): + pytest.importorskip('statsmodels') + elevation, d8, area = smoothing_inputs + theta = dem.ThetaFromChiWithSmoothing( + elevation=elevation, area=area, flow_direction=d8, + vertical_interval=15.0, min_area=2000.0, verbose=False) + fitted = theta._griddata[~np.isnan(theta._griddata)] + assert fitted.size > 0 + assert np.all(np.abs(fitted) <= 10.0), 'the optimiser is bounded at +/-10' + + +def test_geographic_theta_inherits_the_theta_class(): + """It used to inherit KsFromChiWithSmoothing and compute ks instead.""" + assert issubclass(dem.GeographicThetaFromChiWithSmoothing, + dem.ThetaFromChiWithSmoothing) + assert issubclass(dem.GeographicKsFromChiWithSmoothing, + dem.KsFromChiWithSmoothing) + + +# --------------------------------------------------------------------------- +# Profile tools +# --------------------------------------------------------------------------- + + +def test_hypsometric_integral_detects_nodata(): + """`np.nan in array` is always False, so NaNs slipped through.""" + from TopoAnalysis import demRecursionTools + + tree = {'elevation': np.nan, 'dA': 1.0, + 'next': [{'elevation': 5.0, 'dA': 1.0}]} + assert demRecursionTools.hi_list(tree) == 0.0 + + +def test_tributary_step_length_multiplies_the_scale(): + """`trb_ld['de'] + trb_ld['distance_scale']` should have been a product.""" + from TopoAnalysis import demRecursionTools + + source = open(demRecursionTools.__file__).read() + assert "trb_ld['de'] * trb_ld['distance_scale']" in source + + +# --------------------------------------------------------------------------- +# Backend equivalence at the library level +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not kernels.have_extension(), + reason='the compiled extension is not built') +def test_pure_python_backend_gives_the_same_pipeline(tmp_path): + """The whole pipeline must agree between the two backends.""" + script = ''' +import os +os.environ["TOPOANALYSIS_PURE_PYTHON"] = "1" +import numpy as np +import TopoAnalysis.kernels as kernels +assert kernels.backend() == "python" +import TopoAnalysis.dem as dem + +rng = np.random.default_rng(99) +n = 25 +y, x = np.mgrid[0:n, 0:n] / n +z = 100*(1 - ((x-0.5)**2 + (y-0.5)**2)) + rng.normal(0, 0.5, (n, n)) +e = dem.Elevation(dx=30.0, grid=z) +f = dem.FilledElevation(elevation=e) +d8 = dem.FlowDirectionD8(flooded_dem=f) +a = dem.Area(flow_direction=d8) +l = dem.FlowLength(flow_direction=d8) +np.save(os.environ["OUT"], np.stack([f._griddata, d8._griddata.astype(float), + a._griddata, l._griddata])) +''' + out = str(tmp_path / 'python_backend.npy') + result = run_in_subprocess(script, OUT=out) + assert result.returncode == 0, result.stderr + + rng = np.random.default_rng(99) + n = 25 + y, x = np.mgrid[0:n, 0:n] / n + z = 100 * (1 - ((x - 0.5) ** 2 + (y - 0.5) ** 2)) + rng.normal(0, 0.5, (n, n)) + e = dem.Elevation(dx=30.0, grid=z) + f = dem.FilledElevation(elevation=e) + d8 = dem.FlowDirectionD8(flooded_dem=f) + a = dem.Area(flow_direction=d8) + l = dem.FlowLength(flow_direction=d8) + + expected = np.stack([f._griddata, d8._griddata.astype(float), + a._griddata, l._griddata]) + assert np.allclose(np.load(out), expected) + + +def test_kernels_expose_the_barnes_reference(): + assert 'Barnes' in kernels.PRIORITY_FLOOD_REFERENCE + assert '2014' in kernels.PRIORITY_FLOOD_REFERENCE + assert kernels.backend() in ('c++', 'python') + + +def test_fastops_and_kernels_expose_the_same_api(): + shared = [name for name in fastops.__all__ if not name.startswith('D8_')] + for name in shared: + assert hasattr(kernels, name), name + + +# --------------------------------------------------------------------------- +# Defects found by the post-rewrite review +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not kernels.have_extension(), + reason='the compiled extension is not built') +def test_kernels_are_thread_safe(): + """Every binding must resolve its buffers before releasing the GIL. + + ``py::array::request()`` calls ``PyObject_GetBuffer``, which touches + reference counts. Calling it inside a ``gil_scoped_release`` block + corrupted the interpreter -- leaked references, and reproducible aborts -- + as soon as two threads ran a kernel at once. + """ + import threading + + rng = np.random.default_rng(1) + z = np.ascontiguousarray(rng.random((40, 40)) * 10) + kernels.priority_flood(z, mode='epsilon', epsilon=1e-9) + codes = kernels.flow_directions(z, cellsize=1.0) + recv = np.ascontiguousarray(kernels.receivers(codes)) + order = np.ascontiguousarray(kernels.topological_order(recv)[0]) + weights = np.ones_like(z) + streams = np.ones(z.shape, dtype=np.uint8) + outlets = np.flatnonzero(recv.reshape(-1) < 0).astype(np.int64) + + watched = (recv, order, weights, streams, outlets) + before = [sys.getrefcount(a) for a in watched] + + def hammer(): + for _ in range(400): + kernels.accumulate(recv, order, weights) + kernels.downstream_distance(recv, order, weights) + kernels.stream_order(recv, order, streams, 'strahler') + kernels.flow_length(recv, order, weights) + kernels.drainage_basins(recv, order, outlets, None) + kernels.upstream_mask(recv, outlets) + + threads = [threading.Thread(target=hammer) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert [sys.getrefcount(a) for a in watched] == before + + +@pytest.mark.skipif(not kernels.have_extension(), + reason='the compiled extension is not built') +def test_kernels_reject_a_mismatched_order_array(): + """A wrong-length ``order`` must raise, not read off the end of it.""" + recv = np.full((4, 4), -1, dtype=np.int64) + weights = np.ones((4, 4)) + with pytest.raises((ValueError, RuntimeError)): + kernels.accumulate(recv, np.arange(3, dtype=np.int64), weights) + + +def test_strahler_order_does_not_depend_on_the_topological_order(): + """Both backends, and any valid ordering, must agree. + + The incremental rule gives {2, 2, 3} an order of 4 or 3 according to + which donor arrives first; the junction is now resolved as a whole. + """ + recv = np.array([[5, 6, 1, -1], [1, -1, 5, 6], [5, 10, 7, 6]], dtype=np.int64) + streams = np.ones(recv.shape, dtype=np.uint8) + + cpp_order, _ = kernels.topological_order(recv) + py_order, _ = fastops.topological_order(recv) + results = [ + np.asarray(kernels.stream_order(recv, cpp_order, streams, 'strahler')), + np.asarray(kernels.stream_order(recv, py_order, streams, 'strahler')), + np.asarray(fastops.stream_order(recv, cpp_order, streams, 'strahler')), + np.asarray(fastops.stream_order(recv, py_order, streams, 'strahler')), + ] + for other in results[1:]: + assert np.array_equal(results[0], other) + + +def test_strahler_of_a_mixed_junction(): + """Orders {2, 2, 3} meeting give 3, not 4: only the largest can pair.""" + # Four headwaters feeding one cell: two combine to 2, another two to 2, + # those combine to 3, and a bare order-1 also arrives. + recv = np.array([ + [4, 4, 4, 4], + [-1, 4, 4, 4], + ], dtype=np.int64) + streams = np.ones(recv.shape, dtype=np.uint8) + order, _ = kernels.topological_order(recv) + result = np.asarray(kernels.stream_order(recv, order, streams, 'strahler')) + # Cell 4 receives four order-1 donors, so it is order 2. + assert result.reshape(-1)[4] == 2 + + +def test_drainage_basins_honours_valid_with_outlets(): + """The C++ path used to drop the mask whenever outlets were supplied.""" + z = np.add.outer(np.arange(9.0), np.arange(9.0)) + codes = kernels.flow_directions(z, cellsize=1.0) + recv = kernels.receivers(codes) + order, _ = kernels.topological_order(recv) + + valid = np.zeros((9, 9), dtype=np.uint8) + valid[:4, :4] = 1 + outlets = np.array([0], dtype=np.int64) + + cpp = np.asarray(kernels.drainage_basins(recv, order, outlets, valid)) + py = np.asarray(fastops.drainage_basins(recv, order, outlets, valid)) + assert np.array_equal(cpp, py) + assert cpp[valid == 0].sum() == 0 + assert (cpp > 0).sum() <= int(valid.sum()) + + +def test_priority_flood_rejects_a_non_contiguous_grid(): + """reshape(-1) copies, so the fill would be silently thrown away.""" + base = np.array([[5., 5., 5., 5., 5.], + [5., 0., 0., 0., 5.], + [5., 0., 0., 0., 5.], + [5., 0., 0., 0., 5.], + [5., 5., 5., 5., 5.]]) + for awkward in (np.asfortranarray(base), np.repeat(base, 2, axis=0)[::2]): + with pytest.raises(TypeError): + fastops.priority_flood(awkward) + if kernels.have_extension(): + with pytest.raises(TypeError): + kernels.priority_flood(awkward) + + +def test_imposemin_rejects_a_non_contiguous_grid(): + recv = np.full((4, 4), -1, dtype=np.int64) + order = np.arange(16, dtype=np.int64) + step = np.ones((4, 4)) + awkward = np.asfortranarray(np.zeros((4, 4))) + with pytest.raises(TypeError): + fastops.imposemin(recv, order, step, 1e-3, awkward) + + +def test_default_seeds_of_an_empty_grid(): + assert fastops.default_seeds(np.zeros((0, 5))).size == 0 + assert fastops.default_seeds(np.zeros((5, 0))).size == 0 + + +def test_create_from_geotransform_places_the_grid_correctly(): + """yllcenter was read from ny before ny was assigned.""" + gt = (1000.0, 10.0, 0.0, 2000.0, 0.0, -10.0) + grid = dem.Elevation(nx=15, ny=12, projection='', geo_transform=gt, + grid=np.zeros((12, 15))) + + # The south edge of a 12-row, 10 m grid whose north edge is 2000 is 1880, + # so the southern row of cell centres sits at 1885. + assert grid._georef_info.yllcenter == pytest.approx(1885.0) + assert grid._rowscols_to_xy(((0, 0),))[0] == pytest.approx((1005.0, 1995.0)) + assert grid._rowscols_to_xy(((11, 0),))[0] == pytest.approx((1005.0, 1885.0)) + + +@pytest.mark.gdal +def test_create_from_geotransform_round_trips_through_gdal(tmp_path): + """A grid built from a geotransform must agree with the file it writes.""" + gt = (1000.0, 10.0, 0.0, 2000.0, 0.0, -10.0) + rng = np.random.default_rng(6) + grid = dem.Elevation(nx=15, ny=12, projection='', geo_transform=gt, + grid=rng.random((12, 15))) + + path = str(tmp_path / 'grid.tif') + grid.save(path) + back = dem.Elevation.load(path) + + assert back._georef_info.yllcenter == pytest.approx(grid._georef_info.yllcenter) + assert back._rowscols_to_xy(((0, 0),))[0] == pytest.approx( + grid._rowscols_to_xy(((0, 0),))[0]) + assert back._rowscols_to_xy(((11, 14),))[0] == pytest.approx( + grid._rowscols_to_xy(((11, 14),))[0]) + + +def test_editing_a_flow_code_invalidates_the_cached_network(): + """Routing is cached now, so a write has to drop the cache.""" + rng = np.random.default_rng(2) + grid = dem.Elevation(dx=10.0, grid=rng.random((20, 20)) * 10) + fd = dem.FlowDirectionD8(flooded_dem=dem.FilledElevation(elevation=grid)) + + before = dem.Area(flow_direction=fd)._griddata.copy() + + current = int(fd._griddata[10, 10]) + fd[10, 10] = 1 if current != 1 else 16 + assert not np.array_equal(dem.Area(flow_direction=fd)._griddata, before) + + after_setitem = dem.Area(flow_direction=fd)._griddata.copy() + current = int(fd._griddata[5, 5]) + fd.set_value_at_rowscols(1 if current != 1 else 16, [(5, 5)]) + assert not np.array_equal(dem.Area(flow_direction=fd)._griddata, after_setitem) + + +def test_geographic_laplacian_uses_metres_not_degrees(): + """dx is in DEGREES on a lat/lon grid; dividing by it was 1e10 wrong.""" + n = 11 + y, x = np.mgrid[0:n, 0:n].astype(float) + grid = dem.GeographicElevation(dx=0.01, grid=0.25 * (x ** 2 + y ** 2)) + grid._georef_info.geoTransform = (-120.0, 0.01, 0.0, 40.11, 0.0, -0.01) + grid._georef_info.xllcenter = -119.995 + grid._georef_info.yllcenter = 40.005 + + curvature = dem.GeographicLaplacian(elevation=grid)._griddata[5, 5] + cell = grid._mean_pixel_dimension()[5, 5] + # The Laplacian of 0.25*(x^2 + y^2) is 1 per cell squared. + assert curvature == pytest.approx(1.0 / cell ** 2, rel=1e-6) + assert curvature < 1e-4, 'a degree-scaled result would be about 1e4' + + +def test_tile_with_zero_padding_does_not_duplicate_the_edges(): + """`griddata[:, -0:]` is the whole array, not an empty slice.""" + data = np.arange(100, dtype=float).reshape(10, 10) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + tiles = grid.tile(tile_xdim=5, tile_ydim=5, tile_xpadding=0, tile_ypadding=0) + assert [t._griddata.shape for t in tiles] == [(5, 5)] * 4 + rebuilt = dem.BaseSpatialGrid.mosaic([t.remove_padding(0, 0) for t in tiles]) + assert np.array_equal(rebuilt._griddata, data) + + +def test_star_import_names_only_things_that_exist(): + import TopoAnalysis + + missing = [name for name in TopoAnalysis.__all__ + if not hasattr(TopoAnalysis, name)] + assert missing == [] + + namespace = {} + exec('from TopoAnalysis import *', namespace) # noqa: S102 - that is the test + assert 'Elevation' in namespace and 'FlowDirectionD8' in namespace + + +def test_the_test_suite_runs_from_a_differently_named_checkout(tmp_path): + """A GitHub zip unpacks to TopoAnalysis-main, not TopoAnalysis.""" + import shutil + + target = tmp_path / 'TopoAnalysis-main' + shutil.copytree(PACKAGE_DIR, str(target), + ignore=shutil.ignore_patterns('__pycache__', 'build', '.git', + '.pytest_cache', 'docs')) + + env = dict(os.environ) + env.pop('PYTHONPATH', None) + result = subprocess.run( + [sys.executable, '-m', 'pytest', '-q', 'tests/test_grid.py', + '-p', 'no:cacheprovider'], + capture_output=True, text=True, cwd=str(target), env=env) + assert result.returncode == 0, result.stdout[-4000:] + result.stderr[-2000:] + + +def test_extension_is_loaded_only_from_the_package_directory(tmp_path): + """A stray `_topoanalysis` elsewhere on sys.path must not be picked up. + + The flat-import fallback used a bare ``import _topoanalysis``, which + searches all of sys.path; a stale or unrelated module of that name would + be loaded instead of the package's own. + """ + decoy = tmp_path / '_topoanalysis.py' + decoy.write_text('MARKER = "decoy"\n') + + script = ( + 'import sys, os;' + 'sys.path.insert(0, {0!r});' + 'sys.path.insert(0, {1!r});' + 'import kernels;' + 'print(getattr(kernels._ext, "MARKER", "not-the-decoy"))' + ).format(str(tmp_path), PACKAGE_DIR) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, + text=True, cwd=str(tmp_path)) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == 'not-the-decoy' diff --git a/tests/test_terrain.py b/tests/test_terrain.py new file mode 100644 index 0000000..ddc7d55 --- /dev/null +++ b/tests/test_terrain.py @@ -0,0 +1,175 @@ +"""Tests for the terrain-attribute grids, against analytic answers.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from TopoAnalysis import dem + + +# --------------------------------------------------------------------------- +# Slope and hillshade +# --------------------------------------------------------------------------- + + +def test_max_slope_on_a_plane(east_plane): + slope = dem.MaxSlope(elevation=east_plane) + # Centred differences reproduce a plane exactly in the interior. + assert np.allclose(slope._griddata[1:-1, 1:-1], 1.0) + + +def test_max_slope_scales_with_cell_size(): + data = np.tile(np.arange(7, dtype=float), (7, 1)) + fine = dem.MaxSlope(elevation=dem.Elevation(dx=1.0, grid=data)) + coarse = dem.MaxSlope(elevation=dem.Elevation(dx=2.0, grid=data)) + assert np.allclose(fine._griddata[1:-1, 1:-1], + 2.0 * coarse._griddata[1:-1, 1:-1]) + + +def test_hillshade_lights_the_slope_that_faces_the_sun(east_plane): + """An east-facing slope is lit from the east and shadowed from the west.""" + lit = dem.Hillshade(elevation=east_plane, azimuth=90, inclination=45) + shadowed = dem.Hillshade(elevation=east_plane, azimuth=270, inclination=45) + assert lit._griddata[2, 2] == 255 + assert shadowed._griddata[2, 2] == 0 + + +def test_hillshade_of_a_south_facing_slope(south_plane): + lit = dem.Hillshade(elevation=south_plane, azimuth=180, inclination=45) + shadowed = dem.Hillshade(elevation=south_plane, azimuth=0, inclination=45) + assert lit._griddata[2, 2] > shadowed._griddata[2, 2] + + +def test_hillshade_is_a_byte_grid_without_wraparound(rough_dome): + hs = dem.Hillshade(elevation=rough_dome, azimuth=315, inclination=45) + assert hs._griddata.dtype == np.uint8 + # Negative cosines used to wrap round the byte into bright speckle. + assert hs._griddata.min() >= 0 and hs._griddata.max() <= 255 + + +def test_hillshade_of_a_flat_surface_is_the_sine_of_the_altitude(): + flat = dem.Elevation(dx=1.0, grid=np.zeros((7, 7))) + hs = dem.Hillshade(elevation=flat, azimuth=315, inclination=30) + assert hs._griddata[3, 3] == pytest.approx(255 * np.cos(np.radians(60)), abs=1) + + +# --------------------------------------------------------------------------- +# Curvature +# --------------------------------------------------------------------------- + + +def test_laplacian_of_a_quadratic_bowl(): + """The Laplacian of a(x^2 + y^2) is 4a.""" + n = 21 + y, x = np.mgrid[0:n, 0:n].astype(float) + a = 0.25 + grid = dem.Elevation(dx=1.0, grid=a * (x ** 2 + y ** 2)) + + curvature = dem.Laplacian(elevation=grid) + assert np.allclose(curvature._griddata[2:-2, 2:-2], 4 * a) + + +def test_laplacian_scales_with_cell_size(): + n = 15 + y, x = np.mgrid[0:n, 0:n].astype(float) + data = x ** 2 + fine = dem.Laplacian(elevation=dem.Elevation(dx=1.0, grid=data)) + coarse = dem.Laplacian(elevation=dem.Elevation(dx=2.0, grid=data)) + assert np.allclose(fine._griddata[2:-2, 2:-2], + 4.0 * coarse._griddata[2:-2, 2:-2]) + + +def test_local_relief_window_is_centred(): + """A single peak must raise the relief symmetrically about itself.""" + data = np.zeros((21, 21)) + data[10, 10] = 100.0 + grid = dem.Elevation(dx=1.0, grid=data) + + relief = dem.LocalRelief(elevation=grid, pixel_radius=3) + # np.ogrid[-r:r] gave a window one cell short on the far side, so the + # footprint was off-centre. + assert relief._griddata[10, 7] == pytest.approx(relief._griddata[10, 13]) + assert relief._griddata[7, 10] == pytest.approx(relief._griddata[13, 10]) + assert relief._griddata[10, 10] == pytest.approx(100.0) + + +def test_local_relief_equals_the_range_over_the_disc(rough_dome): + """The value is exactly the range over the disc, not over its bounding box.""" + radius = 2 + relief = dem.LocalRelief(elevation=rough_dome, pixel_radius=radius) + + yy, xx = np.ogrid[-radius:radius + 1, -radius:radius + 1] + disc = (xx * xx + yy * yy) <= radius * radius + for (i, j) in ((10, 10), (20, 15), (31, 27)): + window = rough_dome._griddata[i - radius:i + radius + 1, + j - radius:j + radius + 1][disc] + assert relief._griddata[i, j] == pytest.approx(window.max() - window.min()) + + # The bounding box is strictly larger, so it is a different answer. + box = rough_dome._griddata[8:13, 8:13] + assert relief._griddata[10, 10] <= box.max() - box.min() + 1e-9 + assert relief._griddata.min() >= 0.0 + + +def test_log_area_is_a_float_grid(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + + log_area = dem.LogArea(area=area) + # Declaring this uint8 truncated log10(area) to a handful of integers. + assert log_area._griddata.dtype == np.float64 + finite = np.isfinite(log_area._griddata) + assert np.allclose(10 ** log_area._griddata[finite], area._griddata[finite]) + + +def test_gradient_grid_components(east_plane): + gradient = dem.Gradient(elevation=east_plane) + assert np.allclose(gradient._gx[1:-1, 1:-1], -1.0) + assert np.allclose(gradient._gy[1:-1, 1:-1], 0.0) + + +# --------------------------------------------------------------------------- +# Geographic grids +# --------------------------------------------------------------------------- + + +def test_geographic_cell_area_shrinks_polewards(geographic_grid): + areas = geographic_grid._area_per_pixel() + # Row 0 is the north edge, so its cells are the smallest. + assert areas[0, 0] < areas[-1, 0] + assert np.allclose(areas[:, 0], areas[:, -1]) + + +def test_geographic_cell_area_matches_the_spherical_formula(geographic_grid): + areas = geographic_grid._area_per_pixel() + re = geographic_grid.earth_radius + dlon = dlat = 0.01 + lat = geographic_grid._georef_info.yllcenter + 9 * dlat # northernmost row + expected = (re ** 2) * np.radians(dlon) * ( + np.sin(np.radians(lat + dlat / 2)) - np.sin(np.radians(lat - dlat / 2))) + assert areas[0, 0] == pytest.approx(expected, rel=1e-9) + + +def test_geographic_mean_pixel_dimension_is_the_area_root(geographic_grid): + assert np.allclose(geographic_grid._mean_pixel_dimension() ** 2, + geographic_grid._area_per_pixel()) + + +def test_geographic_area_uses_spherical_cells(): + """Drainage area on a lat/lon grid must use the true cell areas.""" + n = 12 + z = np.tile(np.arange(n, 0, -1, dtype=float), (n, 1)) + grid = dem.GeographicElevation(dx=0.01, grid=z) + grid._georef_info.geoTransform = (-120.0, 0.01, 0.0, 40.12, 0.0, -0.01) + grid._georef_info.xllcenter = -119.995 + grid._georef_info.yllcenter = 40.005 + + d8 = dem.FlowDirectionD8(flooded_dem=dem.FilledElevation(elevation=grid)) + area = dem.GeographicArea(flow_direction=d8) + + per_pixel = grid._area_per_pixel() + assert area._griddata[5, 0] == pytest.approx(per_pixel[5, 0]) + # Cells accumulate the true, latitude-dependent areas. + assert area._griddata[5, 4] == pytest.approx(per_pixel[5, 0:5].sum()) diff --git a/tests/test_topotoolbox.py b/tests/test_topotoolbox.py new file mode 100644 index 0000000..560d0dd --- /dev/null +++ b/tests/test_topotoolbox.py @@ -0,0 +1,751 @@ +"""Tests for the TopoToolbox-equivalent functions. + +Without MATLAB on hand, parity is established three ways: + +* against closed-form answers on surfaces whose value is known exactly + (planes, quadratics, spherical cells); +* against the documented definitions transcribed from the MATLAB source + (stencils, constants, conventions); and +* against TopoAnalysis' own results where the two are meant to agree, with + the documented conversion applied. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from TopoAnalysis import dem, error, topotoolbox as tt + + +# --------------------------------------------------------------------------- +# Filling and flats +# --------------------------------------------------------------------------- + + +def test_fillsinks_is_a_flat_fill(single_pit): + filled = tt.fillsinks(single_pit) + # TopoToolbox's fillsinks adds no epsilon: the depression becomes an + # exactly flat plateau at the spill elevation. + assert np.all(filled._griddata == 10.0) + assert single_pit._griddata[2, 2] == 1.0, 'the input must not be modified' + + +def test_fillsinks_respects_maxdepth(single_pit): + filled = tt.fillsinks(single_pit, maxdepth=2.0) + assert filled._griddata[2, 2] == pytest.approx(1.0) + + +def test_fillsinks_treats_nodata_as_an_outlet(): + z = np.array([ + [10, 10, 10, 10, 10], + [10, 5, 5, 5, 10], + [np.nan, 5, 1, 5, 10], + [10, 5, 5, 5, 10], + [10, 10, 10, 10, 10]], dtype=float) + filled = tt.fillsinks(dem.Elevation(dx=1.0, grid=z)) + assert filled._griddata[2, 1] == pytest.approx(5.0) + assert np.isnan(filled._griddata[2, 0]) + + +def test_identifyflats_finds_the_plateau_and_its_sills(plateau): + flats, sills = tt.identifyflats(plateau) + # Every plateau cell has an equal neighbour and no lower one ... + assert flats._griddata[1, 1] + assert flats._griddata[1, 2] + # ... except those beside the 2 m outlet, which do have a lower one. + assert not flats._griddata[3, 2] + assert sills._griddata[3, 2] + assert not sills._griddata[1, 1] + + +def test_identifyflats_single_output(plateau): + only_flats = tt.identifyflats(plateau, output='flats') + both = tt.identifyflats(plateau) + assert np.array_equal(only_flats._griddata, both[0]._griddata) + + +def test_imposemin_only_lowers(v_valley): + fd = tt.flowobj(v_valley) + carved = tt.imposemin(fd, v_valley, sl=1e-3) + assert np.all(carved._griddata <= v_valley._griddata + 1e-12) + + +def test_imposemin_enforces_the_gradient(v_valley): + fd = tt.flowobj(v_valley) + sl = 1e-3 + carved = tt.imposemin(fd, v_valley, sl=sl) + + z = carved._griddata.reshape(-1) + step = fd.step_lengths().reshape(-1) + for k, r in enumerate(fd.receivers.reshape(-1)): + if r >= 0: + assert z[k] - z[r] >= sl * step[k] - 1e-9 + + +# --------------------------------------------------------------------------- +# Flow +# --------------------------------------------------------------------------- + + +def test_flowacc_counts_cells_not_area(east_plane): + fd = tt.flowobj(east_plane) + counts = tt.flowacc(fd) + # TopoToolbox's flowacc has unit weight per cell and counts the cell + # itself, so the minimum is 1 -- not dx**2. + assert counts._griddata.min() == 1.0 + assert counts._griddata[2].tolist() == [1.0, 2.0, 3.0, 4.0, 5.0] + + +def test_flowacc_relates_to_Area_by_the_cell_area(rough_dome): + fd = tt.flowobj(rough_dome) + counts = tt.flowacc(fd) + area = dem.Area(flow_direction=fd) + cell = rough_dome._georef_info.dx ** 2 + assert np.allclose(counts._griddata * cell, area._griddata) + + +def test_flowacc_accepts_weights(east_plane): + fd = tt.flowobj(east_plane) + weights = np.full(east_plane._griddata.shape, 2.0) + assert tt.flowacc(fd, weights)._griddata[2, 4] == pytest.approx(10.0) + + +def test_flowdistance_upstream_is_zero_at_the_outlet(east_plane): + fd = tt.flowobj(east_plane) + distance = tt.flowdistance(fd, 'upstream') + # Distance to the outlet: 4 cells away at the western edge, 0 at the east. + assert distance._griddata[2].tolist() == [4.0, 3.0, 2.0, 1.0, 0.0] + + +def test_flowdistance_downstream_is_the_flow_length(east_plane): + fd = tt.flowobj(east_plane) + tt_length = tt.flowdistance(fd, 'downstream') + ta_length = dem.FlowLength(flow_direction=fd) + assert np.allclose(tt_length._griddata, ta_length._griddata) + + +def test_flowdistance_rejects_an_unknown_direction(east_plane): + fd = tt.flowobj(east_plane) + with pytest.raises(error.InputError): + tt.flowdistance(fd, 'sideways') + + +def test_drainagebasins_partition_matches_the_flow_network(rough_dome): + fd = tt.flowobj(rough_dome) + labels = tt.drainagebasins(fd)._griddata + + flat = labels.reshape(-1) + for k, r in enumerate(fd.receivers.reshape(-1)): + if r >= 0: + assert flat[k] == flat[r] + assert labels.min() >= 1 + + +def test_drainagebasins_from_outlets(rough_dome): + fd = tt.flowobj(rough_dome) + outlets = fd._rowscols_to_xy(((20, 20), (10, 30))) + labels = tt.drainagebasins(fd, outlets=outlets)._griddata + + assert set(np.unique(labels).tolist()) <= {0, 1, 2} + assert labels[20, 20] == 1 + assert labels[10, 30] == 2 + + +def test_streamorder_of_a_confluence(): + z = np.array([[3.0, 9.0, 3.0], + [9.0, 2.0, 9.0], + [9.0, 1.0, 9.0]]) + grid = dem.Elevation(dx=1.0, grid=z) + fd = dem.FlowDirectionD8(elevation=grid) + streams = np.array([[1, 0, 1], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) + + strahler = tt.streamorder(fd, streams)._griddata + assert strahler[0, 0] == 1 and strahler[1, 1] == 2 + shreve = tt.streamorder(fd, streams, 'shreve')._griddata + assert shreve[1, 1] == 2 + + +def test_streamorder_rejects_an_unknown_type(east_plane): + fd = tt.flowobj(east_plane) + with pytest.raises(error.InputError): + tt.streamorder(fd, np.ones((5, 5)), 'horton') + + +def test_flowobj_rejects_carve(east_plane): + """'carve' is not implemented and must say so rather than guess.""" + with pytest.raises(error.InputError): + tt.flowobj(east_plane, preprocess='carve') + + +# --------------------------------------------------------------------------- +# Terrain attributes against closed-form answers +# --------------------------------------------------------------------------- + + +def test_gradient8_on_a_cardinal_plane(east_plane): + assert tt.gradient8(east_plane)._griddata[2, 2] == pytest.approx(1.0) + + +def test_gradient8_on_a_diagonal_plane(): + """A 2 m drop over a sqrt(2) m step is a gradient of sqrt(2).""" + z = np.add.outer(np.arange(5, 0, -1, dtype=float), + np.arange(5, 0, -1, dtype=float)) + grid = dem.Elevation(dx=1.0, grid=z) + assert tt.gradient8(grid)._griddata[2, 2] == pytest.approx(np.sqrt(2.0)) + + +def test_gradient8_is_never_negative(rough_dome): + """The structuring elements include the centre, so pits give 0, not < 0.""" + assert tt.gradient8(rough_dome)._griddata.min() >= 0.0 + + +def test_gradient8_is_zero_on_a_flat(): + flat = dem.Elevation(dx=1.0, grid=np.full((5, 5), 7.0)) + assert np.all(tt.gradient8(flat)._griddata == 0.0) + + +@pytest.mark.parametrize('unit,expected', [ + ('tangent', 1.0), + ('degree', 45.0), + ('radian', np.pi / 4), + ('sine', np.sqrt(0.5)), + ('percent', 100.0), +]) +def test_gradient8_units(east_plane, unit, expected): + assert tt.gradient8(east_plane, unit)._griddata[2, 2] == pytest.approx(expected) + + +def test_gradient8_rejects_an_unknown_unit(east_plane): + with pytest.raises(error.InputError): + tt.gradient8(east_plane, 'furlongs') + + +def test_arcslope_on_a_plane(east_plane): + """The Horn 1-2-1 stencil reproduces a plane exactly.""" + assert tt.arcslope(east_plane)._griddata[2, 2] == pytest.approx(1.0) + + +def test_arcslope_has_no_sqrt2_factor(): + """The 8*cellsize denominator carries all the geometry.""" + z = np.add.outer(np.arange(5, 0, -1, dtype=float), + np.arange(5, 0, -1, dtype=float)) + grid = dem.Elevation(dx=1.0, grid=z) + # dz/dx = dz/dy = 1, so the Horn magnitude is sqrt(2). + assert tt.arcslope(grid)._griddata[2, 2] == pytest.approx(np.sqrt(2.0)) + + +@pytest.mark.parametrize('descent,expected', [ + ('north', 0.0), + ('east', 90.0), + ('south', 180.0), + ('west', 270.0), +]) +def test_aspect_points_down_slope(descent, expected): + """Aspect is the compass bearing the surface descends towards.""" + n = 7 + y, x = np.mgrid[0:n, 0:n].astype(float) + surface = { + 'north': y, # elevation grows with row, i.e. downhill to north + 'south': -y, + 'east': -x, + 'west': x, + }[descent] + grid = dem.Elevation(dx=1.0, grid=surface) + assert tt.aspect(grid)._griddata[3, 3] == pytest.approx(expected) + + +def test_aspect_of_a_flat_cell_is_ninety(): + """atan2(0, 0) is 0, so TopoToolbox reports flats as east-facing.""" + flat = dem.Elevation(dx=1.0, grid=np.zeros((5, 5))) + assert tt.aspect(flat)._griddata[2, 2] == pytest.approx(90.0) + + +def test_aspect_classification(): + n = 7 + y, x = np.mgrid[0:n, 0:n].astype(float) + grid = dem.Elevation(dx=1.0, grid=-x) # descends east, aspect 90 + classes = tt.aspect(grid, classify=True)._griddata + assert classes.dtype == np.uint8 + assert classes[3, 3] == 5 # [90, 135) -> class 5 (E) + + +def test_aspect_is_independent_of_cell_size(): + n = 7 + y, x = np.mgrid[0:n, 0:n].astype(float) + fine = dem.Elevation(dx=1.0, grid=-x) + coarse = dem.Elevation(dx=30.0, grid=-x) + assert np.allclose(tt.aspect(fine)._griddata, tt.aspect(coarse)._griddata) + + +def test_curvature_of_a_plane_is_zero(east_plane): + for ctype in ('profc', 'planc', 'tangc', 'meanc', 'total'): + curv = tt.curvature(east_plane, ctype)._griddata + assert np.allclose(curv[1:-1, 1:-1], 0.0, atol=1e-12), ctype + + +def test_total_curvature_of_a_quadratic(): + """'total' is fxx^2 + 2 fxy^2 + fyy^2, with no minus and no denominator.""" + n = 15 + y, x = np.mgrid[0:n, 0:n].astype(float) + a = 0.02 + grid = dem.Elevation(dx=1.0, grid=a * x ** 2) + curv = tt.curvature(grid, 'total')._griddata + # fxx = 2a, fyy = fxy = 0. + assert np.allclose(curv[2:-2, 2:-2], (2 * a) ** 2) + + +def test_curvature_rejects_an_unknown_type(east_plane): + with pytest.raises(error.InputError): + tt.curvature(east_plane, 'gaussian') + + +def test_curvature_zeroes_the_zero_gradient_cells(): + """0/0 becomes 0, not NaN -- matching TopoToolbox.""" + flat = dem.Elevation(dx=1.0, grid=np.zeros((7, 7))) + assert np.all(tt.curvature(flat, 'profc')._griddata == 0.0) + + +def test_hillshade_is_a_cosine_not_a_byte(east_plane): + shade = tt.hillshade(east_plane, azimuth=90, altitude=45)._griddata + # TopoToolbox scales to 0-255 only when plotting. + assert shade.dtype == np.float64 + assert -1.0 - 1e-9 <= shade.min() and shade.max() <= 1.0 + 1e-9 + + +def test_hillshade_is_brightest_facing_the_light(east_plane): + lit = tt.hillshade(east_plane, azimuth=90, altitude=45)._griddata[2, 2] + shadowed = tt.hillshade(east_plane, azimuth=270, altitude=45)._griddata[2, 2] + assert lit == pytest.approx(1.0) + assert shadowed == pytest.approx(0.0, abs=1e-9) + + +def test_hillshade_defaults_match_topotoolbox(rough_dome): + """azimuth 315, altitude 60 -- the code's defaults, not the docstring's.""" + default = tt.hillshade(rough_dome)._griddata + explicit = tt.hillshade(rough_dome, azimuth=315, altitude=60)._griddata + assert np.array_equal(default, explicit) + + +def test_hillshade_of_a_flat_surface_is_the_sine_of_the_altitude(): + flat = dem.Elevation(dx=1.0, grid=np.zeros((7, 7))) + for altitude in (30.0, 45.0, 60.0): + shade = tt.hillshade(flat, altitude=altitude)._griddata[3, 3] + assert shade == pytest.approx(np.sin(np.radians(altitude))) + + +def test_hillshade_agrees_in_sign_with_the_esri_version(rough_dome): + """The two formulations must at least light the same slopes.""" + tt_shade = tt.hillshade(rough_dome, azimuth=315, altitude=45)._griddata + esri = dem.Hillshade(elevation=rough_dome, azimuth=315, + inclination=45)._griddata.astype(float) + interior = (slice(2, -2), slice(2, -2)) + correlation = np.corrcoef(tt_shade[interior].ravel(), esri[interior].ravel())[0, 1] + assert correlation > 0.97 + + +def test_localtopography_range_matches_local_relief(rough_dome): + tt_relief = tt.localtopography(rough_dome, radius=60.0, type='range')._griddata + ta_relief = dem.LocalRelief(elevation=rough_dome, pixel_radius=2)._griddata + assert np.allclose(tt_relief, ta_relief) + + +# --------------------------------------------------------------------------- +# Cell area +# --------------------------------------------------------------------------- + + +def test_cellarea_uses_the_topotoolbox_sphere(geographic_grid): + areas = tt.cellarea(geographic_grid)._griddata + radius_km = np.sqrt(tt.EARTH_SURFACE_AREA_KM2 / (4 * np.pi)) + dlon = dlat = 0.01 + lat = geographic_grid._georef_info.yllcenter + 9 * dlat + expected = (radius_km * 1000.0) ** 2 * np.radians(dlon) * ( + np.sin(np.radians(lat + dlat / 2)) - np.sin(np.radians(lat - dlat / 2))) + assert areas[0, 0] == pytest.approx(expected, rel=1e-9) + + +def test_cellarea_km_units(geographic_grid): + metres = tt.cellarea(geographic_grid, 'm')._griddata + kilometres = tt.cellarea(geographic_grid, 'km')._griddata + assert np.allclose(metres, kilometres * 1e6) + + +def test_cellarea_matches_topoanalysis_to_the_radius_difference(geographic_grid): + """The two differ only through their Earth radii, by ~1.5e-5.""" + tt_area = tt.cellarea(geographic_grid)._griddata + ta_area = geographic_grid._area_per_pixel() + ratio = (np.sqrt(tt.EARTH_SURFACE_AREA_KM2 / (4 * np.pi)) * 1000.0 + / geographic_grid.earth_radius) ** 2 + # TopoAnalysis evaluates the longitude difference per cell, so its + # areas vary in the last few bits along a row; TopoToolbox's do not. + assert np.allclose(tt_area, ta_area * ratio, rtol=1e-9) + + +def test_cellarea_rejects_an_unknown_unit(geographic_grid): + with pytest.raises(error.InputError): + tt.cellarea(geographic_grid, 'miles') + + +# --------------------------------------------------------------------------- +# Chi and steepness +# --------------------------------------------------------------------------- + + +def test_chitransform_is_zero_at_the_outlet(v_valley): + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + chi = tt.chitransform(fd, counts, mn=0.45, a0=1e6)._griddata + + outlets = fd.receivers < 0 + assert np.allclose(chi[outlets], 0.0) + assert np.nanmax(chi) > 0.0 + + +def test_chitransform_correctcellsize(v_valley): + """With correctcellsize the area is taken to be in cells, not m^2.""" + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + area = dem.Area(flow_direction=fd) # already m^2 + + corrected = tt.chitransform(fd, counts, correctcellsize=True)._griddata + direct = tt.chitransform(fd, area, correctcellsize=False)._griddata + assert np.allclose(corrected, direct, equal_nan=True) + + +def test_chitransform_scales_with_a0(v_valley): + """chi is proportional to a0**mn.""" + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + mn = 0.45 + one = tt.chitransform(fd, counts, mn=mn, a0=1.0)._griddata + million = tt.chitransform(fd, counts, mn=mn, a0=1e6)._griddata + valid = np.isfinite(one) & (one > 0) + assert np.allclose(million[valid] / one[valid], 1e6 ** mn) + + +def test_chitransform_uses_the_trapezoid_rule(v_valley): + """It must agree with Chi(trapezoid=True), not with the default rule.""" + fd = tt.flowobj(v_valley) + area = dem.Area(flow_direction=fd) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlets = area._rowscols_to_xy((outlet_rc,)) + + tt_chi = tt.chitransform(fd, area, mn=0.45, a0=1e6, outlets=outlets, + correctcellsize=False)._griddata + ta_chi = dem.Chi(area=area, flow_direction=fd, theta=0.45, Ao=1e6, + outlets=outlets, trapezoid=True)._griddata + + finite = np.isfinite(tt_chi) + assert np.allclose(tt_chi[finite], ta_chi[finite]) + + +def test_ksn_is_slope_times_area_to_theta(v_valley): + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + theta = 0.45 + k = tt.ksn(fd, v_valley, counts, theta=theta)._griddata + + carved = tt.imposemin(fd, v_valley, sl=1e-5) + z = carved._griddata.reshape(-1) + step = fd.step_lengths().reshape(-1) + receivers = fd.receivers.reshape(-1) + area = (counts._griddata * fd._georef_info.dx ** 2).reshape(-1) + + donors = np.flatnonzero(receivers >= 0) + gradient = (z[donors] - z[receivers[donors]]) / step[donors] + assert np.allclose(k.reshape(-1)[donors], gradient * area[donors] ** theta) + + +def test_mchi_is_the_elevation_gradient_in_chi_space(v_valley): + """M_chi is dz/dchi, a plain forward difference with no carving.""" + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + chi = tt.chitransform(fd, counts, mn=0.45, a0=1.0) + mc = tt.mchi(fd, v_valley, chi)._griddata.reshape(-1) + + z = np.asarray(v_valley._griddata, dtype=float).reshape(-1) + c = chi._griddata.reshape(-1) + receivers = fd.receivers.reshape(-1) + donors = np.flatnonzero(receivers >= 0) + expected = (z[donors] - z[receivers[donors]]) / (c[donors] - c[receivers[donors]]) + + finite = np.isfinite(expected) + assert np.allclose(mc[donors][finite], expected[finite]) + + +def test_mchi_scales_inversely_with_a0(v_valley): + """M_chi depends on the a0 baked into chi: mchi(a0) * a0**mn is invariant.""" + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + mn = 0.45 + + one = tt.mchi(fd, v_valley, tt.chitransform(fd, counts, mn=mn, a0=1.0))._griddata + million = tt.mchi(fd, v_valley, tt.chitransform(fd, counts, mn=mn, a0=1e6))._griddata + + finite = np.isfinite(one) & np.isfinite(million) & (np.abs(one) > 1e-12) + assert np.allclose(million[finite] * (1e6 ** mn), one[finite], rtol=1e-9) + + +def test_mchi_and_ksn_are_close_but_not_identical(v_valley): + """TopoToolbox's docstring claims they are the same; the code differs. + + ``ksn`` carves the DEM first and takes the integrand at the upstream + cell, while ``mchi`` uses the raw DEM and chi's trapezoidal averaging. + """ + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + mn = 0.45 + + carved = tt.imposemin(fd, v_valley, sl=1e-5) + mc = tt.mchi(fd, carved, tt.chitransform(fd, counts, mn=mn, a0=1.0))._griddata + k = tt.ksn(fd, v_valley, counts, theta=mn)._griddata + + receivers = fd.receivers.reshape(-1) + donors = np.flatnonzero(receivers >= 0) + a = mc.reshape(-1)[donors] + b = k.reshape(-1)[donors] + finite = np.isfinite(a) & np.isfinite(b) & (b > 0) + + ratio = a[finite] / b[finite] + assert np.all(ratio > 0.5) and np.all(ratio < 2.0) + assert not np.allclose(a[finite], b[finite]) + + +# --------------------------------------------------------------------------- +# Referencing +# --------------------------------------------------------------------------- + + +def test_getcoordinates_runs_north_to_south(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((4, 6))) + x, y = tt.getcoordinates(grid) + assert x.size == 6 and y.size == 4 + assert y[0] > y[-1], 'row 0 is the north edge' + assert x[1] - x[0] == pytest.approx(10.0) + + +def test_getextent_is_cell_centres_not_edges(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((4, 6))) + centres = tt.getextent(grid) + edges = grid.extent() + assert centres[0] == pytest.approx(edges[0] + 5.0) + assert centres[1] == pytest.approx(edges[1] - 5.0) + + +def test_coord2sub_and_sub2coord_round_trip(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((7, 5))) + for row in range(7): + for col in range(5): + x, y = tt.sub2coord(grid, row, col) + r, c = tt.coord2sub(grid, x, y) + assert (int(r[0]), int(c[0])) == (row, col) + + +def test_coord2sub_returns_nan_outside_the_grid(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + r, c = tt.coord2sub(grid, [1000.0], [1000.0]) + assert np.isnan(r[0]) and np.isnan(c[0]) + + +def test_coord2sub_rounds_halves_away_from_zero(): + """MATLAB's round, not Python's round-half-to-even.""" + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((5, 5))) + info = grid._georef_info + # Exactly half a cell east of column 0. + r, c = tt.coord2sub(grid, [info.xllcenter + 0.5], [info.yllcenter]) + assert int(c[0]) == 1 + + +# --------------------------------------------------------------------------- +# Checks against literal ports of the MATLAB source +# +# These transcribe the .m files directly rather than restating the intent, so +# they catch a drift in the Python that a behavioural test would not. +# --------------------------------------------------------------------------- + + +def matlab_surfnorm(z): + """A literal port of MATLAB's ``surfnorm(Z)`` (graph3d/surfnorm.m). + + The boundary is expanded by the quadratic ghost cell ``3*a1 - 3*a2 + a3`` + -- *not* the linear ``2*a1 - a2`` -- which makes the edge derivative the + second-order one-sided difference. + """ + m, n = z.shape + x, y = np.meshgrid(np.arange(1.0, n + 1), np.arange(1.0, m + 1)) + + def expand(a): + a = np.vstack([3 * a[0:1] - 3 * a[1:2] + a[2:3], a, + 3 * a[m - 1:m] - 3 * a[m - 2:m - 1] + a[m - 3:m - 2]]) + return np.hstack([3 * a[:, 0:1] - 3 * a[:, 1:2] + a[:, 2:3], a, + 3 * a[:, n - 1:n] - 3 * a[:, n - 2:n - 1] + a[:, n - 3:n - 2]]) + + xx, yy, zz = expand(x), expand(y), expand(z) + + def filter_row(a): # filter2([1 0 -1]/2, a) -- correlation + out = np.zeros_like(a) + out[:, 1:-1] = 0.5 * (a[:, :-2] - a[:, 2:]) + return out + + def filter_col(a): # filter2([-1;0;1]/2, a) + out = np.zeros_like(a) + out[1:-1, :] = 0.5 * (a[2:, :] - a[:-2, :]) + return out + + rows, cols = slice(1, m + 1), slice(1, n + 1) + ax, ay, az = (filter_row(v)[rows, cols] for v in (xx, yy, zz)) + bx, by, bz = (filter_col(v)[rows, cols] for v in (xx, yy, zz)) + + nx = -(ay * bz - az * by) + ny = -(az * bx - ax * bz) + nz = -(ax * by - ay * bx) + magnitude = np.sqrt(nx * nx + ny * ny + nz * nz) + magnitude[magnitude == 0] = np.finfo(float).eps + return nx / magnitude, ny / magnitude, nz / magnitude + + +@pytest.mark.parity +@pytest.mark.parametrize('shape', [(7, 8), (12, 12), (5, 20)]) +def test_surfnorm_matches_matlab_including_the_border(shape): + from TopoAnalysis.topotoolbox import _surfnorm + + z = np.random.default_rng(7).random(shape) * 100 + for ours, theirs in zip(_surfnorm(z), matlab_surfnorm(z)): + assert np.allclose(ours, theirs, atol=1e-12) + + +def matlab_identifyflats(z): + """A literal port of ``@GRIDobj/identifyflats.m``.""" + from scipy.ndimage import binary_dilation, grey_dilation, grey_erosion + + log_nans = np.isnan(z) + dem_ = np.where(log_nans, -np.inf, z) + + flats = grey_erosion(dem_, size=(3, 3), mode='constant', cval=np.inf) == dem_ + if log_nans.any(): + flats &= ~log_nans + flats[:, [0, -1]] = False + flats[[0, -1], :] = False + if log_nans.any(): + flats[binary_dilation(log_nans, structure=np.ones((3, 3), bool))] = False + + marker = np.full(dem_.shape, -np.inf) + marker[flats] = dem_[flats] + sills = (grey_dilation(marker, size=(3, 3), mode='constant', + cval=-np.inf) == dem_) & ~flats + if log_nans.any(): + sills[log_nans] = False + return flats, sills + + +@pytest.mark.parity +@pytest.mark.parametrize('seed', [0, 1, 2, 3]) +def test_identifyflats_matches_matlab(seed): + """Including the cleared rim, the NaN halo, and isolated pits.""" + from scipy.ndimage import gaussian_filter + + rng = np.random.default_rng(seed) + n = int(rng.integers(20, 45)) + # Rounded, so plateaus and ties actually occur. + z = np.round(gaussian_filter(rng.random((n, n)) * 50, 2.0), 1) + if seed % 2: + z[rng.random(z.shape) < 0.05] = np.nan + + filled = tt.fillsinks(dem.Elevation(dx=1.0, grid=z)) + expected_flats, expected_sills = matlab_identifyflats(filled._griddata) + flats, sills = tt.identifyflats(filled) + + assert np.array_equal(flats._griddata, expected_flats) + assert np.array_equal(sills._griddata, expected_sills) + + +@pytest.mark.parity +def test_identifyflats_clears_the_rim(): + """A plateau reaching the grid edge is flat only in the interior.""" + z = np.full((6, 6), 3.0) + z[5, 5] = 2.0 # the plateau's only outlet, at a corner + flats = tt.identifyflats(dem.Elevation(dx=1.0, grid=z), output='flats')._griddata + + # The rim is cleared unconditionally ... + assert not flats[0, :].any() and not flats[-1, :].any() + assert not flats[:, 0].any() and not flats[:, -1].any() + # ... and interior cells away from the outlet are flat. + assert flats[1, 1] and flats[2, 3] + # (4, 4) touches the 2, so it has a lower neighbour and is not flat. + assert not flats[4, 4] + + +@pytest.mark.parity +def test_identifyflats_clears_the_nodata_halo(): + """MATLAB drops the 3x3 dilation of the no-data mask from the flats.""" + z = np.full((7, 7), 3.0) + z[3, 3] = np.nan + flats = tt.identifyflats(dem.Elevation(dx=1.0, grid=z), output='flats')._griddata + + for di in (-1, 0, 1): + for dj in (-1, 0, 1): + assert not flats[3 + di, 3 + dj], 'the NaN halo must be cleared' + assert flats[1, 1], 'the rest of the plateau is still flat' + + +@pytest.mark.parity +def test_identifyflats_counts_an_isolated_pit_as_a_flat(): + """MATLAB's test is 'no strictly lower neighbour', with no equal-neighbour rule.""" + z = np.full((5, 5), 99.0) + z[1:4, 1:4] = 5.0 + z[2, 2] = 1.0 # lower than all eight neighbours + flats = tt.identifyflats(dem.Elevation(dx=1.0, grid=z), output='flats')._griddata + assert flats[2, 2] + + +@pytest.mark.parity +def test_closed_basins_exclude_the_grid_edge(): + z = np.full((7, 7), 50.0) + z[3, 3] = 10.0 # an interior closed basin + z[0, 3] = 1.0 # touches the edge, so not closed + closed = tt.identifyflats(dem.Elevation(dx=1.0, grid=z), + output='closedbasins')._griddata + assert closed[3, 3] + assert not closed[0, 3] + + +@pytest.mark.parity +def test_fillsinks_maxdepth_is_all_or_nothing_per_depression(): + """MATLAB restores the whole depression, not just its deepest cells.""" + z = np.full((7, 7), 10.0) + z[1:6, 1:6] = 9.0 + z[2:5, 2:5] = 6.0 + z[3, 3] = 4.0 # one depression, spill 10, depth 6 + grid = dem.Elevation(dx=1.0, grid=z) + + # Below the depression's depth: nothing is touched at all. + for maxdepth in (1.0, 3.0, 5.0): + assert np.array_equal(tt.fillsinks(grid, maxdepth=maxdepth)._griddata, z), \ + 'maxdepth={0} must leave the whole depression alone'.format(maxdepth) + + # At or above it: the depression fills completely. + for maxdepth in (6.0, 7.0): + filled = tt.fillsinks(grid, maxdepth=maxdepth)._griddata + assert np.all(filled[1:6, 1:6] == 10.0) + + +@pytest.mark.parity +def test_localtopography_mean_uses_the_disc_not_its_bounding_box(): + data = np.zeros((15, 15)) + data[4, 4] = 100.0 # 4.24 cells from the centre + grid = dem.Elevation(dx=1.0, grid=data) + # A radius-3 disc excludes it; the 7x7 bounding box does not. + assert tt.localtopography(grid, radius=3.0, type='mean')._griddata[7, 7] == \ + pytest.approx(0.0) + assert tt.localtopography(grid, radius=6.0, type='mean')._griddata[7, 7] > 0.0 + + +@pytest.mark.parity +def test_localtopography_std_is_the_sample_standard_deviation(): + """MATLAB's stdfilt normalises by N-1.""" + values = np.arange(25.0).reshape(5, 5) + grid = dem.Elevation(dx=1.0, grid=values) + window = [7.0, 11.0, 12.0, 13.0, 17.0] # the radius-1 disc at (2, 2) + assert tt.localtopography(grid, radius=1.0, type='std')._griddata[2, 2] == \ + pytest.approx(np.std(window, ddof=1)) diff --git a/topotoolbox.py b/topotoolbox.py new file mode 100644 index 0000000..1b79469 --- /dev/null +++ b/topotoolbox.py @@ -0,0 +1,1039 @@ +"""TopoToolbox-equivalent terrain analysis. + +Every function here reproduces the numerical result of the identically named +function in TopoToolbox 2 (Schwanghart & Scherler 2014), the MATLAB package, +so that a result computed in TopoAnalysis can be checked against one computed +in MATLAB. + +The rest of TopoAnalysis keeps its own idioms -- drainage area in square +metres, ESRI-style hillshade, chi integrated with a left-endpoint rule. This +module exists for the cases where you need the *same number*, not just the +same quantity. :doc:`docs/topotoolbox_parity` tabulates the correspondence +and lists the places where exact agreement is not achievable. + +Example +------- +:: + + from TopoAnalysis import dem, topotoolbox as tt + + z = dem.Elevation(gdal_filename='srtm.tif') + zf = tt.fillsinks(z) # == GRIDobj/fillsinks + fd = tt.flowobj(z, preprocess='fill') # == FLOWobj(DEM,'preprocess','fill') + a = tt.flowacc(fd) # == flowacc(FD), in CELLS + c = tt.chitransform(fd, a, mn=0.45) # == chitransform(S,A) + +Conventions +----------- +* Grids are TopoAnalysis grid objects; results are returned as grid objects + of the same shape and georeferencing. +* Drainage area is in **number of cells**, as TopoToolbox reports it -- not + in square metres, which is what :class:`TopoAnalysis.dem.Area` returns. +* Row 0 is the north edge, matching both packages. + +References +---------- +Schwanghart, W. and Scherler, D. (2014). "TopoToolbox 2 -- MATLAB-based +software for topographic analysis and modeling in Earth surface sciences." +*Earth Surface Dynamics* 2, 1-7. +""" + +from __future__ import annotations + +import numpy as np + +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as _dem + from . import error as Error + from . import kernels +except ImportError: # pragma: no cover + import dem as _dem + import error as Error + import kernels + +__all__ = [ + "fillsinks", + "identifyflats", + "imposemin", + "flowobj", + "flowacc", + "flowdistance", + "drainagebasins", + "gradient8", + "arcslope", + "hillshade", + "aspect", + "curvature", + "cellarea", + "localtopography", + "streamorder", + "chitransform", + "ksn", + "mchi", + "getcoordinates", + "getextent", + "coord2sub", + "sub2coord", + "STREAM_UNITS", +] + +#: Unit conversions shared by ``gradient8`` and ``arcslope``. +_UNIT_FUNCS = { + 'tangent': lambda g: g, + 'degree': lambda g: np.degrees(np.arctan(g)), + 'radian': lambda g: np.arctan(g), + 'sine': lambda g: np.sin(np.arctan(g)), + 'percent': lambda g: g * 100.0, +} + +STREAM_UNITS = tuple(sorted(_UNIT_FUNCS)) + + +def _like(source, values, cls=None): + """A new grid with ``source``'s georeferencing and ``values`` as data.""" + target = (cls or _dem.BaseSpatialGrid)() + target._copy_info_from_grid(source, set_zeros=True) + target._griddata = values + return target + + +def _convert_unit(values, unit): + try: + return _UNIT_FUNCS[unit](values) + except KeyError: + raise Error.InputError( + 'unit', "unit must be one of {0}; got {1!r}".format(STREAM_UNITS, unit)) + + +# --------------------------------------------------------------------------- +# Depression handling +# --------------------------------------------------------------------------- + + +def fillsinks(elevation, maxdepth=None): + """Fill topographic depressions to a flat surface. + + Equivalent to ``GRIDobj/fillsinks(DEM)`` and, with ``maxdepth``, to + ``fillsinks(DEM, maxdepth)``. + + Unlike :class:`TopoAnalysis.dem.FilledElevation`, whose default adds a + tiny gradient across filled areas so that D8 routing has a direction to + follow, this fills to an exactly flat plateau -- which is what + TopoToolbox does, and what ``identifyflats`` then relies on. + + Parameters + ---------- + elevation : BaseSpatialGrid + The DEM. ``NaN`` is no-data and, as in TopoToolbox, acts as an + outlet: water reaching a NaN cell leaves the grid. + maxdepth : float, optional + Leave depressions deeper than this unfilled. + + Returns + ------- + Elevation + A new grid; the input is not modified. + """ + from scipy.ndimage import generate_binary_structure, label + + original = np.array(elevation._griddata, dtype=np.float64, copy=True) + + filled = _dem.Elevation() + filled._copy_info_from_grid(elevation) + # An explicit copy: np.ascontiguousarray returns the *same* array when the + # input is already float64 and contiguous, and the flood works in place. + filled._griddata = original.copy() + + if maxdepth is None: + kernels.priority_flood(filled._griddata, mode='flat') + return filled + + # MATLAB's depth-limited branch is all-or-nothing per *connected + # depression*, not per cell: whenever a depression's deepest point + # exceeds maxdepth, that cell is added to the marker set, which restores + # the whole depression to its original elevations. Unblocking it can + # split the depression into shallower ones, which then do get filled, so + # the process repeats until nothing exceeds the limit. + maxdepth = float(maxdepth) + structure = generate_binary_structure(2, 2) + seeds = list(kernels.default_seeds(original).tolist()) + + for _ in range(original.size + 1): + surface = original.copy() + kernels.priority_flood(surface, seeds=np.array(seeds, dtype=np.int64), + mode='flat') + depth = surface - original + labels, count = label(depth > 0, structure=structure) + if count == 0: + break + + added = [] + for component in range(1, count + 1): + inside = labels == component + if depth[inside].max() <= maxdepth: + continue + # Seed the deepest cell of the component. + flat_index = int(np.flatnonzero((labels == component).reshape(-1))[ + int(np.argmax(depth[inside]))]) + added.append(flat_index) + if not added: + break + seeds.extend(added) + + filled._griddata = surface + return filled + + +def identifyflats(elevation, output='both'): + """Locate flat areas, their sills and the closed basins. + + Equivalent to ``GRIDobj/identifyflats``. + + * A **flat** is a cell with no strictly lower neighbour -- the MATLAB test + is ``imerode(dem, ones(3)) == dem``. An isolated pit therefore counts + as a flat. The grid's outer ring is cleared, as is the one-cell halo + around any no-data. + * A **sill** is a non-flat cell at the same elevation as an adjacent + flat: the point a flat drains through. + * A **closed basin** is a regional minimum that does not touch the grid + edge. + + Parameters + ---------- + elevation : BaseSpatialGrid + Normally the output of :func:`fillsinks`. + output : {'both', 'flats', 'sills', 'closedbasins', 'all'} + ``'both'`` returns ``(flats, sills)``; ``'all'`` adds the closed + basins. + + Returns + ------- + BaseSpatialGrid or tuple + Boolean grids. + """ + from scipy.ndimage import binary_dilation, generate_binary_structure, label + + z = np.asarray(elevation._griddata, dtype=np.float64) + nodata = np.isnan(z) + work = np.where(nodata, -np.inf, z) + structure = generate_binary_structure(2, 2) # ones(3) + + # imerode with a 3x3 of ones is the local minimum over the 3x3 INCLUDING + # the centre, padded with +inf; `== dem` therefore means "no strictly + # lower neighbour". There is deliberately no requirement of an *equal* + # neighbour, so a cell lower than all eight of its neighbours is a flat. + local_min = work.copy() + for d in range(8): + di, dj = int(kernels.D8_DI[d]), int(kernels.D8_DJ[d]) + local_min = np.minimum(local_min, _shift(work, di, dj, np.inf)) + flats = (local_min == work) + if nodata.any(): + flats &= ~nodata + + # Regional minima, before the rim is cleared: connected components of + # `flats` are equal-valued plateaus with no lower neighbour outside them. + regional_minima = flats.copy() + + flats[0, :] = False + flats[-1, :] = False + flats[:, 0] = False + flats[:, -1] = False + if nodata.any(): + flats[binary_dilation(nodata, structure=structure)] = False + + # Sills: the 3x3 maximum taken over the flat cells only, compared with + # the cell's own elevation, excluding the flats themselves. + marker = np.full(z.shape, -np.inf) + marker[flats] = work[flats] + dilated = marker.copy() + for d in range(8): + di, dj = int(kernels.D8_DI[d]), int(kernels.D8_DJ[d]) + dilated = np.maximum(dilated, _shift(marker, di, dj, -np.inf)) + sills = (dilated == work) & ~flats + if nodata.any(): + sills[nodata] = False + + if output in ('closedbasins', 'all'): + # imclearborder: drop the components that touch the grid edge. + labels, count = label(regional_minima, structure=structure) + on_border = set(labels[0, :].tolist()) | set(labels[-1, :].tolist()) \ + | set(labels[:, 0].tolist()) | set(labels[:, -1].tolist()) + on_border.discard(0) + closed = regional_minima & ~np.isin(labels, list(on_border)) + if nodata.any(): + closed |= nodata + labels, _ = label(closed, structure=structure) + on_border = set(labels[0, :].tolist()) | set(labels[-1, :].tolist()) \ + | set(labels[:, 0].tolist()) | set(labels[:, -1].tolist()) + on_border.discard(0) + closed &= ~np.isin(labels, list(on_border)) + closed[nodata] = False + + flats_grid = _like(elevation, flats) + sills_grid = _like(elevation, sills) + if output == 'flats': + return flats_grid + if output == 'sills': + return sills_grid + if output == 'closedbasins': + return _like(elevation, closed) + if output == 'all': + return flats_grid, sills_grid, _like(elevation, closed) + return flats_grid, sills_grid + + +def _shift(grid, di, dj, fill): + """``grid`` translated so element (i, j) holds ``grid[i+di, j+dj]``.""" + out = np.full_like(grid, fill) + ny, nx = grid.shape + src_i = slice(max(di, 0), ny + min(di, 0)) + src_j = slice(max(dj, 0), nx + min(dj, 0)) + dst_i = slice(max(-di, 0), ny + min(-di, 0)) + dst_j = slice(max(-dj, 0), nx + min(-dj, 0)) + out[dst_i, dst_j] = grid[src_i, src_j] + return out + + +def imposemin(flow_direction, elevation, sl=0.0): + """Carve elevations so every flow path descends at gradient >= ``sl``. + + Equivalent to ``FLOWobj/imposemin(FD, DEM, sl)``. Cells are only ever + lowered, so channels are cut through obstructions instead of being + drowned by them. + + Returns + ------- + Elevation + A new grid. + """ + carved = _dem.Elevation() + carved._copy_info_from_grid(elevation) + carved._griddata = np.array(elevation._griddata, dtype=np.float64, copy=True) + kernels.imposemin( + flow_direction.receivers, + flow_direction.topological_order, + np.ascontiguousarray(flow_direction.step_lengths(), dtype=np.float64), + float(sl), + carved._griddata) + return carved + + +# --------------------------------------------------------------------------- +# Flow +# --------------------------------------------------------------------------- + + +def flowobj(elevation, preprocess='fill'): + """Build a D8 flow-direction grid the way ``FLOWobj`` does. + + Parameters + ---------- + elevation : BaseSpatialGrid + preprocess : {'fill', 'none'} + ``'fill'`` removes depressions first (TopoToolbox's + ``'preprocess','fill'``). ``'carve'`` is not implemented; see the + note below. + + Returns + ------- + FlowDirectionD8 + + Notes + ----- + TopoToolbox routes flow across the flat areas left by filling using an + auxiliary "geodesic" topography, so that flat cells drain towards the + nearest sill. TopoAnalysis instead adds a small gradient while filling + (:attr:`~TopoAnalysis.dem.PriorityQueueMixIn.aggradation_slope`). Both + produce a valid, connected drainage network with identical basin + boundaries wherever the terrain is not flat, but **inside flats the two + networks differ**. TopoToolbox's ``'carve'`` preprocessing lowers the + DEM instead of raising it and is a different surface again. + """ + if preprocess not in ('fill', 'none'): + raise Error.InputError( + 'preprocess', "preprocess must be 'fill' or 'none'; 'carve' is not " + "implemented (see the module docstring)") + + if preprocess == 'fill': + # A small gradient across filled flats, so every cell has a defined + # downslope neighbour. A pure flat fill would leave plateaus with + # no direction at all. + surface = _dem.FilledElevation(elevation=elevation) + else: + surface = elevation + + return _dem.FlowDirectionD8(flooded_dem=surface) + + +def flowacc(flow_direction, weights=None): + """Flow accumulation in **number of cells**. + + Equivalent to ``FLOWobj/flowacc(FD)``: every cell counts itself, so the + minimum value is 1. + + Parameters + ---------- + flow_direction : FlowDirectionD8 + weights : ndarray or BaseSpatialGrid, optional + Per-cell weight; defaults to 1. + + Returns + ------- + BaseSpatialGrid + + See Also + -------- + TopoAnalysis.dem.Area : the same accumulation weighted by cell area. + """ + shape = flow_direction._griddata.shape + if weights is None: + w = np.ones(shape, dtype=np.float64) + else: + w = np.ascontiguousarray( + getattr(weights, '_griddata', weights), dtype=np.float64) + + accumulated = kernels.accumulate( + flow_direction.receivers, flow_direction.topological_order, w) + return _like(flow_direction, np.asarray(accumulated)) + + +def flowdistance(flow_direction, direction='upstream'): + """Along-flow distance in map units. + + Equivalent to ``FLOWobj/flowdistance(FD, direction)`` with no seed + points. + + Parameters + ---------- + direction : {'upstream', 'downstream', 'maxdownstream'} + ``'upstream'`` gives the distance from each cell down to its outlet. + ``'downstream'`` (a synonym for ``'maxdownstream'``) gives the length + of the longest flow path reaching each cell -- the quantity + :class:`TopoAnalysis.dem.FlowLength` computes. + """ + step = np.ascontiguousarray(flow_direction.step_lengths(), dtype=np.float64) + receivers = flow_direction.receivers + order = flow_direction.topological_order + + if direction == 'upstream': + distance = kernels.downstream_distance(receivers, order, step) + elif direction in ('downstream', 'maxdownstream'): + distance, _ = kernels.flow_length(receivers, order, step) + else: + raise Error.InputError( + 'direction', + "direction must be 'upstream', 'downstream' or 'maxdownstream'") + + return _like(flow_direction, np.asarray(distance)) + + +def drainagebasins(flow_direction, outlets=None, valid=None): + """Label drainage basins. + + Equivalent to ``FLOWobj/drainagebasins``. Without ``outlets`` every + terminal cell that something drains into seeds its own basin; with them, + each cell is labelled by the 1-based position of the outlet it drains to. + + Parameters + ---------- + outlets : sequence of (x, y), optional + valid : ndarray or BaseSpatialGrid, optional + Restrict labelling to these cells (e.g. ``~np.isnan(dem._griddata)``). + + Returns + ------- + BaseSpatialGrid + Integer labels; 0 means "reaches none of the outlets". + + Notes + ----- + TopoToolbox numbers basins in the order its edge list happens to meet + them, so the *partition* matches but the integer labels generally do + not. Compare partitions, not label values. + """ + outlet_indices = None + if outlets is not None: + flat = [] + for (row, col) in flow_direction._xy_to_rowscols(outlets): + if row is not None: + flat.append(row * flow_direction._georef_info.nx + col) + outlet_indices = np.array(flat, dtype=np.int64) + + valid_mask = None + if valid is not None: + valid_mask = np.ascontiguousarray( + (np.asarray(getattr(valid, '_griddata', valid)) != 0).astype(np.uint8)) + + labels = kernels.drainage_basins( + flow_direction.receivers, flow_direction.topological_order, + outlet_indices, valid_mask) + return _like(flow_direction, np.asarray(labels)) + + +def streamorder(flow_direction, stream_mask, order_type='strahler'): + """Strahler or Shreve stream order. + + Equivalent to ``FLOWobj/streamorder`` / ``STREAMobj/streamorder``. + + Parameters + ---------- + stream_mask : ndarray or BaseSpatialGrid + Non-zero on the channel network, e.g. ``flowacc(fd)._griddata > 1000``. + order_type : {'strahler', 'shreve'} + + Returns + ------- + BaseSpatialGrid + Integer orders; 0 off the network. + """ + if order_type not in ('strahler', 'shreve'): + raise Error.InputError('order_type', "order_type must be 'strahler' or 'shreve'") + + mask = np.ascontiguousarray( + (np.asarray(getattr(stream_mask, '_griddata', stream_mask)) != 0).astype(np.uint8)) + orders = kernels.stream_order( + flow_direction.receivers, flow_direction.topological_order, mask, order_type) + return _like(flow_direction, np.asarray(orders)) + + +# --------------------------------------------------------------------------- +# Terrain attributes +# --------------------------------------------------------------------------- + + +def gradient8(elevation, unit='tangent'): + """Steepest downward gradient over the 8 neighbours. + + Equivalent to ``GRIDobj/gradient8``. The cardinal and diagonal drops are + divided by ``cellsize`` and ``sqrt(2)*cellsize`` respectively, and the + result is clamped at 0, so pits and flats give exactly 0 rather than a + negative gradient. + + Parameters + ---------- + unit : {'tangent', 'degree', 'radian', 'sine', 'percent'} + """ + z = np.asarray(elevation._griddata, dtype=np.float64) + cs = elevation._georef_info.dx + nodata = np.isnan(z) + work = np.where(nodata, np.inf, z) + + gradient = np.zeros(z.shape, dtype=np.float64) + for d in range(8): + di, dj = int(kernels.D8_DI[d]), int(kernels.D8_DJ[d]) + # +inf outside the grid, so an off-grid neighbour never wins the + # minimum (TopoToolbox erodes with an infinite pad for the same + # reason). + neighbour = _shift(work, di, dj, np.inf) + distance = cs * kernels.D8_DIST[d] + gradient = np.maximum(gradient, (work - neighbour) / distance) + + gradient[nodata] = np.nan + return _like(elevation, _convert_unit(gradient, unit)) + + +def arcslope(elevation, unit='tangent'): + """ArcGIS/Horn 3x3 slope magnitude. + + Equivalent to ``GRIDobj/arcslope``. Uses the 1-2-1 weighted 3x3 stencil + with an ``8 * cellsize`` denominator -- there is no ``sqrt(2)`` anywhere; + the diagonals enter through the weights. + + ``NaN`` holes are filled from their nearest valid neighbour before + differencing, then restored, which is how TopoToolbox handles them here + (and is the opposite of what :func:`gradient8` does). + """ + from scipy.ndimage import distance_transform_edt + from scipy.signal import convolve2d + + z = np.asarray(elevation._griddata, dtype=np.float64) + cs = elevation._georef_info.dx + nodata = np.isnan(z) + + padded = np.pad(z, 1, mode='edge') + padded_nodata = np.pad(nodata, 1, mode='edge') + if padded_nodata.any(): + _, indices = distance_transform_edt(padded_nodata, return_indices=True) + padded = padded[tuple(indices)] + + kernel_y = np.array([[-1.0, -2.0, -1.0], [0.0, 0.0, 0.0], [1.0, 2.0, 1.0]]) + kernel_x = kernel_y.T + gx = convolve2d(padded, kernel_x, mode='valid') / (8.0 * cs) + gy = convolve2d(padded, kernel_y, mode='valid') / (8.0 * cs) + + slope = np.hypot(gx, gy) + slope[nodata] = np.nan + return _like(elevation, _convert_unit(slope, unit)) + + +def _surfnorm(z): + """MATLAB ``surfnorm(Z)`` for the single-argument case. + + Returns ``(Nx, Ny, Nz)``, the unit surface normal with derivatives taken + per cell index: ``Nx`` positive where elevation decreases eastward, + ``Ny`` positive where it decreases southward (row-increasing), ``Nz`` + always positive. + + The edge treatment is the second-order one-sided difference, which is + what MATLAB's quadratic ghost-cell extrapolation reduces to. + """ + z = np.asarray(z, dtype=np.float64) + if min(z.shape) < 3: + raise Error.InputError('surfnorm', 'the grid must be at least 3x3') + + dz_drow, dz_dcol = np.gradient(z, edge_order=2) + nx = -dz_dcol + ny = -dz_drow + nz = np.ones_like(z) + + magnitude = np.sqrt(nx * nx + ny * ny + nz * nz) + magnitude[magnitude == 0] = np.finfo(np.float64).eps + return nx / magnitude, ny / magnitude, nz / magnitude + + +def hillshade(elevation, azimuth=315.0, altitude=60.0, exaggerate=1.0): + """Shaded relief as the cosine of the illumination incidence angle. + + Equivalent to ``GRIDobj/hillshade`` with its default ``'surfnorm'`` + method, including its defaults of ``azimuth=315`` and ``altitude=60``. + + Returns + ------- + BaseSpatialGrid + Values in ``[-1, 1]``. TopoToolbox only scales to 0-255 when it is + plotting, so neither does this. + + See Also + -------- + TopoAnalysis.dem.Hillshade : the ESRI formulation, returning 0-255. + """ + cs = elevation._georef_info.dx + z = np.asarray(elevation._griddata, dtype=np.float64) + + # The compass bearing is converted by subtracting 90, which is correct + # here because surfnorm's +y axis points along increasing row (south). + azimuth_rad = np.radians(azimuth - 90.0) + altitude_rad = np.radians(altitude) + cos_elevation = np.cos(altitude_rad) + sx = cos_elevation * np.cos(azimuth_rad) + sy = cos_elevation * np.sin(azimuth_rad) + sz = np.sin(altitude_rad) + + nx, ny, nz = _surfnorm(z / cs * exaggerate) + return _like(elevation, nx * sx + ny * sy + nz * sz) + + +def aspect(elevation, classify=False): + """Down-slope direction in degrees clockwise from north. + + Equivalent to ``GRIDobj/aspect``. 0 = north, 90 = east, 180 = south, + 270 = west. Perfectly flat cells come out as 90, exactly as they do in + TopoToolbox. + + Parameters + ---------- + classify : bool + Return the 8-class Gomez-Plaza wetness ranking (1 = N, 3 = NE, + 5 = E, 7 = SE, 8 = S, 6 = SW, 4 = W, 2 = NW) instead of degrees. + """ + nx, ny, _ = _surfnorm(np.asarray(elevation._griddata, dtype=np.float64)) + degrees = np.mod(90.0 + np.degrees(np.arctan2(ny, nx)), 360.0) + # atan2(0, 0) is 0 in MATLAB, so a flat cell comes out as 90 (east). + # Reproducing that through NumPy depends on the sign of zero, which + # np.gradient does not preserve, so set it explicitly. + degrees[(nx == 0.0) & (ny == 0.0)] = 90.0 + + if not classify: + return _like(elevation, degrees) + + edges = np.arange(0, 361, 45) + classes = np.array([1, 3, 5, 7, 8, 6, 4, 2], dtype=np.uint8) + bins = np.digitize(degrees, edges, right=False) - 1 + out = np.zeros(degrees.shape, dtype=np.uint8) + inside = (bins >= 0) & (bins < classes.size) & ~np.isnan(degrees) + out[inside] = classes[bins[inside]] + return _like(elevation, out) + + +def curvature(elevation, ctype='profc'): + """Surface curvature from the Evans 3x3 quadratic fit. + + Equivalent to ``GRIDobj/curvature``. + + Parameters + ---------- + ctype : {'profc', 'planc', 'tangc', 'meanc', 'total'} + Profile, plan, tangential, mean, or total curvature. ``'profc'`` + (the default) is positive where a slope steepens downhill. + + Returns + ------- + BaseSpatialGrid + Units are 1/length, except ``'total'`` which is 1/length^2. + + Notes + ----- + Cells where the gradient vanishes give 0/0; TopoToolbox sets those to 0, + and so does this. Cells adjacent to no-data therefore come out as 0 + rather than ``NaN`` -- again matching TopoToolbox. + """ + from scipy.signal import convolve2d + + valid_types = ('profc', 'planc', 'tangc', 'meanc', 'total') + if ctype not in valid_types: + raise Error.InputError('ctype', 'ctype must be one of {0}'.format(valid_types)) + + z = np.asarray(elevation._griddata, dtype=np.float64) + cs = elevation._georef_info.dx + padded = np.pad(z, 1, mode='symmetric') + + def conv(kernel): + return convolve2d(padded, kernel, mode='valid') + + fx = conv(np.array([[-1.0, 0, 1.0]] * 3) / (6 * cs)) + fy = conv(np.array([[1.0, 1.0, 1.0], [0, 0, 0], [-1.0, -1.0, -1.0]]) / (6 * cs)) + kxx = np.array([[1.0, -2.0, 1.0]] * 3) / (3 * cs ** 2) + fxx = conv(kxx) + fyy = conv(kxx.T) + fxy = conv(np.array([[-1.0, 0, 1.0], [0, 0, 0], [1.0, 0, -1.0]]) / (4 * cs ** 2)) + + p2 = fx ** 2 + fy ** 2 + with np.errstate(divide='ignore', invalid='ignore'): + if ctype == 'profc': + curv = -(fx ** 2 * fxx + 2 * fx * fy * fxy + fy ** 2 * fyy) / ( + p2 * (1 + p2) ** 1.5) + elif ctype == 'tangc': + curv = -(fy ** 2 * fxx - 2 * fx * fy * fxy + fx ** 2 * fyy) / ( + p2 * (1 + p2) ** 0.5) + elif ctype == 'planc': + curv = -(fy ** 2 * fxx - 2 * fx * fy * fxy + fx ** 2 * fyy) / (p2 ** 1.5) + elif ctype == 'meanc': + curv = -((1 + fy ** 2) * fxx - 2 * fxy * fx * fy + (1 + fx ** 2) * fyy) / ( + 2 * (p2 + 1) ** 1.5) + else: + curv = fxx ** 2 + 2 * fxy ** 2 + fyy ** 2 + + # Order matters: zero the 0/0 cells first, then restore NaN only where + # the DEM itself was no-data. + curv[~np.isfinite(curv)] = 0.0 + curv[np.isnan(z)] = np.nan + return _like(elevation, curv) + + +#: Earth's total surface area in km^2, the constant TopoToolbox's ``cellarea`` +#: uses. It implies a spherical radius of 6 371 047 m. +EARTH_SURFACE_AREA_KM2 = 510072000.0 + + +def cellarea(grid, unit='m'): + """Area of each cell of a geographic (lat/lon) grid. + + Equivalent to ``GRIDobj/cellarea``. Cells are treated as spherical + quadrangles on a sphere whose total area is + :data:`EARTH_SURFACE_AREA_KM2`. + + Parameters + ---------- + unit : {'m', 'km'} + Square metres (the default) or square kilometres. + + Returns + ------- + BaseSpatialGrid + Constant along each row, since area depends only on latitude. + """ + unit = unit.lower() + if unit not in ('m', 'km'): + raise Error.InputError('unit', "unit must be 'm' or 'km'") + scale = 1e6 if unit == 'm' else 1.0 + + info = grid._georef_info + dlon = abs(info.geoTransform[1]) + dlat = abs(info.geoTransform[5]) + + # Row 0 is the north edge, so latitudes descend with row index. + lat_center = info.yllcenter + (info.ny - 1 - np.arange(info.ny)) * dlat + lat_north = np.radians(lat_center + dlat / 2.0) + lat_south = np.radians(lat_center - dlat / 2.0) + + fraction = (np.radians(dlon) * (np.sin(lat_north) - np.sin(lat_south))) / (4 * np.pi) + per_row = EARTH_SURFACE_AREA_KM2 * fraction * scale + return _like(grid, np.repeat(per_row[:, None], info.nx, axis=1)) + + +def localtopography(elevation, radius=5000.0, type='range'): + """Statistic of elevation within a circular window. + + Equivalent to ``GRIDobj/localtopography``. The window radius is given in + map units and rounded up to a whole number of cells. + + Parameters + ---------- + type : {'range', 'max', 'min', 'mean', 'median', 'std'} + + Notes + ----- + TopoToolbox's default disk (``strel('disk', r)`` with ``N=8``) is a + periodic-line approximation whose exact shape is an internal MATLAB + detail. This uses an exact disc -- TopoToolbox's ``N=0`` -- so values + can differ slightly near the window scale. + """ + from scipy.ndimage import (convolve, distance_transform_edt, generic_filter, + maximum_filter, median_filter, minimum_filter) + + z = np.asarray(elevation._griddata, dtype=np.float64) + cs = elevation._georef_info.dx + radius_px = int(np.ceil(radius / cs)) + yy, xx = np.ogrid[-radius_px:radius_px + 1, -radius_px:radius_px + 1] + footprint = (xx * xx + yy * yy) <= radius_px * radius_px + + nodata = np.isnan(z) + if nodata.any() and type in ('mean', 'median', 'std'): + _, indices = distance_transform_edt(nodata, return_indices=True) + work = z[tuple(indices)] + else: + work = z + + if type == 'max': + result = maximum_filter(np.where(nodata, -np.inf, work), footprint=footprint) + elif type == 'min': + result = minimum_filter(np.where(nodata, np.inf, work), footprint=footprint) + elif type == 'range': + result = (maximum_filter(np.where(nodata, -np.inf, work), footprint=footprint) + - minimum_filter(np.where(nodata, np.inf, work), footprint=footprint)) + elif type in ('mean', 'average'): + # The disc, not the bounding square: uniform_filter(size=2r+1) would + # include the four corners, which lie outside the window. + kernel = footprint.astype(np.float64) + kernel /= kernel.sum() + result = convolve(work, kernel, mode='reflect') + elif type == 'median': + result = median_filter(work, footprint=footprint, mode='reflect') + elif type == 'std': + # ddof=1: MATLAB's stdfilt is the sample standard deviation. + result = generic_filter(work, lambda v: np.std(v, ddof=1), + footprint=footprint, mode='reflect') + else: + raise Error.InputError( + 'type', "type must be one of 'range', 'max', 'min', 'mean', 'median', 'std'") + + result = np.asarray(result, dtype=np.float64) + result[nodata] = np.nan + return _like(elevation, result) + + +# --------------------------------------------------------------------------- +# Chi and channel steepness +# --------------------------------------------------------------------------- + + +def chitransform(flow_direction, area, mn=0.45, a0=1e6, outlets=None, + correctcellsize=True, stream_mask=None): + r"""The chi integral transform. + + Equivalent to ``STREAMobj/chitransform``: + + .. math:: + \chi = \int_{\mathrm{outlet}}^{x} + \left(\frac{A_0}{A(x')}\right)^{m/n}\,\mathrm{d}x' + + integrated upstream with the trapezoidal rule, so chi is 0 at each outlet + and increases upstream. + + Parameters + ---------- + area : BaseSpatialGrid + Flow accumulation. With ``correctcellsize=True`` (the default, as in + TopoToolbox) it is taken to be in **number of cells** and multiplied + by ``cellsize**2``; pass ``False`` if it is already in m^2, e.g. if + it came from :class:`TopoAnalysis.dem.Area`. + mn : float + The m/n concavity ratio. TopoToolbox's default is 0.45. + a0 : float + Reference area in m^2. TopoToolbox's default is 1e6 (1 km^2). + outlets : sequence of (x, y), optional + Integration starts here. Defaults to every terminal cell of the + network, which is what a STREAMobj built over the whole grid gives. + stream_mask : ndarray or BaseSpatialGrid, optional + Restrict the integration to the channel network. + + Returns + ------- + BaseSpatialGrid + Chi in map units; ``NaN`` away from the integrated network. + """ + a = np.asarray(area._griddata, dtype=np.float64) + if correctcellsize: + # TopoToolbox multiplies by S.cellsize^2 unconditionally; feeding it + # an area that is already in m^2 squares the cell size twice. + a = a * (flow_direction._georef_info.dx ** 2) + + receivers = flow_direction.receivers + order = flow_direction.topological_order + step = np.ascontiguousarray(flow_direction.step_lengths(), dtype=np.float64) + + if outlets is None: + outlet_indices = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + else: + flat = [] + for (row, col) in flow_direction._xy_to_rowscols(outlets): + if row is not None: + flat.append(row * flow_direction._georef_info.nx + col) + outlet_indices = np.array(flat, dtype=np.int64) + + mask = None + if stream_mask is not None: + mask = np.ascontiguousarray( + (np.asarray(getattr(stream_mask, '_griddata', stream_mask)) != 0).astype(np.uint8)) + + chi_values, _ = kernels.chi( + receivers, order, np.ascontiguousarray(a), step, outlet_indices, + float(a0), float(mn), True, 0.0, mask) + return _like(flow_direction, np.asarray(chi_values)) + + +def ksn(flow_direction, elevation, area, theta=0.45, correctcellsize=True, + min_gradient=1e-5): + r"""Normalized channel steepness index, :math:`k_{sn} = S A^{\theta}`. + + Equivalent to ``STREAMobj/ksn``, including its unconditional minima + imposition at a gradient of 1e-5 before the slope is measured. + + Parameters + ---------- + area : BaseSpatialGrid + Flow accumulation in cells (see ``correctcellsize``). + theta : float + Reference concavity; TopoToolbox's default is 0.45. + min_gradient : float + Gradient imposed by the carving step. TopoToolbox hard-codes 1e-5. + + Returns + ------- + BaseSpatialGrid + ``ksn`` with a reference area of 1 m^2 -- TopoToolbox's ``ksn`` has + no ``a0``. Multiply by ``a0**theta`` to compare against + ``chiplot``'s ``ks``. + """ + a = np.asarray(area._griddata, dtype=np.float64) + if correctcellsize: + a = a * (flow_direction._georef_info.dx ** 2) + + carved = imposemin(flow_direction, elevation, sl=min_gradient) + + receivers = flow_direction.receivers.reshape(-1) + z = np.asarray(carved._griddata, dtype=np.float64).reshape(-1) + step = flow_direction.step_lengths().reshape(-1) + + gradient = np.zeros(z.shape, dtype=np.float64) + donors = np.flatnonzero(receivers >= 0) + gradient[donors] = (z[donors] - z[receivers[donors]]) / step[donors] + + shape = flow_direction._griddata.shape + with np.errstate(divide='ignore', invalid='ignore'): + values = gradient.reshape(shape) / (a ** (-theta)) + return _like(flow_direction, values) + + +def mchi(flow_direction, elevation, chi): + r"""Gradient of elevation in chi space, :math:`M_{\chi}`. + + Equivalent to ``STREAMobj/mchi``: a forward difference assigned to the + upstream cell, with no carving and no smoothing. + + ``M_chi`` scales with the ``a0`` baked into ``chi``: computing chi with + ``a0=1`` puts it in the same units as :func:`ksn`, and + ``mchi(a0) * a0**mn`` is invariant. + + It is *close to* but not equal to :func:`ksn`, despite what TopoToolbox's + own docstring says: ``ksn`` carves the DEM first and evaluates the + integrand at the upstream cell, while ``mchi`` uses the raw elevations + and inherits chi's trapezoidal averaging. + """ + receivers = flow_direction.receivers.reshape(-1) + z = np.asarray(elevation._griddata, dtype=np.float64).reshape(-1) + c = np.asarray(getattr(chi, '_griddata', chi), dtype=np.float64).reshape(-1) + + values = np.zeros(z.shape, dtype=np.float64) + donors = np.flatnonzero(receivers >= 0) + with np.errstate(divide='ignore', invalid='ignore'): + values[donors] = ((z[donors] - z[receivers[donors]]) + / (c[donors] - c[receivers[donors]])) + return _like(flow_direction, values.reshape(flow_direction._griddata.shape)) + + +# --------------------------------------------------------------------------- +# Referencing +# --------------------------------------------------------------------------- + + +def getcoordinates(grid): + """``(x, y)`` cell-centre coordinate vectors. + + Equivalent to ``GRIDobj/getcoordinates``. ``x`` runs west to east; + ``y`` runs **north to south**, matching the row order of the grid. + """ + info = grid._georef_info + x = info.xllcenter + np.arange(info.nx) * info.dx + y = info.yllcenter + (info.ny - 1 - np.arange(info.ny)) * info.dx + return x, y + + +def getextent(grid): + """``(xmin, xmax, ymin, ymax)`` of the outermost **cell centres**. + + Equivalent to ``GRIDobj/getextent``. Note this is half a cell inside + :meth:`TopoAnalysis.dem.BaseSpatialGrid.extent`, which returns the pixel + edges that matplotlib wants. + """ + x, y = getcoordinates(grid) + return (float(x.min()), float(x.max()), float(y.min()), float(y.max())) + + +def _round_half_away(values): + """MATLAB's ``round``: halves go away from zero, not to even.""" + values = np.asarray(values, dtype=np.float64) + return np.where(values >= 0, np.floor(values + 0.5), np.ceil(values - 0.5)) + + +def coord2sub(grid, x, y): + """Map coordinates to 0-based ``(row, col)`` subscripts. + + Equivalent to ``GRIDobj/coord2sub`` apart from the index base. Points + outside the grid give ``NaN``. + + Notes + ----- + Halves are rounded away from zero, as MATLAB does. Python's built-in + ``round`` -- which + :meth:`TopoAnalysis.dem.BaseSpatialGrid._xy_to_rowscols` uses -- rounds + halves to even, so the two disagree for points landing exactly on a cell + boundary. + """ + info = grid._georef_info + x = np.atleast_1d(np.asarray(x, dtype=np.float64)) + y = np.atleast_1d(np.asarray(y, dtype=np.float64)) + + north = info.yllcenter + (info.ny - 1) * info.dx + col = _round_half_away((x - info.xllcenter) / info.dx) + row = _round_half_away((north - y) / info.dx) + + outside = (col < 0) | (col > info.nx - 1) | (row < 0) | (row > info.ny - 1) \ + | np.isnan(x) | np.isnan(y) + row = np.where(outside, np.nan, row) + col = np.where(outside, np.nan, col) + return row, col + + +def sub2coord(grid, row, col): + """0-based ``(row, col)`` subscripts to cell-centre map coordinates. + + Equivalent to ``GRIDobj/sub2coord``. Subscripts outside the grid are + extrapolated rather than rejected, as in TopoToolbox. + """ + info = grid._georef_info + row = np.asarray(row, dtype=np.float64) + col = np.asarray(col, dtype=np.float64) + x = info.xllcenter + col * info.dx + y = info.yllcenter + (info.ny - 1 - row) * info.dx + return x, y diff --git a/utils.py b/utils.py index 96523e8..ff8209b 100644 --- a/utils.py +++ b/utils.py @@ -1,27 +1,60 @@ +"""Convenience wrappers for fitting channel steepness at a single outlet.""" + +import numpy as np + +try: # pragma: no cover - exercised by whichever import style is in use + from . import demRecursionTools +except ImportError: # pragma: no cover + import demRecursionTools + + def calc_ks_for_outlet(outlet, theta, **kwargs): - fd = kwargs['flow_direction'] - xo = kwargs.get('xo') if kwargs.get('xo') is not None else 500.0 - kwargs.pop('xo') + """Fit channel steepness to the basin draining to ``outlet``. + + Parameters + ---------- + outlet : (float, float) + Map coordinates of the outlet. + theta : float + Reference concavity. + flow_direction : FlowDirectionD8 + Required, passed by keyword. + xo : float, optional + Reference length; the reference area is ``xo**2``. Default 500. + plot : float, optional + If equal to ``theta``, also draw the chi-elevation plot. + **kwargs + Any other keyword is a grid to sample along the profiles, e.g. + ``elevation=dem, area=area``. + + Returns + ------- + (ndarray, float) + ``(ks, r_squared)``. + """ + kwargs = dict(kwargs) + fd = kwargs.pop('flow_direction') + # kwargs.pop('xo') without a default raised KeyError whenever xo was + # left at its default. + xo = kwargs.pop('xo', None) + if xo is None: + xo = 500.0 plot = kwargs.pop('plot', None) - + de = fd._mean_pixel_dimension() ld_list = fd.map_values_to_recursive_list(outlet, **kwargs) - - from demRecursionTools import best_ks_with_r2_list - from numpy import array - ret = best_ks_with_r2_list(ld_list, de, array([theta]), xo = xo) + + ret = demRecursionTools.best_ks_with_r2_list(ld_list, de, np.array([theta]), xo=xo) if plot is not None and plot == theta: import matplotlib.pylab as plt - from demRecursionTools import chi_elevation - import numpy as np - e, c = chi_elevation(ld_list, de, array([theta]), xo=xo) + + e, c = demRecursionTools.chi_elevation(ld_list, de, np.array([theta]), xo=xo) plt.figure() plt.plot(c, e, 'k.') - plt.hold(True) + # Matplotlib 3.0 removed the explicit hold call; overlaying is now + # the default, so no equivalent is needed here. (ks, _) = ret - plt.plot([0, np.max(c)],[0, ks*np.max(c)], 'k-') - + plt.plot([0, np.max(c)], [0, ks * np.max(c)], 'k-') + return ret - - \ No newline at end of file From b0635f3218d54170592f4532bbbc5e43be75cc00 Mon Sep 17 00:00:00 2001 From: George Hilley Date: Tue, 22 Sep 2026 08:33:06 -0700 Subject: [PATCH 2/2] CI: test GDAL on Windows, and let a label run the source build The Windows route is the only GDAL strategy with nothing behind it: there is no gdal-config and no source build on Windows, so a prebuilt wheel that bundles the C library is the whole plan. It had never been exercised anywhere. The new job installs GDAL through it and asserts that ReadAsArray actually works rather than that osgeo merely imports. The from-source job stays off the default path -- it is about forty minutes -- but a pull request labelled `test-source-build` now runs it, which is what a change to the build itself needs. Waiting for the weekly schedule means finding out on Monday. `labeled` joins the pull_request event types so applying the label is enough to start it. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 51 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a51b7e..75c35ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,10 @@ on: push: branches: [master, main] pull_request: + # `labeled` is here so that adding the `test-source-build` label to an + # open pull request is enough to start the long job below, without + # needing a dummy commit to wake the workflow up. + types: [opened, synchronize, reopened, labeled] workflow_dispatch: # The from-source GDAL build takes about half an hour, so it runs on a # schedule rather than on every push -- but it has to run *somewhere*, @@ -185,10 +189,55 @@ jobs: working-directory: TopoAnalysis run: python -m pytest -q + # Windows has no gdal-config and no source build, so the only practical + # route is a wheel that bundles the C library. That makes this the one + # strategy with no fallback behind it, and worth testing on its own. + gdal-on-windows: + name: GDAL on Windows (prebuilt wheel) + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + # The wheels are published for recent CPythons only. + python-version: '3.12' + - name: Install TopoAnalysis + working-directory: TopoAnalysis + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test]" + - name: Install GDAL + working-directory: TopoAnalysis + run: python -m TopoAnalysis.gdal_setup install + - name: The doctor must agree that it worked + working-directory: TopoAnalysis + run: python -m TopoAnalysis.gdal_setup doctor + - name: ReadAsArray must work, not merely import + shell: python + run: | + import numpy as np + from osgeo import gdal, gdal_array # noqa: F401 + ds = gdal.GetDriverByName('MEM').Create('', 4, 3, 1, gdal.GDT_Float32) + ds.GetRasterBand(1).WriteArray(np.arange(12, dtype='float32').reshape(3, 4)) + assert ds.ReadAsArray().shape == (3, 4) + print('gdal', gdal.__version__, 'numpy bridge ok') + - name: Test, including the GDAL-marked cases + working-directory: TopoAnalysis + run: python -m pytest -q + # The no-root route. Slow, so it is scheduled or triggered by hand. gdal-from-source: name: GDAL built from source - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + # Half an hour of compute, so not on every push. Runs on the weekly + # schedule, on demand, or on a pull request labelled `test-source-build` + # -- which is how to exercise it on a change that touches the build + # itself, rather than finding out the following Monday. + if: >- + github.event_name == 'schedule' + || github.event_name == 'workflow_dispatch' + || contains(github.event.pull_request.labels.*.name, 'test-source-build') runs-on: ${{ matrix.os }} timeout-minutes: 120 strategy: