diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..75c35ae --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,285 @@ +name: tests + +on: + push: + branches: [master, main] + pull_request: + # `labeled` is here so that adding the `test-source-build` label to an + # open pull request is enough to start the long job below, without + # needing a dummy commit to wake the workflow up. + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + # The from-source GDAL build takes about half an hour, so it runs on a + # schedule rather than on every push -- but it has to run *somewhere*, + # because it is the route that is hardest to keep working. + schedule: + - cron: '0 5 * * 1' + +jobs: + # The main matrix: no GDAL, so it runs everywhere and proves that the + # library works without it. + test: + name: py${{ matrix.python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python: ['3.8', '3.11', '3.12'] + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install + working-directory: TopoAnalysis + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test,stats]" + + - name: Report the active backend + working-directory: TopoAnalysis + run: python -c "import TopoAnalysis; print(TopoAnalysis.backend())" + + - name: Fail if the C++ kernels did not build + working-directory: TopoAnalysis + run: python -c "import sys, TopoAnalysis; sys.exit(0 if TopoAnalysis.backend()=='c++' else 1)" + + - name: Test + working-directory: TopoAnalysis + run: python -m pytest -q + + - name: Test the pure-NumPy kernels + working-directory: TopoAnalysis + env: + TOPOANALYSIS_PURE_PYTHON: '1' + run: python -m pytest -q + + # A second job with GDAL from conda-forge, which exercises the file I/O + # paths the matrix above skips. + test-with-gdal: + name: with GDAL (ubuntu) + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + + - uses: conda-incubator/setup-miniconda@v3 + with: + environment-file: TopoAnalysis/environment.yml + activate-environment: topoanalysis + auto-activate-base: false + + - name: Install + working-directory: TopoAnalysis + run: python -m pip install -e ".[test]" + + - name: Test + working-directory: TopoAnalysis + run: python -m pytest -q + + # Prove the package still installs when there is no compiler: the + # extension is optional and the NumPy kernels must take over. + test-without-compiler: + name: no compiler + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install with the compiler hidden + working-directory: TopoAnalysis + env: + CC: /bin/false + CXX: /bin/false + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test]" + + - name: The NumPy kernels must be active + working-directory: TopoAnalysis + run: python -c "import sys, TopoAnalysis; print(TopoAnalysis.backend()); sys.exit(0 if TopoAnalysis.backend()=='python' else 1)" + + - name: Test + working-directory: TopoAnalysis + run: python -m pytest -q + + # The advertised one-command install, on every platform. GDAL is skipped + # here so the job stays fast; the jobs below cover it. + install-script: + name: install.py on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install everything from a clean checkout + working-directory: TopoAnalysis + run: python install.py --no-gdal + - name: The installed environment must work + working-directory: TopoAnalysis + shell: bash + run: | + if [ -f .venv/bin/python ]; then PY=.venv/bin/python; else PY=.venv/Scripts/python.exe; fi + "$PY" -c "import sys, TopoAnalysis; print(TopoAnalysis.backend()); sys.exit(0 if TopoAnalysis.backend()=='c++' else 1)" + "$PY" -m TopoAnalysis.gdal_setup doctor || true + + # Ubuntu's own GDAL is older than the version whose sdist declares numpy as + # a build dependency, so this job is the regression test for the + # gdal_array trap: installing the bindings the naive way here produces an + # osgeo that imports fine and cannot ReadAsArray. + gdal-from-system-packages: + name: GDAL from apt (the gdal_array trap) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install the GDAL C library only + run: | + sudo apt-get update + sudo apt-get install -y gdal-bin libgdal-dev + gdal-config --version + - name: Install TopoAnalysis + working-directory: TopoAnalysis + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test]" + - name: The bindings are missing, and the doctor should say so + working-directory: TopoAnalysis + run: | + python -m TopoAnalysis.gdal_setup doctor && exit 1 || true + - name: Install the bindings against the system library + working-directory: TopoAnalysis + run: python -m TopoAnalysis.gdal_setup install + - name: ReadAsArray must work, not merely import + run: | + python - <<'EOF' + import numpy as np + from osgeo import gdal, gdal_array # noqa: F401 + ds = gdal.GetDriverByName('MEM').Create('', 4, 3, 1, gdal.GDT_Float32) + ds.GetRasterBand(1).WriteArray(np.arange(12, dtype='float32').reshape(3, 4)) + assert ds.ReadAsArray().shape == (3, 4) + print('gdal', gdal.__version__, 'numpy bridge ok') + EOF + - name: Test, including the GDAL-marked cases + working-directory: TopoAnalysis + run: python -m pytest -q + + # Windows has no gdal-config and no source build, so the only practical + # route is a wheel that bundles the C library. That makes this the one + # strategy with no fallback behind it, and worth testing on its own. + gdal-on-windows: + name: GDAL on Windows (prebuilt wheel) + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + # The wheels are published for recent CPythons only. + python-version: '3.12' + - name: Install TopoAnalysis + working-directory: TopoAnalysis + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test]" + - name: Install GDAL + working-directory: TopoAnalysis + run: python -m TopoAnalysis.gdal_setup install + - name: The doctor must agree that it worked + working-directory: TopoAnalysis + run: python -m TopoAnalysis.gdal_setup doctor + - name: ReadAsArray must work, not merely import + shell: python + run: | + import numpy as np + from osgeo import gdal, gdal_array # noqa: F401 + ds = gdal.GetDriverByName('MEM').Create('', 4, 3, 1, gdal.GDT_Float32) + ds.GetRasterBand(1).WriteArray(np.arange(12, dtype='float32').reshape(3, 4)) + assert ds.ReadAsArray().shape == (3, 4) + print('gdal', gdal.__version__, 'numpy bridge ok') + - name: Test, including the GDAL-marked cases + working-directory: TopoAnalysis + run: python -m pytest -q + + # The no-root route. Slow, so it is scheduled or triggered by hand. + gdal-from-source: + name: GDAL built from source + # Half an hour of compute, so not on every push. Runs on the weekly + # schedule, on demand, or on a pull request labelled `test-source-build` + # -- which is how to exercise it on a change that touches the build + # itself, rather than finding out the following Monday. + if: >- + github.event_name == 'schedule' + || github.event_name == 'workflow_dispatch' + || contains(github.event.pull_request.labels.*.name, 'test-source-build') + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Cache the source tarballs + uses: actions/cache@v4 + with: + path: ~/.cache/TopoAnalysis/downloads + key: gdal-sources-${{ hashFiles('TopoAnalysis/_gdal_source.py') }} + - name: Install everything, building GDAL from source + working-directory: TopoAnalysis + run: python install.py --gdal-strategy source + - name: Test against the GDAL we built + working-directory: TopoAnalysis + run: | + .venv/bin/python -m TopoAnalysis.gdal_setup doctor + .venv/bin/python -m pytest -q + + sdist: + name: sdist rebuilds the extension + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: TopoAnalysis + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Build and install from an sdist + working-directory: TopoAnalysis + run: | + python -m pip install --upgrade pip build + python -m build --sdist + python -m pip install dist/*.tar.gz + cd / + python -c "import sys, TopoAnalysis; print(TopoAnalysis.backend()); sys.exit(0 if TopoAnalysis.backend()=='c++' else 1)" diff --git a/.gitignore b/.gitignore index 614013b..dd67ad1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,36 @@ +# Build artefacts +build/ +dist/ +*.egg-info/ +*.so +*.pyd +*.o + +# Python +__pycache__/ +*.py[cod] +.pytest_cache/ +.coverage +htmlcov/ + +# Environments. install.py creates .venv here, because this directory is +# itself the package. +.venv/ +venv/ + +# Where a from-source GDAL build installs when it has nowhere better +share/topoanalysis/ + +# Docs +docs/_build/ + +# Editors +.idea/ +.vscode/ .metadata -*.pyc -.idea -.idea/* +.project +.pydevproject +.settings/ + +# macOS +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5bb9c9b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,366 @@ +# Changelog + +## 1.0.0 + +The first release with a test suite, an install story, and compiled kernels. + +The **public interface is unchanged**: every class, method and keyword that +existed before still exists and still means the same thing. What changed is +that a lot of it now works. + +--- + +## Results that change + +Read this section before comparing new output against old. Each item is a +place where the previous answer was wrong, so the new number differs on +purpose. + +### Chi (`Ksi`) + +The integrand was `(Ao / (A - Ao))**theta`; it is now `(Ao / A)**theta`. The +old form diverges as drainage area approaches the threshold `Ao`, so chi was +inflated near every channel head and the error propagated downstream. +Chi is defined as ∫(A₀/A)^θ dx (Perron & Royden 2013). + +### Chi (`Chi`) + +Chi at the outlet is now exactly 0. It used to be one cell-step of the +integrand, so every chi value in a basin was offset by that amount. + +`Chi` also accepts `trapezoid=True`, which averages the integrand across each +step instead of taking it at the upstream cell. That is the rule TopoToolbox +uses; the default remains the original left-endpoint rule. + +### Hillshade + +The aspect term was `atan2(dz/dy, dz/dx)` where the ESRI definition the code +claimed to implement is `atan2(dz/dy, -dz/dx)`. Illumination was mirrored +east–west: slopes that should have been lit were shadowed and vice versa. +Hillshades are now clipped to 0–255 and stored as `uint8` rather than +wrapping negative values around the byte into bright speckle. + +### D8 flow directions + +Three separate problems, all fixed by moving to a single shared kernel: + +- The windowed comparison stopped two columns and two rows short, so the + last row and the last column of every grid received **no flow direction at + all**. +- The diagonal distance factor was `1.41`, not `sqrt(2)`. +- Ties between equally steep neighbours resolved towards the north-east + because of the order the codes were written; they now resolve + deterministically in the order E, SE, S, SW, W, NW, N, NE. + +`FlowDirectionD8.update_flow_codes_in_mask`, the per-cell re-router, had the +scaling **inverted** — it divided the cardinal drops by 1.41 and left the +diagonals alone — which biased routing towards the diagonals. + +### Flow accumulation (`Area`) + +The accumulation order came from `flow_direction.sort()`. That returns an +elevation sort only when the grid was built from a filled DEM in the same +session; for a flow-direction grid **loaded from a file** it sorted the cells +by their direction *code*, which is meaningless, and the resulting areas were +wrong. Accumulation now derives its own topological order from the flow +network, so it is correct however the grid was obtained. + +The `mask` keyword had no effect: the test was `mask[i, j] is not None`, and +`__getitem__` returns the value, so it was true even for a masked-out cell. +Masks now actually mask. + +### `MainstemValleyArea` + +Its `__calcD8Area` override was name-mangled to +`_MainstemValleyArea__calcD8Area` while the inherited caller looked for +`_Area__calcD8Area`, so it never ran: the class silently behaved exactly +like `ValleyArea`. It now accumulates along the longest-flow-length path as +intended. (The method also compared a whole grid object against a scalar, +which would have raised the moment it did run.) + +### `FlowLength` direction codes + +`FlowLength` stored its main-stem directions in a **row-flipped** convention: +internally consistent, but contradicting the ArcGIS convention used by +`FlowDirectionD8` and by the rest of the library. It now uses the same +convention as everything else. + +**`*_directions` side-files written by earlier versions must be +regenerated.** `FlowLength.load` cannot detect which convention a file uses. + +### Recursive-profile `distance_scale` + +In the nested dictionaries from `map_values_to_recursive_list`, +`distance_scale` was written onto the *parent* node and overwritten by each +child in turn, so it ended up describing whichever child was visited last. +Chi profiles built from these dictionaries were wrong at every confluence +where a diagonal and a cardinal tributary met. Each node now carries the +step length from its own parent. + +The root node's `index` is now a plain `(row, col)` pair like every other +node's, instead of the nested `((row, col),)` the caller happened to pass. + +### Flexural deflection + +`(2 * 3.141)**4` is 0.2% low; it is now `(2 * np.pi)**4`. The wavenumber +axes were built from half-width ranges whose length did not have to match +the grid, so the spectral multiply could fail outright; they now come from +`np.fft.fftfreq`. + +### Georeferencing of `nx/ny/projection/geo_transform` grids + +`__populate_georef_info_using_geoTransform` read `ny` two lines before it +assigned it, so a grid built from an explicit geotransform picked up the +default of 0 and its `yllcenter` came out `ny * dx` too far north. Every +coordinate derived from it — `_rowscols_to_xy`, `_xy_to_rowscols`, +`extent()`, the ArcInfo ASCII header, outlets given in map coordinates — +inherited the error, and the same grid round-tripped through GDAL disagreed +with itself. The other four construction paths were always correct. + +### Geographic curvature + +`GeographicLaplacian` divided by `dx`, which on a lat/lon grid is in +**degrees**: curvatures came out about 10¹⁰ too large. It now uses the true +metric cell size, like every other geographic class. + +### Strahler stream order + +The order was applied one donor at a time, which makes a junction of orders +{2, 2, 3} come out as 4 or 3 depending on which donor arrives first — and +the two backends emit different (equally valid) topological orders, so they +disagreed on about 3% of networks. Junctions are now resolved as a whole: +the largest incoming order, plus one if two or more donors carry it. + +Note this also differs from TopoToolbox, whose incremental form has the same +order-dependence. + +### Smaller behavioural changes + +**`search_down_flow_direction_with_length`** now includes the terminal cell +of the path, and follows a path that merely *starts* on the grid perimeter. +The stop condition is "no downstream neighbour inside the grid" rather than +"this cell is on the perimeter", so a cell on the western edge draining east +is now followed instead of returning an empty path. + +**`FlowDirectionD8.divides()`** now evaluates the perimeter as well as the +interior. Border cells with no upstream neighbour are divides, and used to +be reported as 0 regardless. + +**`Chi(mask=...)`** treats the mask as a boolean gate on the integration +rather than multiplying the finished chi grid by the mask's *values*. The +old behaviour scaled chi by whatever the mask happened to contain, and +carried a hard-coded `area > 1e4` escape hatch. + +**`_xy_to_rowscols`** rejects a point exactly one cell beyond the eastern or +southern edge. It used to return the subscript `nx` or `ny`, which raised an +`IndexError` somewhere else entirely. + +**`FlowDirectionD8` caches its flow network** (`receivers`, +`topological_order`) because every downstream computation now reads it. +Writing a flow code with `fd[i, j] = ...` or `set_value_at_rowscols` drops +the cache, so hand edits take effect as they always did; call +`fd.invalidate_network()` after modifying `_griddata` directly. + +**`tile(..., tile_xpadding=0)`** no longer duplicates the edge tiles' data. +`griddata[:, -0:]` selects the whole array rather than nothing, so a zero +padding produced double-width edge tiles. + +**`resample`** warns when SciPy cannot provide the requested interpolation +order and it falls back to linear, instead of quietly changing the answer. + +**`Elevation.track_flow_downhill`** and +**`DiscreteFlowAccumulation`** stop on a `NaN` neighbour as before, but the +test is now a real `NaN` check: `None not in numpy_array` never matched, so +off-grid neighbours did not stop the walk the way the code intended. + +### Other corrected numerics + +| What | Was | Now | +|---|---|---| +| `assignBCs` south-east corner | copied the south-west corner | copies the south-east corner | +| `get_XY_matricies` | returned `n-1` coordinates per axis | returns `n` | +| `resample` | dropped `xllcenter` from the axis stop, and flipped the grid left-right unconditionally | keeps the south-west cell centre fixed; uses `RegularGridInterpolator` | +| `sort(mask=...)` | indexed the mask with positions in the sorted array | selects the sorted positions inside the mask | +| `average_over_distance` | kernel centred half a cell off; `ifftshift` where `fftshift` was needed | kernel centred on cell (0,0), no shift needed | +| `extent_of_data` | scanned rows for the column limits | scans the right axis for each | +| gradient / Laplacian over a length scale | divided by `(2N+1)dx`; float slice indices | divides by `2N·dx`; integer indices | +| `LocalRelief` window | `np.ogrid[-r:r]`, one cell short on the far side | symmetric about the centre | +| `LogArea` | declared `uint8`, truncating log₁₀(area) to integers | `float64` | +| `tile` geotransform | north edge one cell too far north | consistent with the reader | +| `CircularMovingWindow` | squared the row offset twice, giving a vertical band | a disc | +| `datasets` `slope_y` | multiplied the relief by a ramp, zeroing row 0 | adds a ramp | +| geographic cell area | `np.float64(range(n))` — a `TypeError` on modern NumPy | `np.arange` | +| ArcInfo ASCII reader | ignored `NODATA_value`, turning −9999 flags into elevations | maps it to `NaN` | +| ArcInfo ASCII writer | wrote the literal text `nan` | writes −9999 by default | + +--- + +## Crashes fixed + +These raised immediately on any current scientific-Python stack. + +**Removed upstream APIs.** `np.NAN`, `np.NaN`, `np.float`, +`scipy.ndimage.morphology`, `scipy.ndimage.filters`, +`scipy.interpolate.interp2d`, `np.int`, and `matplotlib.pyplot.hold` were all +in use. A regression test now fails if any of them reappears. + +**Python 3 iterators.** `zip()` objects were used as NumPy subscripts and +subscripted directly in `ValueGrid.set_value_at_indexes`, +`CrossDivideDChi`, `NormalizedCrossDivideDChi` and +`FlowDirectionD8.divides_for_outlets`. Several functions returned a `zip` +that callers iterated twice, getting nothing the second time. + +**Comparisons with `None`.** `bounds_of_basin_for_outlet` evaluated +`lon > bounds[0][1] or bounds[0][1] is None` — the comparison runs first, so +it raised on its own first iteration. `areas_between` compared `None` with a +float when a cell had no upstream neighbour. + +**Names that did not exist.** + +| Call | Problem | +|---|---| +| `PriorityQueueMixIn.randomize_subbasins_with_mask` | called `self.__flood`, mangled to `_PriorityQueueMixIn__flood`; the method is `_flood` | +| `MovingWindow.apply_moving_window` | called `self.__adjustKernel`; the method is `__adjust_kernel` | +| `ScaledRelief(flooded_dem=...)` | called `_create_from_flow_direction_flooded_dem_and_elevation`, never defined | +| `ChannelSlopeWithSmoothing(horizontal_interval=...)` | dispatched to `_create_from_elevation_flow_direction`, never defined | +| `BaseSpatialGrid.apply_moving_window` | `out_grid = None` then `out_grid._georef_info = ...` | +| `convert_rivertools_directions_to_arc` | `int()` of an array; referenced an undefined `self.noData` | +| `dem.plot(xlabel=...)` | the local name was only bound when the argument was *absent* | +| `utils.calc_ks_for_outlet` | `kwargs.pop('xo')` with no default | +| `MovingWindow` subclasses | read `self.window_radius`, which the constructor never set | +| `DiscreteFlowAccumulation._mean_pixel_dimension` | required a `kwargs['elevation']` its callers did not pass | +| `GeographicThetaFromChiWithSmoothing` | inherited `KsFromChiWithSmoothing`, so it fitted ks instead of theta | + +**Stack overflows.** `get_indexes_of_upstream_cells`, +`map_values_to_recursive_list`, `Chi.__recurse_chi`, +`RestoredElevation.__fill_upstream_points` and +`FlowLength.__get_upstream_indexes` were all recursive, which is why the +module raised the recursion limit to 1 000 000 at import — a limit CPython +segfaults long before reaching. All of them are now iterative, and the +`setrecursionlimit` call is gone. + +**Undercounting.** `self._n[rows, cols] += 1` with repeated subscripts +applies each increment only once; the profile-crossing counts in +`KsFromChiWithSmoothing` and `ThetaFromChiWithSmoothing` now use +`np.add.at`. + +--- + +## Thread safety + +The compiled kernels release the GIL, but eleven of the thirteen bindings +resolved their NumPy buffers *inside* the released region. +`py::array::request()` calls `PyObject_GetBuffer`, which touches reference +counts, so running a kernel from two threads at once leaked references and +could abort the interpreter. Every binding now resolves and validates its +buffers while the GIL is held and runs nothing but plain C++ afterwards. +`test_kernels_are_thread_safe` pins it down. + +The bindings also validate array sizes, so a mismatched `order` array raises +instead of reading past the end of it. + +## Added + +- **`install.py`** — one command that takes a fresh clone to a working + installation: it creates a virtual environment, installs the dependencies, + compiles the kernels, installs GDAL, and then proves the result works by + round-tripping a georeferenced GeoTIFF. Standard library only, so it runs + on a machine where nothing is set up yet. `--dry-run` reports what it would + do without doing it. +- **`TopoAnalysis.gdal_setup`** and the `topoanalysis-gdal` command — find, + diagnose and install GDAL, meaning *both* the C library and the Python + bindings. There is no GDAL wheel on PyPI at any version for any platform, + so pip alone can never finish the job. `doctor` reports what is installed + and what is wrong with it; `plan` shows what an install would do; `install` + does it, choosing between an existing C library, conda, a prebuilt Windows + wheel, the system package manager, or a from-source build. + + Two failure modes it exists to catch: + + - **The silent `gdal_array` trap.** GDAL sdists only declared numpy as a + build dependency from 3.9 on. Below that, pip's build isolation leaves + numpy out and the numpy bridge is quietly omitted — `osgeo` imports + cleanly and `Band.ReadAsArray()` fails at run time. Ubuntu 22.04 (GDAL + 3.4) and 24.04 (3.8) are both on the wrong side of that line. The + bindings are now installed with `--no-build-isolation` against an + interpreter that already has numpy. + - **Architecture mismatch**, such as a Homebrew arm64 GDAL under an x86_64 + Python running via Rosetta. This is detected before anything is built, + and the package manager that caused it is not asked to fix it. + - **A stale `PROJ_LIB` or `GDAL_DATA`.** Activating a conda environment + exports both, and they then outrank the paths compiled into every other + GDAL on the machine; a PROJ 9 library handed PROJ 8's `proj.db` refuses + every coordinate lookup. `doctor` names the offending variable and its + value instead of proposing a reinstall, which would not have helped. + + Repairing broken bindings rebuilds them rather than re-requesting them: + the usual fault leaves `osgeo` at exactly the right version, so plain + `pip install gdal==X.Y.Z` reports "already satisfied", and pip's wheel + cache holds the very build that is broken. + + `install.py` exits non-zero when GDAL was asked for and did not arrive, + while saying plainly that the rest of the installation is fine. `--no-gdal` + is how to say it was never wanted. +- **`TopoAnalysis._gdal_source`** — builds SQLite, PROJ and GDAL from pinned, + SHA-256-checked tarballs into a single prefix, by default the active + virtual environment. It needs only a compiler and CMake: no root, no + package manager, and no system `-dev` packages, because GDAL is built + against its own bundled libtiff, libgeotiff, libpng, libjpeg and zlib. + Pins can be moved with `TOPOANALYSIS_GDAL_SOURCE_*` environment variables, + which require a checksum alongside any new URL. +- **`TopoAnalysis.kernels`** — compute kernels in C++ (`TopoAnalysis._cpp`) + with an equivalent NumPy implementation in `TopoAnalysis.fastops`. The + test suite asserts the two agree exactly. Set + `TOPOANALYSIS_PURE_PYTHON=1` to force the NumPy path. +- **`TopoAnalysis.topotoolbox`** — functions that reproduce TopoToolbox 2 + results: `fillsinks`, `identifyflats`, `imposemin`, `flowobj`, `flowacc`, + `flowdistance`, `drainagebasins`, `streamorder`, `gradient8`, `arcslope`, + `hillshade`, `aspect`, `curvature`, `cellarea`, `localtopography`, + `chitransform`, `ksn`, `mchi`, and the referencing helpers. +- **`topoanalysis-process`** — a command-line pipeline. `--info` reports the + active backend. +- **`python -m TopoAnalysis.benchmark`** — times both backends. +- `FilledElevation(aggradation_slope=0.0)` for a flat fill, and a + `fill_report` attribute recording how much was filled. +- `FlowDirectionD8.receivers`, `.topological_order`, `.cycle_count`, + `.basin_mask()` and `.step_lengths()`, which expose the flow network + directly. +- `FlowDirectionD8(elevation=...)` as an alias for `flooded_dem=`, for DEMs + conditioned elsewhere. +- Every grid class now accepts the in-memory constructors `dx=..., grid=...` + and `nx=..., ny=..., dx=...`, not just `BaseSpatialGrid`. +- `BaseSpatialGrid.extent()`, `.coordinate_vectors()` and the correctly + spelled `.get_XY_matrices()` alias. +- A circular flow path in a hand-edited flow-direction grid is now reported + (`cycle_count`, and a warning) instead of producing silently wrong results. +- `identifyflats(..., output='all')` also returns the closed basins. +- `FilledElevation`'s flood reports which cells a `maximum_pit_depth` left + unfilled, so `clip_to_fill` still excises them — the idiom for masking out + quarries and lakes, which the traversal fix would otherwise have broken. +- `FlowDirectionD8.invalidate_network()`. + +## Changed + +- **GDAL is imported lazily.** The package imports, and most of it runs, + without GDAL installed. `gdal_is_available()` reports whether it is there. + GDAL exceptions are enabled, so a bad path raises instead of returning + `None` and failing three frames later. +- **statsmodels is imported lazily** and only by the classes that fit + regressions; it used to print a warning at import time. +- **rasterio is imported lazily** by `analysis.Quadrats.load_data`. +- Both import styles work: `from TopoAnalysis import dem` and, with the + package directory on `sys.path`, `import dem`. +- `save()` writes `NaN` as the GeoTIFF no-data value and flushes before + returning. +- `mosaicFolder` uses the GDAL Python API instead of shelling out to + `gdal_merge.py`, which is not on most installations' `PATH`. +- Progress reporting is quieter and accepts `verbose=False`. +- Bare `except:` clauses — which swallowed `KeyboardInterrupt` and real + programming errors, and were used for bounds checking in the hot loops — + are gone. +- The NumPy kernels reject a non-C-contiguous grid rather than filling a + throwaway copy and reporting success. The compiled kernels always did. +- `from TopoAnalysis import *` no longer names classes that were never + imported into the package namespace. +- The test suite runs from a checkout under any name — a GitHub zip unpacks + to `TopoAnalysis-main` — as well as from an installed copy. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..aac2c04 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,43 @@ +# A reproducible TopoAnalysis image. +# +# docker build -t topoanalysis . +# docker run --rm topoanalysis topoanalysis-gdal doctor +# docker run --rm -v "$PWD:/data" topoanalysis \ +# topoanalysis-process /data/srtm.tif -o /data/out +# +# The base image already carries a matched GDAL C library and Python +# bindings, which is the one thing that is genuinely hard to arrange. +# ``install.py`` therefore finds GDAL already working and does nothing to it; +# everything else it does normally. +FROM ghcr.io/osgeo/gdal:ubuntu-small-3.9.3 + +# build-essential and python3-dev are for the Priority-Flood kernels; without +# them the package still installs and falls back to its NumPy kernels. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential python3-dev python3-pip python3-venv \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/TopoAnalysis +COPY . . + +# --system because the image *is* the environment: there is nothing here to +# protect from a pip install, and a virtualenv would only hide the GDAL that +# the base image went to the trouble of providing. +# +# --break-system-packages because this base image is built on Ubuntu 24.04, +# whose Python carries the PEP 668 "externally managed" marker. That marker +# exists to stop pip from fighting the host's package manager; inside a +# single-purpose container there is no host to protect, and overriding it is +# the normal and intended thing to do. +RUN python3 install.py --system --break-system-packages --extras test + +# Fail the build rather than ship an image whose kernels silently did not +# compile or whose GDAL does not work. +RUN python3 -c "import sys, TopoAnalysis; \ + print('backend:', TopoAnalysis.backend()); \ + sys.exit(0 if TopoAnalysis.backend() == 'c++' else 1)" \ + && topoanalysis-gdal doctor + +WORKDIR /data +CMD ["topoanalysis-process", "--info"] diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9019295 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,29 @@ +# Sources needed to rebuild the C++ extension from an sdist. +include _cpp/*.hpp +include _cpp/*.cpp + +include pyproject.toml +include setup.py +include README.md +include CHANGELOG.md +include environment.yml +include requirements.txt +include requirements-dev.txt +include Makefile +include install.py +include Dockerfile + +recursive-include docs *.md +recursive-include docs *.py +recursive-include tests *.py + +# install.py creates .venv here, because this directory is the package. +prune .venv +prune venv +prune build +prune dist + +exclude .DS_Store +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] +recursive-exclude * .DS_Store diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ca57585 --- /dev/null +++ b/Makefile @@ -0,0 +1,74 @@ +# Common tasks. `make help` lists them. + +PYTHON ?= python3 +PIP ?= $(PYTHON) -m pip + +.PHONY: help bootstrap gdal gdal-doctor gdal-source install develop build test \ + test-python bench docs clean distclean lint + +help: + @echo "bootstrap everything from a fresh clone, GDAL included" + @echo "gdal install GDAL (C library and Python bindings)" + @echo "gdal-doctor report what GDAL is installed and what is wrong" + @echo "gdal-source build GDAL from source, no root or package manager" + @echo "install install the package and build the C++ kernels" + @echo "develop editable install for working on the source" + @echo "build compile the C++ kernels in place" + @echo "test run the test suite" + @echo "test-python run the test suite against the pure-NumPy kernels" + @echo "bench compare the two backends on a synthetic DEM" + @echo "docs regenerate docs/api.md from the docstrings" + @echo "clean remove build artefacts" + @echo "distclean also remove the compiled extension" + +# The one-command path: creates an environment, installs everything, +# installs GDAL, and checks the result by round-tripping a GeoTIFF. +bootstrap: + $(PYTHON) install.py + +# Run by path rather than as `python -m TopoAnalysis.gdal_setup`: this +# directory *is* the package, so the dotted form only resolves once the +# package is installed, and these targets need to work before that. +gdal: + $(PYTHON) gdal_setup.py install + +gdal-doctor: + $(PYTHON) gdal_setup.py doctor + +gdal-source: + $(PYTHON) gdal_setup.py install --strategy source + +install: + $(PIP) install . + @$(PYTHON) -c "import TopoAnalysis; print('backend:', TopoAnalysis.backend())" + +develop: + $(PIP) install -e ".[test]" + @$(PYTHON) -c "import TopoAnalysis; print('backend:', TopoAnalysis.backend())" + +build: + $(PYTHON) setup.py build_ext --inplace + +test: + $(PYTHON) -m pytest -q + +test-python: + TOPOANALYSIS_PURE_PYTHON=1 $(PYTHON) -m pytest -q + +bench: + $(PYTHON) -m TopoAnalysis.benchmark + +docs: + $(PYTHON) docs/generate_api.py + @echo "docs/api.md regenerated; the rest of docs/ is hand-written" + +lint: + $(PYTHON) -m compileall -q . >/dev/null && echo "all modules compile" + +clean: + rm -rf build dist *.egg-info .pytest_cache docs/_build + find . -name '__pycache__' -type d -prune -exec rm -rf {} + + find . -name '*.py[co]' -delete + +distclean: clean + rm -f _topoanalysis*.so _topoanalysis*.pyd diff --git a/MovingWindow.py b/MovingWindow.py index 2042f9a..1e3c780 100644 --- a/MovingWindow.py +++ b/MovingWindow.py @@ -1,88 +1,148 @@ -import error +"""Moving-window filters over a grid. + +A :class:`MovingWindow` applies a reducing function to the cells inside a +window centred on every cell. Subclasses choose the window shape; a +concrete class supplies the reduction as :attr:`~MovingWindow.function`:: + + class WindowMean(CircularMovingWindow): + function = staticmethod(np.mean) + + grid.apply_moving_window(WindowMean(window_radius=200.0)) + +Cells outside the grid, and no-data cells, are dropped from the window +rather than filled, so the function always sees real data. + +For the common reductions (min, max, mean, median, standard deviation) +:func:`TopoAnalysis.topotoolbox.localtopography` is far faster, because it +uses SciPy's separable filters instead of a Python loop. +""" + +from __future__ import annotations + import numpy as np +try: # pragma: no cover - exercised by whichever import style is in use + from . import error +except ImportError: # pragma: no cover + import error + + class MovingWindow(object): - + """Abstract base for moving-window filters. + + Parameters + ---------- + window_radius : float + Radius of the window in map units. ``window_dimension`` is accepted + as a synonym for backward compatibility. + """ + + #: The reduction applied to the values inside each window. function = None - + def __init__(self, *args, **kwargs): - if kwargs.get('window_dimension') is None: - raise error.InputError('Window dimension', 'is a required parameter') - self.window_dimension = kwargs.get('window_dimension') - + radius = kwargs.get('window_radius', kwargs.get('window_dimension')) + if radius is None: + raise error.InputError('Window radius', + 'window_radius is a required parameter') + # Both names are kept: window_dimension is what the original + # constructor took, window_radius is what _build_search_kernel reads. + self.window_radius = radius + self.window_dimension = radius + if self.__class__ is MovingWindow: raise error.Error('MovingWindow is an abstract base class') - - def _build_search_kernel(self, dx): - return None, None - - def __adjust_kernel(self, row,col,grid,searchKernelRows,searchKernelCols): - # Function to turn the rows and columns of the search kernel specified by searchKernelRows & ...Cols as just indices of the - # data that is actually within the grid - - #Get the current absolute positions within the grid - theseRows = searchKernelRows+row - theseCols = searchKernelCols+col - - #Do any points in searchKernel extend to before the start of the grid? or off the end? - gdIndices = (theseCols < grid.shape[1]) & (theseRows < grid.shape[0]) & (theseRows >= 0) & (theseCols >= 0) - - #Which rows are within the grid - goodRows = theseRows[gdIndices] - goodCols = theseCols[gdIndices] - - #Now that we are all within the grid, of those points within the grid, do any include - #Do any points in searchKernel include nans ? - gdIndices = np.logical_not(np.isnan(grid[goodRows, goodCols])) - - return goodRows[gdIndices], goodCols[gdIndices] + + def _build_search_kernel(self, dx): + """Relative ``(row, col)`` offsets of the window, as flat arrays.""" + raise NotImplementedError + + def _adjust_kernel(self, row, col, grid, searchKernelRows, searchKernelCols): + """Clip the window to the grid and drop no-data cells.""" + theseRows = searchKernelRows + row + theseCols = searchKernelCols + col + + inside = ((theseCols < grid.shape[1]) & (theseRows < grid.shape[0]) + & (theseRows >= 0) & (theseCols >= 0)) + + goodRows = theseRows[inside] + goodCols = theseCols[inside] + + valid = np.logical_not(np.isnan(grid[goodRows, goodCols])) + + return goodRows[valid], goodCols[valid] def apply_moving_window(self, grid, dx, dtype): - # Function to scan the moving window specified by Kernel across the dem, a numpy grid, and apply the specified function - # to each location. function is a method that returns a single value given any number of inputs, e.g. - # movingWindow(i) = function(dem[Kernel@i]) - - outgrid = np.zeros_like(grid, dtype = dtype) - - search_kernel_rows, search_kernel_cols = self._build_search_kernel(grid) - + """Run the window over every cell of ``grid``. + + Parameters + ---------- + grid : ndarray + 2-D data; ``NaN`` is no-data. + dx : float + Cell size, used to convert the window radius to cells. + dtype : numpy dtype + Type of the output array. + + Returns + ------- + ndarray + """ + if self.function is None: + raise error.Error( + '{0} has no bound function'.format(self.__class__.__name__)) + + outgrid = np.zeros_like(grid, dtype=dtype) + + # The kernel depends on the cell size, not on the data; the original + # passed the grid itself here, so window_radius/dx divided a float by + # an array. + search_kernel_rows, search_kernel_cols = self._build_search_kernel(dx) + for i in range(grid.shape[0]): for j in range(grid.shape[1]): - these_rows,these_cols = self.__adjustKernel(i,j,grid,search_kernel_rows,search_kernel_cols) #trim the searchKernel and return it as a list of indices - outgrid[i,j] = self.function(grid[these_rows, these_cols]) #apply the specified function to the dem values for this search location - + # The original called self.__adjustKernel, which does not + # exist -- the method is __adjust_kernel. + these_rows, these_cols = self._adjust_kernel( + i, j, grid, search_kernel_rows, search_kernel_cols) + if these_rows.size == 0: + outgrid[i, j] = np.nan if np.issubdtype(dtype, np.floating) else 0 + continue + outgrid[i, j] = self.function(grid[these_rows, these_cols]) + return outgrid + class RectangularMovingWindow(MovingWindow): - + """A square window of side ``2 * round(window_radius / dx) + 1`` cells.""" + def __init__(self, *args, **kwargs): super(RectangularMovingWindow, self).__init__(*args, **kwargs) if self.__class__ is RectangularMovingWindow: raise error.Error('RectangularMovingWindow has no bound function') - + def _build_search_kernel(self, dx): - # Function to build a search kernel that is either square or circular - - pxlRadius = int(round(self.window_radius/dx)) #Is this right... - relCoords = np.arange(1 + 2*pxlRadius)-pxlRadius #Relative coordinates of neighbors within this row, col distance - searchKernelRow, serchKernelCol = np.meshgrid(relCoords,relCoords) - return searchKernelRow.flatten(), serchKernelCol.flatten() - + pxlRadius = int(round(self.window_radius / dx)) + relCoords = np.arange(1 + 2 * pxlRadius) - pxlRadius + searchKernelRow, searchKernelCol = np.meshgrid(relCoords, relCoords) + return searchKernelRow.flatten(), searchKernelCol.flatten() + + class CircularMovingWindow(MovingWindow): - + """A disc-shaped window of radius ``round(window_radius / dx)`` cells.""" + def __init__(self, *args, **kwargs): super(CircularMovingWindow, self).__init__(*args, **kwargs) if self.__class__ is CircularMovingWindow: raise error.Error('CircularMovingWindow has no bound function') - + def _build_search_kernel(self, dx): - # Function to build a search kernel that is either square or circular - - pxlRadius = int(round(self.window_radius/dx)) #Is this right... - relCoords = np.arange(1 + 2*pxlRadius)-pxlRadius #Relative coordinates of neighbors within this row, col distance - searchKernelRow, serchKernelCol = np.meshgrid(relCoords,relCoords) - dists = np.sqrt(searchKernelRow**2 + searchKernelRow**2) - searchKernelRow = searchKernelRow[dists +cd TopoAnalysis +python3 install.py +``` + +That is the whole procedure. It creates a virtual environment, installs the +dependencies, compiles the kernels, installs **GDAL — both the C library and +the Python bindings** — and then checks the result by writing a georeferenced +GeoTIFF and reading it back. + +GDAL needs its own installer because pip cannot install it: there is no GDAL +wheel on PyPI for any platform at any version, only an sdist that compiles +against a C library which must already exist. `install.py` supplies both +halves, using conda, the system package manager, a prebuilt wheel on Windows, +or — when none of those is available — by building PROJ and GDAL from pinned, +checksummed source into the virtual environment, which needs no +administrator rights at all. + +```bash +topoanalysis-gdal # what GDAL is installed, and what is wrong with it +topoanalysis-gdal install # install or repair it +python3 install.py --dry-run # show the plan for this machine, change nothing +``` + +Neither GDAL nor a C++ compiler is required: without a compiler the library +falls back to NumPy kernels that produce identical numbers, and without GDAL +everything except file I/O works. See +[docs/installation.md](docs/installation.md) for platform notes, offline +installs, and troubleshooting. + +## A first analysis + +```python +from TopoAnalysis import Elevation, FilledElevation, FlowDirectionD8, Area, FlowLength + +dem = Elevation(gdal_filename='srtm.tif') +filled = FilledElevation(elevation=dem) # Priority-Flood +d8 = FlowDirectionD8(flooded_dem=filled) # steepest-descent D8 +area = Area(flow_direction=d8) # contributing area, m^2 +length = FlowLength(flow_direction=d8) # longest upstream path + +area.save('area.tif') +``` + +Grids are built **by keyword**, and which keywords you pass decides how the +grid is made: + +| What you pass | What you get | +|---|---| +| `gdal_filename='dem.tif'` | read from a file | +| `ai_ascii_filename=..., EPSGprojectionCode=32611` | read an ArcInfo ASCII grid | +| `dx=30.0, grid=array` | wrap an array already in memory | +| `nx=..., ny=..., projection=..., geo_transform=...` | an empty georeferenced grid | +| `flow_direction=d8`, `elevation=dem`, … | derive from another grid | + +Every class lists the combinations it accepts in +`required_inputs_and_actions`; passing something else raises an error that +tells you what the class does accept. + +## Chi and channel steepness + +```python +from TopoAnalysis import Chi, Ksi, ScaledRelief + +outlets = area.areas_greater_than(1e8) # big basins + +chi = Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e6, outlets=outlets) +ksi = Ksi(area=area, flow_direction=d8, flow_length=length, theta=0.45, Ao=1e6) +relief = ScaledRelief(flow_direction=d8, elevation=dem, flow_length=length, + Ao=1e6, theta=0.45, area=area) +``` + +`Chi` integrates upstream from the outlets you name; `Ksi` integrates along +longest-flow-length paths across the whole grid. Plotting `ScaledRelief` +against `Ksi` gives a chi-elevation plot whose slope is the steepness index. + +## Cross-checking against TopoToolbox + +```python +from TopoAnalysis import topotoolbox as tt + +zf = tt.fillsinks(dem) # flat fill, == GRIDobj/fillsinks +fd = tt.flowobj(dem) # == FLOWobj(DEM,'preprocess','fill') +a = tt.flowacc(fd) # == flowacc(FD) -- in CELLS, not m^2 +c = tt.chitransform(fd, a) # == chitransform(S,A), mn=0.45, a0=1e6 +k = tt.ksn(fd, dem, a) # == ksn(S,DEM,A) + +tt.gradient8(dem) # == GRIDobj/gradient8 +tt.curvature(dem, 'profc') # == GRIDobj/curvature +tt.hillshade(dem) # == GRIDobj/hillshade (cosine, not 0-255) +tt.aspect(dem) # == GRIDobj/aspect +``` + +The conventions differ between the two packages in a few places that matter — +drainage area in cells versus square metres, chi integrated with the +trapezoidal rule versus the left-endpoint rule, hillshade as a cosine versus +ESRI's 0–255. [docs/topotoolbox_parity.md](docs/topotoolbox_parity.md) +tabulates all of them and says where exact agreement is and is not +achievable. + +## Command line + +```bash +topoanalysis-process srtm.tif -o results --products filled,flowdir,area,length +topoanalysis-process --info # which backend is active, is GDAL present +``` + +## Performance + +`python -m TopoAnalysis.benchmark 1024` — a 1024 × 1024 DEM, about a million +cells, on a 2019 laptop: + +| Stage | C++ | NumPy | +|---|---:|---:| +| Priority-Flood fill | 0.10 s | 5.5 s | +| D8 flow directions | 0.02 s | 0.04 s | +| Flow accumulation | 0.01 s | 0.41 s | +| Flow length | 0.02 s | 0.83 s | +| Chi | 0.04 s | 2.6 s | +| **whole pipeline** | **0.20 s** | **9.4 s** | + +The compiled kernels release the GIL, so several grids can be processed in +parallel from threads. + +## What changed in 1.0 + +Version 1.0 repairs about 70 defects — several of which stopped the library +importing at all on current NumPy and SciPy — and replaces the recursive +basin traversals, which needed a million-frame recursion limit and still +overflowed on real DEMs, with iterative kernels. + +**The public interface is unchanged**: every class, method and keyword still +exists and still means what it did. A handful of results are *different +because they were wrong*; they are listed in [CHANGELOG.md](CHANGELOG.md) +under "Results that change". Read that section before comparing new output +with old. + +## Documentation + +- [Installation](docs/installation.md) — including offline and no-compiler cases +- [User guide](docs/guide.md) — the grid model, the analysis chain, worked examples +- [API reference](docs/api.md) — every public class and function +- [Algorithms](docs/algorithms.md) — what each computation actually does, and why +- [TopoToolbox parity](docs/topotoolbox_parity.md) — the correspondence table +- [Changelog](CHANGELOG.md) — fixed defects and behavioural changes + +## References + +Barnes, R., Lehman, C. and Mulla, D. (2014). Priority-flood: An optimal +depression-filling and watershed-labeling algorithm for digital elevation +models. *Computers & Geosciences* 62, 117–127. + +Perron, J.T. and Royden, L. (2013). An integral approach to bedrock river +profile analysis. *Earth Surface Processes and Landforms* 38, 570–576. + +Schwanghart, W. and Scherler, D. (2014). TopoToolbox 2 — MATLAB-based +software for topographic analysis and modeling in Earth surface sciences. +*Earth Surface Dynamics* 2, 1–7. + +Willett, S.D., McCoy, S.W., Perron, J.T., Goren, L. and Chen, C.-Y. (2014). +Dynamic reorganization of river basins. *Science* 343, 1248765. + +## Credits + +Originally written by Sam Johnstone (January 2015); maintained since by the +Stanford geomorphology group. diff --git a/__init__.py b/__init__.py index e69de29..bb50802 100644 --- a/__init__.py +++ b/__init__.py @@ -0,0 +1,156 @@ +"""TopoAnalysis -- digital elevation model analysis for tectonic geomorphology. + +A typical analysis chains grid classes together, each one built from the +last:: + + from TopoAnalysis import Elevation, FilledElevation, FlowDirectionD8, Area + + dem = Elevation(gdal_filename='srtm.tif') + filled = FilledElevation(elevation=dem) + d8 = FlowDirectionD8(flooded_dem=filled) + area = Area(flow_direction=d8) + +Grids are constructed by keyword: which keywords you pass decides how the +grid is built. ``Elevation(gdal_filename=...)`` reads a file, +``Elevation(dx=..., grid=...)`` wraps an array already in memory, and +``Area(flow_direction=...)`` derives one grid from another. Each class +lists its accepted combinations in ``required_inputs_and_actions``. + +Sub-modules +----------- +:mod:`TopoAnalysis.dem` + The grid classes. Everything public is re-exported here. +:mod:`TopoAnalysis.topotoolbox` + Functions that reproduce TopoToolbox 2 (MATLAB) results exactly. +:mod:`TopoAnalysis.kernels` + The compute kernels, in C++ where available and NumPy otherwise. +:mod:`TopoAnalysis.plotting`, :mod:`TopoAnalysis.demRecursionTools` + Longitudinal and chi profiles, and steepness fitting along them. +:mod:`TopoAnalysis.datasets` + Synthetic landscapes for testing. + +Depression filling uses the Priority-Flood algorithms of Barnes, Lehman and +Mulla (2014); see :mod:`TopoAnalysis.kernels`. +""" + +from __future__ import annotations + +__version__ = "1.0.0" + +# The package is importable both as ``TopoAnalysis`` and, with this directory +# on sys.path, flat -- the way the original scripts in this repository load +# it. The same shim appears in every module here. +try: # pragma: no cover - exercised by whichever import style is in use + from . import error, fastops, gdal_setup, kernels # noqa: F401 + from . import dem +except ImportError: # pragma: no cover + # The relative form failed, so this file was executed without a parent + # package. That happens whenever something imports it by path under a + # name other than ``TopoAnalysis`` -- notably pytest, which names the + # module after the checkout *directory*, and so only gets a valid package + # name when the directory happens to be called TopoAnalysis. + # + # Falling back to flat imports only works if this directory is on the + # path, which used to mean "only when it is also the working directory". + # Put it there explicitly instead. Appended rather than prepended: these + # names only need to be findable, and nothing here should ever take + # precedence over the standard library or an installed package. + import os as _os + import sys as _sys + + _here = _os.path.dirname(_os.path.abspath(__file__)) + if _here not in _sys.path: + _sys.path.append(_here) + del _os, _sys, _here + + import error, fastops, gdal_setup, kernels # noqa: F401 + import dem + +#: Everything :mod:`TopoAnalysis.dem` exports through this package. Listed +#: explicitly rather than filtered out of ``dir(dem)`` so that a name +#: disappearing from ``dem`` breaks the import here, loudly, instead of +#: leaving ``__all__`` promising something that is not there. +_FROM_DEM = ( + "Area", + "AlongFlowSmoothing", + "BaseSpatialGrid", + "BaseSpatialShape", + "CalculationMixin", + "ChannelDownSlopeWithSmoothing", + "ChannelSlope", + "ChannelSlopeWithSmoothing", + "ChannelUpSlopeWithSmoothing", + "Chi", + "ChiScaledRelief", + "CrossDivideDChi", + "Deflection", + "DiscreteFlowAccumulation", + "Elevation", + "FilledElevation", + "FlowDirection", + "FlowDirectionD8", + "FlowLength", + "GDALMixin", + "GeographicArea", + "GeographicChi", + "GeographicDeflection", + "GeographicDiscreteFlowAccumulation", + "GeographicElevation", + "GeographicFlowLength", + "GeographicGridMixin", + "GeographicHillshade", + "GeographicKsFromChiWithSmoothing", + "GeographicKsi", + "GeographicLaplacian", + "GeographicMainstemValleyArea", + "GeographicMaxSlope", + "GeographicRestoredElevation", + "GeographicThetaFromChiWithSmoothing", + "GeographicValleyArea", + "Georef_info", + "Gradient", + "Hillshade", + "Ksi", + "KsFromChiWithSmoothing", + "Laplacian", + "LocalRelief", + "LogArea", + "MainstemValleyArea", + "Mask", + "MaxFlowLengthTrackingMixin", + "MaxSlope", + "MultiscaleCurvatureValleyWidth", + "NormalizedCrossDivideDChi", + "PriorityFillGrid", + "PriorityQueueMixIn", + "Relief", + "RestoredElevation", + "ScaledRelief", + "ScarpWavelet", + "ThetaFromChiWithSmoothing", + "ValleyArea", + "ValueGrid", + "gdal_is_available", + "mosaicFolder", + "plot", +) + +for _name in _FROM_DEM: + globals()[_name] = getattr(dem, _name) +del _name + + +def backend() -> str: + """Which kernel backend is in use: ``'c++'`` or ``'python'``.""" + return kernels.backend() + + +__all__ = sorted(_FROM_DEM) + [ + "__version__", + "backend", + "dem", + "error", + "fastops", + "gdal_setup", + "kernels", +] diff --git a/_cpp/flow_routing.hpp b/_cpp/flow_routing.hpp new file mode 100644 index 0000000..772f30e --- /dev/null +++ b/_cpp/flow_routing.hpp @@ -0,0 +1,625 @@ +// flow_routing.hpp -- D8 flow routing kernels +// +// These replace the per-cell Python loops and the deeply recursive traversals +// that used to dominate TopoAnalysis' runtime. Everything here is iterative, +// so there is no recursion limit to raise and no stack to overflow on large +// basins. +// +// Flow-direction codes follow the ArcGIS convention used throughout +// TopoAnalysis, with row 0 at the north edge of the grid: +// +// | 32 64 128 | | (i-1,j-1) (i-1,j) (i-1,j+1) | +// | 16 X 1 | == | (i ,j-1) X (i ,j+1) | +// | 8 4 2 | | (i+1,j-1) (i+1,j) (i+1,j+1) | +// +// A code of 0 means "no downstream neighbour" (a pit, an outlet, or no-data). + +#ifndef TOPOANALYSIS_FLOW_ROUTING_HPP +#define TOPOANALYSIS_FLOW_ROUTING_HPP + +#include +#include +#include +#include +#include +#include + +#include "priority_flood.hpp" + +namespace topoanalysis { + +// --------------------------------------------------------------------------- +// Flow direction +// --------------------------------------------------------------------------- + +// Steepest-descent D8 flow directions. +// +// The drop to each neighbour is divided by the centre-to-centre distance, so +// diagonal neighbours compete on gradient rather than on raw elevation drop. +// This is the standard D8 rule (O'Callaghan & Mark 1984) and matches +// TopoToolbox's ``gradient8``/FLOWobj steepest-neighbour selection. +// +// ``cellsize`` may be a single value (uniform grid) or a per-cell array of +// mean pixel dimensions, which is what geographic (lat/lon) grids need. +// Ties are broken by neighbour order (E, SE, S, SW, W, NW, N, NE), which is +// deterministic and reproducible. +inline void d8_flow_directions(const double* elevations, + std::size_t ny, + std::size_t nx, + const double* cellsize, // ny*nx, or nullptr + double uniform_cellsize, + std::uint8_t* out_codes) { + const std::int64_t rows = static_cast(ny); + const std::int64_t cols = static_cast(nx); + + for (std::int64_t i = 0; i < rows; ++i) { + for (std::int64_t j = 0; j < cols; ++j) { + const std::int64_t k = i * cols + j; + const double z = elevations[k]; + if (std::isnan(z)) { + out_codes[k] = 0; + continue; + } + const double de = cellsize ? cellsize[k] : uniform_cellsize; + + double best_slope = 0.0; + int best_dir = -1; + for (int d = 0; d < 8; ++d) { + const std::int64_t ni = i + kD8Di[d]; + const std::int64_t nj = j + kD8Dj[d]; + if (ni < 0 || nj < 0 || ni >= rows || nj >= cols) continue; + const double zn = elevations[ni * cols + nj]; + if (std::isnan(zn)) continue; + const double slope = (z - zn) / (de * kD8Dist[d]); + if (slope > best_slope) { + best_slope = slope; + best_dir = d; + } + } + out_codes[k] = (best_dir < 0) ? 0 : static_cast(1u << best_dir); + } + } +} + +// Translate a flow-direction code grid into an explicit receiver index per +// cell (-1 where flow leaves the grid or the cell has no receiver). +inline void d8_receivers(const std::uint8_t* codes, + std::size_t ny, + std::size_t nx, + std::int64_t* out_receivers) { + const std::int64_t rows = static_cast(ny); + const std::int64_t cols = static_cast(nx); + + for (std::int64_t i = 0; i < rows; ++i) { + for (std::int64_t j = 0; j < cols; ++j) { + const std::int64_t k = i * cols + j; + const std::uint8_t code = codes[k]; + out_receivers[k] = -1; + if (code == 0) continue; + for (int d = 0; d < 8; ++d) { + if (code != static_cast(1u << d)) continue; + const std::int64_t ni = i + kD8Di[d]; + const std::int64_t nj = j + kD8Dj[d]; + if (ni < 0 || nj < 0 || ni >= rows || nj >= cols) break; + out_receivers[k] = ni * cols + nj; + break; + } + } + } +} + +// --------------------------------------------------------------------------- +// Topological ordering +// --------------------------------------------------------------------------- + +// Order the cells so that every cell appears before the cell it drains into +// (Kahn's algorithm on the donor counts). This is the ordering flow +// accumulation needs, and unlike sorting by elevation it is derived from the +// flow network itself -- so it stays correct for flow-direction grids that +// were loaded from disk, hand-edited, or derived from a different DEM. +// +// Returns the number of cells that could NOT be ordered because they lie on a +// cycle. Those cells are appended at the end in index order so that callers +// still see every cell exactly once. +inline std::size_t d8_topological_order(const std::int64_t* receivers, + std::size_t n, + std::int64_t* out_order) { + std::vector indegree(n, 0); + for (std::size_t k = 0; k < n; ++k) { + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && static_cast(r) != k) { + ++indegree[static_cast(r)]; + } + } + + std::vector stack; + stack.reserve(n); + for (std::size_t k = 0; k < n; ++k) { + if (indegree[k] == 0) stack.push_back(static_cast(k)); + } + + std::size_t written = 0; + while (!stack.empty()) { + const std::int64_t k = stack.back(); + stack.pop_back(); + out_order[written++] = k; + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && r != k) { + if (--indegree[static_cast(r)] == 0) stack.push_back(r); + } + } + + const std::size_t unordered = n - written; + if (unordered > 0) { + // Cells on a cycle: emit them in index order so the output is still a + // permutation of all cells. + std::vector emitted(n, 0); + for (std::size_t p = 0; p < written; ++p) emitted[static_cast(out_order[p])] = 1; + for (std::size_t k = 0; k < n; ++k) { + if (!emitted[k]) out_order[written++] = static_cast(k); + } + } + return unordered; +} + +// --------------------------------------------------------------------------- +// Flow accumulation +// --------------------------------------------------------------------------- + +// Accumulate ``weights`` downstream. ``weights`` is normally the per-cell +// area (dx*dy, or the true spherical cell area for geographic grids); pass an +// array of ones to get the number of contributing cells, which is what +// TopoToolbox's ``flowacc`` returns by default. +// +// ``accumulate_from`` is an optional 0/1 mask: cells with a zero entry +// contribute nothing and do not pass their accumulated value downstream. +// This reproduces the ``evaluate_at`` option of the original Python code. +inline void d8_accumulate(const std::int64_t* receivers, + const std::int64_t* order, + const double* weights, + const std::uint8_t* accumulate_from, // may be nullptr + std::size_t n, + double* out_accumulation) { + for (std::size_t k = 0; k < n; ++k) { + const bool active = (accumulate_from == nullptr) || (accumulate_from[k] != 0); + out_accumulation[k] = active ? weights[k] : 0.0; + } + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + if (accumulate_from != nullptr && accumulate_from[k] == 0) continue; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + out_accumulation[r] += out_accumulation[k]; + } +} + +// --------------------------------------------------------------------------- +// Flow length +// --------------------------------------------------------------------------- + +// Longest upstream flow distance reaching each cell, plus the code of the +// upstream neighbour that supplied it. ``step_length`` is the length of the +// step from each cell to its receiver. +// +// ``out_from_codes`` records, for each cell, the direction code *pointing back +// at* the donor that lies on the longest path. ``FlowLength`` uses it to walk +// the main stem back upstream. +// +// When two donors tie on length the one with the lower flat index wins. That +// tie-break matters: without it the "main stem" would depend on which valid +// topological order happened to be used, and two runs of the same analysis +// could disagree. +inline void d8_flow_length(const std::int64_t* receivers, + const std::int64_t* order, + const double* step_length, + std::size_t ny, + std::size_t nx, + double* out_length, + std::uint8_t* out_from_codes) { + const std::size_t n = ny * nx; + const std::int64_t cols = static_cast(nx); + + std::vector from_index(n, -1); + for (std::size_t k = 0; k < n; ++k) out_length[k] = 0.0; + + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + const double candidate = out_length[k] + step_length[k]; + const std::int64_t incumbent = from_index[static_cast(r)]; + const bool better = candidate > out_length[r] || + (candidate == out_length[r] && (incumbent < 0 || k < incumbent)); + if (better) { + out_length[r] = candidate; + from_index[static_cast(r)] = k; + } + } + + if (!out_from_codes) return; + for (std::size_t r = 0; r < n; ++r) { + out_from_codes[r] = 0; + const std::int64_t k = from_index[r]; + if (k < 0) continue; + const std::int64_t di = (k / cols) - (static_cast(r) / cols); + const std::int64_t dj = (k % cols) - (static_cast(r) % cols); + for (int d = 0; d < 8; ++d) { + if (kD8Di[d] == di && kD8Dj[d] == dj) { + out_from_codes[r] = static_cast(1u << d); + break; + } + } + } +} + +// Propagate a value downstream, but only along the longest-flow-length path. +// ``Relief`` uses mode=Carry (each cell inherits the value of its main-stem +// donor) and ``Ksi`` uses mode=Sum (each cell adds its main-stem donor's +// value). ``gate`` optionally suppresses the transfer for cells where the +// gate is zero, which is how ``Relief`` implements its ``Ao`` cut-off. +enum class PropagateMode { Carry = 0, Sum = 1 }; + +inline void d8_propagate_along_main_stem(const std::int64_t* receivers, + const std::int64_t* order, + const std::uint8_t* main_stem_from_codes, + const std::uint8_t* gate, // may be nullptr + std::size_t ny, + std::size_t nx, + PropagateMode mode, + double* values) { + const std::size_t n = ny * nx; + const std::int64_t cols = static_cast(nx); + + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + + // Is k the donor that defines r's longest flow path? + const std::int64_t di = (k / cols) - (r / cols); + const std::int64_t dj = (k % cols) - (r % cols); + std::uint8_t code = 0; + for (int d = 0; d < 8; ++d) { + if (kD8Di[d] == di && kD8Dj[d] == dj) { + code = static_cast(1u << d); + break; + } + } + if (code == 0 || main_stem_from_codes[r] != code) continue; + if (gate != nullptr && gate[k] == 0) continue; + + if (mode == PropagateMode::Sum) { + values[r] += values[k]; + } else { + values[r] = values[k]; + } + } +} + +// --------------------------------------------------------------------------- +// Upstream traversal +// --------------------------------------------------------------------------- + +// Mark every cell that drains (directly or indirectly) into any of +// ``outlets``. Iterative, so basin size is limited only by memory. +inline void d8_upstream_mask(const std::int64_t* receivers, + std::size_t n, + const std::int64_t* outlets, + std::size_t n_outlets, + std::uint8_t* out_mask) { + // Build the donor adjacency in CSR form so the traversal is O(n). + std::vector counts(n + 1, 0); + for (std::size_t k = 0; k < n; ++k) { + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && static_cast(r) != k) { + ++counts[static_cast(r) + 1]; + } + } + for (std::size_t k = 0; k < n; ++k) counts[k + 1] += counts[k]; + std::vector donors(static_cast(counts[n])); + std::vector cursor(counts.begin(), counts.end() - 1); + for (std::size_t k = 0; k < n; ++k) { + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && static_cast(r) != k) { + donors[static_cast(cursor[static_cast(r)]++)] = + static_cast(k); + } + } + + std::fill(out_mask, out_mask + n, static_cast(0)); + std::vector stack; + for (std::size_t o = 0; o < n_outlets; ++o) { + const std::int64_t k = outlets[o]; + if (k < 0 || static_cast(k) >= n) continue; + if (out_mask[k]) continue; + out_mask[k] = 1; + stack.push_back(k); + } + while (!stack.empty()) { + const std::int64_t k = stack.back(); + stack.pop_back(); + for (std::int64_t p = counts[static_cast(k)]; + p < counts[static_cast(k) + 1]; ++p) { + const std::int64_t donor = donors[static_cast(p)]; + if (out_mask[donor]) continue; + out_mask[donor] = 1; + stack.push_back(donor); + } + } +} + +// --------------------------------------------------------------------------- +// Chi +// --------------------------------------------------------------------------- + +// Integrate the chi coordinate upstream from a set of outlets: +// +// chi(cell) = chi(receiver) + integrand * step_length(cell) +// +// with ``integrand = (A0 / A)^theta``. ``trapezoid`` switches from the +// left-endpoint rule used by the original TopoAnalysis recursion to the +// trapezoidal rule used by TopoToolbox's ``chitransform``: +// +// chi(cell) = chi(receiver) +// + 0.5 * ((A0/A_cell)^theta + (A0/A_recv)^theta) * step_length +// +// Cells outside the outlets' basins, and cells where the drainage area is +// not positive, are left as NaN. ``max_length`` (if > 0) stops the +// integration once the along-stream distance from the outlet exceeds it. +inline void d8_chi(const std::int64_t* receivers, + const std::int64_t* order, + const double* area, + const double* step_length, + std::size_t n, + const std::int64_t* outlets, + std::size_t n_outlets, + double A0, + double theta, + bool trapezoid, + double max_length, + const std::uint8_t* mask, // may be nullptr + double* out_chi, + double* out_distance) { + const double nan = std::numeric_limits::quiet_NaN(); + for (std::size_t k = 0; k < n; ++k) { + out_chi[k] = nan; + if (out_distance) out_distance[k] = nan; + } + for (std::size_t o = 0; o < n_outlets; ++o) { + const std::int64_t k = outlets[o]; + if (k < 0 || static_cast(k) >= n) continue; + if (mask != nullptr && mask[k] == 0) continue; + out_chi[k] = 0.0; + if (out_distance) out_distance[k] = 0.0; + } + + // ``order`` runs donors-before-receivers, so walking it backwards visits + // every receiver before its donors. + for (std::size_t p = n; p-- > 0;) { + const std::int64_t k = order[p]; + if (!std::isnan(out_chi[k])) continue; // an outlet, already seeded + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + if (std::isnan(out_chi[r])) continue; + if (mask != nullptr && mask[k] == 0) continue; + + const double a = area[k]; + if (!(a > 0.0)) continue; + + const double dl = step_length[k]; + if (max_length > 0.0 && out_distance) { + if (out_distance[r] + dl > max_length) continue; + } + + double integrand = std::pow(A0 / a, theta); + if (trapezoid) { + const double ar = area[r]; + const double integrand_r = (ar > 0.0) ? std::pow(A0 / ar, theta) : integrand; + integrand = 0.5 * (integrand + integrand_r); + } + out_chi[k] = out_chi[r] + integrand * dl; + if (out_distance) out_distance[k] = out_distance[r] + dl; + } +} + +// --------------------------------------------------------------------------- +// Distance to the outlet +// --------------------------------------------------------------------------- + +// Along-flow distance from each cell down to the end of its flow path. +// This is TopoToolbox's ``flowdistance(FD,'upstream')``: 0 at every outlet +// and at every cell with no receiver, increasing upstream. +inline void d8_downstream_distance(const std::int64_t* receivers, + const std::int64_t* order, + const double* step_length, + std::size_t n, + double* out_distance) { + std::fill(out_distance, out_distance + n, 0.0); + // Receivers before donors, so a cell's receiver is final when it is read. + for (std::size_t p = n; p-- > 0;) { + const std::int64_t k = order[p]; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + out_distance[k] = out_distance[r] + step_length[k]; + } +} + +// --------------------------------------------------------------------------- +// Minima imposition (carving) +// --------------------------------------------------------------------------- + +// Lower each receiver until the flow path descends at a gradient of at least +// ``sl``. This is TopoToolbox's ``imposemin``: unlike depression filling it +// only ever *lowers* cells, so the channel network is carved through +// obstructions rather than drowned by them. +// +// ``elevations`` is modified in place. +inline void d8_imposemin(const std::int64_t* receivers, + const std::int64_t* order, + const double* step_length, + std::size_t n, + double sl, + double* elevations) { + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + const double candidate = elevations[k] - sl * step_length[k]; + if (candidate < elevations[r]) elevations[r] = candidate; + } +} + +// --------------------------------------------------------------------------- +// Stream order +// --------------------------------------------------------------------------- + +// Strahler stream order over the cells marked in ``is_stream``. +// +// Order 1 at channel heads; where two or more streams of the highest incoming +// order meet, the order increases by one; a smaller tributary joining a +// larger stream leaves the order unchanged. Non-stream cells stay 0. +// +// The rule is applied to each junction as a whole -- the largest incoming +// order and how many donors carry it -- rather than donor by donor. The +// incremental form gives a different answer depending on the sequence the +// donors arrive in (orders {2, 2, 3} yield 4 or 3 according to whether the 3 +// is seen first), which would make the result depend on which valid +// topological order the backend produced. +inline void d8_strahler(const std::int64_t* receivers, + const std::int64_t* order, + const std::uint8_t* is_stream, + std::size_t n, + std::int32_t* out_order) { + std::fill(out_order, out_order + n, 0); + + std::vector best(n, 0); // largest incoming order + std::vector ties(n, 0); // how many donors carry it + + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + if (!is_stream[k]) continue; + + // Donors are all settled by now, so this cell's own order is final. + out_order[k] = (ties[static_cast(k)] >= 2) + ? best[static_cast(k)] + 1 + : std::max(best[static_cast(k)], 1); + + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + if (!is_stream[r]) continue; + + const std::size_t ru = static_cast(r); + if (out_order[k] > best[ru]) { + best[ru] = out_order[k]; + ties[ru] = 1; + } else if (out_order[k] == best[ru]) { + ++ties[ru]; + } + } +} + +// Shreve magnitude: the number of channel heads upstream of each cell. +inline void d8_shreve(const std::int64_t* receivers, + const std::int64_t* order, + const std::uint8_t* is_stream, + std::size_t n, + std::int32_t* out_order) { + std::vector has_donor(n, 0); + for (std::size_t k = 0; k < n; ++k) { + if (!is_stream[k]) continue; + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && r != static_cast(k) + && is_stream[r]) { + has_donor[static_cast(r)] = 1; + } + } + for (std::size_t k = 0; k < n; ++k) { + out_order[k] = (is_stream[k] && !has_donor[k]) ? 1 : 0; + } + for (std::size_t p = 0; p < n; ++p) { + const std::int64_t k = order[p]; + if (!is_stream[k]) continue; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + if (!is_stream[r]) continue; + out_order[r] += out_order[k]; + } +} + +// --------------------------------------------------------------------------- +// Drainage basins +// --------------------------------------------------------------------------- + +// Label every cell with the index (1-based) of the outlet it drains to. +// Cells that reach no listed outlet are labelled 0. +inline void d8_drainage_basins(const std::int64_t* receivers, + const std::int64_t* order, + std::size_t n, + const std::int64_t* outlets, + std::size_t n_outlets, + const std::uint8_t* valid, // may be nullptr + std::int32_t* out_labels) { + std::fill(out_labels, out_labels + n, 0); + for (std::size_t o = 0; o < n_outlets; ++o) { + const std::int64_t k = outlets[o]; + if (k < 0 || static_cast(k) >= n) continue; + if (valid != nullptr && valid[k] == 0) continue; + out_labels[k] = static_cast(o + 1); + } + for (std::size_t p = n; p-- > 0;) { + const std::int64_t k = order[p]; + if (out_labels[k] != 0) continue; + if (valid != nullptr && valid[k] == 0) continue; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + out_labels[k] = out_labels[r]; + } +} + +// Label each cell with the id of the terminal cell (pit or edge outlet) that +// its flow path ends at. This is the ``drainagebasins(FD)`` behaviour of +// TopoToolbox when no outlets are supplied. +// +// A terminal cell only seeds a basin if something drains into it. A cell +// that is neither a giver nor a receiver -- an isolated no-data cell, say -- +// stays 0, which is also what TopoToolbox does (such a cell never appears in +// its edge list). +inline void d8_drainage_basins_all(const std::int64_t* receivers, + const std::int64_t* order, + const std::uint8_t* valid, // may be nullptr + std::size_t n, + std::int32_t* out_labels) { + std::fill(out_labels, out_labels + n, 0); + + std::vector has_donor(n, 0); + for (std::size_t k = 0; k < n; ++k) { + if (valid != nullptr && valid[k] == 0) continue; + const std::int64_t r = receivers[k]; + if (r >= 0 && static_cast(r) < n && r != static_cast(k)) { + has_donor[static_cast(r)] = 1; + } + } + + std::int32_t next_label = 0; + for (std::size_t k = 0; k < n; ++k) { + if (valid != nullptr && valid[k] == 0) continue; + const std::int64_t r = receivers[k]; + const bool terminal = + (r < 0 || static_cast(r) >= n || r == static_cast(k)); + if (terminal && has_donor[k]) { + out_labels[k] = ++next_label; + } + } + for (std::size_t p = n; p-- > 0;) { + const std::int64_t k = order[p]; + if (out_labels[k] != 0) continue; + if (valid != nullptr && valid[k] == 0) continue; + const std::int64_t r = receivers[k]; + if (r < 0 || static_cast(r) >= n || r == k) continue; + out_labels[k] = out_labels[r]; + } +} + +} // namespace topoanalysis + +#endif // TOPOANALYSIS_FLOW_ROUTING_HPP diff --git a/_cpp/module.cpp b/_cpp/module.cpp new file mode 100644 index 0000000..b166574 --- /dev/null +++ b/_cpp/module.cpp @@ -0,0 +1,540 @@ +// module.cpp -- pybind11 bindings for the TopoAnalysis C++ kernels. +// +// Everything exposed here has a pure-NumPy counterpart in +// ``TopoAnalysis.fastops``; the extension is an accelerator, never a hard +// requirement. Arrays are accepted as C-contiguous float64/uint8/int64. +// +// One rule governs every function below: **resolve every buffer and validate +// every size while the GIL is still held, then release it and run nothing but +// plain C++**. ``py::array::request()`` calls ``PyObject_GetBuffer``, which +// touches reference counts; calling it inside a ``gil_scoped_release`` block +// corrupts the interpreter as soon as two threads do it at once. The +// ``Resolved*`` helpers exist so that rule is enforced by construction rather +// than remembered at each call site. + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "priority_flood.hpp" +#include "flow_routing.hpp" + +namespace py = pybind11; +using namespace topoanalysis; + +namespace { + +using F64Array = py::array_t; +using U8Array = py::array_t; +using I64Array = py::array_t; +using I32Array = py::array_t; + +// --------------------------------------------------------------------------- +// GIL-safe buffer resolution +// --------------------------------------------------------------------------- + +// Holds an array alive together with the raw pointer into it. Construct one +// while the GIL is held; use ``.data`` after releasing it. +template +struct Resolved { + Array array; + T* data = nullptr; + std::size_t size = 0; + std::vector shape; + + Resolved() = default; + + explicit Resolved(Array a) : array(std::move(a)) { + auto info = array.request(); + data = static_cast(info.ptr); + size = static_cast(info.size); + shape.assign(info.shape.begin(), info.shape.end()); + } + + bool present() const { return data != nullptr; } + + void require_2d(const char* name) const { + if (shape.size() != 2) { + throw std::invalid_argument(std::string(name) + " must be a 2-D array"); + } + } + + void require_size(std::size_t expected, const char* name) const { + if (size != expected) { + throw std::invalid_argument( + std::string(name) + " has " + std::to_string(size) + + " elements; expected " + std::to_string(expected)); + } + } + + std::size_t ny() const { return static_cast(shape[0]); } + std::size_t nx() const { return static_cast(shape[1]); } +}; + +using ResolvedF64 = Resolved; +using ResolvedConstF64 = Resolved; +using ResolvedU8 = Resolved; +using ResolvedI64 = Resolved; +using ResolvedI32 = Resolved; + +// Optional arguments that may be None. +ResolvedU8 optional_u8(const py::object& obj) { + if (obj.is_none()) return ResolvedU8(); + return ResolvedU8(obj.cast()); +} + +ResolvedI64 optional_i64(const py::object& obj) { + if (obj.is_none()) return ResolvedI64(); + return ResolvedI64(obj.cast()); +} + +ResolvedF64 optional_f64(const py::object& obj) { + if (obj.is_none()) return ResolvedF64(); + return ResolvedF64(obj.cast()); +} + +// An output array of the given 2-D shape, resolved up front. +ResolvedF64 make_f64(std::size_t ny, std::size_t nx) { + return ResolvedF64(F64Array({static_cast(ny), static_cast(nx)})); +} + +ResolvedU8 make_u8(std::size_t ny, std::size_t nx) { + return ResolvedU8(U8Array({static_cast(ny), static_cast(nx)})); +} + +ResolvedI32 make_i32(std::size_t ny, std::size_t nx) { + return ResolvedI32(I32Array({static_cast(ny), static_cast(nx)})); +} + +// --------------------------------------------------------------------------- +// Depression filling +// --------------------------------------------------------------------------- + +py::dict py_priority_flood(F64Array elevations, + py::object closed_obj, + py::object seeds_obj, + const std::string& mode, + double epsilon, + double cellsize, + double max_pit_depth, + bool track_visited, + bool reference_algorithm) { + ResolvedF64 z(std::move(elevations)); + z.require_2d("elevations"); + const std::size_t ny = z.ny(), nx = z.nx(), n = ny * nx; + + std::vector closed(n, 0); + ResolvedU8 closed_arg = optional_u8(closed_obj); + if (closed_arg.present()) { + closed_arg.require_size(n, "closed"); + closed.assign(closed_arg.data, closed_arg.data + n); + } + for (std::size_t k = 0; k < n; ++k) { + if (std::isnan(z.data[k])) closed[k] = 1; + } + + std::vector seeds; + ResolvedI64 seeds_arg = optional_i64(seeds_obj); + if (!seeds_arg.present()) { + seeds = default_seeds(z.data, ny, nx); + // Honour a caller-supplied mask: a seed that is masked out would + // otherwise silently disappear and leave the grid unflooded. + std::vector kept; + kept.reserve(seeds.size()); + for (std::int64_t k : seeds) { + if (!closed[static_cast(k)]) kept.push_back(k); + } + if (closed_arg.present() && kept.empty()) { + // The mask excluded the whole grid perimeter. Fall back to the + // boundary of the masked region itself. + for (std::size_t i = 0; i < ny; ++i) { + for (std::size_t j = 0; j < nx; ++j) { + const std::size_t k = i * nx + j; + if (closed[k]) continue; + bool boundary = (i == 0 || j == 0 || i + 1 == ny || j + 1 == nx); + for (int d = 0; d < 8 && !boundary; ++d) { + const std::int64_t ni = static_cast(i) + kD8Di[d]; + const std::int64_t nj = static_cast(j) + kD8Dj[d]; + if (ni < 0 || nj < 0 || static_cast(ni) >= ny || + static_cast(nj) >= nx) { + continue; + } + if (closed[static_cast(ni) * nx + + static_cast(nj)]) { + boundary = true; + } + } + if (boundary) kept.push_back(static_cast(k)); + } + } + } + seeds.swap(kept); + } else { + seeds.assign(seeds_arg.data, seeds_arg.data + seeds_arg.size); + } + + FloodOptions options; + if (mode == "epsilon") { + options.mode = FillMode::Epsilon; + } else if (mode == "flat") { + options.mode = FillMode::Flat; + } else { + throw std::invalid_argument("mode must be 'flat' or 'epsilon'"); + } + options.epsilon = epsilon; + options.cellsize = cellsize; + options.max_pit_depth = max_pit_depth; + options.track_visited = track_visited; + options.track_unfilled = max_pit_depth > 0.0; + + FloodResult result; + { + py::gil_scoped_release release; + result = reference_algorithm + ? priority_flood_original(z.data, ny, nx, closed, seeds, options) + : priority_flood(z.data, ny, nx, closed, seeds, options); + } + + py::dict out; + out["cells_filled"] = result.cells_filled; + out["max_fill_depth"] = result.max_fill_depth; + if (track_visited) { + ResolvedU8 visited = make_u8(ny, nx); + std::copy(result.visited.begin(), result.visited.end(), visited.data); + out["visited"] = visited.array; + } else { + out["visited"] = py::none(); + } + if (options.track_unfilled) { + ResolvedU8 unfilled = make_u8(ny, nx); + std::copy(result.unfilled.begin(), result.unfilled.end(), unfilled.data); + out["unfilled"] = unfilled.array; + } else { + out["unfilled"] = py::none(); + } + return out; +} + +// --------------------------------------------------------------------------- +// Flow routing +// --------------------------------------------------------------------------- + +U8Array py_flow_directions(F64Array elevations, py::object cellsize_obj, + double uniform_cellsize) { + ResolvedF64 z(std::move(elevations)); + z.require_2d("elevations"); + const std::size_t ny = z.ny(), nx = z.nx(); + + ResolvedF64 cellsize = optional_f64(cellsize_obj); + if (cellsize.present()) cellsize.require_size(ny * nx, "cellsize_grid"); + + ResolvedU8 out = make_u8(ny, nx); + { + py::gil_scoped_release release; + d8_flow_directions(z.data, ny, nx, cellsize.data, uniform_cellsize, out.data); + } + return out.array; +} + +I64Array py_receivers(U8Array codes) { + ResolvedU8 c(std::move(codes)); + c.require_2d("codes"); + const std::size_t ny = c.ny(), nx = c.nx(); + + ResolvedI64 out(I64Array({static_cast(ny), static_cast(nx)})); + { + py::gil_scoped_release release; + d8_receivers(c.data, ny, nx, out.data); + } + return out.array; +} + +py::tuple py_topological_order(I64Array receivers) { + ResolvedI64 recv(std::move(receivers)); + const std::size_t n = recv.size; + + ResolvedI64 out(I64Array(static_cast(n))); + std::size_t cycles = 0; + { + py::gil_scoped_release release; + cycles = d8_topological_order(recv.data, n, out.data); + } + return py::make_tuple(out.array, cycles); +} + +F64Array py_accumulate(I64Array receivers, I64Array order, F64Array weights, + py::object gate_obj) { + ResolvedI64 recv(std::move(receivers)); + ResolvedI64 ord(std::move(order)); + ResolvedF64 w(std::move(weights)); + const std::size_t n = recv.size; + w.require_size(n, "weights"); + ord.require_size(n, "order"); + + ResolvedU8 gate = optional_u8(gate_obj); + if (gate.present()) gate.require_size(n, "gate"); + + ResolvedF64 out(F64Array(w.shape)); + { + py::gil_scoped_release release; + d8_accumulate(recv.data, ord.data, w.data, gate.data, n, out.data); + } + return out.array; +} + +py::tuple py_flow_length(I64Array receivers, I64Array order, F64Array step_length) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedF64 step(std::move(step_length)); + step.require_size(n, "step_length"); + + ResolvedF64 length = make_f64(ny, nx); + ResolvedU8 from_codes = make_u8(ny, nx); + { + py::gil_scoped_release release; + d8_flow_length(recv.data, ord.data, step.data, ny, nx, length.data, from_codes.data); + } + return py::make_tuple(length.array, from_codes.array); +} + +F64Array py_propagate_along_main_stem(I64Array receivers, I64Array order, + U8Array main_stem_from_codes, py::object gate_obj, + F64Array values, const std::string& mode) { + if (mode != "sum" && mode != "carry") { + throw std::invalid_argument("mode must be 'carry' or 'sum'"); + } + + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedU8 codes(std::move(main_stem_from_codes)); + codes.require_size(n, "main_stem_from_codes"); + ResolvedF64 vals(std::move(values)); + vals.require_size(n, "values"); + ResolvedU8 gate = optional_u8(gate_obj); + if (gate.present()) gate.require_size(n, "gate"); + + ResolvedF64 out = make_f64(ny, nx); + std::copy(vals.data, vals.data + n, out.data); + { + py::gil_scoped_release release; + d8_propagate_along_main_stem(recv.data, ord.data, codes.data, gate.data, ny, nx, + mode == "sum" ? PropagateMode::Sum : PropagateMode::Carry, + out.data); + } + return out.array; +} + +U8Array py_upstream_mask(I64Array receivers, I64Array outlets) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(); + ResolvedI64 out_idx(std::move(outlets)); + + ResolvedU8 out = make_u8(ny, nx); + { + py::gil_scoped_release release; + d8_upstream_mask(recv.data, ny * nx, out_idx.data, out_idx.size, out.data); + } + return out.array; +} + +py::tuple py_chi(I64Array receivers, I64Array order, F64Array area, F64Array step_length, + I64Array outlets, double A0, double theta, bool trapezoid, double max_length, + py::object mask_obj) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedF64 a(std::move(area)); + a.require_size(n, "area"); + ResolvedF64 step(std::move(step_length)); + step.require_size(n, "step_length"); + ResolvedI64 out_idx(std::move(outlets)); + ResolvedU8 mask = optional_u8(mask_obj); + if (mask.present()) mask.require_size(n, "mask"); + + ResolvedF64 chi_out = make_f64(ny, nx); + ResolvedF64 distance = make_f64(ny, nx); + { + py::gil_scoped_release release; + d8_chi(recv.data, ord.data, a.data, step.data, n, out_idx.data, out_idx.size, + A0, theta, trapezoid, max_length, mask.data, chi_out.data, distance.data); + } + return py::make_tuple(chi_out.array, distance.array); +} + +F64Array py_downstream_distance(I64Array receivers, I64Array order, F64Array step_length) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedF64 step(std::move(step_length)); + step.require_size(n, "step_length"); + + ResolvedF64 out = make_f64(ny, nx); + { + py::gil_scoped_release release; + d8_downstream_distance(recv.data, ord.data, step.data, n, out.data); + } + return out.array; +} + +void py_imposemin(I64Array receivers, I64Array order, F64Array step_length, double sl, + py::array_t elevations) { + ResolvedI64 recv(std::move(receivers)); + const std::size_t n = recv.size; + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedF64 step(std::move(step_length)); + step.require_size(n, "step_length"); + + auto einfo = elevations.request(); + if (static_cast(einfo.size) != n) { + throw std::invalid_argument("elevations must match the receiver grid"); + } + double* z = static_cast(einfo.ptr); + + py::gil_scoped_release release; + d8_imposemin(recv.data, ord.data, step.data, n, sl, z); +} + +I32Array py_stream_order(I64Array receivers, I64Array order, U8Array is_stream, + const std::string& kind) { + if (kind != "strahler" && kind != "shreve") { + throw std::invalid_argument("kind must be 'strahler' or 'shreve'"); + } + + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedU8 streams(std::move(is_stream)); + streams.require_size(n, "is_stream"); + + ResolvedI32 out = make_i32(ny, nx); + { + py::gil_scoped_release release; + if (kind == "shreve") { + d8_shreve(recv.data, ord.data, streams.data, n, out.data); + } else { + d8_strahler(recv.data, ord.data, streams.data, n, out.data); + } + } + return out.array; +} + +I32Array py_drainage_basins(I64Array receivers, I64Array order, py::object outlets_obj, + py::object valid_obj) { + ResolvedI64 recv(std::move(receivers)); + recv.require_2d("receivers"); + const std::size_t ny = recv.ny(), nx = recv.nx(), n = ny * nx; + + ResolvedI64 ord(std::move(order)); + ord.require_size(n, "order"); + ResolvedU8 valid = optional_u8(valid_obj); + if (valid.present()) valid.require_size(n, "valid"); + ResolvedI64 outlets = optional_i64(outlets_obj); + + ResolvedI32 out = make_i32(ny, nx); + { + py::gil_scoped_release release; + if (!outlets.present()) { + d8_drainage_basins_all(recv.data, ord.data, valid.data, n, out.data); + } else { + // `valid` applies here too; dropping it made the two backends + // disagree whenever both arguments were supplied. + d8_drainage_basins(recv.data, ord.data, n, outlets.data, outlets.size, + valid.data, out.data); + } + } + return out.array; +} + +} // namespace + +PYBIND11_MODULE(_topoanalysis, m) { + m.doc() = + "Compiled kernels for TopoAnalysis.\n\n" + "Depression filling implements the Priority-Flood family of algorithms of\n" + "Barnes, R., Lehman, C. and Mulla, D. (2014), 'Priority-flood: An optimal\n" + "depression-filling and watershed-labeling algorithm for digital elevation\n" + "models', Computers & Geosciences 62, 117-127."; + + m.attr("__algorithm_reference__") = + "Barnes, R., Lehman, C., Mulla, D. (2014). Priority-flood: An optimal " + "depression-filling and watershed-labeling algorithm for digital elevation " + "models. Computers & Geosciences 62, 117-127. doi:10.1016/j.cageo.2013.04.024"; + + m.def("priority_flood", &py_priority_flood, py::arg("elevations").noconvert(), + py::arg("closed") = py::none(), py::arg("seeds") = py::none(), + py::arg("mode") = "flat", py::arg("epsilon") = 0.0, py::arg("cellsize") = 1.0, + py::arg("max_pit_depth") = 0.0, py::arg("track_visited") = false, + py::arg("reference_algorithm") = false, + "Fill depressions in place using Priority-Flood (Barnes et al. 2014)."); + + m.def("flow_directions", &py_flow_directions, py::arg("elevations"), + py::arg("cellsize_grid") = py::none(), py::arg("cellsize") = 1.0, + "Steepest-descent D8 flow directions as ArcGIS codes."); + + m.def("receivers", &py_receivers, py::arg("codes"), + "Flat receiver index for each cell (-1 where flow leaves the grid)."); + + m.def("topological_order", &py_topological_order, py::arg("receivers"), + "Donors-before-receivers ordering; returns (order, n_cells_on_cycles)."); + + m.def("accumulate", &py_accumulate, py::arg("receivers"), py::arg("order"), + py::arg("weights"), py::arg("gate") = py::none(), + "Accumulate weights downstream along the D8 network."); + + m.def("flow_length", &py_flow_length, py::arg("receivers"), py::arg("order"), + py::arg("step_length"), + "Longest upstream flow distance and the main-stem donor direction codes."); + + m.def("propagate_along_main_stem", &py_propagate_along_main_stem, py::arg("receivers"), + py::arg("order"), py::arg("main_stem_from_codes"), py::arg("gate"), py::arg("values"), + py::arg("mode"), "Carry or sum values downstream along the longest flow path."); + + m.def("upstream_mask", &py_upstream_mask, py::arg("receivers"), py::arg("outlets"), + "Mask of every cell draining to any of the given outlets."); + + m.def("chi", &py_chi, py::arg("receivers"), py::arg("order"), py::arg("area"), + py::arg("step_length"), py::arg("outlets"), py::arg("A0"), py::arg("theta"), + py::arg("trapezoid") = false, py::arg("max_length") = 0.0, + py::arg("mask") = py::none(), "Integrate chi upstream from the given outlets."); + + m.def("drainage_basins", &py_drainage_basins, py::arg("receivers"), py::arg("order"), + py::arg("outlets") = py::none(), py::arg("valid") = py::none(), + "Label drainage basins, either for given outlets or for every terminal cell."); + + m.def("downstream_distance", &py_downstream_distance, py::arg("receivers"), py::arg("order"), + py::arg("step_length"), + "Along-flow distance from each cell to the end of its flow path."); + + m.def("imposemin", &py_imposemin, py::arg("receivers"), py::arg("order"), + py::arg("step_length"), py::arg("sl"), py::arg("elevations").noconvert(), + "Carve elevations in place so flow paths descend at gradient >= sl."); + + m.def("stream_order", &py_stream_order, py::arg("receivers"), py::arg("order"), + py::arg("is_stream"), py::arg("kind") = "strahler", + "Strahler or Shreve stream order over the marked stream cells."); +} diff --git a/_cpp/priority_flood.hpp b/_cpp/priority_flood.hpp new file mode 100644 index 0000000..92670ed --- /dev/null +++ b/_cpp/priority_flood.hpp @@ -0,0 +1,378 @@ +// priority_flood.hpp -- Priority-Flood depression filling +// +// Implements the Priority-Flood family of depression-filling algorithms +// described in +// +// Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal +// depression-filling and watershed-labeling algorithm for digital +// elevation models." Computers & Geosciences 62, 117-127. +// doi:10.1016/j.cageo.2013.04.024 +// +// Three variants from that paper are provided: +// +// * Algorithm 1 -- Priority-Flood. Every cell passes through the priority +// queue. O(n log n). Kept as a reference implementation used to test +// the faster variants. +// * Algorithm 2 -- Improved Priority-Flood. Cells that need no filling are +// handled through a FIFO ("pit") queue instead of the priority queue, +// which reduces the practical cost to O(n) for most DEMs. This is the +// default. +// * Algorithm 4 -- Priority-Flood+epsilon. As Algorithm 2, but filled +// cells are raised by a small increment so that the filled surface has a +// strictly monotonic downhill path and contains no flat areas. +// +// The implementation is self-contained (standard library only) and templated +// on the elevation type so it can be instantiated for float and double. +// +// Conventions +// ----------- +// * Grids are row-major, ``ny`` rows by ``nx`` columns; index ``k = i*nx + j``. +// * ``NaN`` marks no-data. No-data cells are never filled and never enqueued. +// * Neighbourhood is D8 (8-connected), matching Barnes et al. and TopoToolbox. + +#ifndef TOPOANALYSIS_PRIORITY_FLOOD_HPP +#define TOPOANALYSIS_PRIORITY_FLOOD_HPP + +#include +#include +#include +#include +#include +#include +#include + +namespace topoanalysis { + +// D8 neighbour offsets. Order matches the ArcGIS code order used elsewhere in +// TopoAnalysis: E, SE, S, SW, W, NW, N, NE. +constexpr int kD8Di[8] = {0, 1, 1, 1, 0, -1, -1, -1}; +constexpr int kD8Dj[8] = {1, 1, 0, -1, -1, -1, 0, 1}; +// Centre-to-centre distance multiples for the same ordering. +const double kD8Dist[8] = {1.0, 1.4142135623730951, 1.0, 1.4142135623730951, + 1.0, 1.4142135623730951, 1.0, 1.4142135623730951}; + +enum class FillMode { + Flat = 0, // Algorithm 2: fill depressions to a flat surface + Epsilon = 1, // Algorithm 4: fill with a small upslope increment +}; + +// A priority-queue entry. ``order`` makes the ordering total, so ties are +// broken first-in-first-out. Without it std::priority_queue would resolve +// ties arbitrarily and the filled surface would not be reproducible. +template +struct PqCell { + T z; + std::int64_t order; + std::int64_t idx; +}; + +template +struct PqGreater { + bool operator()(const PqCell& a, const PqCell& b) const { + if (a.z != b.z) return a.z > b.z; + return a.order > b.order; + } +}; + +// Options controlling a Priority-Flood run. +struct FloodOptions { + FillMode mode = FillMode::Flat; + + // Epsilon increment. Interpreted as a *slope*: a cell is raised to + // parent_elevation + epsilon * distance_to_parent + // where the distance is in the same units as ``cellsize``. A value of 0 + // with ``mode == Epsilon`` falls back to the smallest representable + // increment (std::nextafter), which is what Barnes et al. Algorithm 4 + // prescribes. + double epsilon = 0.0; + double cellsize = 1.0; + + // Depressions deeper than this are left unfilled (0 or negative disables + // the check). Cells inside such a depression keep their original + // elevation but are still traversed, so the rest of the grid is reached. + double max_pit_depth = 0.0; + + // Record which cells the flood reached. + bool track_visited = false; + + // Record which cells a depth limit left unfilled. Without this the + // caller cannot tell a cell the flood never reached from one it reached + // but deliberately declined to raise. + bool track_unfilled = false; +}; + +struct FloodResult { + std::vector visited; // empty unless options.track_visited + std::vector unfilled; // empty unless options.track_unfilled + std::size_t cells_filled = 0; + double max_fill_depth = 0.0; +}; + +namespace detail { + +template +inline bool is_nodata(T v) { + return std::isnan(v); +} + +// Raise ``value`` by the smallest amount that is representable at that +// magnitude. Used by Algorithm 4 when no explicit epsilon is supplied. +template +inline T next_up(T value) { + return std::nextafter(value, std::numeric_limits::infinity()); +} + +} // namespace detail + +// --------------------------------------------------------------------------- +// Algorithm 2 of Barnes et al. (2014), with the epsilon variant of Algorithm 4 +// folded in. ``elevations`` is modified in place. +// +// ``closed`` must be a pre-initialised ny*nx byte array: a non-zero entry +// marks a cell that must never be processed (no-data, or outside a caller +// supplied mask). ``seeds`` lists the flat indices the flood starts from; +// they are usually the grid perimeter plus every valid cell adjacent to +// no-data, but callers may supply outlet cells instead. +// --------------------------------------------------------------------------- +template +FloodResult priority_flood(T* elevations, + std::size_t ny, + std::size_t nx, + std::vector& closed, + const std::vector& seeds, + const FloodOptions& options) { + if (ny == 0 || nx == 0) return FloodResult{}; + const std::size_t n = ny * nx; + if (closed.size() != n) { + throw std::invalid_argument("closed mask has the wrong size"); + } + + FloodResult result; + if (options.track_visited) result.visited.assign(n, 0); + if (options.track_unfilled) result.unfilled.assign(n, 0); + + std::priority_queue, std::vector>, PqGreater> open; + std::queue pit; + + std::int64_t order = 0; + for (std::int64_t k : seeds) { + if (k < 0 || static_cast(k) >= n) continue; + if (closed[static_cast(k)]) continue; + if (detail::is_nodata(elevations[k])) continue; + closed[static_cast(k)] = 1; + open.push(PqCell{elevations[k], order++, k}); + } + + const bool use_epsilon = (options.mode == FillMode::Epsilon); + const bool limit_depth = options.max_pit_depth > 0.0; + // The FIFO shortcut is only sound when every cell it holds sits at exactly + // the current flood level. Two options break that invariant: an epsilon + // fill raises each cell by its own increment, and a depth limit re-queues + // unfilled cells *below* the current level. In either case the priority + // queue handles everything, i.e. the run degrades to Algorithm 1. + const bool use_pit = !use_epsilon && !limit_depth; + + while (!open.empty() || !pit.empty()) { + std::int64_t cur; + T cur_z; + // Draining the FIFO first is what makes Algorithm 2 faster than + // Algorithm 1 while producing an identical surface: cells already at + // or below the current flood level cannot be reordered by the + // priority queue. When both queues hold the same elevation the + // priority queue wins, preserving Algorithm 1's visit order. + if (!pit.empty() && !open.empty() && open.top().z <= elevations[pit.front()]) { + cur = open.top().idx; + cur_z = open.top().z; + open.pop(); + } else if (!pit.empty()) { + cur = pit.front(); + pit.pop(); + cur_z = elevations[cur]; + } else { + cur = open.top().idx; + cur_z = open.top().z; + open.pop(); + } + + if (options.track_visited) result.visited[static_cast(cur)] = 1; + + const std::size_t i = static_cast(cur) / nx; + const std::size_t j = static_cast(cur) % nx; + + for (int d = 0; d < 8; ++d) { + const std::int64_t ni = static_cast(i) + kD8Di[d]; + const std::int64_t nj = static_cast(j) + kD8Dj[d]; + if (ni < 0 || nj < 0 || static_cast(ni) >= ny || + static_cast(nj) >= nx) { + continue; + } + const std::int64_t nk = ni * static_cast(nx) + nj; + if (closed[static_cast(nk)]) continue; + if (detail::is_nodata(elevations[nk])) { + closed[static_cast(nk)] = 1; + continue; + } + closed[static_cast(nk)] = 1; + + T target; + if (use_epsilon) { + if (options.epsilon > 0.0) { + target = static_cast(cur_z + options.epsilon * options.cellsize * kD8Dist[d]); + if (target <= cur_z) target = detail::next_up(cur_z); + } else { + target = detail::next_up(cur_z); + } + } else { + target = cur_z; + } + + if (elevations[nk] < target) { + const double depth = static_cast(target) - static_cast(elevations[nk]); + if (limit_depth && depth > options.max_pit_depth) { + // Leave this depression alone, but keep traversing so the + // terrain beyond it is still reached. + if (options.track_unfilled) result.unfilled[static_cast(nk)] = 1; + open.push(PqCell{elevations[nk], order++, nk}); + continue; + } + elevations[nk] = target; + ++result.cells_filled; + if (depth > result.max_fill_depth) result.max_fill_depth = depth; + if (use_pit) { + pit.push(nk); + } else { + open.push(PqCell{elevations[nk], order++, nk}); + } + } else { + open.push(PqCell{elevations[nk], order++, nk}); + } + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// Algorithm 1 of Barnes et al. (2014). Every cell goes through the priority +// queue. Retained because it is the simplest statement of the method and +// therefore a good oracle for testing the improved variant. +// --------------------------------------------------------------------------- +template +FloodResult priority_flood_original(T* elevations, + std::size_t ny, + std::size_t nx, + std::vector& closed, + const std::vector& seeds, + const FloodOptions& options) { + if (ny == 0 || nx == 0) return FloodResult{}; + const std::size_t n = ny * nx; + if (closed.size() != n) { + throw std::invalid_argument("closed mask has the wrong size"); + } + + FloodResult result; + if (options.track_visited) result.visited.assign(n, 0); + if (options.track_unfilled) result.unfilled.assign(n, 0); + + std::priority_queue, std::vector>, PqGreater> open; + std::int64_t order = 0; + for (std::int64_t k : seeds) { + if (k < 0 || static_cast(k) >= n) continue; + if (closed[static_cast(k)]) continue; + if (detail::is_nodata(elevations[k])) continue; + closed[static_cast(k)] = 1; + open.push(PqCell{elevations[k], order++, k}); + } + + const bool use_epsilon = (options.mode == FillMode::Epsilon); + const bool limit_depth = options.max_pit_depth > 0.0; + + while (!open.empty()) { + const PqCell c = open.top(); + open.pop(); + if (options.track_visited) result.visited[static_cast(c.idx)] = 1; + + const std::size_t i = static_cast(c.idx) / nx; + const std::size_t j = static_cast(c.idx) % nx; + + for (int d = 0; d < 8; ++d) { + const std::int64_t ni = static_cast(i) + kD8Di[d]; + const std::int64_t nj = static_cast(j) + kD8Dj[d]; + if (ni < 0 || nj < 0 || static_cast(ni) >= ny || + static_cast(nj) >= nx) { + continue; + } + const std::int64_t nk = ni * static_cast(nx) + nj; + if (closed[static_cast(nk)]) continue; + if (detail::is_nodata(elevations[nk])) { + closed[static_cast(nk)] = 1; + continue; + } + closed[static_cast(nk)] = 1; + + T target; + if (use_epsilon) { + if (options.epsilon > 0.0) { + target = static_cast(c.z + options.epsilon * options.cellsize * kD8Dist[d]); + if (target <= c.z) target = detail::next_up(c.z); + } else { + target = detail::next_up(c.z); + } + } else { + target = c.z; + } + + if (elevations[nk] < target) { + const double depth = static_cast(target) - static_cast(elevations[nk]); + if (limit_depth && depth > options.max_pit_depth) { + if (options.track_unfilled) result.unfilled[static_cast(nk)] = 1; + open.push(PqCell{elevations[nk], order++, nk}); + continue; + } + elevations[nk] = target; + ++result.cells_filled; + if (depth > result.max_fill_depth) result.max_fill_depth = depth; + } + open.push(PqCell{elevations[nk], order++, nk}); + } + } + + return result; +} + +// --------------------------------------------------------------------------- +// Default seed set: the grid perimeter plus every valid cell that touches a +// no-data cell. This matches TopoToolbox's ``fillsinks``, whose marker image +// keeps the original elevation on the perimeter and around NaN holes. +// --------------------------------------------------------------------------- +template +std::vector default_seeds(const T* elevations, + std::size_t ny, + std::size_t nx) { + std::vector seeds; + if (ny == 0 || nx == 0) return seeds; + + const std::int64_t rows = static_cast(ny); + const std::int64_t cols = static_cast(nx); + + for (std::int64_t i = 0; i < rows; ++i) { + for (std::int64_t j = 0; j < cols; ++j) { + const std::int64_t k = i * cols + j; + if (detail::is_nodata(elevations[k])) continue; + + bool is_seed = (i == 0 || j == 0 || i + 1 == rows || j + 1 == cols); + for (int d = 0; d < 8 && !is_seed; ++d) { + const std::int64_t ni = i + kD8Di[d]; + const std::int64_t nj = j + kD8Dj[d]; + if (ni < 0 || nj < 0 || ni >= rows || nj >= cols) continue; + if (detail::is_nodata(elevations[ni * cols + nj])) is_seed = true; + } + if (is_seed) seeds.push_back(k); + } + } + return seeds; +} + +} // namespace topoanalysis + +#endif // TOPOANALYSIS_PRIORITY_FLOOD_HPP diff --git a/_gdal_source.py b/_gdal_source.py new file mode 100644 index 0000000..65d0268 --- /dev/null +++ b/_gdal_source.py @@ -0,0 +1,562 @@ +"""Download, verify and build the GDAL C library from source. + +This is the last resort of :mod:`TopoAnalysis.gdal_setup`, for machines with +no conda, no system package manager and no administrator rights. Everything +here installs into a single prefix that the caller chooses -- normally the +active virtualenv -- so nothing outside it is touched and removing the prefix +undoes the whole thing. + +Three components, built in order, because each needs the one before it: + + sqlite3 only when the system copy is unusable. PROJ needs both the + library *and* the ``sqlite3`` command, which it runs during its + own build to assemble ``proj.db``. + PROJ mandatory for GDAL 3.x. Built without TIFF and curl support, + which are only needed for downloading and reading the optional + high-accuracy datum grids. + GDAL built against its own bundled copies of libtiff, libgeotiff, + libpng, libjpeg and zlib (``GDAL_USE_INTERNAL_LIBS``), so the + only external dependency left is PROJ. That is what makes an + unattended build realistic: no hunting for system ``-dev`` + packages. The result reads GeoTIFF, ArcInfo ASCII, ENVI, HFA + and the rest of the built-in drivers, which is everything + TopoAnalysis uses. + +The Python bindings are deliberately *not* built here. ``cmake`` would build +them for whichever interpreter it happens to find; :mod:`TopoAnalysis.gdal_setup` +installs them with pip instead, against the interpreter we actually care +about. + +Every tarball is pinned by SHA-256. The pins below were taken from bytes +checked against what the projects themselves publish -- GDAL's and PROJ's +``.md5`` sidecars, SQLite's ``PRODUCT`` line -- so a download that does not +match them is a download that should not be built. +""" + +from __future__ import annotations + +import hashlib +import os +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request + +__all__ = [ + "SOURCES", + "apply_environment_overrides", + "build_one", + "components_needed", + "default_cache_dir", + "missing_build_tools", + "system_sqlite_is_usable", +] + + +class BuildError(RuntimeError): + """A download, configure, compile or install step failed.""" + + +# --------------------------------------------------------------------------- +# What we build, and exactly which bytes +# --------------------------------------------------------------------------- + +SOURCES = { + "sqlite": { + "version": "3.53.4", + "url": "https://www.sqlite.org/2026/sqlite-autoconf-3530400.tar.gz", + "sha256": "0e9483900e92cd5de8fd48d16bf9200145a61f7fd5be542a5ac81d8a9516eb9c", + "minutes": 2, + }, + "proj": { + "version": "9.4.1", + "url": "https://download.osgeo.org/proj/proj-9.4.1.tar.gz", + "sha256": "ffe20170ee2b952207adf8a195e2141eab12cda181e49fdeb54425d98c7171d7", + "minutes": 6, + }, + "gdal": { + # Not merely "a recent GDAL": the version matters for this build. + # GDAL bundles its own zlib, libpng, libtiff and friends, and the + # copies shipped up to about 3.9 do not compile against a current + # Apple SDK -- both zlib's ``zutil.h`` and libpng's ``pngpriv.h`` + # take their "Mac OS classic" branch, because that is selected on + # ``TARGET_OS_MAC``, which modern SDKs define on every Apple + # platform. zlib then redefines ``fdopen`` to NULL and the system + # ``_stdio.h`` fails to parse. Later GDAL releases carry repaired + # copies, so the pin has to stay reasonably current. + "version": "3.13.3", + "url": "https://github.com/OSGeo/gdal/releases/download/v3.13.3/gdal-3.13.3.tar.gz", + "sha256": "5e0c388d83da2d686cc00a40272882432cdb54edff43d4af173e532844a0a0ea", + "minutes": 20, + }, +} + +#: Order matters: each entry links against the ones before it. +BUILD_ORDER = ("sqlite", "proj", "gdal") + +#: Seconds to wait for any single read from a download before giving up. +DOWNLOAD_TIMEOUT = 60.0 + +#: The floor declared by both pinned projects: PROJ 9.4's CMakeLists and +#: GDAL 3.13's ``GdalCMakeMinimumRequired.cmake`` each say 3.16. +MIN_CMAKE_VERSION = (3, 16) + + +def _cmake_version(cmake): + """``(major, minor)`` reported by ``cmake --version``, or None.""" + try: + result = subprocess.run( + [cmake, "--version"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + except OSError: + return None + if result.returncode != 0: + return None + text = result.stdout.decode("utf8", "replace") + match = re.search(r"(\d+)\.(\d+)", text) + return (int(match.group(1)), int(match.group(2))) if match else None + + +def apply_environment_overrides(): + """Let ``TOPOANALYSIS_GDAL_SOURCE_*`` environment variables move a pin. + + A pinned version is a dead end the day it stops being suitable -- a + platform that needs a newer GDAL, a site that mirrors its own tarballs. + Overriding one means supplying its checksum too, so the integrity check + cannot be turned off by accident:: + + TOPOANALYSIS_GDAL_SOURCE_GDAL_VERSION=3.12.4 + TOPOANALYSIS_GDAL_SOURCE_GDAL_URL=https://example.org/gdal-3.12.4.tar.gz + TOPOANALYSIS_GDAL_SOURCE_GDAL_SHA256=<64 hex digits> + """ + for component in SOURCES: + prefix = "TOPOANALYSIS_GDAL_SOURCE_%s_" % component.upper() + url = os.environ.get(prefix + "URL") + sha256 = os.environ.get(prefix + "SHA256") + version = os.environ.get(prefix + "VERSION") + if url or sha256: + if not (url and sha256): + raise BuildError( + "%sURL and %sSHA256 must be set together" % (prefix, prefix) + ) + SOURCES[component]["url"] = url + SOURCES[component]["sha256"] = sha256 + if version: + SOURCES[component]["version"] = version + + +def default_cache_dir(): + """Where downloaded tarballs are kept between runs.""" + root = os.environ.get("XDG_CACHE_HOME") or os.path.join( + os.path.expanduser("~"), ".cache" + ) + return os.path.join(root, "TopoAnalysis", "downloads") + + +# --------------------------------------------------------------------------- +# Prerequisites +# --------------------------------------------------------------------------- + + +def missing_build_tools(): + """Names of build tools that are needed for a source build and absent. + + Returns an empty list when the machine can build. ``ninja`` is not + required -- CMake falls back to makefiles -- so it is not listed. + """ + missing = [] + cmake = shutil.which("cmake") + if cmake is None: + missing.append("cmake") + else: + version = _cmake_version(cmake) + # Both PROJ 9 and GDAL 3.13 declare this floor. Checking it here + # turns a puzzling failure part-way through `cmake -S` into a + # sentence naming the version that is needed. + if version is not None and version < MIN_CMAKE_VERSION: + missing.append( + "cmake %d.%d or newer (found %s)" + % (MIN_CMAKE_VERSION[0], MIN_CMAKE_VERSION[1], + ".".join(str(part) for part in version)) + ) + # ninja is not an alternative here: no -G is passed, so CMake generates + # makefiles, and SQLite's tarball is a plain autoconf build that calls + # make directly. + if shutil.which("make") is None: + missing.append("make") + if _c_compiler() is None: + missing.append("a C compiler") + # PROJ and GDAL are C++17 projects, and a machine can genuinely have cc + # without a C++ compiler -- a bare gcc package without g++ is the usual + # way. Checking only for a C compiler would let such a machine start a + # build that CMake aborts a few seconds later. + if _cxx_compiler() is None: + missing.append("a C++ compiler") + return missing + + +def _find_compiler(variable, candidates): + """The compiler named by ``variable``, else the first of ``candidates``.""" + explicit = os.environ.get(variable) + if explicit: + # The variable may carry arguments ("ccache gcc"); only the program + # matters here. + return shutil.which(explicit.split()[0]) + for name in candidates: + found = shutil.which(name) + if found is not None: + return found + return None + + +def _c_compiler(): + """The C compiler to probe with, honouring ``CC`` as the toolchain does.""" + return _find_compiler("CC", ("cc", "gcc", "clang")) + + +def _cxx_compiler(): + """The C++ compiler, honouring ``CXX``.""" + return _find_compiler("CXX", ("c++", "g++", "clang++")) + + +def system_sqlite_is_usable(): + """True when the system SQLite can satisfy PROJ's build. + + PROJ needs three things, and a machine can easily have one or two of + them: the shared library, the ``sqlite3.h`` header (often packaged + separately as ``libsqlite3-dev``), and the ``sqlite3`` command-line shell, + which PROJ runs to build ``proj.db``. Checking the header by compiling + against it is the only honest test -- the file can be present but + unreachable from the compiler's default include path, which is the normal + situation inside a conda environment. + """ + if shutil.which("sqlite3") is None: + return False + compiler = _c_compiler() + if compiler is None: + return False + + program = ( + "#include \n" + "#if SQLITE_VERSION_NUMBER < 3011000\n" + '#error "PROJ needs SQLite 3.11 or newer"\n' + "#endif\n" + "int main(void) { return sqlite3_libversion_number() > 0 ? 0 : 1; }\n" + ) + with tempfile.TemporaryDirectory() as tmp: + source = os.path.join(tmp, "probe.c") + with open(source, "w") as handle: + handle.write(program) + # Compile only: linking would additionally need -lsqlite3 to resolve, + # and a missing library is a different (rarer) problem than a missing + # header. PROJ's CMake will find the library if the header is there. + result = subprocess.run( + [compiler, "-c", source, "-o", os.path.join(tmp, "probe.o")], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + return result.returncode == 0 + + +def components_needed(): + """Which of :data:`BUILD_ORDER` this machine actually has to build.""" + needed = list(BUILD_ORDER) + if system_sqlite_is_usable(): + needed.remove("sqlite") + return needed + + +# --------------------------------------------------------------------------- +# Download and unpack +# --------------------------------------------------------------------------- + + +def _download(url, sha256, cache_dir, log): + """Fetch ``url`` into ``cache_dir``, verifying its SHA-256. + + A cached file that fails the check is deleted and fetched again: that is + far more often a truncated download than anything sinister, and leaving it + in place would make every later run fail the same way. + """ + os.makedirs(cache_dir, exist_ok=True) + target = os.path.join(cache_dir, url.rsplit("/", 1)[-1]) + + if os.path.exists(target): + if _sha256(target) == sha256: + log(" using cached %s" % os.path.basename(target)) + return target + log(" cached %s failed its checksum; re-downloading" % os.path.basename(target)) + os.remove(target) + + log(" downloading %s" % url) + partial = target + ".part" + try: + # With no timeout a stalled connection blocks the install for ever, + # with nothing on screen to say why. This bounds each individual + # read, not the whole transfer, so a slow-but-alive mirror still + # works. + with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT) as response: + with open(partial, "wb") as handle: + shutil.copyfileobj(response, handle) + except Exception as exc: # network, DNS, TLS, HTTP -- all equally fatal here + if os.path.exists(partial): + os.remove(partial) + raise BuildError("could not download %s: %s" % (url, exc)) + + got = _sha256(partial) + if got != sha256: + os.remove(partial) + raise BuildError( + "checksum mismatch for %s\n expected %s\n got %s\n" + "Refusing to build code that is not the code we pinned." % (url, sha256, got) + ) + os.replace(partial, target) + return target + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _extract(tarball, into): + """Unpack ``tarball`` and return the directory it created.""" + with tarfile.open(tarball) as archive: + names = archive.getnames() + # ``filter="data"`` rejects absolute paths, parent-directory escapes, + # links out of the tree and device nodes. It is the default from + # Python 3.14; ask for it explicitly wherever it exists. + try: + archive.extractall(into, filter="data") + except TypeError: # pragma: no cover - Python < 3.11.4 + _check_members(archive, names) + archive.extractall(into) + roots = {name.split("/", 1)[0] for name in names if name and not name.startswith("/")} + if len(roots) != 1: + raise BuildError("unexpected archive layout in %s" % tarball) + return os.path.join(into, roots.pop()) + + +def _check_members(archive, names): # pragma: no cover - old Python only + """The safety half of ``filter='data'`` for interpreters that lack it.""" + for name in names: + if name.startswith("/") or os.path.isabs(name) or ".." in name.split("/"): + raise BuildError("refusing to extract unsafe path %r" % name) + for member in archive.getmembers(): + if member.isdev(): + raise BuildError("refusing to extract device node %r" % member.name) + + +# --------------------------------------------------------------------------- +# Running the build +# --------------------------------------------------------------------------- + + +def _build_env(prefix): + """Environment that makes each component find the ones built before it.""" + env = dict(os.environ) + bindir = os.path.join(prefix, "bin") + env["PATH"] = bindir + os.pathsep + env.get("PATH", "") + + libdir = os.path.join(prefix, "lib") + pkgconfig = os.pathsep.join( + [os.path.join(libdir, "pkgconfig"), os.path.join(prefix, "share", "pkgconfig")] + ) + existing = env.get("PKG_CONFIG_PATH") + env["PKG_CONFIG_PATH"] = pkgconfig + (os.pathsep + existing if existing else "") + + # So the freshly built tools (PROJ's own, and sqlite3) can run during the + # build before anything has been relinked. + loader = "DYLD_LIBRARY_PATH" if sys.platform == "darwin" else "LD_LIBRARY_PATH" + env[loader] = libdir + (os.pathsep + env[loader] if env.get(loader) else "") + return env + + +def _run(command, cwd, env, log, logfile): + """Run a build command, tee-ing its output to ``logfile``.""" + log(" $ %s" % " ".join(command)) + with open(logfile, "a") as handle: + handle.write("\n$ %s\n" % " ".join(command)) + handle.flush() + result = subprocess.run( + command, cwd=cwd, env=env, stdout=handle, stderr=subprocess.STDOUT + ) + if result.returncode != 0: + raise BuildError( + "%s failed (exit %d).\nThe full build log is at:\n %s" + % (command[0], result.returncode, logfile) + ) + + +def _jobs(requested): + if requested: + return max(1, int(requested)) + return max(1, (os.cpu_count() or 2)) + + +def _cmake_common(prefix): + """CMake settings shared by PROJ and GDAL.""" + settings = [ + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_INSTALL_PREFIX=" + prefix, + "-DCMAKE_PREFIX_PATH=" + prefix, + "-DBUILD_TESTING=OFF", + "-DBUILD_SHARED_LIBS=ON", + # Pin the library directory instead of letting GNUInstallDirs choose. + # On Fedora, RHEL and openSUSE it defaults to lib64 on 64-bit hosts, + # and everything else here -- the RPATH below, the loader path in + # _build_env, the -L handed to the bindings build -- assumes one + # known location. A private prefix has no distro convention to + # honour, so "lib" everywhere is both simpler and correct. + "-DCMAKE_INSTALL_LIBDIR=lib", + # Without an install RPATH the shared libraries only resolve each + # other when LD_LIBRARY_PATH happens to be set, which it will not be + # in the user's next shell. Baking the path in makes the prefix work + # on its own. + "-DCMAKE_INSTALL_RPATH=" + os.path.join(prefix, "lib"), + "-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON", + "-DCMAKE_INSTALL_NAME_DIR=" + os.path.join(prefix, "lib"), + ] + if sys.platform == "darwin": + # Build for the architecture the *interpreter* runs as. On an Apple + # Silicon machine running an x86_64 Python under Rosetta, the default + # would be arm64 and the bindings would refuse to link. + settings.append("-DCMAKE_OSX_ARCHITECTURES=" + platform.machine()) + return settings + + +def build_one(component, prefix, jobs=None, log=None, cache_dir=None, keep_build=False): + """Download, build and install one component into ``prefix``. + + Parameters + ---------- + component : {'sqlite', 'proj', 'gdal'} + prefix : str + Installation prefix. Created if it does not exist. + jobs : int, optional + Parallel compile jobs; defaults to the CPU count. + log : callable, optional + Receives progress lines. Defaults to printing. + cache_dir : str, optional + Where tarballs are cached; defaults to :func:`default_cache_dir`. + keep_build : bool + Leave the unpacked source tree behind, for debugging a failure. + + Returns + ------- + str + The path of the build log, which is worth showing the user whether or + not the build succeeded. + """ + if component not in SOURCES: + raise ValueError("unknown component %r" % component) + log = log or (lambda line: print(line, flush=True)) + cache_dir = cache_dir or default_cache_dir() + apply_environment_overrides() + spec = SOURCES[component] + prefix = os.path.abspath(prefix) + os.makedirs(prefix, exist_ok=True) + + logdir = os.path.join(prefix, "var", "log", "topoanalysis") + os.makedirs(logdir, exist_ok=True) + logfile = os.path.join(logdir, "build-%s.log" % component) + # Start each attempt with a clean log so a failure message points at the + # failure, not at the tail of some earlier successful run. + open(logfile, "w").close() + + log(" %s %s" % (component, spec["version"])) + tarball = _download(spec["url"], spec["sha256"], cache_dir, log) + + workdir = tempfile.mkdtemp(prefix="topoanalysis-%s-" % component) + try: + source = _extract(tarball, workdir) + env = _build_env(prefix) + builder = { + "sqlite": _build_sqlite, + "proj": _build_proj, + "gdal": _build_gdal, + }[component] + builder(source, prefix, env, _jobs(jobs), log, logfile) + log(" %s installed into %s" % (component, prefix)) + finally: + if keep_build: + log(" build tree kept at %s" % workdir) + else: + shutil.rmtree(workdir, ignore_errors=True) + return logfile + + +def _build_sqlite(source, prefix, env, jobs, log, logfile): + """SQLite's amalgamation tarball: plain configure/make/install. + + Only flags that both of SQLite's build systems understand are used here. + Releases up to 3.48 shipped classic autoconf; 3.49 and later ship + autosetup, which has no ``--enable-shared`` at all (shared is the default + and only the negative form exists). Passing the positive form would make + the newer releases fail on an unknown option. + """ + _run( + [ + os.path.join(source, "configure"), + "--prefix=" + prefix, + "--disable-static", + # PROJ needs the core library and the shell, nothing else. + "--disable-readline", + ], + source, + env, + log, + logfile, + ) + _run(["make", "-j%d" % jobs], source, env, log, logfile) + _run(["make", "install"], source, env, log, logfile) + + +def _build_proj(source, prefix, env, jobs, log, logfile): + build = os.path.join(source, "build") + os.makedirs(build, exist_ok=True) + settings = _cmake_common(prefix) + [ + # Both are only used for the optional high-accuracy grid files, which + # need a network at run time anyway. Leaving them out removes the + # need for libtiff and libcurl development packages. + "-DENABLE_TIFF=OFF", + "-DENABLE_CURL=OFF", + # The command-line tools are not needed by GDAL, but projinfo is + # useful enough when diagnosing a projection problem to keep. + "-DBUILD_PROJSYNC=OFF", + "-DBUILD_TESTING=OFF", + ] + _run(["cmake", "-S", source, "-B", build] + settings, source, env, log, logfile) + _run(["cmake", "--build", build, "-j", str(jobs)], source, env, log, logfile) + _run(["cmake", "--install", build], source, env, log, logfile) + + +def _build_gdal(source, prefix, env, jobs, log, logfile): + build = os.path.join(source, "build") + os.makedirs(build, exist_ok=True) + settings = _cmake_common(prefix) + [ + # The point of the whole exercise: use GDAL's vendored copies of + # libtiff, libgeotiff, libpng, libjpeg, libz and friends rather than + # requiring system development packages we have no way to install. + "-DGDAL_USE_INTERNAL_LIBS=ON", + "-DGDAL_USE_EXTERNAL_LIBS=OFF", + # PROJ is the exception: it is mandatory and we just built it. + "-DGDAL_USE_PROJ=ON", + "-DPROJ_ROOT=" + prefix, + # Bindings are pip's job, against the interpreter we mean to use. + "-DBUILD_PYTHON_BINDINGS=OFF", + "-DBUILD_JAVA_BINDINGS=OFF", + "-DBUILD_CSHARP_BINDINGS=OFF", + "-DGDAL_BUILD_OPTIONAL_DRIVERS=ON", + "-DOGR_BUILD_OPTIONAL_DRIVERS=ON", + ] + _run(["cmake", "-S", source, "-B", build] + settings, source, env, log, logfile) + _run(["cmake", "--build", build, "-j", str(jobs)], source, env, log, logfile) + _run(["cmake", "--install", build], source, env, log, logfile) diff --git a/analysis.py b/analysis.py index a2281ec..8459b82 100644 --- a/analysis.py +++ b/analysis.py @@ -1,10 +1,10 @@ -import numpy as np -import rasterio - -import matplotlib.pyplot as plt +"""Quadrat sampling of a raster.""" from itertools import product +import numpy as np +import matplotlib.pyplot as plt + class Quadrats(object): @@ -22,6 +22,18 @@ def __init__(self, filename=None, data=None, band=1, dx=None, dy=None): self.quadrats = [] def load_data(self, filename, band=1): + """Read a raster band with rasterio. + + rasterio is imported here rather than at module scope so that the + rest of this module works without it. + """ + try: + import rasterio + except ImportError as exc: # pragma: no cover - environment dependent + raise ImportError( + "Quadrats.load_data needs rasterio; install it with " + "`pip install rasterio`, or pass data=... instead." + ) from exc with rasterio.open(filename, 'r') as src: self.data = src.read(band) diff --git a/benchmark.py b/benchmark.py new file mode 100644 index 0000000..dc5b5fa --- /dev/null +++ b/benchmark.py @@ -0,0 +1,121 @@ +"""Compare the compiled and pure-NumPy kernels on a synthetic DEM. + +Run it with:: + + python -m TopoAnalysis.benchmark # 512 x 512 + python -m TopoAnalysis.benchmark 2048 # a bigger grid + python -m TopoAnalysis.benchmark 2048 --skip-python +""" + +from __future__ import annotations + +import argparse +import sys +import time + +import numpy as np + +try: # pragma: no cover - exercised by whichever import style is in use + from . import fastops, kernels +except ImportError: # pragma: no cover + import fastops + import kernels + + +def synthetic_dem(n, seed=1): + """A noisy dome: a radial network with plenty of small depressions.""" + rng = np.random.default_rng(seed) + y, x = np.mgrid[0:n, 0:n] / float(n) + z = (500.0 * np.exp(-((x - 0.5) ** 2 + (y - 0.5) ** 2) / 0.1) + + 25.0 * np.sin(12 * np.pi * x) * np.sin(12 * np.pi * y)) + return z + rng.normal(0.0, 2.0, (n, n)) + + +def _time(label, function, *args, **kwargs): + start = time.perf_counter() + result = function(*args, **kwargs) + elapsed = time.perf_counter() - start + print(' {0:<22s} {1:9.1f} ms'.format(label, elapsed * 1000.0)) + return result, elapsed + + +def run(module, z, cellsize): + """Time the full pipeline with one backend; returns the totals.""" + total = 0.0 + + filled = np.ascontiguousarray(z, dtype=np.float64) + _, dt = _time('fill (epsilon)', module.priority_flood, filled, + mode='epsilon', epsilon=1e-9, cellsize=cellsize) + total += dt + + codes, dt = _time('flow directions', module.flow_directions, filled, + cellsize=cellsize) + total += dt + + receivers, dt = _time('receivers', module.receivers, codes) + total += dt + + (order, _), dt = _time('topological order', module.topological_order, receivers) + total += dt + + weights = np.full(z.shape, cellsize ** 2) + area, dt = _time('accumulate', module.accumulate, receivers, order, weights) + total += dt + + step = np.full(z.shape, cellsize) + (length, main_stem), dt = _time('flow length', module.flow_length, + receivers, order, step) + total += dt + + outlets = np.flatnonzero(np.asarray(receivers).reshape(-1) < 0).astype(np.int64) + _, dt = _time('chi', module.chi, receivers, order, np.asarray(area), step, + outlets, 1e6, 0.45, True, 0.0, None) + total += dt + + print(' {0:<22s} {1:9.1f} ms'.format('TOTAL', total * 1000.0)) + return total + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog='python -m TopoAnalysis.benchmark', + description='Time the TopoAnalysis kernels on a synthetic DEM.') + parser.add_argument('size', nargs='?', type=int, default=512, + help='grid side length in cells (default: 512)') + parser.add_argument('--cellsize', type=float, default=30.0) + parser.add_argument('--skip-python', action='store_true', + help='do not time the pure-NumPy kernels') + args = parser.parse_args(argv) + + n = args.size + z = synthetic_dem(n) + print('{0} x {1} cells ({2:,} total), dx = {3}'.format( + n, n, n * n, args.cellsize)) + + if not kernels.have_extension(): + print('\nThe compiled extension is not built; only the NumPy kernels ' + 'are available.\nBuild it with `pip install -e .` from the ' + 'TopoAnalysis directory.') + print('\nNumPy kernels:') + run(fastops, z, args.cellsize) + return 0 + + print('\nC++ kernels:') + cpp_total = run(kernels, z, args.cellsize) + + if args.skip_python: + return 0 + if n > 1024: + print('\nSkipping the NumPy kernels: they would take minutes at this ' + 'size. Pass a smaller size to compare them.') + return 0 + + print('\nNumPy kernels:') + python_total = run(fastops, z, args.cellsize) + + print('\nSpeed-up: {0:.0f}x'.format(python_total / cpp_total)) + return 0 + + +if __name__ == '__main__': # pragma: no cover + sys.exit(main()) diff --git a/cli.py b/cli.py new file mode 100644 index 0000000..f8da17c --- /dev/null +++ b/cli.py @@ -0,0 +1,179 @@ +"""Command-line entry point: ``topoanalysis-process``. + +Runs the standard pipeline over a DEM and writes the derived grids as +GeoTIFFs beside it:: + + topoanalysis-process srtm.tif --outdir results --theta 0.45 --a0 1e6 + +Use ``topoanalysis-process --info`` to report which compute backend is +active and whether GDAL is available. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time + +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as _dem + from . import kernels + from .__init__ import __version__ +except ImportError: # pragma: no cover + import dem as _dem + import kernels + __version__ = "unknown" + + +#: Derived grids the pipeline can produce, in dependency order. +PRODUCTS = ('filled', 'flowdir', 'area', 'length', 'slope', 'hillshade', + 'curvature', 'chi', 'ksi', 'relief') + + +def _build_parser(): + parser = argparse.ArgumentParser( + prog='topoanalysis-process', + description='Derive flow routing and channel metrics from a DEM.') + parser.add_argument('dem', nargs='?', help='input DEM readable by GDAL') + parser.add_argument('-o', '--outdir', default='.', + help='directory for the output grids (default: .)') + parser.add_argument('-p', '--prefix', default=None, + help="output filename prefix (default: the DEM's basename)") + parser.add_argument('--products', default='filled,flowdir,area,length', + help='comma-separated subset of: ' + ','.join(PRODUCTS)) + parser.add_argument('--theta', type=float, default=0.45, + help='reference concavity for chi and ksi (default: 0.45)') + parser.add_argument('--a0', type=float, default=1e6, + help='reference drainage area in m^2 (default: 1e6)') + parser.add_argument('--min-area', type=float, default=1e6, + help='drainage area defining the channel network, m^2') + parser.add_argument('--geographic', action='store_true', + help='treat the grid as lat/lon and use spherical cell areas') + parser.add_argument('--flat-fill', action='store_true', + help='fill depressions to a flat surface (as TopoToolbox ' + 'does) instead of adding a small gradient') + parser.add_argument('--info', action='store_true', + help='report the active backend and exit') + parser.add_argument('--version', action='version', + version='TopoAnalysis {0}'.format(__version__)) + return parser + + +def _report_environment(): + print('TopoAnalysis {0}'.format(__version__)) + print(' compute backend : {0}'.format(kernels.backend())) + if not kernels.have_extension(): + print(' (build the C++ extension with `pip install -e .` ' + 'for a large speed-up)') + print(' GDAL available : {0}'.format(_dem.gdal_is_available())) + print(' fill algorithm : {0}'.format(kernels.PRIORITY_FLOOD_REFERENCE)) + + +def main(argv=None): + """Run the CLI. Returns a process exit status.""" + parser = _build_parser() + args = parser.parse_args(argv) + + if args.info: + _report_environment() + return 0 + + if not args.dem: + parser.error('a DEM is required unless --info is given') + + requested = [p.strip() for p in args.products.split(',') if p.strip()] + unknown = [p for p in requested if p not in PRODUCTS] + if unknown: + parser.error('unknown product(s): {0}'.format(', '.join(unknown))) + + os.makedirs(args.outdir, exist_ok=True) + prefix = args.prefix or os.path.splitext(os.path.basename(args.dem))[0] + + def out(name): + return os.path.join(args.outdir, '{0}_{1}.tif'.format(prefix, name)) + + elevation_cls = _dem.GeographicElevation if args.geographic else _dem.Elevation + area_cls = _dem.GeographicArea if args.geographic else _dem.Area + length_cls = _dem.GeographicFlowLength if args.geographic else _dem.FlowLength + ksi_cls = _dem.GeographicKsi if args.geographic else _dem.Ksi + + started = time.time() + + def step(label): + print('[{0:6.1f}s] {1}'.format(time.time() - started, label)) + sys.stdout.flush() + + step('reading {0}'.format(args.dem)) + elevation = elevation_cls(gdal_filename=args.dem) + print(' {0} x {1} cells, dx = {2}'.format( + elevation._georef_info.nx, elevation._georef_info.ny, elevation._georef_info.dx)) + + step('filling depressions') + fill_kwargs = {'elevation': elevation} + if args.flat_fill: + fill_kwargs['aggradation_slope'] = 0.0 + filled = _dem.FilledElevation(**fill_kwargs) + if 'filled' in requested: + filled.save(out('filled')) + + step('routing flow') + flow_direction = _dem.FlowDirectionD8(flooded_dem=filled) + if 'flowdir' in requested: + flow_direction.save(out('flowdir')) + + area = None + if {'area', 'chi', 'ksi', 'relief'} & set(requested): + step('accumulating drainage area') + area = area_cls(flow_direction=flow_direction) + if 'area' in requested: + area.save(out('area')) + + flow_length = None + if {'length', 'ksi', 'relief'} & set(requested): + step('measuring flow length') + flow_length = length_cls(flow_direction=flow_direction) + if 'length' in requested: + flow_length.save(out('length')) + + if 'slope' in requested: + step('computing slope') + slope_cls = _dem.GeographicMaxSlope if args.geographic else _dem.MaxSlope + slope_cls(elevation=elevation).save(out('slope')) + + if 'hillshade' in requested: + step('computing hillshade') + hs_cls = _dem.GeographicHillshade if args.geographic else _dem.Hillshade + hs_cls(elevation=elevation, azimuth=315, inclination=45).save(out('hillshade')) + + if 'curvature' in requested: + step('computing curvature') + lap_cls = _dem.GeographicLaplacian if args.geographic else _dem.Laplacian + lap_cls(elevation=elevation).save(out('curvature')) + + if 'chi' in requested: + step('integrating chi') + outlets = area.areas_greater_than(args.min_area) + edge_outlets = [xy for xy in outlets + if not flow_direction.get_flow_to_cell( + *flow_direction._xy_to_rowscols((xy,))[0])[2]] + _dem.Chi(area=area, flow_direction=flow_direction, theta=args.theta, + Ao=args.a0, outlets=edge_outlets or outlets).save(out('chi')) + + if 'ksi' in requested: + step('integrating ksi') + ksi_cls(area=area, flow_direction=flow_direction, flow_length=flow_length, + theta=args.theta, Ao=args.a0).save(out('ksi')) + + if 'relief' in requested: + step('computing scaled relief') + _dem.ScaledRelief(flow_direction=flow_direction, elevation=elevation, + flow_length=flow_length, Ao=args.a0, + theta=args.theta, area=area).save(out('relief')) + + step('done') + return 0 + + +if __name__ == '__main__': # pragma: no cover + raise SystemExit(main()) diff --git a/datasets.py b/datasets.py index 1280c4b..d0f3c74 100644 --- a/datasets.py +++ b/datasets.py @@ -6,7 +6,10 @@ from scipy.signal import sawtooth -from dem import Elevation +try: # pragma: no cover - exercised by whichever import style is in use + from .dem import Elevation +except ImportError: # pragma: no cover + from dem import Elevation def triangle_grid(ny, nx, width, amp=1, sig=0, slope_y=None): @@ -45,12 +48,11 @@ def triangle_grid(ny, nx, width, amp=1, sig=0, slope_y=None): if slope_y: y = np.linspace(0, ny, num=ny).reshape(ny, 1) X = np.tile(y, (1, nx)) - tilt = slope_y * X - triangle *= tilt + # Added, not multiplied: a slope is a ramp superimposed on the + # relief, whereas multiplying scaled the relief and zeroed row 0. + triangle = triangle + slope_y * X - out_obj = Elevation(nx = nx, ny = ny, dx = 1.0) - out_obj._griddata = triangle - return out_obj + return Elevation(dx=1.0, grid=triangle) def sinusoid_grid(ny, nx, width, amp=1, sig=0, slope_y=None): @@ -89,9 +91,6 @@ def sinusoid_grid(ny, nx, width, amp=1, sig=0, slope_y=None): if slope_y: y = np.linspace(0, ny, num=ny).reshape(ny, 1) X = np.tile(y, (1, nx)) - tilt = slope_y * X - sinusoid *= tilt + sinusoid = sinusoid + slope_y * X - out_obj = Elevation(nx = nx, ny = ny, dx = 1.0) - out_obj._griddata = sinusoid - return out_obj + return Elevation(dx=1.0, grid=sinusoid) diff --git a/dem.py b/dem.py index 15119b6..d685a33 100644 --- a/dem.py +++ b/dem.py @@ -1,30 +1,184 @@ -#Functions for demMethods calculations typical of geomorphology -#Developed by Sam Johnstone, January 2015, samuelj@stanford.edu , johnsamstone@gmail.com -#Not all functions are benchmarked -#If you somehow have this, and have never spoken to me, please reach out -#- I'd be curious to hear what you are doing (maybe I can even help with something!) - -from osgeo import gdal, ogr, osr # Used to load gis in files -import os # Used to join file paths, iteract with os in other ways.. -import glob # Used for finding files that I want to mosaic (by allowing wildcard searches of the filesystem) -import heapq # Used for constructing priority queue, which is used for filling dems -import numpy as np # Used for tons o stuff, keeping most data stored as numpy arrays -import subprocess # Used to run gdal_merge.py from the command line -from numpy import uint8, int8, float64 -from matplotlib import pyplot as plt +"""Raster grids and DEM analysis for tectonic geomorphology. + +Originally developed by Sam Johnstone (January 2015); maintained since by the +Stanford geomorphology group. + +Every raster is a :class:`BaseSpatialGrid` or one of its subclasses. Grids +are constructed by keyword, and the keywords you supply select how the grid is +built -- see :attr:`BaseSpatialGrid.required_inputs_and_actions`:: + + elevation = Elevation(gdal_filename='dem.tif') + filled = FilledElevation(elevation=elevation) + d8 = FlowDirectionD8(flooded_dem=filled) + area = Area(flow_direction=d8) + +Conventions +----------- +* ``_griddata`` is a 2-D NumPy array with row 0 at the **north** edge. +* ``NaN`` marks no-data in floating-point grids. +* D8 flow-direction codes follow the ArcGIS convention:: + + | 32 64 128 | + | 16 X 1 | + | 8 4 2 | + +* Depression filling uses the Priority-Flood algorithms of Barnes, Lehman and + Mulla (2014); see :mod:`TopoAnalysis.kernels`. +""" + +import glob # Used for finding files that I want to mosaic +import os +import subprocess # Used to run gdal_merge.py from the command line import sys -from scipy import stats -from . import error as Error +import warnings -try: - import statsmodels.api as sm -except: - print('Warning: no statsmodels present. If you want to compute steepness, you will need to install this.') +import numpy as np +from numpy import float64, uint8 +from matplotlib import pyplot as plt + +# These modules are importable both as a package (``from TopoAnalysis import +# dem``) and flat (``import dem`` with this directory on sys.path). The flat +# form is how the original scripts in this repository load them, so it has to +# keep working. +try: # pragma: no cover - exercised by whichever import style is in use + from . import error as Error + from . import kernels +except ImportError: # pragma: no cover + import error as Error + import kernels + + +class _LazyGDALModule(object): + """Defer ``from osgeo import ...`` until something actually touches GDAL. + + GDAL is only needed to read and write files. Importing it lazily keeps + the rest of the library usable -- and testable -- where GDAL is absent, + which is common because it cannot be installed reliably from PyPI. When + it really is needed and really is missing, the error says what to do + about it instead of surfacing as a bare ``ImportError`` at import time. + """ + + __slots__ = ("_submodule", "_module") + + def __init__(self, submodule): + self._submodule = submodule + self._module = None + + def _load(self): + if self._module is None: + # Point GDAL and PROJ at the data files of a prefix TopoAnalysis + # built for itself, if there is one. A no-op otherwise, and it + # never overrides a setting the user made. + try: + from . import gdal_setup + except ImportError: # pragma: no cover - flat import style + try: + import gdal_setup + except ImportError: + gdal_setup = None + if gdal_setup is not None: + gdal_setup.activate_environment() + + try: + from osgeo import gdal, ogr, osr + except ImportError as exc: # pragma: no cover - environment dependent + raise ImportError( + "TopoAnalysis needs the GDAL Python bindings to read and " + "write raster files, and they are not installed.\n" + "\n" + " topoanalysis-gdal install\n" + "\n" + "installs both the GDAL C library and the bindings, " + "choosing whichever route suits this machine; run " + "`topoanalysis-gdal doctor` first to see what it would " + "do. Grids built in memory (dx=..., grid=...) do not " + "require GDAL at all." + ) from exc + # Without this GDAL signals failure by returning None, which turns + # a bad file path into an AttributeError three frames later. + gdal.UseExceptions() + ogr.UseExceptions() + osr.UseExceptions() + self._module = {"gdal": gdal, "ogr": ogr, "osr": osr}[self._submodule] + return self._module + + def __getattr__(self, name): + return getattr(self._load(), name) + + +gdal = _LazyGDALModule("gdal") +ogr = _LazyGDALModule("ogr") +osr = _LazyGDALModule("osr") + + +def gdal_is_available(): + """True when the GDAL Python bindings can be imported.""" + try: + gdal._load() + except ImportError: + return False + return True + + +def _require_statsmodels(): + """Import statsmodels on demand; only the regression grids need it.""" + try: + import statsmodels.api as sm + except ImportError as exc: # pragma: no cover - depends on the environment + raise ImportError( + "Fitting channel steepness needs statsmodels. Install it with " + "`pip install statsmodels`." + ) from exc + return sm + + +SQRT2 = np.sqrt(2.0) + + +class _Progress(object): + """Percent-complete ticker for the long per-cell regression loops. + + Prints ``Percent completion... 10...20...`` on one line and finishes with + ``100``. Pass ``verbose=False`` to silence it. + """ + + def __init__(self, total, verbose=True, stream=None): + self.total = int(total) + self.verbose = verbose and self.total > 0 + self.stream = stream if stream is not None else sys.stdout + self.count = 0 + self.next_readout = 10 + if self.verbose: + self.stream.write('Percent completion...') + self.stream.flush() + + def tick(self): + if not self.verbose: + return + self.count += 1 + # Integer arithmetic: accumulating 1.0/total in a float drifted and + # could skip or repeat a readout. + percent = (100 * self.count) // self.total + while percent >= self.next_readout and self.next_readout < 100: + self.stream.write(str(self.next_readout) + '...') + self.stream.flush() + self.next_readout += 10 + + def done(self): + if not self.verbose: + return + self.stream.write('100\n') + self.stream.flush() -sys.setrecursionlimit(1000000) - class GDALMixin(object): + """Reading and writing rasters through GDAL. + + Mixed into :class:`BaseSpatialGrid`; nothing here is meant to be called + directly. Every method reaches GDAL through the module-level lazy + proxies, so the mixin can be present without GDAL being installed. + """ + def _get_projection_from_EPSG_projection_code(self, EPSGprojectionCode): # Get raster projection @@ -76,19 +230,33 @@ def _get_numpy_type_for_gdal_type(self, gdal_type): def _readGDALFile(self, filename, dtype): gdal_file = gdal.Open(filename) - geoTransform, nx, ny, data = self._read_GDAL_dataset(gdal_file, dtype) - gdal_file = None - return geoTransform, nx, ny, data - + try: + return self._read_GDAL_dataset(gdal_file, dtype) + finally: + # GDAL only flushes and releases the file handle when the last + # reference goes away, so drop it explicitly rather than waiting + # for the garbage collector. + gdal_file = None + + @staticmethod + def _read_band(band, dtype): + """Read one band, converting its no-data value to ``NaN``. + + Integer grids have no NaN to convert to, so their no-data value is + left as-is. + """ + data = band.ReadAsArray().astype(dtype) + nodata = band.GetNoDataValue() + if nodata is not None and np.issubdtype(np.dtype(dtype), np.floating): + if np.isnan(nodata): + pass # already NaN in the array + else: + data[data == nodata] = np.nan + return data + def _read_GDAL_dataset(self, gdal_dataset, dtype): band = gdal_dataset.GetRasterBand(1) - nodata = band.GetNoDataValue() - data = band.ReadAsArray().astype(dtype) - nodata_elements = np.where(data == nodata) - from numpy import uint8 - if dtype is not uint8: - data[nodata_elements] = np.NAN - + data = self._read_band(band, dtype) geoTransform = gdal_dataset.GetGeoTransform() nx = gdal_dataset.RasterXSize ny = gdal_dataset.RasterYSize @@ -106,6 +274,7 @@ def _getGeoRefInfo(self, gdalDataset): return nx, ny, projection, geoTransform def getDEMcoords(self, GdalData, dx): + """Cell-centre x and y coordinate lists for an open GDAL dataset.""" #Get grid size nx, ny = GdalData.RasterXSize, GdalData.RasterYSize @@ -121,30 +290,44 @@ def getDEMcoords(self, GdalData, dx): return xcoordinates, ycoordinates - def _create_gdal_representation_from_array(self, georef_info, GDALDRIVERNAME, array_data, dtype, outfile_path='name', dst_options = [], multiple_bands = False): - #A function to write the data in the numpy array arrayData into a georeferenced dataset of type - # specified by GDALDRIVERNAME, a string, options here: http://www.gdal.org/formats_list.html - # This is accomplished by copying the georeferencing information from an existing GDAL dataset, - # provided by createDataSetFromArray - - #Initialize new data - drvr = gdal.GetDriverByName(GDALDRIVERNAME) # Get the desired driver - bands = 1 if multiple_bands is False else len(array_data) - outRaster = drvr.Create(outfile_path, georef_info.nx, georef_info.ny, bands , self._get_gdal_type_for_numpy_type(dtype), dst_options) # Open the file - - #Write geographic information - outRaster.SetGeoTransform(georef_info.geoTransform) # Steal the coordinate system from the old dataset - if georef_info.projection != 0: - outRaster.SetProjection(georef_info.projection) # Steal the Projections from the old dataset - - #Write the array - if not multiple_bands: - outRaster.GetRasterBand(1).WriteArray(array_data) # Writes my array to the raster - else: - for i in range(len(array_data)): - print('writing band ' + str(i) + "/" + str(bands)) - outRaster.GetRasterBand(i+1).WriteArray(array_data[i]) - + def _create_gdal_representation_from_array(self, georef_info, GDALDRIVERNAME, array_data, dtype, + outfile_path='name', dst_options=None, + multiple_bands=False, verbose=False): + """Write a NumPy array out as a georeferenced GDAL dataset. + + Driver names are listed at https://gdal.org/drivers/raster/. The + returned dataset is still open; let it go out of scope (or set it to + ``None``) to flush it to disk. + """ + if dst_options is None: + dst_options = [] + + drvr = gdal.GetDriverByName(GDALDRIVERNAME) + if drvr is None: + raise Error.InputError( + 'GDAL driver', "no GDAL driver named '{0}'".format(GDALDRIVERNAME)) + + bands = len(array_data) if multiple_bands else 1 + outRaster = drvr.Create(outfile_path, georef_info.nx, georef_info.ny, bands, + self._get_gdal_type_for_numpy_type(dtype), dst_options) + + outRaster.SetGeoTransform(georef_info.geoTransform) + # An unset projection is stored as the integer 0 by Georef_info. + if georef_info.projection != 0 and georef_info.projection is not None: + outRaster.SetProjection(georef_info.projection) + + is_float = np.issubdtype(np.dtype(dtype), np.floating) + band_arrays = array_data if multiple_bands else [array_data] + for index, band_array in enumerate(band_arrays): + if verbose: + print('writing band {0}/{1}'.format(index + 1, bands)) + band = outRaster.GetRasterBand(index + 1) + if is_float: + # Recording NaN as the no-data value means other GIS software + # renders the holes as holes instead of as real elevations. + band.SetNoDataValue(float('nan')) + band.WriteArray(np.asarray(band_array)) + return outRaster def _clipRasterToRaster(self, input_gdal_dataset, clipping_gdal_dataset, dtype): @@ -252,38 +435,59 @@ def _getRasterGeoTransformFromAsciiRaster(self, fileName): return (xUL, dx, 0, yUL, 0, -dx), nx, ny - def _writeArcAsciiRaster(self, georef_info, outfile_path, np_array_data, nodata_value, format_string): - #A function to write the data stored in the numpy array npArrayData to a ArcInfo Text grid. Gdal doesn't - #allow creation of these types of data for whatever reason - + def _writeArcAsciiRaster(self, georef_info, outfile_path, np_array_data, nodata_value, + format_string): + """Write an ArcInfo ASCII grid. + + GDAL's AAIGrid driver cannot ``Create()`` -- only ``CreateCopy()`` -- + so the file is written directly. ``NaN`` cells are replaced by + ``nodata_value`` on the way out, because most GIS software does not + recognise the literal text ``nan`` in an ASCII grid. + """ + data = np.asarray(np_array_data) + if np.issubdtype(data.dtype, np.floating): + if nodata_value is None or (isinstance(nodata_value, float) and np.isnan(nodata_value)): + nodata_value = -9999.0 + data = np.where(np.isnan(data), nodata_value, data) + header = "ncols %s\n" % georef_info.nx header += "nrows %s\n" % georef_info.ny header += "xllcenter %s\n" % georef_info.xllcenter header += "yllcenter %s\n" % georef_info.yllcenter header += "cellsize %s\n" % georef_info.dx header += "NODATA_value %s" % nodata_value - - np.savetxt(outfile_path, np_array_data, header=header, fmt=format_string, comments='') - - def _asciiRasterToMemory(self, fileName): - # the geotransfrom structured as (xUL, dx, skewX, yUL, scewY, -dy) + np.savetxt(outfile_path, data, header=header, fmt=format_string, comments='') + + def _asciiRasterToMemory(self, fileName): + """Read an ArcInfo ASCII grid, honouring its ``NODATA_value``.""" + # The geotransform is (xUL, dx, skewX, yUL, skewY, -dy). It is parsed + # from the header rather than taken from GDAL so that grids written + # with xllcenter (rather than xllcorner) land in the right place. gt, nx, ny = self._getRasterGeoTransformFromAsciiRaster(fileName) - - #Open gdal dataset + ds = gdal.Open(fileName) - - #Prep output - memDrv = gdal.GetDriverByName('MEM') # Create a gdal driver in memory - dataOut = memDrv.CreateCopy('name', ds, 0) #Copy data to gdal driver - - data = dataOut.ReadAsArray() - dataOut = None - - data = self.dtype(data) + try: + # Ignoring NODATA_value here is what used to turn -9999 flags into + # real elevations several thousand metres below sea level. + data = self._read_band(ds.GetRasterBand(1), self.dtype) + finally: + ds = None + return gt, nx, ny, data class GeographicGridMixin(object): + """Spherical geometry for grids in degrees of latitude and longitude. + + Mix it in *before* the grid class to override the cell area and cell + dimension with their true, latitude-dependent values:: + + class GeographicArea(GeographicGridMixin, Area): + pass + + Every ``Geographic*`` class in this module is exactly that. + """ + def _getUTMZone(self, dataset): #Function to get the approximate UTM zone %NOTE: I need to check how east and west are handled... @@ -335,40 +539,56 @@ def _approximateDxFromGeographicData(self, geoTransform): return int(dTheta*metersPerDegree) #convert degrees to meters + #: Earth radius in metres used for spherical cell areas. TopoToolbox's + #: ``cellarea`` instead derives its radius from a total surface area of + #: 510 072 000 km^2, i.e. 6 371 047 m, which makes its areas larger by + #: about 1.5e-5 relative. Override this attribute to match exactly. + earth_radius = 6371.0 * 1000.0 + def _area_per_pixel(self, *args, **kwargs): + """Area of each cell in square metres, on a sphere. - if hasattr(self, '_GeographicGridMixin__app'): + Cells shrink towards the poles, so this varies with row but not with + column. The result is cached because the trigonometry is not free + and several derived grids ask for it. + """ + if getattr(self, '_GeographicGridMixin__app', None) is not None: return self._GeographicGridMixin__app - - #Returns a grid of the area of each pixel within the domain specified by the gdalDataset - - re = 6371.0 * 1000.0 #radius of the earth in meters - + + re = self.earth_radius + dLong = np.abs(self._georef_info.geoTransform[1]) dLat = np.abs(self._georef_info.geoTransform[5]) - - lats = self._georef_info.yllcenter + np.float64(range(self._georef_info.ny))*dLat - longs = self._georef_info.xllcenter + np.float64(range(self._georef_info.nx))*dLong - - #Get the size of pixels in the lat and long direction + + # np.float64(range(n)) is a TypeError on modern NumPy -- np.float64 is + # a scalar type, not an array constructor. + lats = self._georef_info.yllcenter + np.arange(self._georef_info.ny, dtype=float64) * dLat + longs = self._georef_info.xllcenter + np.arange(self._georef_info.nx, dtype=float64) * dLong [LONG, LAT] = np.meshgrid(longs, lats) LAT1 = np.radians(LAT - dLat / 2.0) LAT2 = np.radians(LAT + dLat / 2.0) LONG1 = np.radians(LONG - dLong / 2.0) LONG2 = np.radians(LONG + dLong / 2.0) - + + # lats ascends northwards; flip so row 0 is the north edge again. initialAreas = np.flipud(np.abs((re**2)*(LONG1 - LONG2)*(np.sin(LAT2) - np.sin(LAT1)))) - + self.__app = initialAreas return self.__app - + def _mean_pixel_dimension(self, *args, **kwargs): - + """Square root of the cell area: an equivalent isotropic cell size.""" return np.sqrt(self._area_per_pixel()) class CalculationMixin(object): + """Finite-difference terrain derivatives. + + Supplies the slope, curvature and boundary-padding helpers that + :class:`Elevation`, :class:`Hillshade` and :class:`MaxSlope` share. + """ + def _calcFiniteSlopes(self, grid, dx, nx, ny): # sx,sy = calcFiniteDiffs(elevGrid,dx) @@ -388,6 +608,11 @@ def _calcFiniteSlopes(self, grid, dx, nx, ny): return Sx, Sy def assignBCs(self, grid, nx, ny): + """Pad a grid by one cell, replicating the edge values. + + Returns an array two rows and two columns larger, so that a centred + difference can be taken everywhere including the perimeter. + """ # Pads the boundaries of a grid # Boundary condition pads the boundaries with equivalent values # to the data margins, e.g. x[-1,1] = x[1,1] @@ -406,11 +631,12 @@ def assignBCs(self, grid, nx, ny): Zbc[0, 0] = grid[0, 0] Zbc[0, -1] = grid[0, -1] Zbc[-1, 0] = grid[-1, 0] - Zbc[-1, -1] = grid[-1, 0] - + Zbc[-1, -1] = grid[-1, -1] + return Zbc def calcFiniteCurv(self, grid, dx): + """Laplacian from a 5-point centred stencil, same shape as the input.""" #C = calcFiniteCurv(elevGrid, dx) #calculates finite differnces in X and Y direction using the centered difference method. #Applies a boundary condition such that the size and location of the grids in is the same as that out. @@ -425,6 +651,11 @@ def calcFiniteCurv(self, grid, dx): return Cx+Cy def calcContourCurvature(self, grid,dx): + """Contour (plan) curvature: the curvature of the contour lines. + + Positive where contours are convex downslope, i.e. on noses; negative + in hollows. The one-cell border comes back as ``NaN``. + """ # kt = (fxx*fy^2 - 2*fxyfxfy + fyy*fx^2)/((fx^2 + fy^2)*sqrt((fx^2 + fy^2)+1) #Preallocate @@ -448,20 +679,41 @@ def calcContourCurvature(self, grid,dx): return Kt def calcAverageSlopeOfGridSubset(self, gridSubset,dx): + """Least-squares plane through a block of cells; returns its slopes. + + Returns ``(dz/dx, dz/dy)`` of the best-fit plane ``z = ax + by + c``. + """ ## Sx,Sy = calcAverageSlopeOfGridSubset(numpy matrix, dx) #Compute the average slope over a subset of a grid (or a whole grid if you're into that), #by fitting a plane to the elevation data stored in grid subset - nx,ny = len(gridSubset[1,:]), len(gridSubset[:,1]) + ny, nx = gridSubset.shape xs = (0.5+np.arange(nx))*dx - (nx*dx)/2.0 ys = (0.5+np.arange(ny))*dx - (ny*dx)/2.0 X,Y = np.meshgrid(xs,ys) #Fit a plane of the form z = ax + by + c, where beta = [a b c] - M=np.vstack((X.flatten(),Y.flatten(),np.ones((1,nx*ny)))).T - beta = np.linalg.lstsq(M,gridSubset.flatten())[0] - - return beta[0], beta[1] #Return the slope in the x and y directions respecticely + M=np.vstack((X.flatten(),Y.flatten(),np.ones(nx*ny))).T + beta = np.linalg.lstsq(M, gridSubset.flatten(), rcond=None)[0] + + return beta[0], beta[1] #Return the slope in the x and y directions respectively class Georef_info(object): + """Where a grid sits on the ground. + + Attributes + ---------- + geoTransform : tuple + The GDAL 6-tuple ``(x_origin, dx, 0, y_origin, 0, -dy)``, whose + origin is the *outer corner* of the north-west cell. + projection : str or int + WKT, or the integer 0 when no projection is set. + dx : float + Cell size in map units. Cells are assumed square. + xllcenter, yllcenter : float + Centre of the south-west cell. + nx, ny : int + Columns and rows. + """ + def __init__(self): self.geoTransform = 0 self.projection = 0 @@ -482,13 +734,23 @@ def __deepcopy__(self, memo): class BaseSpatialShape(object): - # Wrapper for GDAL shapes. + """A vector layer, used to cut rasters to a boundary. + + :: + + shape = BaseSpatialShape(shapefile_name='basin.shp') + mask = shape.createMaskFromShape(dem._georef_info, + dem._georef_info.projection, + gdal.GDT_Byte) + """ + def __init__(self, *args, **kwargs): if kwargs.get('shapefile_name') is None: raise Error.InputError('Input Error', 'Inputs not satisfied') self.shapedata = ogr.Open(kwargs.get('shapefile_name')) def createMaskFromShape(self, geoRefInfo, projection, dtype, noDataValue = 0): + """Rasterise the shape onto ``geoRefInfo``'s frame, 1 inside and 0 out.""" #Open Shapefile source_ds = self.shapedata @@ -507,7 +769,28 @@ def createMaskFromShape(self, geoRefInfo, projection, dtype, noDataValue = 0): class BaseSpatialGrid(GDALMixin): - + """A georeferenced raster, and the base class of every grid here. + + Construct one by keyword; the combination you pass selects how it is + built (see :attr:`required_inputs_and_actions`):: + + BaseSpatialGrid(gdal_filename='grid.tif') + BaseSpatialGrid(dx=30.0, grid=array) + BaseSpatialGrid(nx=100, ny=80, dx=30.0) # random values + BaseSpatialGrid(nx=100, ny=80, projection=wkt, geo_transform=gt) + + The data live in ``_griddata``, a 2-D NumPy array with row 0 at the + **north** edge and ``NaN`` for no-data; the georeferencing lives in + ``_georef_info`` (a :class:`Georef_info`). + + Indexing is by ``(row, col)`` and is forgiving: reading outside the grid + returns ``None`` and writing outside it is ignored, which is what the + neighbourhood traversals in this module rely on. + """ + + #: Keyword combinations this class accepts, each paired with the method + #: that builds the grid from them. Subclasses extend the list; the + #: entries here are available to every grid class. required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), @@ -527,57 +810,88 @@ def __deepcopy__(self, memo): return result def __init__(self, *args, **kwargs): - + super(BaseSpatialGrid,self).__init__() self._georef_info = Georef_info() self._sorted = False - + if len(kwargs.keys()) == 0: return - + evaluative_action = self.__get_evaluative_action(*args, **kwargs) - - if evaluative_action == None: - raise Error.InputError('Input Error', 'Inputs not satisfied') - - eval('self.' + evaluative_action + '(*args, **kwargs)') - + + if evaluative_action is None: + raise Error.InputError( + 'Input Error', + '{0} could not be built from keywords {1}. Supported ' + 'combinations are: {2}'.format( + self.__class__.__name__, + sorted(kwargs), + '; '.join(', '.join(keys) + for keys, _ in self.required_inputs_and_actions))) + + # Resolve the method by name rather than by building and executing a + # source string: the action comes from class data, but compiling text + # to call a method is both slower and a needless way to run code. + getattr(self, evaluative_action)(*args, **kwargs) + def __setitem__(self, key, value): + """Set a cell, ignoring writes that fall outside the grid. + + Out-of-range writes are silently dropped rather than raising: several + of the neighbourhood traversals in this module rely on being able to + write past an edge without checking first. + """ i, j = key - if i < 0 or j < 0 or i > self._georef_info.ny or j > self._georef_info.nx: + if i < 0 or j < 0 or i >= self._georef_info.ny or j >= self._georef_info.nx: return self._griddata[i, j] = value - + def __getitem__(self, key): + """Return a cell value, or ``None`` if the index is outside the grid. + + The ``None`` return -- rather than an ``IndexError`` -- is part of the + interface: neighbourhood code throughout this module tests the result + against ``None`` instead of bounds-checking every access. + """ i, j = key i = int(i) j = int(j) - - if i >= 0 and j >= 0 and i < self._georef_info.ny and j < self._georef_info.nx: - try: - return self._griddata[i, j] - except: - print('error: '+ str(i) + str(j)) - + + if 0 <= i < self._georef_info.ny and 0 <= j < self._georef_info.nx: + return self._griddata[i, j] + return None - + def __get_evaluative_action(self, *args, **kwargs): - - for required_input_set, evaluative_action in self.required_inputs_and_actions: - these_kw = set(kwargs.keys()) - required_kw = set(required_input_set) - if required_kw.issubset(these_kw): - return evaluative_action - + """Pick the constructor that the supplied keywords satisfy.""" + these_kw = set(kwargs.keys()) + + # Subclass-specific combinations win, then the generic ones every + # grid supports. Falling back to the base list is what lets + # `Mask(dx=..., grid=...)` and friends be built in memory without + # every subclass having to restate the common entries. + for source in (self.required_inputs_and_actions, + BaseSpatialGrid.required_inputs_and_actions): + for required_input_set, evaluative_action in source: + if set(required_input_set).issubset(these_kw): + return evaluative_action + return None def __populate_georef_info_using_geoTransform(self, nx, ny): - self._georef_info.dx = self._georef_info.geoTransform[1] - self._georef_info.xllcenter = self._georef_info.geoTransform[0]+self._georef_info.dx/2.0 - self._georef_info.yllcenter = self._georef_info.geoTransform[3]-(self._georef_info.dx*(self._georef_info.ny-0.5)) + """Derive dx and the south-west cell centre from the geotransform.""" + # nx and ny are assigned FIRST: yllcenter depends on ny, and reading + # self._georef_info.ny before setting it picked up the default of 0, + # which put the grid ny*dx too far north for anyone building one with + # nx/ny/projection/geo_transform. self._georef_info.nx = nx self._georef_info.ny = ny + self._georef_info.dx = self._georef_info.geoTransform[1] + self._georef_info.xllcenter = self._georef_info.geoTransform[0] + self._georef_info.dx / 2.0 + self._georef_info.yllcenter = (self._georef_info.geoTransform[3] + - self._georef_info.dx * (ny - 0.5)) def _create(self, *args, **kwargs): @@ -589,23 +903,44 @@ def _create(self, *args, **kwargs): else: self._griddata = kwargs.get('grid') + def _geoTransform_from_corner(self): + """Derive the GDAL geotransform from dx and the lower-left centre. + + Grids built in memory used to leave ``geoTransform`` at its default + of 0, which made ``save()`` fail with an unhelpful GDAL error. + """ + dx = self._georef_info.dx + return (self._georef_info.xllcenter - 0.5 * dx, + dx, + 0.0, + self._georef_info.yllcenter + (self._georef_info.ny - 0.5) * dx, + 0.0, + -dx) + def _create_random_grid(self, *args, **kwargs): self._georef_info.dx = kwargs['dx'] self._georef_info.nx = kwargs['nx'] self._georef_info.ny = kwargs['ny'] - self._georef_info.xllcenter = 0 - self._georef_info.yllcenter = 0 + self._georef_info.xllcenter = kwargs.get('xllcenter', 0.0) + self._georef_info.yllcenter = kwargs.get('yllcenter', 0.0) + self._griddata = np.zeros((self._georef_info.ny, self._georef_info.nx), + dtype=self.dtype) self._randomize_grid_values(*args, **kwargs) + self._georef_info.geoTransform = self._geoTransform_from_corner() def _create_from_grid(self, *args, **kwargs): self._georef_info.dx = kwargs['dx'] - (ny, nx) = kwargs['grid'].shape + grid = np.asarray(kwargs['grid']) + (ny, nx) = grid.shape self._georef_info.nx = nx self._georef_info.ny = ny - self._georef_info.xllcenter = 0 - self._georef_info.yllcenter = 0 - self._griddata = kwargs['grid'] - + self._georef_info.xllcenter = kwargs.get('xllcenter', 0.0) + self._georef_info.yllcenter = kwargs.get('yllcenter', 0.0) + self._griddata = grid + self._georef_info.geoTransform = self._geoTransform_from_corner() + if kwargs.get('projection') is not None: + self._georef_info.projection = kwargs['projection'] + def _randomize_grid_values(self, *args, **kwargs): if kwargs.get('mask') is not None: i = np.where(kwargs.get('mask')._griddata == 1) @@ -658,7 +993,10 @@ def _xy_to_rowscols(self, v): col = int(round((x-self._georef_info.xllcenter)/self._georef_info.dx)) row = int((self._georef_info.ny - 1) - round((y-self._georef_info.yllcenter)/self._georef_info.dx)) - if col > self._georef_info.nx or row > self._georef_info.ny or col < 0 or row < 0: + # ``>=``, not ``>``: a point one cell past the right or bottom + # edge used to come back as a valid subscript and then raise an + # IndexError somewhere else entirely. + if col >= self._georef_info.nx or row >= self._georef_info.ny or col < 0 or row < 0: l.append((None, None)) else: l.append((row,col)) @@ -678,113 +1016,209 @@ def _area_per_pixel(self, *args, **kwargs): def _mean_pixel_dimension(self, *args, **kwargs): return self._georef_info.dx * np.ones_like(self._griddata, dtype = float64) + def coordinate_vectors(self): + """``(x, y)`` coordinates of the cell centres, west-to-east and + south-to-north. + + Returns + ------- + (ndarray, ndarray) + ``x`` has ``nx`` entries and ``y`` has ``ny`` entries. Note that + ``y`` ascends northwards, i.e. ``y[0]`` is the *last* row of + ``_griddata``. + """ + info = self._georef_info + x = info.xllcenter + np.arange(info.nx) * info.dx + y = info.yllcenter + np.arange(info.ny) * info.dx + return x, y + def get_XY_matricies(self): - xllc, yllc, nx, ny, dx = (self._georef_info.xllcenter, self._georef_info.yllcenter, self._georef_info.nx, self._georef_info.ny, self._georef_info.dx) - x = np.arange(xllc, xllc+(nx-1)*dx, dx) - y = np.arange(yllc, yllc+(ny-1)*dx, dx) - return np.meshgrid(x,y) + """Meshgrid of cell-centre coordinates. - def resample(self, de, interpolation = 'quintic'): - return_grid = self.__class__() - return_grid._copy_info_from_grid(self, set_zeros=True) + The name's spelling is preserved for backward compatibility; + :meth:`get_XY_matrices` is the same method. + """ + # np.arange(start, start + (n-1)*dx, dx) yields n-1 points, not n, so + # the returned grids used to be one column and one row short. + x, y = self.coordinate_vectors() + return np.meshgrid(x, y) + + #: Correctly spelled alias for :meth:`get_XY_matricies`. + get_XY_matrices = get_XY_matricies + + def resample(self, de, interpolation='quintic'): + """Resample onto a grid of spacing ``de``. + + Parameters + ---------- + de : float + Target cell size, in the grid's own units. + interpolation : str + ``'nearest'``, ``'linear'``, ``'cubic'`` or ``'quintic'``. The + last two use spline interpolation of order 3 and 5. + + Returns + ------- + BaseSpatialGrid + A new grid of the same class. + """ import copy - geoTransform = (self._georef_info.geoTransform[0], - np.sign(self._georef_info.geoTransform[1])*de, - self._georef_info.geoTransform[2], - self._georef_info.geoTransform[3], - self._georef_info.geoTransform[4], - np.sign(self._georef_info.geoTransform[5])*de) - - georef = copy.deepcopy(self._georef_info) - georef.geoTransform = geoTransform + from scipy.interpolate import RegularGridInterpolator + + order = {'nearest': 0, 'linear': 1, 'cubic': 3, 'quintic': 5}.get(interpolation) + if order is None: + raise Error.InputError( + 'resample', "interpolation must be one of 'nearest', 'linear', " + "'cubic' or 'quintic'; got {0!r}".format(interpolation)) + + info = self._georef_info + georef = copy.deepcopy(info) georef.dx = de - georef.nx = int(np.round(self._georef_info.nx * self._georef_info.dx / de)) - georef.ny = int(np.round(self._georef_info.ny * self._georef_info.dx / de)) + georef.nx = max(int(np.round(info.nx * info.dx / de)), 1) + georef.ny = max(int(np.round(info.ny * info.dx / de)), 1) + # Keep the south-west cell centre fixed, then rebuild the transform + # from it -- the old code kept the source transform's origin and + # merely swapped in the new spacing, which shifted the whole grid. + georef.xllcenter = info.xllcenter + georef.yllcenter = info.yllcenter + georef.geoTransform = (georef.xllcenter - 0.5 * de, + de, + info.geoTransform[2] if info.geoTransform != 0 else 0.0, + georef.yllcenter + (georef.ny - 0.5) * de, + info.geoTransform[4] if info.geoTransform != 0 else 0.0, + -de) + + # Interpolate in south-to-north order so the y axis is increasing, + # which is what RegularGridInterpolator requires. + source = np.flipud(np.asarray(self._griddata, dtype=float64)) + xo = info.xllcenter + np.arange(info.nx) * info.dx + yo = info.yllcenter + np.arange(info.ny) * info.dx + + method = {0: 'nearest', 1: 'linear', 3: 'cubic', 5: 'quintic'}[order] + try: + interpolator = RegularGridInterpolator( + (yo, xo), source, method=method, bounds_error=False, fill_value=None) + except ValueError: + # 'cubic' and 'quintic' need SciPy >= 1.10 and enough points per + # axis. Fall back rather than fail, but say so: silently + # changing the interpolation order changes the answer. + warnings.warn( + "SciPy cannot do {0!r} interpolation on this grid (it needs " + "SciPy >= 1.10 and at least {1} points per axis); falling " + "back to 'linear'.".format(interpolation, order + 1), + RuntimeWarning, stacklevel=2) + interpolator = RegularGridInterpolator( + (yo, xo), source, method='linear', bounds_error=False, fill_value=None) + + xf = georef.xllcenter + np.arange(georef.nx) * de + yf = georef.yllcenter + np.arange(georef.ny) * de + YF, XF = np.meshgrid(yf, xf, indexing='ij') + resampled = interpolator(np.column_stack((YF.ravel(), XF.ravel()))) + resampled = resampled.reshape(georef.ny, georef.nx) + + return_grid = self.__class__() return_grid._georef_info = georef - xo = np.linspace(self._georef_info.xllcenter,(self._georef_info.nx-1)*(self._georef_info.dx), num = self._georef_info.nx) - yo = np.linspace(self._georef_info.yllcenter,(self._georef_info.ny-1)*(self._georef_info.dx), num = self._georef_info.ny) - xf = np.linspace(georef.xllcenter,(georef.nx-1)*georef.dx, num = georef.nx) - yf = np.linspace(georef.yllcenter,(georef.ny-1)*georef.dx, num = georef.ny) - from scipy.interpolate import interp2d - interp_grid = copy.deepcopy(self._griddata) - if geoTransform[1] > 0: - interp_grid = np.fliplr(interp_grid) - if geoTransform[5] < 0: - interp_grid = np.flipud(interp_grid) - f = interp2d(xo, yo, interp_grid, kind=interpolation) - return_grid._griddata = f(xf, yf) + return_grid._griddata = np.flipud(resampled).astype(self.dtype) return return_grid - + def location_in_grid(self, xo): + """True when ``xo = (x, y)`` lands on a cell holding real data. + + Real means: inside the grid, not ``NaN``, and not zero. + """ index = self._xy_to_rowscols((xo,))[0] return index[0] is not None and index[1] is not None and self[index[0], index[1]] is not None and self[index[0], index[1]] != 0 and not np.isnan(self[index[0], index[1]]) def tile(self, tile_xdim = 400, tile_ydim=400, tile_xpadding = 10, tile_ypadding = 10): - - num_xtiles = int(np.ceil(float(self._georef_info.nx) / float(tile_xdim))) - num_ytiles = int(np.ceil(float(self._georef_info.ny) / float(tile_ydim))) - + """Split the grid into overlapping tiles. + + Interior tiles overlap their neighbours by the padding width; tiles on + the outside of the grid are mirrored outwards instead, so every tile + has the same amount of context around its core. Analyses that need a + neighbourhood (filling, curvature at a scale) can then be run + tile-by-tile and reassembled with :meth:`remove_padding` followed by + :meth:`mosaic`. + + Returns + ------- + list + Tiles, each an independent grid of this class. + """ + dx = self._georef_info.dx + grid_nx, grid_ny = self._georef_info.nx, self._georef_info.ny + + num_xtiles = int(np.ceil(float(grid_nx) / float(tile_xdim))) + num_ytiles = int(np.ceil(float(grid_ny) / float(tile_ydim))) + tiles = [] - import copy - + for x_left in range(num_xtiles): for y_top in range(num_ytiles): - - left = x_left*tile_xdim-tile_xpadding if (x_left*tile_xdim-tile_xpadding) >= 0 else 0 - right = (x_left+1)*tile_xdim+tile_xpadding if ((x_left+1)*tile_xdim+tile_xpadding) <= (self._georef_info.nx-1) else self._georef_info.nx - top = y_top*tile_ydim-tile_ypadding if (y_top*tile_ydim-tile_ypadding) >= 0 else 0 - bottom = (y_top+1)*tile_ydim+tile_ypadding if ((y_top+1)*tile_ydim+tile_ypadding) <= (self._georef_info.ny-1) else self._georef_info.ny - - new_tile = self.__class__() - + + left = max(x_left * tile_xdim - tile_xpadding, 0) + right = min((x_left + 1) * tile_xdim + tile_xpadding, grid_nx) + top = max(y_top * tile_ydim - tile_ypadding, 0) + bottom = min((y_top + 1) * tile_ydim + tile_ypadding, grid_ny) + needs_left_padding = (x_left == 0) - needs_right_padding = (x_left == (num_xtiles-1)) - + needs_right_padding = (x_left == (num_xtiles - 1)) needs_top_padding = (y_top == 0) - needs_bottom_padding = (y_top == (num_ytiles-1)) - - griddata = self._griddata[top:bottom,left:right] - - xllcenter = self._georef_info.xllcenter + left*self._georef_info.dx - yllcenter = self._georef_info.yllcenter + (self._georef_info.ny - bottom)*self._georef_info.dx - nx = right-left + needs_bottom_padding = (y_top == (num_ytiles - 1)) + + griddata = self._griddata[top:bottom, left:right].copy() + + xllcenter = self._georef_info.xllcenter + left * dx + yllcenter = self._georef_info.yllcenter + (grid_ny - bottom) * dx + nx = right - left ny = bottom - top - - if needs_left_padding: - griddata = np.concatenate((np.fliplr(griddata[:,0:tile_xpadding]), griddata), axis = 1) - xllcenter -= tile_xpadding*self._georef_info.dx + + # Each `and tile_?padding` guard matters: with zero padding + # `griddata[:, -0:]` is the whole array, so the edge tiles + # came back with their data duplicated. + if needs_left_padding and tile_xpadding: + griddata = np.concatenate((np.fliplr(griddata[:, 0:tile_xpadding]), griddata), axis=1) + xllcenter -= tile_xpadding * dx nx += tile_xpadding - - if needs_right_padding: - griddata = np.concatenate((griddata, np.fliplr(griddata[:,-tile_xpadding:])), axis = 1) + + if needs_right_padding and tile_xpadding: + griddata = np.concatenate((griddata, np.fliplr(griddata[:, -tile_xpadding:])), axis=1) nx += tile_xpadding - - if needs_top_padding: - griddata = np.concatenate((np.flipud(griddata[0:tile_ypadding,:]), griddata), axis = 0) + + if needs_top_padding and tile_ypadding: + griddata = np.concatenate((np.flipud(griddata[0:tile_ypadding, :]), griddata), axis=0) ny += tile_ypadding - - if needs_bottom_padding: - griddata = np.concatenate((griddata, np.flipud(griddata[-tile_ypadding:,:])), axis = 0) + + if needs_bottom_padding and tile_ypadding: + griddata = np.concatenate((griddata, np.flipud(griddata[-tile_ypadding:, :])), axis=0) ny += tile_ypadding - yllcenter -= tile_ypadding*self._georef_info.dx - - new_tile._georef_info.geoTransform = (xllcenter - 0.5*self._georef_info.dx, self._georef_info.dx, 0, yllcenter + (ny + 0.5)*self._georef_info.dx, 0, -self._georef_info.dx) + yllcenter -= tile_ypadding * dx + + new_tile = self.__class__() + # (ny - 0.5), not (ny + 0.5): the north edge of the transform + # has to invert yllcenter = gt[3] - dx*(ny - 0.5), otherwise + # every tile is georeferenced one cell too far north. + new_tile._georef_info.geoTransform = ( + xllcenter - 0.5 * dx, dx, 0, yllcenter + (ny - 0.5) * dx, 0, -dx) + new_tile._georef_info.projection = self._georef_info.projection new_tile._georef_info.xllcenter = xllcenter new_tile._georef_info.yllcenter = yllcenter - new_tile._georef_info.dx = self._georef_info.dx + new_tile._georef_info.dx = dx new_tile._georef_info.nx = nx new_tile._georef_info.ny = ny new_tile._griddata = griddata - - tiles += [copy.deepcopy(new_tile)] - + + tiles.append(new_tile) + return tiles - + def remove_padding(self, xpadding = 10, ypadding = 10): - + """Strip the overlap that :meth:`tile` added, ready for :meth:`mosaic`.""" import copy unpadded = copy.deepcopy(self) - unpadded._griddata = unpadded._griddata[ypadding:-ypadding,xpadding:-xpadding] + # `[-0:]`-style slices select everything, so guard the zero case. + rows = slice(ypadding, -ypadding if ypadding else None) + cols = slice(xpadding, -xpadding if xpadding else None) + unpadded._griddata = unpadded._griddata[rows, cols] unpadded._georef_info.xllcenter += xpadding*unpadded._georef_info.dx unpadded._georef_info.yllcenter += ypadding*unpadded._georef_info.dx unpadded._georef_info.nx -= 2*xpadding @@ -794,92 +1228,161 @@ def remove_padding(self, xpadding = 10, ypadding = 10): return unpadded def sort(self, reverse=True, force = False, mask = None): + """Flat indices of the cells, ordered by value. + + Parameters + ---------- + reverse : bool + ``True`` (the default) returns highest-value first. + force : bool + Recompute even if a sort is already cached. + mask : BaseSpatialGrid, optional + Keep only cells where the mask equals 1. + + Returns + ------- + ndarray + Flat (C-order) indices into ``_griddata``. + + Notes + ----- + The result is cached, so passing a different ``mask`` on a later call + has no effect unless ``force=True`` is also given. + """ if force: self._sorted = False if not self._sorted: + sort_indexes = self._griddata.argsort(axis=None) if mask is not None: - sort_indexes = self._griddata.argsort(axis = None) - unraveled_sort_indexes = np.unravel_index(sort_indexes, mask.shape) - query_indexes = np.where(mask._griddata[unraveled_sort_indexes] == 1) - query_indexes_raveled = np.ravel_multi_index(query_indexes,mask.shape) - self._sort_indexes = sort_indexes[query_indexes_raveled] - else: - self._sort_indexes = self._griddata.argsort(axis = None) + # Select the sorted positions whose cell is inside the mask. + # The previous version fed positions-in-the-sorted-array to + # ravel_multi_index as if they were (row, col) subscripts, + # which produced indices unrelated to the mask. + mask_flat = np.asarray(mask._griddata).reshape(-1) + sort_indexes = sort_indexes[mask_flat[sort_indexes] == 1] + self._sort_indexes = sort_indexes self._sorted = True if reverse: return self._sort_indexes[::-1] else: return self._sort_indexes - + def apply_moving_window(self, moving_window): - out_grid = None - out_grid._georef_info = self._georef_info - out_grid._griddata = moving_window.apply_moving_window(self._griddata, self._georef_info.dx, self.dtype) + """Apply a :class:`~TopoAnalysis.MovingWindow.MovingWindow` to the grid.""" + out_grid = self.__class__() + out_grid._copy_info_from_grid(self, set_zeros=True) + out_grid._griddata = moving_window.apply_moving_window( + self._griddata, self._georef_info.dx, self.dtype) return out_grid - - def average_over_distance(self, distance, grid = None): - (nx, ny) = (self._georef_info.nx, self._georef_info.ny) - (dx, dy) = (self._georef_info.dx, self._georef_info.dx) - (xmin, xmax) = (self._georef_info.xllcenter, nx*dx+self._georef_info.xllcenter) - (ymin, ymax) = (self._georef_info.yllcenter, ny*dy+self._georef_info.yllcenter) - (xcenter, ycenter) = ((xmin+xmax) / 2.0, (ymin+ymax) / 2.0) - [x,y] = np.meshgrid(np.arange(xmin, xmax, dx),np.arange(ymin, ymax, dy)) - D = np.sqrt(np.power((x - xcenter),2) + np.power((y - ycenter), 2)) - template = (D <= distance).astype(float) - template /= np.sum(template[:]) + + def average_over_distance(self, distance, grid=None): + """Mean of the grid within ``distance`` of each cell. + + The averaging kernel is a disc of radius ``distance`` in map units, + normalised to unit sum, applied by FFT convolution. Edges wrap, as + they always do for an FFT convolution. + """ + info = self._georef_info + dx = info.dx if grid is None: grid = self._griddata - - from numpy.fft import fft2 as fft2 - from numpy.fft import ifft2 as ifft2 - from numpy.fft import ifftshift - - return np.real(ifftshift(ifft2(fft2(grid)*fft2(template)))) - + grid = np.asarray(grid, dtype=float64) + ny, nx = grid.shape + + # Build the kernel about index (0, 0) with wrap-around distances. + # That is exactly the origin a circular convolution assumes, so no + # fftshift is needed afterwards -- and unlike a centre-of-array + # kernel it stays correct for even grid dimensions. (Building the + # distances with np.arange(xmin, xmax, dx) could also produce + # nx +/- 1 samples and make the FFT multiply below fail outright.) + di = np.minimum(np.arange(ny), ny - np.arange(ny)) * dx + dj = np.minimum(np.arange(nx), nx - np.arange(nx)) * dx + D = np.hypot(di[:, None], dj[None, :]) + template = (D <= distance).astype(float64) + total = template.sum() + if total == 0: + raise Error.InputError( + 'average_over_distance', + 'distance {0} is smaller than one cell ({1})'.format(distance, dx)) + template /= total + + from numpy.fft import fft2, ifft2 + + return np.real(ifft2(fft2(grid) * fft2(template))) + def clip_to_bounds(self, bounds): + """Clip to ``((xmin, xmax), (ymin, ymax))``.""" extent = (bounds[0][0], bounds[0][1], bounds[1][0], bounds[1][1]) return self.clip_to_extent(extent) - + def clip_to_extent(self, extent): + """Clip to ``(xmin, xmax, ymin, ymax)``, keeping every cell inside it. + + Returns + ------- + BaseSpatialGrid + A new grid. The original is untouched. + """ import copy - return_grid = copy.deepcopy(self) - return_grid._georef_info = copy.deepcopy(self._georef_info) - lower_left = (extent[0]+self._georef_info.dx, extent[2]+self._georef_info.dx) - upper_right = (extent[1]-self._georef_info.dx, extent[3]-self._georef_info.dx) - idx = self._xy_to_rowscols((lower_left, upper_right)) - return_grid._griddata = return_grid._griddata[idx[1][0]+1:idx[0][0]+1,idx[0][1]:idx[1][1]] + + xmin, xmax, ymin, ymax = extent + info = self._georef_info + if xmax < xmin or ymax < ymin: + raise Error.InputError('clip_to_extent', + 'extent must be (xmin, xmax, ymin, ymax)') + + # Work in index space and clamp, rather than round-tripping through + # _xy_to_rowscols (which returns None outside the grid) and then + # nudging the result by a cell in each direction. + col_min = int(np.ceil((xmin - info.xllcenter) / info.dx - 1e-9)) + col_max = int(np.floor((xmax - info.xllcenter) / info.dx + 1e-9)) + row_from_y_min = int(np.ceil((ymin - info.yllcenter) / info.dx - 1e-9)) + row_from_y_max = int(np.floor((ymax - info.yllcenter) / info.dx + 1e-9)) + + col_min = max(col_min, 0) + col_max = min(col_max, info.nx - 1) + row_from_y_min = max(row_from_y_min, 0) + row_from_y_max = min(row_from_y_max, info.ny - 1) + + if col_max < col_min or row_from_y_max < row_from_y_min: + raise Error.InputError('clip_to_extent', + 'extent does not overlap the grid') + + # Row 0 is the north edge, so the largest y is the smallest row. + top = (info.ny - 1) - row_from_y_max + bottom = (info.ny - 1) - row_from_y_min + + return_grid = self.__class__() + return_grid._georef_info = copy.deepcopy(info) + return_grid._griddata = self._griddata[top:bottom + 1, col_min:col_max + 1].copy() return_grid._georef_info.nx = return_grid._griddata.shape[1] return_grid._georef_info.ny = return_grid._griddata.shape[0] - return_grid._georef_info.xllcenter = self._rowscols_to_xy((idx[0],))[0][0] - return_grid._georef_info.yllcenter = self._rowscols_to_xy((idx[0],))[0][1] - return_grid._georef_info.geoTransform = (return_grid._georef_info.xllcenter, return_grid._georef_info.geoTransform[1], return_grid._georef_info.geoTransform[2], return_grid._georef_info.yllcenter + (return_grid._georef_info.dx*(return_grid._georef_info.ny-1)), return_grid._georef_info.geoTransform[4], return_grid._georef_info.geoTransform[5],) - + return_grid._georef_info.xllcenter = info.xllcenter + col_min * info.dx + return_grid._georef_info.yllcenter = info.yllcenter + row_from_y_min * info.dx + return_grid._georef_info.geoTransform = return_grid._geoTransform_from_corner() + return return_grid - + def extent_of_data(self): - - top = 0 - bottom = self._georef_info.ny - 1 - left = 0 - right = self._georef_info.nx - 1 - - while np.sum(np.isnan(self._griddata[top,:])) == self._georef_info.nx: - top = top + 1 - - while np.sum(np.isnan(self._griddata[bottom,:])) == self._georef_info.nx: - bottom = bottom - 1 - - while np.sum(np.isnan(self._griddata[left,:])) == self._georef_info.ny: - left = left + 1 - - while np.sum(np.isnan(self._griddata[right,:])) == self._georef_info.ny: - right = right - 1 - - (ll, ur) = self._rowscols_to_xy(((bottom, left), (top, right))) + """``(xmin, xmax, ymin, ymax)`` of the smallest box holding all valid data.""" + valid = ~np.isnan(np.asarray(self._griddata, dtype=float64)) + if not valid.any(): + raise Error.InputError('extent_of_data', 'the grid is entirely no-data') + + rows = np.flatnonzero(valid.any(axis=1)) + # Columns need any() along axis 0; the original indexed rows for both, + # so a grid with empty left columns reported the wrong extent (and + # ran off the end for non-square grids). + cols = np.flatnonzero(valid.any(axis=0)) + top, bottom = int(rows[0]), int(rows[-1]) + left, right = int(cols[0]), int(cols[-1]) + + (ll, ur) = self._rowscols_to_xy(((bottom, left), (top, right))) return (ll[0], ur[0], ll[1], ur[1]) - - + + def clip_to_mask_grid(self, mask_grid): + """Reproject and resample this grid onto ``mask_grid``'s frame, in place.""" gdal_source = self._create_gdal_representation_from_array(self._georef_info, 'MEM', self._griddata, self.dtype) gdal_mask = mask_grid._create_gdal_representation_from_array(mask_grid._georef_info, 'MEM', mask_grid._griddata, mask_grid.dtype) gdal_clip = self._clipRasterToRaster(gdal_source, gdal_mask, self.dtype) @@ -887,861 +1390,1148 @@ def clip_to_mask_grid(self, mask_grid): self.__populate_georef_info_using_geoTransform(self._georef_info.nx, self._georef_info.ny) def clip_to_shapefile(self, shapefile): + """Cut the grid to a vector boundary, in place. + + .. warning:: + ``_clipRasterToShape`` is a stub, so this currently does nothing. + """ gdal_source = self._create_gdal_representation_from_array(self._georef_info, 'MEM', self._griddata, self.dtype) gdal_clip = shapefile.shapedata gdal_result = self._clipRasterToShape(gdal_source, gdal_clip) self._georef_info.geoTransform, self._georef_info.nx, self._georef_info.ny, self._griddata = self._read_GDAL_dataset(gdal_result, self.dtype) self.__populate_georef_info_using_geoTransform(self._georef_info.nx, self._georef_info.ny) - def calculate_gradient_over_length_scale(self,length_scale): - # sx,sy = calcFiniteDiffs(elevGrid,dx) - # calculates finite differences in X and Y direction using a finite difference - # kernel that extends N cells from the center. The width of the kernel is then - # 2N + 1 by 2N + 1. Applies a boundary condition such that the size and location - # of the grids in is the same as that out. However, the larger N is, the more NoData - #Will be around the edges . - - #Compute finite differences - elevGrid = self._griddata + def calculate_gradient_over_length_scale(self, length_scale): + """Centred finite-difference gradient measured over ``length_scale``. + + The stencil reaches ``N = ceil(length_scale / dx)`` cells either side + of the centre, so the two sampled cells are ``2*N*dx`` apart. A band + ``N`` cells wide around the grid edge comes back as ``NaN``. + + Returns + ------- + (ndarray, ndarray) + ``(dz/dx, dz/dy)`` with y increasing northwards. + """ + elevGrid = np.asarray(self._griddata, dtype=float64) dx = self._georef_info.dx - N = np.ceil(length_scale / dx) - - Sx = (elevGrid[N:-N, (2*N):] - elevGrid[N:-N, :-(2*N)])/(((2*N)+1)*dx) - Sy = (elevGrid[(2*N):,N:-N] - elevGrid[:-(2*N), N:-N])/(((2*N)+1)*dx) - - #Create two new arrays of the original DEMs size - SxPadded = np.empty(elevGrid.shape) - SxPadded[:] = np.NAN - SyPadded = np.empty(elevGrid.shape) - SyPadded[:] = np.NAN - - SyPadded[N:-N, N:-N] = Sy + # int(), not a bare np.ceil(): NumPy returns a float and Python 3 + # refuses to slice with one. + N = int(np.ceil(length_scale / dx)) + if N < 1: + raise Error.InputError('calculate_gradient_over_length_scale', + 'length_scale must be at least one cell') + if 2 * N >= min(elevGrid.shape): + raise Error.InputError('calculate_gradient_over_length_scale', + 'length_scale is larger than the grid') + + # The separation between the sampled cells is 2*N*dx, not (2*N+1)*dx. + span = 2.0 * N * dx + Sx = (elevGrid[N:-N, (2 * N):] - elevGrid[N:-N, :-(2 * N)]) / span + # Row index increases southwards, so negate to get dz/dy in map space. + Sy = -(elevGrid[(2 * N):, N:-N] - elevGrid[:-(2 * N), N:-N]) / span + + SxPadded = np.full(elevGrid.shape, np.nan) + SyPadded = np.full(elevGrid.shape, np.nan) SxPadded[N:-N, N:-N] = Sx - + SyPadded[N:-N, N:-N] = Sy + return SxPadded, SyPadded - + def calculate_laplacian_over_length_scale(self, length_scale): - #C = calcFiniteCurv(elevGrid, dx) - #calculates finite differnces in X and Y direction using the centered difference method. - #Applies a boundary condition such that the size and location of the grids in is the same as that out. - + """Laplacian measured over ``length_scale`` with a centred stencil. + + A band ``N = ceil(length_scale / dx)`` cells wide around the grid edge + comes back as ``NaN``. + """ dx = self._georef_info.dx - grid = self._griddata - winRad = np.ceil(length_scale / dx) - #Assign boundary conditions - Curv = np.zeros_like(grid)*np.nan - - #Compute finite differences - Cx = (grid[winRad:-winRad, (2*winRad):] - 2*grid[winRad:-winRad, winRad:-winRad] + grid[winRad:-winRad, :-(2*winRad)])/(2*dx*winRad)**2 - Cy = (grid[(2*winRad):, winRad:-winRad] - 2*grid[winRad:-winRad, winRad:-winRad] + grid[:-(2*winRad), winRad:-winRad])/(2*dx*winRad)**2 - - Curv[winRad:-winRad,winRad:-winRad] = Cx+Cy + grid = np.asarray(self._griddata, dtype=float64) + N = int(np.ceil(length_scale / dx)) + if N < 1: + raise Error.InputError('calculate_laplacian_over_length_scale', + 'length_scale must be at least one cell') + if 2 * N >= min(grid.shape): + raise Error.InputError('calculate_laplacian_over_length_scale', + 'length_scale is larger than the grid') + + Curv = np.full(grid.shape, np.nan) + h2 = (N * dx) ** 2 + centre = grid[N:-N, N:-N] + Cx = (grid[N:-N, (2 * N):] - 2 * centre + grid[N:-N, :-(2 * N)]) / h2 + Cy = (grid[(2 * N):, N:-N] - 2 * centre + grid[:-(2 * N), N:-N]) / h2 + + Curv[N:-N, N:-N] = Cx + Cy return Curv def principal_curvatures(self): - Zy, Zx = np.gradient(self._griddata, self._georef_info.dx) - Zxy, Zxx = np.gradient(Zx, self._georef_info.dx) - Zyy, _ = np.gradient(Zy, self._georef_info.dx) - - H = (Zx**2 + 1)*Zyy - 2*Zx*Zy*Zxy + (Zy**2 + 1)*Zxx - H = -H/(2*(Zx**2 + Zy**2 + 1)**(1.5)) - - K = (Zxx * Zyy - (Zxy ** 2)) / (1 + (Zx ** 2) + (Zy **2)) ** 2 - - k1 = H + np.sqrt(np.power(H,2) - K) - k2 = H - np.sqrt(np.power(H,2) - K) - - k1re = BaseSpatialGrid() + """Maximum and minimum principal curvature of the surface. + + Returns + ------- + (BaseSpatialGrid, BaseSpatialGrid) + ``(k1, k2)`` with ``k1 >= k2``. Positive curvature is convex-up. + """ + dx = self._georef_info.dx + grid = np.asarray(self._griddata, dtype=float64) + # np.gradient returns the derivative along axis 0 (rows) first, and + # rows increase southwards, so flip its sign to get dz/dy. + dz_drow, Zx = np.gradient(grid, dx) + Zy = -dz_drow + _, Zxx = np.gradient(Zx, dx) + dZy_drow, Zxy = np.gradient(Zy, dx) + Zyy = -dZy_drow + + p = Zx ** 2 + Zy ** 2 + H = (Zx ** 2 + 1) * Zyy - 2 * Zx * Zy * Zxy + (Zy ** 2 + 1) * Zxx + H = -H / (2 * (p + 1) ** 1.5) + + K = (Zxx * Zyy - Zxy ** 2) / (1 + p) ** 2 + + # H**2 - K is non-negative in exact arithmetic; clip so that rounding + # near an umbilic point does not turn both curvatures into NaN. + discriminant = np.sqrt(np.maximum(H ** 2 - K, 0.0)) + k1 = H + discriminant + k2 = H - discriminant + + k1re = BaseSpatialGrid() k1re._copy_info_from_grid(self, True) k2re = BaseSpatialGrid() k2re._copy_info_from_grid(self, True) k1re._griddata = k1 k2re._griddata = k2 - + return (k1re, k2re) - def plot(self, **kwargs): + def extent(self): + """``(xmin, xmax, ymin, ymax)`` of the grid's outer pixel edges. + + These are the bounds matplotlib's ``imshow(extent=...)`` expects. + """ + info = self._georef_info + half = 0.5 * info.dx + return (info.xllcenter - half, + info.xllcenter + (info.nx - 0.5) * info.dx, + info.yllcenter - half, + info.yllcenter + (info.ny - 0.5) * info.dx) + def plot(self, **kwargs): + """Display the grid with ``imshow``, georeferenced in map coordinates. + + Parameters + ---------- + interactive : bool + Leave the figure open without blocking (default ``True``). + colorbar : bool + Draw a colour bar (default ``True``). + ax : matplotlib Axes, optional + Draw into an existing axes instead of a new figure. + **kwargs + Passed through to ``imshow`` (``cmap``, ``vmin``, ``alpha``, ...). + + Returns + ------- + matplotlib.figure.Figure + """ interactive = kwargs.pop('interactive', True) colorbar = kwargs.pop('colorbar', True) - extent = [self._georef_info.xllcenter, self._georef_info.xllcenter+(self._georef_info.nx-0.5)*self._georef_info.dx, self._georef_info.yllcenter, self._georef_info.yllcenter+(self._georef_info.ny-0.5)*self._georef_info.dx] - fig = plt.figure() - plt.axis(extent) - if interactive: - plt.ion() - plt.show(block = False) + ax = kwargs.pop('ax', None) + kwargs.setdefault('extent', self.extent()) + + if ax is None: + fig = plt.figure() + ax = fig.gca() else: - plt.ioff() - im = plt.imshow(self._griddata, extent = extent, **kwargs) + fig = ax.figure + + im = ax.imshow(self._griddata, **kwargs) if colorbar: - plt.colorbar(im) - plt.draw() - plt.pause(0.001) + fig.colorbar(im, ax=ax) if interactive: - plt.show() + plt.ion() + plt.show(block=False) + plt.pause(0.001) return fig - + def _neighbourhood_indices(self, index, pixel_radius): + """Row/column indices of the in-bounds cells within ``pixel_radius``. + + Returns ``(rows, cols)`` as flat integer arrays. The disc is + inclusive of the radius and of the far edge -- the previous + ``range(i - r, i + r)`` stopped one cell short and let negative + indices wrap around to the opposite side of the grid. + """ + i, j = int(index[0]), int(index[1]) + r = int(np.ceil(pixel_radius)) + ny, nx = self._griddata.shape + + rows = np.arange(max(i - r, 0), min(i + r, ny - 1) + 1) + cols = np.arange(max(j - r, 0), min(j + r, nx - 1) + 1) + if rows.size == 0 or cols.size == 0: + return np.empty(0, dtype=int), np.empty(0, dtype=int) + + RR, CC = np.meshgrid(rows, cols, indexing='ij') + inside = np.hypot(RR - i, CC - j) <= pixel_radius + return RR[inside], CC[inside] + def find_nearest_cell_with_value(self, index, value, pixel_radius): + """Index of the cell within ``pixel_radius`` whose value is closest to ``value``. + + Returns ``(None, None)`` when the neighbourhood holds no valid data. + """ + rows, cols = self._neighbourhood_indices(index, pixel_radius) + if rows.size == 0: + return None, None + difference = np.abs(value - self._griddata[rows, cols]) + if np.all(np.isnan(difference)): + return None, None + best = int(np.nanargmin(difference)) + return int(rows[best]), int(cols[best]) - (i, j) = index - nearest = tuple() - minimum_difference = np.nan - for y in range(int(i-pixel_radius),int(i+pixel_radius)): - for x in range(int(j-pixel_radius),int(j+pixel_radius)): - if x is not None and y is not None and np.sqrt( (y-i)**2 + (x-j)**2) <= pixel_radius and x < self._griddata.shape[1] and y < self._griddata.shape[0]: - value_difference = np.abs(value - self._griddata[y,x]) - if np.isnan(minimum_difference) or minimum_difference > value_difference: - minimum_difference = value_difference - nearest = (y,x) - (y,x) = nearest - return y, x - def find_nearest_cell_with_value_greater_than(self, index, value, pixel_radius): + """Index of the cell within ``pixel_radius`` whose value is the smallest + one at or above ``value``. + + If no cell reaches ``value``, falls back to the cell closest to it + from below. Returns ``(None, None)`` when there is no valid data. + """ + rows, cols = self._neighbourhood_indices(index, pixel_radius) + if rows.size == 0: + return None, None + difference = self._griddata[rows, cols] - value + + above = difference >= 0 + if np.any(above & ~np.isnan(difference)): + candidates = np.where(above, difference, np.nan) + else: + candidates = np.where(~np.isnan(difference), -difference, np.nan) + if np.all(np.isnan(candidates)): + return None, None + best = int(np.nanargmin(candidates)) + return int(rows[best]), int(cols[best]) - (i, j) = index - nearest = tuple() - minimum_difference = np.nan - for y in range(int(i-pixel_radius),int(i+pixel_radius)): - for x in range(int(j-pixel_radius),int(j+pixel_radius)): - if x is not None and y is not None and np.sqrt( (y-i)**2 + (x-j)**2) <= pixel_radius and x < self._griddata.shape[1] and y < self._griddata.shape[0]: - value_difference = self._griddata[y,x] - value - if (np.isnan(minimum_difference) or (minimum_difference > value_difference)) and (value_difference >= 0): - minimum_difference = value_difference - nearest = (y,x) - if np.isnan(minimum_difference): - for y in range(int(i-pixel_radius),int(i+pixel_radius)): - for x in range(int(j-pixel_radius),int(j+pixel_radius)): - if x is not None and y is not None and np.sqrt( (y-i)**2 + (x-j)**2) <= pixel_radius and x < self._griddata.shape[1] and y < self._griddata.shape[0]: - value_difference = value - self._griddata[y,x] - if (np.isnan(minimum_difference) or (minimum_difference > value_difference)) : - minimum_difference = value_difference - nearest = (y,x) - (y,x) = nearest - return y, x - def find_nearest_cell_with_greatest_value(self, index, pixel_radius = 5): - - (i, j) = index - nearest = tuple() - maximum_value = np.nan - for y in range(int(i-pixel_radius),int(i+pixel_radius)): - for x in range(int(j-pixel_radius),int(j+pixel_radius)): - if np.sqrt( (y-i)**2 + (x-j)**2) <= pixel_radius: - if np.isnan(maximum_value) or maximum_value < self._griddata[y,x]: - maximum_value = self._griddata[y,x] - nearest = (y,x) - - (y,x) = nearest - return y, x - + """Index of the highest-valued cell within ``pixel_radius``. + + Returns ``(None, None)`` when the neighbourhood holds no valid data. + """ + rows, cols = self._neighbourhood_indices(index, pixel_radius) + if rows.size == 0: + return None, None + values = self._griddata[rows, cols] + if np.all(np.isnan(values)): + return None, None + best = int(np.nanargmax(values)) + return int(rows[best]), int(cols[best]) + def snap_locations_to_greatest_value(self, v, pixel_radius = 5): - - idxs = self._xy_to_rowscols(v) + """Move each ``(x, y)`` to the highest-valued cell within ``pixel_radius``. + + Locations for which no valid cell is found are dropped. + """ snap_idxs = list() - for idx in idxs: + for idx in self._xy_to_rowscols(v): + if idx[0] is None: + continue i, j = self.find_nearest_cell_with_greatest_value(idx, pixel_radius) - snap_idxs.append((i,j)) + if i is not None: + snap_idxs.append((i, j)) return self._rowscols_to_xy(snap_idxs) - + def snap_locations_to_closest_value(self, v, value, pixel_radius = 5): - idxs = self._xy_to_rowscols(v) + """Move each ``(x, y)`` to the nearby cell whose value best matches. + + ``value`` is a sequence with one target value per location. + Locations for which no valid cell is found are dropped. + """ snap_idxs = list() - values_with_idx = zip(idxs, value) - for (idx, value) in values_with_idx: - i, j = self.find_nearest_cell_with_value(idx, value, pixel_radius) - snap_idxs.append((i,j)) + for (idx, target) in zip(self._xy_to_rowscols(v), value): + if idx[0] is None: + continue + i, j = self.find_nearest_cell_with_value(idx, target, pixel_radius) + if i is not None: + snap_idxs.append((i, j)) return self._rowscols_to_xy(snap_idxs) - - def export_arc_e00_grid(self, filename): + def export_arc_e00_grid(self, filename): + """Write an ArcInfo E00 interchange grid.""" self._create_gdal_representation_from_array(self._georef_info, 'E00GRID', self._griddata, self.dtype, filename) def set_value_at_rowscols(self, value, rowscols): - rowscols_array = (np.array(list(zip(*rowscols))).T[:,0],np.array(list(zip(*rowscols))).T[:,1]) - self._griddata[rowscols_array] = value - + """Set ``value`` at each ``(row, col)`` in ``rowscols``.""" + rowscols = list(rowscols) + if not rowscols: + return + rows, cols = zip(*rowscols) + self._griddata[np.asarray(rows, dtype=int), np.asarray(cols, dtype=int)] = value + def save(self, filename): - - self._create_gdal_representation_from_array(self._georef_info, 'GTiff', self._griddata, self.dtype, filename, ['COMPRESS=LZW']) - - def write_to_ai(self, filename): - self._writeArcAsciiRaster(self._georef_info, filename, self._griddata, np.NAN, '%10.2f') - + """Write the grid to an LZW-compressed GeoTIFF.""" + dataset = self._create_gdal_representation_from_array( + self._georef_info, 'GTiff', self._griddata, self.dtype, filename, + ['COMPRESS=LZW']) + # Dropping the reference is what flushes GDAL's write cache; without + # it the file can be truncated if the process exits soon after. + dataset.FlushCache() + del dataset + + def write_to_ai(self, filename, nodata_value=-9999.0): + """Write the grid as an ArcInfo ASCII grid.""" + self._writeArcAsciiRaster(self._georef_info, filename, self._griddata, + nodata_value, '%10.2f') + def vectorize(self, filename): - - import uuid - tmpfilename = str(uuid.uuid4()) - self.save(tmpfilename) - gdal_dataset = gdal.Open(tmpfilename) - srcband = gdal_dataset.GetRasterBand(1) - - dst_layername = filename - drv = ogr.GetDriverByName("ESRI Shapefile") - dst_ds = drv.CreateDataSource( dst_layername + ".shp" ) - dst_layer = dst_ds.CreateLayer(dst_layername, srs = None ) - newField = ogr.FieldDefn('RASTER_VAL', ogr.OFTReal) - dst_layer.CreateField(newField) - gdal.Polygonize( srcband, None, dst_layer, 0, [], callback=None ) - os.remove(tmpfilename) - + """Polygonize the grid into an ESRI shapefile called ``filename.shp``. + + Each polygon carries the raster value in a ``RASTER_VAL`` field. + """ + import tempfile + + handle, tmpfilename = tempfile.mkstemp(suffix='.tif') + os.close(handle) + try: + self.save(tmpfilename) + gdal_dataset = gdal.Open(tmpfilename) + try: + srcband = gdal_dataset.GetRasterBand(1) + drv = ogr.GetDriverByName("ESRI Shapefile") + dst_ds = drv.CreateDataSource(filename + ".shp") + try: + dst_layer = dst_ds.CreateLayer(os.path.basename(filename), srs=None) + dst_layer.CreateField(ogr.FieldDefn('RASTER_VAL', ogr.OFTReal)) + gdal.Polygonize(srcband, None, dst_layer, 0, [], callback=None) + finally: + # The shapefile is only written when the data source is + # released; the original code left it dangling. + dst_ds = None + finally: + gdal_dataset = None + finally: + if os.path.exists(tmpfilename): + os.remove(tmpfilename) + + @classmethod + def _georef_from_dataset(cls, return_object, gdal_dataset): + """Populate a grid's georeferencing from an open GDAL dataset.""" + nx = gdal_dataset.RasterXSize + ny = gdal_dataset.RasterYSize + info = return_object._georef_info + info.geoTransform = gdal_dataset.GetGeoTransform() + info.projection = gdal_dataset.GetProjection() + info.dx = info.geoTransform[1] + info.xllcenter = info.geoTransform[0] + info.dx / 2.0 + info.yllcenter = info.geoTransform[3] - (info.dx * (ny - 0.5)) + info.nx = nx + info.ny = ny + return return_object + @classmethod def load(cls, filename): - + """Read a grid previously written by :meth:`save`.""" return_object = cls() - gdal_dataset = gdal.Open(filename) - band = gdal_dataset.GetRasterBand(1) - nodata = band.GetNoDataValue() - return_object._griddata = band.ReadAsArray().astype(cls.dtype) - if nodata is not None: - nodata_elements = np.where(return_object._griddata == nodata) - from numpy import uint8 - if cls.dtype is not uint8: - return_object._griddata[nodata_elements] = np.NAN - - - geoTransform = gdal_dataset.GetGeoTransform() - nx = gdal_dataset.RasterXSize - ny = gdal_dataset.RasterYSize - - return_object._georef_info.geoTransform = geoTransform - return_object._georef_info.dx = return_object._georef_info.geoTransform[1] - return_object._georef_info.xllcenter = return_object._georef_info.geoTransform[0]+return_object._georef_info.dx/2.0 - return_object._georef_info.yllcenter = return_object._georef_info.geoTransform[3]-(return_object._georef_info.dx*(ny-0.5)) - return_object._georef_info.nx = nx - return_object._georef_info.ny = ny - - - gdal_file = None + try: + return_object._griddata = cls._read_band(gdal_dataset.GetRasterBand(1), cls.dtype) + cls._georef_from_dataset(return_object, gdal_dataset) + finally: + gdal_dataset = None return return_object + @classmethod + def _mosaic_frame(cls, tiles): + """Georeferencing for the smallest grid covering every tile.""" + if not tiles: + raise Error.InputError('mosaic', 'no tiles to mosaic') + + dx = tiles[0]._georef_info.dx + xmin = min(t._georef_info.xllcenter for t in tiles) + ymin = min(t._georef_info.yllcenter for t in tiles) + xmax = max(t._georef_info.xllcenter + (t._georef_info.nx - 1) * t._georef_info.dx + for t in tiles) + ymax = max(t._georef_info.yllcenter + (t._georef_info.ny - 1) * t._georef_info.dx + for t in tiles) + + ny = int(round((ymax - ymin) / dx)) + 1 + nx = int(round((xmax - xmin) / dx)) + 1 + + info = Georef_info() + info.nx = nx + info.ny = ny + info.dx = dx + info.xllcenter = xmin + info.yllcenter = ymin + info.projection = tiles[0]._georef_info.projection + info.geoTransform = (xmin - 0.5 * dx, dx, 0, + ymin + (ny - 0.5) * dx, 0, -dx) + return info + + @staticmethod + def _tile_slice(info, tile): + """Where ``tile`` belongs inside a mosaic described by ``info``.""" + j_min = int(round((tile._georef_info.xllcenter - info.xllcenter) / info.dx)) + i_min = int(round((info.yllcenter + (info.ny - 1) * info.dx + - tile._georef_info.yllcenter + - (tile._georef_info.ny - 1) * tile._georef_info.dx) / info.dx)) + return (slice(i_min, i_min + tile._georef_info.ny), + slice(j_min, j_min + tile._georef_info.nx)) + @classmethod def mosaic(cls, tiles): - - xmin = np.nan - xmax = np.nan - ymin = np.nan - ymax = np.nan - - for tile in tiles: - - xmin = tile._georef_info.xllcenter if (np.isnan(xmin) or (xmin > tile._georef_info.xllcenter)) else xmin - ymin = tile._georef_info.yllcenter if (np.isnan(ymin) or (ymin > tile._georef_info.yllcenter)) else ymin - xmax = tile._georef_info.xllcenter+(tile._georef_info.nx-1)*tile._georef_info.dx if (np.isnan(xmax) or (xmax < tile._georef_info.xllcenter+(tile._georef_info.nx-1)*tile._georef_info.dx)) else xmax - ymax = tile._georef_info.yllcenter+(tile._georef_info.ny-1)*tile._georef_info.dx if (np.isnan(ymax) or (ymax < tile._georef_info.yllcenter+(tile._georef_info.ny-1)*tile._georef_info.dx)) else ymax - - ny = int(round((ymax - ymin) / tiles[0]._georef_info.dx)) + 1 - nx = int(round((xmax - xmin) / tiles[0]._georef_info.dx)) + 1 - + """Reassemble tiles produced by :meth:`tile` into a single grid. + + Tiles must share a cell size and be aligned on a common grid. Where + tiles overlap, the last one in ``tiles`` wins. + """ + tiles = list(tiles) + info = cls._mosaic_frame(tiles) + return_object = cls() - return_object._griddata = np.zeros((ny, nx)) - return_object._griddata[:] = np.nan - return_object._georef_info.nx = nx - return_object._georef_info.ny = ny - return_object._georef_info.dx = tiles[0]._georef_info.dx - return_object._georef_info.xllcenter = xmin - return_object._georef_info.yllcenter = ymin - return_object._georef_info.geoTransform = (return_object._georef_info.xllcenter - 0.5*return_object._georef_info.dx, return_object._georef_info.dx, 0, return_object._georef_info.yllcenter + (float(return_object._georef_info.ny-0.5))*return_object._georef_info.dx, 0, -return_object._georef_info.dx) - + return_object._georef_info = info + return_object._griddata = np.full((info.ny, info.nx), np.nan) + for tile in tiles: - j_min = int(round((tile._georef_info.xllcenter - return_object._georef_info.xllcenter) / return_object._georef_info.dx)) - j_max = j_min + tile._georef_info.nx - i_min = int(round((return_object._georef_info.yllcenter + (return_object._georef_info.ny-1)*return_object._georef_info.dx - tile._georef_info.yllcenter - (tile._georef_info.ny-1)*tile._georef_info.dx) / return_object._georef_info.dx)) - i_max = i_min + tile._georef_info.ny - - return_object._griddata[i_min:i_max, j_min:j_max] = tile._griddata - + rows, cols = cls._tile_slice(info, tile) + return_object._griddata[rows, cols] = tile._griddata + return return_object - + class ValueGrid(BaseSpatialGrid): - + """A plain raster of values, with a helper for scattered writes.""" + + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), (('dx', 'grid'), '_create_from_grid')) - + def set_value_at_indexes(self, indexes, value): - ij = zip(*indexes) - self._griddata[ij] = value - - + """Set ``value`` at each ``(row, col)`` pair in ``indexes``.""" + indexes = list(indexes) + if not indexes: + return + # zip() is a lazy iterator in Python 3; using one directly as a NumPy + # subscript raises rather than doing what it did in Python 2. + rows, cols = zip(*indexes) + self._griddata[np.asarray(rows, dtype=int), np.asarray(cols, dtype=int)] = value + + class FlowDirection(BaseSpatialGrid): + """Base class for flow-direction grids.""" pass class FlowDirectionD8(FlowDirection): - + """Single-direction (D8) flow directions. + + Values are ArcGIS direction codes; 0 means the cell has no downstream + neighbour (a pit, an outlet at the grid edge, or no-data):: + + | 32 64 128 | + | 16 X 1 | + | 8 4 2 | + + Build one from a filled DEM:: + + d8 = FlowDirectionD8(flooded_dem=FilledElevation(elevation=dem)) + + Direction is chosen by steepest *gradient*, so the drop to a diagonal + neighbour is divided by ``sqrt(2) * dx`` before being compared with the + cardinal ones. Ties go to the first direction in the order E, SE, S, SW, + W, NW, N, NE. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), - (('flooded_dem',), '_create_from_flooded_dem')) - - from numpy import uint8 + (('gdal_filename',), '_read_gdal'), + (('flooded_dem',), '_create_from_flooded_dem'), + (('elevation',), '_create_from_elevation')) + dtype = uint8 + def _create_from_elevation(self, *args, **kwargs): + """Route an already-hydrologically-correct DEM. + + Alias for :meth:`_create_from_flooded_dem`; use it when the DEM has + been conditioned elsewhere. + """ + kwargs = dict(kwargs) + kwargs['flooded_dem'] = kwargs.pop('elevation') + self._create_from_flooded_dem(*args, **kwargs) + def _create_from_flooded_dem(self, *args, **kwargs): flooded_dem = kwargs['flooded_dem'] self._copy_info_from_grid(flooded_dem) - self._griddata = np.zeros_like(flooded_dem._griddata, dtype = self.dtype) - - flooded_dem_ijp1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ijp1[:,0:-2] = (flooded_dem._griddata[:,0:-2] - flooded_dem._griddata[:, 1:-1]) - - flooded_dem_ip1jp1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ip1jp1[0:-2,0:-2] = (flooded_dem._griddata[0:-2,0:-2] - flooded_dem._griddata[1:-1, 1:-1]) / 1.41 - - flooded_dem_ip1j = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ip1j[0:-2,:] = (flooded_dem._griddata[0:-2,:] - flooded_dem._griddata[1:-1, :]) - - flooded_dem_ip1jm1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ip1jm1[0:-2,1:-1] = (flooded_dem._griddata[0:-2,1:-1] - flooded_dem._griddata[1:-1, 0:-2]) / 1.41 - - flooded_dem_ijm1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_ijm1[:,1:-1] = (flooded_dem._griddata[:,1:-1] - flooded_dem._griddata[:, 0:-2]) - - flooded_dem_im1jm1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_im1jm1[1:-1,1:-1] = (flooded_dem._griddata[1:-1,1:-1] - flooded_dem._griddata[0:-2, 0:-2]) / 1.41 - - flooded_dem_im1j = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_im1j[1:-1,:] = (flooded_dem._griddata[1:-1,:] - flooded_dem._griddata[0:-2, :]) - - flooded_dem_im1jp1 = -np.ones_like(flooded_dem._griddata) * 1E22 - flooded_dem_im1jp1[1:-1,0:-2] = (flooded_dem._griddata[1:-1,0:-2] - flooded_dem._griddata[0:-2, 1:-1]) / 1.41 - - min_sl = np.fmax(np.fmax(np.fmax(np.fmax(np.fmax(np.fmax(np.fmax(flooded_dem_ijp1, flooded_dem_ip1jp1), flooded_dem_ip1j), flooded_dem_ip1jm1), flooded_dem_ijm1), flooded_dem_im1jm1), flooded_dem_im1j), flooded_dem_im1jp1) - - self._griddata[flooded_dem_ijp1 == min_sl] = 1 - self._griddata[flooded_dem_ip1jp1 == min_sl] = 2 - self._griddata[flooded_dem_ip1j == min_sl] = 4 - self._griddata[flooded_dem_ip1jm1 == min_sl] = 8 - self._griddata[flooded_dem_ijm1 == min_sl] = 16 - self._griddata[flooded_dem_im1jm1 == min_sl] = 32 - self._griddata[flooded_dem_im1j == min_sl] = 64 - self._griddata[flooded_dem_im1jp1 == min_sl] = 128 - - self._griddata[np.isnan(flooded_dem._griddata)] = 0 - + + elevations = np.ascontiguousarray(flooded_dem._griddata, dtype=float64) + kernels.warn_if_slow(elevations.size) + + # A geographic grid's cell size varies with latitude, so ask the grid + # itself rather than assuming a single dx. + cellsize_grid = np.ascontiguousarray( + flooded_dem._mean_pixel_dimension(*args, **kwargs), dtype=float64) + + # The old hand-rolled version compared each cell only with a window + # that stopped two columns/rows short, so the last row and column + # never received a direction; it also scaled diagonals by 1.41 rather + # than sqrt(2), and resolved ties towards NE rather than + # deterministically. + self._griddata = np.asarray( + kernels.flow_directions(elevations, cellsize_grid), dtype=self.dtype) + self._griddata[np.isnan(elevations)] = 0 + mask = kwargs.get('mask') - self._sort_indexes = flooded_dem.sort(reverse = False, force = True, mask = mask) + self._sort_indexes = flooded_dem.sort(reverse=False, force=True, mask=mask) self._sorted = True - + self._invalidate_network() + + # -- flow network ----------------------------------------------------- + # + # ``receivers`` and ``topological_order`` are derived from the direction + # codes and cached. Everything that walks the network downstream uses + # them, which is what lets accumulation, flow length and chi run in a + # single linear pass instead of a per-cell Python loop. + + def _invalidate_network(self): + self._receivers = None + self._topological_order = None + self._cycle_count = 0 + + @property + def receivers(self): + """Flat index of the cell each cell drains to, or -1 for none.""" + if getattr(self, '_receivers', None) is None: + self._receivers = np.asarray(kernels.receivers(self._griddata), dtype=np.int64) + return self._receivers + + @property + def topological_order(self): + """Flat cell indices ordered so every donor precedes its receiver.""" + if getattr(self, '_topological_order', None) is None: + order, cycles = kernels.topological_order(self.receivers) + self._topological_order = np.asarray(order, dtype=np.int64) + self._cycle_count = int(cycles) + if cycles: + warnings.warn( + "{0} cells of this flow-direction grid lie on circular " + "flow paths and cannot be ordered; results that depend on " + "routing through them will be wrong. This usually means " + "the DEM was not filled before routing.".format(cycles), + RuntimeWarning, stacklevel=2) + return self._topological_order + + @property + def cycle_count(self): + """How many cells lie on circular flow paths (0 for a valid network).""" + self.topological_order # populate the cache + return self._cycle_count + + def step_lengths(self, pixel_dimension=None): + """Distance from each cell to its receiver, in map units.""" + if pixel_dimension is None: + pixel_dimension = self._mean_pixel_dimension() + return np.asarray(pixel_dimension, dtype=float64) * self.pixel_scale(float64) + + # -- keeping the cache honest ----------------------------------------- + # + # Everything downstream now reads `receivers`/`topological_order` rather + # than re-deriving routing from `_griddata` on every call, so any write + # to a flow code has to drop the cache. Editing a code and having the + # edit silently ignored would be far worse than the cost of rebuilding. + + def __setitem__(self, key, value): + super(FlowDirectionD8, self).__setitem__(key, value) + self._invalidate_network() + + def set_value_at_rowscols(self, value, rowscols): + super(FlowDirectionD8, self).set_value_at_rowscols(value, rowscols) + self._invalidate_network() + + def invalidate_network(self): + """Drop the cached receivers and ordering. + + Call this after modifying ``_griddata`` directly; the indexed write + paths do it for you. + """ + self._invalidate_network() + def __flow_code_for_position(self, flooded_dem, i, j): - - flow_code = 0 - min_diff = 0 - - if flooded_dem[i,j+1] is not None and ((flooded_dem[i, j] - flooded_dem[i,j+1])) / 1.41 > min_diff: - flow_code = 1 - min_diff = (flooded_dem[i, j] - flooded_dem[i,j+1]) / 1.41 - - if flooded_dem[i+1,j+1] is not None and ((flooded_dem[i, j] - flooded_dem[i+1,j+1])) > min_diff: - flow_code = 2 - min_diff = flooded_dem[i, j] - flooded_dem[i+1,j+1] - - if flooded_dem[i+1,j] is not None and ((flooded_dem[i, j] - flooded_dem[i+1,j])) / 1.41 > min_diff: - flow_code = 4 - min_diff = (flooded_dem[i, j] - flooded_dem[i+1,j]) / 1.41 - - if flooded_dem[i+1,j-1] is not None and ((flooded_dem[i, j] - flooded_dem[i+1,j-1])) > min_diff: - flow_code = 8 - min_diff = flooded_dem[i, j] - flooded_dem[i+1,j-1] - - if flooded_dem[i,j-1] is not None and ((flooded_dem[i, j] - flooded_dem[i,j-1])) / 1.41 > min_diff: - flow_code = 16 - min_diff = (flooded_dem[i, j] - flooded_dem[i,j-1]) / 1.41 - - if flooded_dem[i-1,j-1] is not None and ((flooded_dem[i, j] - flooded_dem[i-1,j-1])) > min_diff: - flow_code = 32 - min_diff = flooded_dem[i, j] - flooded_dem[i-1,j-1] - - if flooded_dem[i-1,j] is not None and ((flooded_dem[i, j] - flooded_dem[i-1,j]) / 1.41) > min_diff: - flow_code = 64 - min_diff = (flooded_dem[i, j] - flooded_dem[i-1,j]) / 1.41 - - if flooded_dem[i-1,j+1] is not None and ((flooded_dem[i, j] - flooded_dem[i-1,j+1])) > min_diff: - flow_code = 128 - min_diff = flooded_dem[i, j] - flooded_dem[i-1,j+1] - - return flow_code - + """Steepest-descent code for a single cell. + + Kept for :meth:`update_flow_codes_in_mask`, which re-routes a handful + of cells after their elevations change. + """ + z = flooded_dem[i, j] + if z is None or np.isnan(z): + return 0 + + best_code = 0 + best_slope = 0.0 + de = self._georef_info.dx + for d, (di, dj) in enumerate(zip(kernels.D8_DI, kernels.D8_DJ)): + neighbour = flooded_dem[i + int(di), j + int(dj)] + if neighbour is None or np.isnan(neighbour): + continue + # The scaling used to be inverted -- cardinal drops were divided + # by 1.41 and diagonal ones were not -- which biased routing + # towards the diagonals. + slope = (z - neighbour) / (de * kernels.D8_DIST[d]) + if slope > best_slope: + best_slope = slope + best_code = int(kernels.D8_CODES[d]) + return best_code + def get_flow_to_cell(self,i,j): - #Function to get the indices of the cell that is drained to based on the flow direction specified in fd - - iOut = None - jOut = None - isGood = False - - if self._griddata[i,j] == 1 and j+1 < self._georef_info.nx: - iOut = i - jOut = j+1 - elif self._griddata[i,j] == 2 and i+1 < self._georef_info.ny and j+1 < self._georef_info.nx: - iOut = i+1 - jOut = j+1 - elif self._griddata[i,j] == 4 and i+1 < self._georef_info.ny: - iOut = i+1 - jOut = j - elif self._griddata[i,j] == 8 and i+1 < self._georef_info.ny and j-1 >= 0: - iOut = i+1 - jOut = j-1 - elif self._griddata[i,j] == 16 and j-1 >= 0: - iOut = i - jOut = j-1 - elif self._griddata[i,j] == 32 and i-1 >= 0 and j-1 >= 0: - iOut = i-1 - jOut = j-1 - elif self._griddata[i,j] == 64 and i-1 >= 0: - iOut = i-1 - jOut = j - elif self._griddata[i,j] == 128 and i-1 >= 0 and j+1 < self._georef_info.nx: - iOut = i-1 - jOut = j+1 - - if not(iOut is None): - isGood = True - - return iOut, jOut, isGood + """Index of the cell that ``(i, j)`` drains into. + + Returns + ------- + (int or None, int or None, bool) + Row, column and a flag that is ``False`` when the cell has no + downstream neighbour inside the grid. + """ + code = self._griddata[i, j] + if code == 0: + return None, None, False + + for d in range(8): + if code != kernels.D8_CODES[d]: + continue + i_out = i + int(kernels.D8_DI[d]) + j_out = j + int(kernels.D8_DJ[d]) + if 0 <= i_out < self._georef_info.ny and 0 <= j_out < self._georef_info.nx: + return i_out, j_out, True + break + + return None, None, False def get_upstream_cell_indexes(self, i, j): - + """Indices of the immediate neighbours that drain into ``(i, j)``.""" options = list() - - if self[i, j+1] == 16: - options += [(i, j+1)] - - if self[i+1, j+1] == 32: - options += [(i+1, j+1)] - - if self[i+1, j] == 64: - options += [(i+1, j)] - - if self[i+1, j-1] == 128: - options += [(i+1, j-1)] - - if self[i, j-1] == 1: - options += [(i, j-1)] - - if self[i-1, j-1] == 2: - options += [(i-1, j-1)] - - if self[i-1, j] == 4: - options += [(i-1, j)] - - if self[i-1, j+1] == 8: - options += [(i-1, j+1)] - + for d in range(8): + i_up = i + int(kernels.D8_DI[d]) + j_up = j + int(kernels.D8_DJ[d]) + # A neighbour in direction d drains here if it points back the + # opposite way, i.e. four bit positions round the compass. + opposite = int(kernels.D8_CODES[(d + 4) % 8]) + if self[i_up, j_up] == opposite: + options.append((i_up, j_up)) return options - - def __get_flow_from_cell(self, i, j, max_recursion_depth=None, depth=0): - - i_source = [i] - j_source = [j] - if max_recursion_depth is not None and depth > max_recursion_depth: - return i_source, j_source + def __get_flow_from_cell(self, i, j, max_recursion_depth=None, depth=0): + """Every cell upstream of ``(i, j)``, including ``(i, j)`` itself. - recursion_depth = depth + 1 + Returns ``(rows, cols)`` as plain lists. ``max_recursion_depth`` + limits how many steps upstream the search goes; ``depth`` is accepted + for backward compatibility and gives the starting depth. + The traversal is iterative. The original recursive version needed + ``sys.setrecursionlimit(1000000)`` and still crashed the interpreter + on basins more than a few tens of thousands of cells long. + """ + if max_recursion_depth is None and depth == 0: + # No depth limit: let the compiled kernel do the whole basin in + # one linear pass over the flow network. + flat = np.int64(i) * self._georef_info.nx + np.int64(j) + mask = kernels.upstream_mask(self.receivers, np.array([flat], dtype=np.int64)) + rows, cols = np.nonzero(mask) + return rows.tolist(), cols.tolist() + + ny, nx = self._georef_info.ny, self._georef_info.nx + limit = None if max_recursion_depth is None else max_recursion_depth - depth + + i_source = [] + j_source = [] + seen = set() + stack = [((int(i), int(j)), 0)] + while stack: + (ci, cj), d = stack.pop() + if (ci, cj) in seen: + continue + seen.add((ci, cj)) + i_source.append(ci) + j_source.append(cj) + if limit is not None and d >= limit: + continue + for k in range(8): + ui = ci + int(kernels.D8_DI[k]) + uj = cj + int(kernels.D8_DJ[k]) + if not (0 <= ui < ny and 0 <= uj < nx): + continue + if self._griddata[ui, uj] == kernels.D8_CODES[(k + 4) % 8]: + stack.append(((ui, uj), d + 1)) - if self[i, j+1] == 16: - [i_append, j_append] = self.__get_flow_from_cell(i,j+1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i+1, j+1] == 32: - [i_append, j_append] = self.__get_flow_from_cell(i+1,j+1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i+1, j] == 64: - [i_append, j_append] = self.__get_flow_from_cell(i+1,j,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i+1, j-1] == 128: - [i_append, j_append] = self.__get_flow_from_cell(i+1,j-1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i, j-1] == 1: - [i_append, j_append] = self.__get_flow_from_cell(i,j-1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i-1, j-1] == 2: - [i_append, j_append] = self.__get_flow_from_cell(i-1,j-1,max_recursion_depth=max_recursion_depth,depth=recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i-1, j] == 4: - [i_append, j_append] = self.__get_flow_from_cell(i-1,j,max_recursion_depth = max_recursion_depth,depth = recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - - if self[i-1, j+1] == 8: - [i_append, j_append] = self.__get_flow_from_cell(i-1,j+1,max_recursion_depth = max_recursion_depth,depth = recursion_depth) - i_source = i_source + i_append - j_source = j_source + j_append - return i_source, j_source def __map_flow_from_cell(self, index, **kwargs): - + """Build the nested upstream dictionary rooted at ``index``. + + Each node is a dict with: + + ``index`` + the ``(row, col)`` of the cell; + ``distance_scale`` + 1 or sqrt(2), the step length from this node's *parent* in cell + widths -- 1 at the root; + ``next`` + the list of upstream child nodes, absent at a headwater; + plus one entry per keyword grid, holding that grid's value at the cell. + + ``assume_valid_routing=False`` guards against cycles in a + flow-direction grid that was not derived from a filled DEM. + + Notes + ----- + ``distance_scale`` used to be written onto the *parent* node and + overwritten by each child in turn, so it ended up describing whichever + child happened to be visited last. Chi profiles built from these + dictionaries were wrong wherever a confluence had a diagonal and a + cardinal tributary. + """ if len(index) == 1: - (i, j) = index[0] + root = tuple(index[0]) else: - (i, j) = index - - return_dict = dict() - - return_dict['index'] = index - for arg in kwargs: - if arg != 'assume_valid_routing': - return_dict[arg] = kwargs[arg][i,j] - - return_dict['next'] = [] - return_dict['distance_scale'] = 1.0 - - if self[i, j+1] == 16: - if kwargs.get('assume_valid_routing', True) or not self.visited[i, j+1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i, j+1] = True - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell((i,j+1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i+1, j+1] == 32: - if kwargs.get('assume_valid_routing', True) or not self.visited[i+1, j+1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i+1, j+1] = True - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell((i+1,j+1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i+1, j] == 64: - if kwargs.get('assume_valid_routing', True) or not self.visited[i+1, j]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i+1, j] = True - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell((i+1,j), **kwargs) - return_dict['next'].append(child_dict) - - if self[i+1, j-1] == 128: - if kwargs.get('assume_valid_routing', True) or not self.visited[i+1, j-1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i+1, j-1] = True - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell((i+1,j-1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i, j-1] == 1: - if kwargs.get('assume_valid_routing', True) or not self.visited[i, j-1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i, j-1] = True - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell((i,j-1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i-1, j-1] == 2: - if kwargs.get('assume_valid_routing', True) or not self.visited[i-1, j-1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i-1, j-1] = True - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell((i-1,j-1), **kwargs) - return_dict['next'].append(child_dict) - - if self[i-1, j] == 4: - if kwargs.get('assume_valid_routing', True) or not self.visited[i-1, j]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i-1, j] = True - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell((i-1,j), **kwargs) - return_dict['next'].append(child_dict) - - if self[i-1, j+1] == 8: - if kwargs.get('assume_valid_routing', True) or not self.visited[i-1, j+1]: - if not kwargs.get('assume_valid_routing', True): - self.visited[i-1, j+1] = True - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell((i-1,j+1), **kwargs) - return_dict['next'].append(child_dict) - - if len(return_dict.get('next', 0)) == 0: - return_dict.pop('next') - - return return_dict + root = tuple(index) + + assume_valid_routing = kwargs.get('assume_valid_routing', True) + value_grids = {name: grid for name, grid in kwargs.items() + if name != 'assume_valid_routing'} + ny, nx = self._georef_info.ny, self._georef_info.nx + + def make_node(cell, scale): + i, j = cell + node = {'index': cell, 'distance_scale': scale} + for name, grid in value_grids.items(): + node[name] = grid[i, j] + return node + + # Every node carries `index` as a plain (row, col) pair, the root + # included. It used to keep the nested form the caller passed, so + # consumers had to special-case the root -- and those that did not + # failed to unpack it. + root_node = make_node(root, 1.0) + # Explicit stack of (node, cell) pairs instead of recursion. + stack = [(root_node, root)] + while stack: + node, (i, j) = stack.pop() + children = [] + for k in range(8): + ui = i + int(kernels.D8_DI[k]) + uj = j + int(kernels.D8_DJ[k]) + if not (0 <= ui < ny and 0 <= uj < nx): + continue + if self._griddata[ui, uj] != kernels.D8_CODES[(k + 4) % 8]: + continue + if not assume_valid_routing: + if self.visited[ui, uj]: + continue + self.visited[ui, uj] = True + child = make_node((ui, uj), float(kernels.D8_DIST[k])) + children.append(child) + stack.append((child, (ui, uj))) + if children: + node['next'] = children + + return root_node def update_flow_codes_in_mask(self, *args, **kwargs): - + """Re-derive flow codes inside a mask after the DEM has changed.""" flooded_dem = args[0] mask = args[1] - + indexes = np.where(mask._griddata == 1) for (i,j) in zip(indexes[0],indexes[1]): flow_code = self.__flow_code_for_position(flooded_dem, i, j) self._griddata[i,j] = flow_code - + self._invalidate_network() + + def basin_mask(self, outlets): + """Boolean array marking every cell draining to any of ``outlets``. + + ``outlets`` is a sequence of ``(x, y)`` map coordinates. + """ + flat = [] + for (row, col) in self._xy_to_rowscols(outlets): + if row is None: + continue + flat.append(row * self._georef_info.nx + col) + mask = kernels.upstream_mask(self.receivers, + np.array(flat, dtype=np.int64)) + return np.asarray(mask).astype(bool) + def divides_for_outlets(self, outlet1, outlet2): - basin1 = BaseSpatialGrid() - basin1._copy_info_from_grid(self, True) - - ind = self.get_indexes_of_upstream_cells_for_location(outlet1[0], outlet1[1]) - i,j = zip(*ind) - basin1._griddata[i,j] = 1 - - basin2 = BaseSpatialGrid() - basin2._copy_info_from_grid(self, True) - - ind = self.get_indexes_of_upstream_cells_for_location(outlet2[0], outlet2[1]) - i,j = zip(*ind) - basin2._griddata[i,j] = 1 - - from scipy.ndimage.morphology import binary_dilation as dilate - - basin3 = BaseSpatialGrid() - basin3._copy_info_from_grid(self, True) - basin3._griddata = basin1._griddata + dilate(basin2._griddata) - - basin4 = BaseSpatialGrid() - basin4._copy_info_from_grid(self, True) - basin4._griddata = basin2._griddata + dilate(basin1._griddata) - - (i, j) = np.where(basin3._griddata == 2) - ind1 = zip(i,j) - - (i,j) = np.where(basin4._griddata == 2) - ind2 = zip(i,j) - - return ind1, ind2 - + """Cells on each side of the divide shared by two basins. + + Returns + ------- + (tuple, tuple) + ``(cells_in_basin1, cells_in_basin2)``, each a tuple of + ``(row, col)`` pairs lying against the shared divide. + """ + from scipy.ndimage import binary_dilation, generate_binary_structure + + in_basin1 = self.basin_mask((outlet1,)) + in_basin2 = self.basin_mask((outlet2,)) + + # An 8-connected structure, so cells meeting only at a corner still + # count as facing each other across the divide. + structure = generate_binary_structure(2, 2) + ind1 = np.argwhere(in_basin1 & binary_dilation(in_basin2, structure=structure)) + ind2 = np.argwhere(in_basin2 & binary_dilation(in_basin1, structure=structure)) + + # Materialise the pairs: the old version returned zip objects, which + # are single-use and silently came back empty on a second read. + return (tuple(map(tuple, ind1)), tuple(map(tuple, ind2))) + def locations_of_paired_hollows(self, outlet1, outlet2, area, Ao=1E5): - - def map_down_to_hollow(ind, fd): - (i,j) = ind - (next_i, next_j, _) = fd.get_flow_to_cell(i,j) - if((area[i,j] < Ao) and (area[next_i, next_j] >= Ao)): - return next_i, next_j - else: - return map_down_to_hollow((next_i, next_j), fd) - + """Channel heads reached by flowing down from a shared divide. + + Walks downstream from each divide cell until drainage area first + exceeds ``Ao``, and returns the distinct cells where that happens on + each side. + """ + def map_down_to_hollow(ind): + """Follow flow down until area first reaches Ao; None if never.""" + (i, j) = ind + seen = set() + while True: + if (i, j) in seen: + return None # circular routing + seen.add((i, j)) + (next_i, next_j, good) = self.get_flow_to_cell(i, j) + if not good: + return None + here = area[i, j] + there = area[next_i, next_j] + if here is None or there is None: + return None + if here < Ao <= there: + return next_i, next_j + (i, j) = (next_i, next_j) + ind1, ind2 = self.divides_for_outlets(outlet1, outlet2) - h1 = set() - for ind in ind1: - h1.add(map_down_to_hollow(ind, self)) - h2 = set() - for ind in ind2: - h2.add(map_down_to_hollow(ind, self)) - - return tuple(h1), tuple(h2) - - + h1 = {h for h in (map_down_to_hollow(ind) for ind in ind1) if h is not None} + h2 = {h for h in (map_down_to_hollow(ind) for ind in ind2) if h is not None} + + return tuple(sorted(h1)), tuple(sorted(h2)) + def get_indexes_of_upstream_cells(self, i, j): - - i,j = self.__get_flow_from_cell(i,j) - return zip(i,j) - + """Tuple of ``(row, col)`` for every cell upstream of ``(i, j)``.""" + rows, cols = self.__get_flow_from_cell(i, j) + # A tuple rather than a zip object: callers iterate the result more + # than once, and a zip is exhausted after the first pass. + return tuple(zip(rows, cols)) + def get_indexes_of_upstream_cells_for_location(self, x, y): - + """As :meth:`get_indexes_of_upstream_cells`, from a map coordinate.""" v = ((x, y), ) ((i, j),) = self._xy_to_rowscols(v) + if i is None: + return tuple() return self.get_indexes_of_upstream_cells(i, j) def search_down_flow_direction_with_length(self, start, search_length = np.inf): - l = list() + """Follow flow downstream from ``start``, recording distance travelled. + + Stops when flow leaves the grid, at a pit, once ``search_length`` is + covered, or if the path revisits a cell. The terminal cell is + included in the result. + + Returns + ------- + (tuple, tuple) + ``(cells, distances)``, where ``cells`` are ``(row, col)`` pairs + and ``distances`` are the along-path distances at each cell. + """ + cells = list() length_list = list() - ((row,col),) = self._xy_to_rowscols((start,)) + ((row, col),) = self._xy_to_rowscols((start,)) + if row is None: + return tuple(), tuple() + + # A set for the revisit test: the original scanned the whole path + # list at every step, making long flow paths quadratic. The edge + # test is now `no in-grid receiver` rather than `on the perimeter`, + # so a path that merely starts on the edge still gets followed. + seen = set() length = 0.0 - while not (row == 0 or row == self._georef_info.ny-1 or col == 0 or col == self._georef_info.nx - 1 or \ - (row,col) in l) and (length < search_length): - l.append((row, col)) + while True: + if (row, col) in seen or length >= search_length: + break + seen.add((row, col)) + cells.append((row, col)) length_list.append(length) - row_n,col_n,inBounds = self.get_flow_to_cell(row, col) - length += self._georef_info.dx * (1.0 if ((row_n == row) or (col_n == col)) else 1.414) - (row, col) = (row_n, col_n) + row_n, col_n, inBounds = self.get_flow_to_cell(row, col) if not inBounds: break - return tuple(l), tuple(length_list) + diagonal = (row_n != row) and (col_n != col) + length += self._georef_info.dx * (SQRT2 if diagonal else 1.0) + (row, col) = (row_n, col_n) + + return tuple(cells), tuple(length_list) def search_down_flow_direction(self, start, search_length = np.inf): + """The cells along the flow path from ``start``, without distances.""" return self.search_down_flow_direction_with_length(start, search_length=search_length)[0] def search_down_flow_direction_from_rowscols_location(self, start, return_rowscols = False, search_length = np.inf): - + """Follow flow downstream from a ``(row, col)`` start. + + Returns map coordinates unless ``return_rowscols`` is true. + """ rcs = self.search_down_flow_direction(self._rowscols_to_xy(((start),))[0], search_length = search_length) return rcs if return_rowscols else self._rowscols_to_xy(rcs) def search_down_flow_direction_from_xy_location(self, start, return_rowscols=False, search_length=np.inf): + """Follow flow downstream from an ``(x, y)`` start. + Returns map coordinates unless ``return_rowscols`` is true. + """ rcs = self.search_down_flow_direction(start, search_length=search_length) return rcs if return_rowscols else self._rowscols_to_xy(rcs) - def convert_rivertools_directions_to_arc(self): - # Function to convert river tools flow directions to arcGisFlowDirections - # ArcGIS convention - # |i-1,j-1 i-1,j i-1,j+1| |32 64 128| - # |i,j-1 i,j i,j+1| |16 X 1 | - # |i+1,j-1 i+1,j i+1,j+1| |8 4 2 | - # In river tools convention the 1 is in the top right - - convertedFlowDir = int(np.log2(self._griddata)) - convertedFlowDir -= 1 - convertedFlowDir[convertedFlowDir == -1] = 7 - convertedFlowDir = 2**convertedFlowDir - convertedFlowDir[self._griddata == self.noData] = self.noData - self._griddata = convertedFlowDir - + def convert_rivertools_directions_to_arc(self, no_data=0): + """Convert RiverTools flow codes in place to the ArcGIS convention. + + RiverTools numbers the same eight directions but starts from the top + right, so every code is rotated by one position:: + + ArcGIS | 32 64 128 | + | 16 X 1 | + | 8 4 2 | + """ + codes = np.asarray(self._griddata) + valid = codes > 0 + # np.log2 of an array cannot be wrapped in int(); use integer + # arithmetic on the exponent instead, and take the no-data value as an + # argument rather than reading an attribute that never existed. + exponent = np.zeros(codes.shape, dtype=np.int16) + exponent[valid] = np.round(np.log2(codes[valid].astype(float64))).astype(np.int16) + exponent -= 1 + exponent[exponent == -1] = 7 + converted = np.zeros(codes.shape, dtype=self.dtype) + converted[valid] = (2 ** exponent[valid]).astype(self.dtype) + converted[codes == no_data] = no_data + self._griddata = converted + self._invalidate_network() + def pixel_scale(self, dtype = np.float32): - + """Step-length multiplier for each cell: 1 cardinal, sqrt(2) diagonal.""" dim = np.ones_like(self._griddata, dtype = dtype) - dim[np.where((self._griddata == 32) | (self._griddata == 128) | (self._griddata == 2) | (self._griddata == 8))] = 1.41421356 + diagonal = np.isin(self._griddata, [2, 8, 32, 128]) + dim[diagonal] = SQRT2 return dim - + def map_values_to_recursive_list(self, outlet, **kwargs): - + """Nested dictionary of the basin upstream of ``outlet``. + + Pass grids as keywords to sample them at every cell, e.g. + ``map_values_to_recursive_list(outlet, elevation=dem, area=area)``. + See :meth:`__map_flow_from_cell` for the node layout. + """ v = (outlet, ) (ij_outlet, ) = self._xy_to_rowscols(v) + if ij_outlet[0] is None: + raise Error.InputError('map_values_to_recursive_list', + 'outlet {0} lies outside the grid'.format(outlet)) if kwargs.get('assume_valid_routing', True) is False: self.visited = np.zeros_like(self._griddata).astype(bool) return self.__map_flow_from_cell((ij_outlet,), **kwargs) - def bounds_of_basin_for_outlet(self, outlet): - + """``((xmin, xmax), (ymin, ymax))`` of the basin draining to ``outlet``. + + ``outlet`` is given as ``(y, x)`` -- latitude then longitude -- which + is the order the rest of this method's callers use. + """ (lat, lon) = outlet indexes = self.get_indexes_of_upstream_cells_for_location(lon, lat) - xy = self._rowscols_to_xy(indexes) - bounds = [[None, None], [None, None]] - for (lon, lat) in xy: - if lon > bounds[0][1] or bounds[0][1] is None: - bounds[0][1] = lon - if lon < bounds[0][0] or bounds[0][0] is None: - bounds[0][0] = lon - if lat > bounds[1][1] or bounds[1][1] is None: - bounds[1][1] = lat - if lat < bounds[1][0] or bounds[1][0] is None: - bounds[1][0] = lat - return ((bounds[0][0], bounds[0][1]), (bounds[1][0], bounds[1][1])) - + xy = self._rowscols_to_xy(indexes) + if not xy: + raise Error.InputError('bounds_of_basin_for_outlet', + 'no cells drain to {0}'.format(outlet)) + # Comparing a float with None raises in Python 3, and `x > None or + # None is None` evaluates the comparison first, so the original could + # not survive its own first iteration. + lons = [p[0] for p in xy] + lats = [p[1] for p in xy] + return ((min(lons), max(lons)), (min(lats), max(lats))) + def divides(self): - + """Grid marking drainage divides: cells with no upstream neighbour.""" divides = BaseSpatialGrid() divides._copy_info_from_grid(self, True) - - ip1j = self._griddata[2:-1,1:-2] - im1j = self._griddata[0:-3,1:-2] - ijp1 = self._griddata[1:-2,2:-1] - ijm1 = self._griddata[1:-2,0:-3] - - ip1jp1 = self._griddata[2:-1, 2:-1] - ip1jm1 = self._griddata[2:-1, 0:-3] - im1jp1 = self._griddata[0:-3, 2:-1] - im1jm1 = self._griddata[0:-3, 0:-3] - - divides._griddata[1:-2,1:-2] = ((ijm1 != 1) & (im1jm1 != 2) & (im1j != 4) & (im1jp1 != 8) & (ijp1 != 16) & (ip1jp1 != 32) & (ip1j != 64) & (ip1jm1 != 128)) + + codes = self._griddata + # A cell is a divide when none of its eight neighbours drains into it. + is_divide = np.ones(codes.shape, dtype=bool) + for d in range(8): + di, dj = int(kernels.D8_DI[d]), int(kernels.D8_DJ[d]) + opposite = kernels.D8_CODES[(d + 4) % 8] + neighbour = np.zeros(codes.shape, dtype=codes.dtype) + src_i = slice(max(di, 0), codes.shape[0] + min(di, 0)) + src_j = slice(max(dj, 0), codes.shape[1] + min(dj, 0)) + dst_i = slice(max(-di, 0), codes.shape[0] + min(-di, 0)) + dst_j = slice(max(-dj, 0), codes.shape[1] + min(-dj, 0)) + neighbour[dst_i, dst_j] = codes[src_i, src_j] + is_divide &= (neighbour != opposite) + + divides._griddata = is_divide.astype(divides.dtype) return divides - + def paired_divides(self, mask = None): - + """Pair each divide cell with the cell directly across the divide. + + The partner is the reflection of the divide cell's downstream + neighbour through the cell itself, i.e. the cell one step *up* the + opposite hillslope. + + Returns + ------- + tuple + ``((xy_from, xy_to), ...)`` in map coordinates. + """ divides = self.divides() if mask is not None: - i = np.where((divides._griddata == 1) & (mask._griddata > 0)) + selected = np.where((divides._griddata == 1) & (mask._griddata > 0)) else: - i = np.where(divides._griddata == 1) + selected = np.where(divides._griddata == 1) - rcs = zip(i[0].tolist(),i[1].tolist()) - pairs = list() - for (i,j) in rcs: - - (i_next, j_next, good) = self.get_flow_to_cell(i,j) - - if good and i_next is not None and j_next is not None: - i_next = 2*i - i_next - j_next = 2*j - j_next - - if i_next >= 0 and i_next < self._georef_info.ny and j_next >= 0 and j_next < self._georef_info.nx: - (xy, ) = self._rowscols_to_xy(((i,j),)) - (xy_next, ) = self._rowscols_to_xy(((i_next, j_next),)) - pairs.append( (xy, xy_next)) - + for (i, j) in zip(selected[0].tolist(), selected[1].tolist()): + + (i_next, j_next, good) = self.get_flow_to_cell(i, j) + + if good: + i_opposite = 2*i - i_next + j_opposite = 2*j - j_next + + if (0 <= i_opposite < self._georef_info.ny + and 0 <= j_opposite < self._georef_info.nx): + (xy, ) = self._rowscols_to_xy(((i, j),)) + (xy_next, ) = self._rowscols_to_xy(((i_opposite, j_opposite),)) + pairs.append((xy, xy_next)) + return tuple(pairs) - + class Elevation(CalculationMixin, BaseSpatialGrid): + """A DEM. + + Adds the terrain-derivative helpers of :class:`CalculationMixin` to the + base raster, plus the edge detection that depression filling starts from. + """ def findDEMedge(self): - #Pad the data so that we can take a windowed max - original = np.zeros((self._georef_info.ny, self._georef_info.nx)) - i = np.where(np.isnan(self._griddata)) - original[i] = 1 - - from scipy.ndimage.morphology import binary_dilation as dilation - from scipy.ndimage import generate_binary_structure as generate_binary_structure - structure = generate_binary_structure(2, 2) - dilated = dilation(original, structure = structure, border_value = 1).astype(int) - edges = dilated - original - - return np.where(edges) - - def outlets_at_coastlines(self): - - import scipy.ndimage.morphology as morph - - mask1 = BaseSpatialGrid() - mask2 = BaseSpatialGrid() - mask3 = BaseSpatialGrid() - mask1._copy_info_from_grid(self, True) - mask1.dtype = np.uint8 - mask1._griddata = mask1._griddata.astype(np.uint8) - mask1._griddata[(self._griddata > 0)] = 1 - mask2._copy_info_from_grid(mask1, False) - mask3._copy_info_from_grid(mask1, False) - mask2._griddata = morph.binary_dilation(mask2._griddata, iterations = 3).astype(uint8) - mask3._griddata = morph.binary_erosion(mask3._griddata, iterations = 3).astype(uint8) - mask3._griddata = (mask3._griddata.astype(float) + mask2._griddata.astype(float)).astype(uint8) - i = np.where(mask3._griddata == 1) - rc = zip(i[0].tolist(),i[1].tolist()) - return self._rowscols_to_xy(rc) - + """Cells where water can leave the DEM. + + Those are the grid perimeter plus every valid cell touching a no-data + cell -- the same seed set TopoToolbox's ``fillsinks`` uses. Priority + flooding starts here. + + Returns + ------- + (ndarray, ndarray) + ``(rows, cols)``, as :func:`numpy.where` returns. + """ + # Delegates to the kernel so that the cells the flood is seeded with + # are, by construction, exactly the cells reported here. + flat = kernels.default_seeds(np.asarray(self._griddata, dtype=float64)) + return np.unravel_index(flat, (self._georef_info.ny, self._georef_info.nx)) + + def outlets_at_coastlines(self, iterations=3): + """Map coordinates of the cells forming the land/sea boundary. + + Land is taken to be every cell with a positive elevation. The + coastline is the band left over when the land mask is dilated and + eroded by ``iterations`` cells and the two are compared. + """ + from scipy.ndimage import binary_dilation, binary_erosion + + land = np.asarray(self._griddata) > 0 + dilated = binary_dilation(land, iterations=iterations) + eroded = binary_erosion(land, iterations=iterations) + # Exactly one of the two is set only in the fringe between them. + coastline = dilated.astype(np.int16) + eroded.astype(np.int16) == 1 + + rows, cols = np.nonzero(coastline) + return self._rowscols_to_xy(list(zip(rows.tolist(), cols.tolist()))) + def track_flow_downhill(self, starting_point, maximum_pit_depth = 20): - - adjust = [(-1, -1), (0, -1), (1,-1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)] - + """Walk downhill from ``starting_point``, hopping over shallow pits. + + At each step the lowest unvisited neighbour is taken, provided it is + no more than ``maximum_pit_depth`` above the current cell. That lets + the path climb out of small closed depressions instead of stopping in + them, without filling the DEM. + + Returns + ------- + (tuple, tuple, tuple) + ``(xy, distance, elevation)`` along the path. + """ + adjust = [(-1, -1), (0, -1), (1, -1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)] + (ij, ) = self._xy_to_rowscols((starting_point,)) - visited = (ij, ) - ij_a = [(ij[0] + x[0], ij[1] + x[1]) for x in adjust] - e_a = np.array([self[i[0], i[1]] for i in ij_a]) - new = True - while None not in e_a and np.sum(np.isnan(e_a)) == 0 and new: - sls = np.argsort(e_a) - new = False - for sl in sls: + if ij[0] is None: + raise Error.InputError('track_flow_downhill', + 'starting point lies outside the grid') + + def neighbour_elevations(cell): + """Elevations of the eight neighbours; NaN where off-grid.""" + # Building this with `self[...] or None` produced an object-dtype + # array, and `None not in array` then silently never matched. + values = np.empty(8, dtype=float64) + for k, (di, dj) in enumerate(adjust): + value = self[cell[0] + di, cell[1] + dj] + values[k] = np.nan if value is None else value + return values + + visited = [tuple(ij)] + seen = {tuple(ij)} + e_a = neighbour_elevations(ij) + + while not np.any(np.isnan(e_a)): + here = self[ij[0], ij[1]] + moved = False + for sl in np.argsort(e_a): this_ij = (ij[0] + adjust[sl][0], ij[1] + adjust[sl][1]) - if (this_ij not in visited) and ( (e_a[sl] - self[ij[0], ij[1]]) <= maximum_pit_depth): - visited += (this_ij, ) - ij = this_ij - new = True - break - if new: - ij_a = [(ij[0] + x[0], ij[1] + x[1]) for x in adjust] - e_a = np.array([self[i[0], i[1]] for i in ij_a]) + if this_ij in seen: + continue + if (e_a[sl] - here) > maximum_pit_depth: + continue + visited.append(this_ij) + seen.add(this_ij) + ij = this_ij + moved = True + break + if not moved: + break + e_a = neighbour_elevations(ij) + xy = self._rowscols_to_xy(visited) - this_xy = xy[0] - l = tuple() - e = tuple() - last_l = 0 - for txy in xy: - (ij, ) = self._xy_to_rowscols((txy,)) - e += (self[ij[0], ij[1]], ) - this_dl = np.sqrt(np.power(txy[0] - this_xy[0], 2) + np.power(txy[1] - this_xy[1], 2)) - last_l = last_l + this_dl - l += (last_l, ) - this_xy = txy - return xy, l, e + elevations = tuple(self[i, j] for (i, j) in visited) + distances = [0.0] + for previous, current in zip(xy[:-1], xy[1:]): + step = np.hypot(current[0] - previous[0], current[1] - previous[1]) + distances.append(distances[-1] + step) + return xy, tuple(distances), elevations class GeographicElevation(GeographicGridMixin, Elevation): + """A DEM on a latitude/longitude grid.""" + pass class Gradient(BaseSpatialGrid): + """The two components of the topographic gradient. + + :: + + gradient = Gradient(elevation=dem) + gradient._gx, gradient._gy + + Stored as two bands rather than one, so direction survives ``save()``. + :meth:`average_gradient` smooths both components over a length scale, + which is how a regional aspect is obtained. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), @@ -1755,6 +2545,7 @@ def _create_from_elevation(self, *args, **kwargs): [self._gy, self._gx] = np.gradient(elevation._griddata,dx) def average_gradient(self, distance): + """Both gradient components averaged over a disc of radius ``distance``.""" outgrid = Gradient() outgrid._copy_info_from_grid(self, True) outgrid._gy = self.average_over_distance(distance, grid = self._gy) @@ -1766,7 +2557,7 @@ def save(self, filename): @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -1774,7 +2565,7 @@ def get_band(gdal_dataset, band_number): if nodata is not None: nodata_elements = np.where(grid == nodata) if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -1822,6 +2613,14 @@ def plot(self, **kwargs): plt.show(block=True) class ScarpWavelet(BaseSpatialGrid): + """Template-matched scarp amplitude, age, orientation and signal-to-noise. + + Holds the four-band output of a scarp template search. Load one with + :meth:`load`; the bands become ``_A``, ``_kt``, ``_orientation`` and + ``_SNR``. :meth:`template_window` needs the separate ``scarplet`` + package. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), @@ -1829,7 +2628,7 @@ class ScarpWavelet(BaseSpatialGrid): @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -1837,7 +2636,7 @@ def get_band(gdal_dataset, band_number): if nodata is not None: nodata_elements = np.where(grid == nodata) if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -1864,10 +2663,16 @@ def get_band(gdal_dataset, band_number): return return_object def load_elevation(self, filename): + """Attach the DEM the templates were matched against.""" self.elevation = Elevation.load(filename) def plot_orientations(self, *args, **kwargs): - + """Draw scarp orientations over a hillshade, faded by signal-to-noise. + + Pass ``elevation`` and ``distance`` to have the regional slope + direction removed first, so what is drawn is the scarp orientation + *relative to* the hillslope it sits on. + """ # pop inputs, create those that are needed: elevation = kwargs.pop('elevation', None) @@ -1935,8 +2740,14 @@ def plot_orientations(self, *args, **kwargs): def calculate_valid_orientations(self, window_size, age, - num=46, + num=46, discard_max_min=True): + """Range of template orientations that stay inside the valid data. + + Returns ``(max_orientation, min_orientation)`` in radians, ``NaN`` + where no orientation fits. Cells within ``window_size`` of an edge + are always ``NaN``. + """ data = self.valid_data() max = -np.inf * np.ones_like(data) min = np.inf * np.ones_like(data) @@ -1985,6 +2796,10 @@ def _convolve_mask(self, data, window_size, age, orientation): return count def template_window(self, window_size, age, orientation, use_pixels=False): + """The scarp template mask at one size, age and orientation. + + Needs the separate ``scarplet`` package. + """ nx = self._georef_info.nx ny = self._georef_info.ny if use_pixels: @@ -1999,6 +2814,7 @@ def template_window(self, window_size, age, orientation, use_pixels=False): return window def valid_data(self): + """1 where the attached DEM has usable, above-sea-level data; NaN elsewhere.""" if self.elevation is None: raise AttributeError('No elevation data! Use load_elevation first') valid = np.ones_like(self.elevation._griddata) @@ -2007,493 +2823,723 @@ def valid_data(self): return valid class LocalRelief(BaseSpatialGrid): - + """Elevation range within a circular window. + + :: + + relief = LocalRelief(elevation=dem, pixel_radius=25) + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation','pixel_radius'), '_create_from_elevation_and_radius')) - + def _create_from_elevation_and_radius(self, *args, **kwargs): + from scipy.ndimage import maximum_filter, minimum_filter + elevation = kwargs['elevation'] - pixel_radius = kwargs['pixel_radius'] - self._copy_info_from_grid(elevation,True) - - import scipy.ndimage.filters as filters - y,x = np.ogrid[-pixel_radius:pixel_radius, -pixel_radius:pixel_radius] - footprint = x*x + y*y <= pixel_radius*pixel_radius - maximum = np.zeros_like(elevation._griddata) - minimum = np.zeros_like(elevation._griddata) - filters.maximum_filter(elevation._griddata, footprint = footprint, output = maximum) - filters.minimum_filter(elevation._griddata, footprint = footprint, output = minimum) + pixel_radius = int(kwargs['pixel_radius']) + self._copy_info_from_grid(elevation, True) + + # np.ogrid[-r:r] yields 2r samples, so the window used to be one cell + # short on the far side and therefore off-centre. + y, x = np.ogrid[-pixel_radius:pixel_radius + 1, -pixel_radius:pixel_radius + 1] + footprint = x * x + y * y <= pixel_radius * pixel_radius + + data = np.asarray(elevation._griddata, dtype=float64) + maximum = maximum_filter(data, footprint=footprint) + minimum = minimum_filter(data, footprint=footprint) self._griddata = maximum - minimum - + class Mask(BaseSpatialGrid): - + """0/1 grid marking the cells that drain to a set of outlets. + + :: + + mask = Mask(flow_direction=d8, outlets=[(x, y), ...]) + """ + dtype = uint8 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction','outlets'), '_create_from_flow_direction_and_outlets')) - + def _create_from_flow_direction_and_outlets(self, *args, **kwargs): flow_direction = kwargs['flow_direction'] outlets = kwargs['outlets'] self._copy_info_from_grid(flow_direction, True) - for (outlet_x, outlet_y) in outlets: - indexes = flow_direction.get_indexes_of_upstream_cells_for_location(outlet_x, outlet_y) - for index in indexes: - self[index[0],index[1]] = 1 + # One linear pass over the flow network for all the outlets at once, + # rather than a separate upstream walk per outlet. + self._griddata = flow_direction.basin_mask(outlets).astype(self.dtype) def perform_opening(self, structure = None, iterations = 1): - from scipy.ndimage.morphology import binary_opening - self._griddata = binary_opening(self._griddata, iterations = iterations, structure = structure) + """Morphological opening: erode then dilate, removing specks.""" + from scipy.ndimage import binary_opening + self._griddata = binary_opening( + self._griddata, iterations=iterations, structure=structure).astype(self.dtype) def perform_erosion(self, structure = None, iterations = 1): - from scipy.ndimage.morphology import binary_erosion - self._griddata = binary_erosion(self._griddata, iterations = iterations, structure = structure) + """Morphological erosion: shrink the mask by ``iterations`` cells.""" + from scipy.ndimage import binary_erosion + self._griddata = binary_erosion( + self._griddata, iterations=iterations, structure=structure).astype(self.dtype) class LogArea(BaseSpatialGrid): - - dtype = uint8 - + """Base-10 logarithm of drainage area.""" + + # float64, not uint8: log10(area) is a real number, and declaring it as a + # byte meant `save()` wrote a grid of 6s and 7s. + dtype = float64 + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('area',), '_create_from_area')) - + def _create_from_area(self, *args, **kwargs): area = kwargs['area'] self._copy_info_from_grid(area, True) - self._griddata = np.log10(area._griddata) - + data = np.asarray(area._griddata, dtype=float64) + with np.errstate(divide='ignore', invalid='ignore'): + self._griddata = np.where(data > 0, np.log10(np.where(data > 0, data, 1.0)), np.nan) + class Hillshade(CalculationMixin, BaseSpatialGrid): - + """Shaded relief, following the ESRI hillshade definition. + + :: + + hs = Hillshade(elevation=dem, azimuth=315, inclination=45) + + ``azimuth`` is the compass bearing of the light source in degrees + (0 = north, 90 = east) and ``inclination`` its height above the horizon. + Values run 0-255. + """ + dtype = uint8 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation','azimuth', 'inclination'), '_create_from_elevation')) - + def _create_from_elevation(self, *args, **kwargs): elevation = kwargs['elevation'] az = kwargs['azimuth'] elev = kwargs['inclination'] - + self._copy_info_from_grid(elevation) self.calcHillshade(az, elev) - - def calcHillshade(self,az,elev): - #Hillshade = calcHillshade(elevGrid,az,elev) - #Esri calculation for generating a hillshade, elevGrid is expected to be a numpy array - - # Convert angular measurements to radians - - azRad, elevRad = (360 - az + 90)*np.pi/180, (90-elev)*np.pi/180 - Sx, Sy = self._calcFiniteSlopes(self._griddata, self._mean_pixel_dimension(), self._georef_info.nx, self._georef_info.ny) # Calculate slope in X and Y directions - - AspectRad = np.arctan2(Sy, Sx) # Angle of aspect - SmagRad = np.arctan(np.sqrt(Sx**2 + Sy**2)) # magnitude of slope in radians - - self._griddata = 255.0 * ((np.cos(elevRad) * np.cos(SmagRad)) + (np.sin(elevRad)* np.sin(SmagRad) * np.cos(azRad - AspectRad))) + + def calcHillshade(self, az, elev, z_factor=1.0): + """Fill the grid with an ESRI-style hillshade. + + Parameters + ---------- + az : float + Illumination azimuth, degrees clockwise from north. + elev : float + Illumination altitude above the horizon, in degrees. + z_factor : float + Vertical exaggeration. + """ + azRad = (360.0 - az + 90.0) * np.pi / 180.0 + zenithRad = (90.0 - elev) * np.pi / 180.0 + + # Sx is dz/dx (positive east); Sy is dz/drow (positive south), which + # is the same sign convention ESRI's [dz/dy] uses. + Sx, Sy = self._calcFiniteSlopes(self._griddata, self._mean_pixel_dimension(), + self._georef_info.nx, self._georef_info.ny) + Sx = Sx * z_factor + Sy = Sy * z_factor + + # ESRI's aspect is atan2(dz/dy, -dz/dx). Without the negation the + # illumination came out mirrored east-west. + AspectRad = np.arctan2(Sy, -Sx) + SmagRad = np.arctan(np.sqrt(Sx**2 + Sy**2)) + + shade = 255.0 * ((np.cos(zenithRad) * np.cos(SmagRad)) + + (np.sin(zenithRad) * np.sin(SmagRad) + * np.cos(azRad - AspectRad))) + # Slopes facing away from the light give a negative value; ESRI + # clamps those to 0 rather than letting them wrap round the byte. + self._griddata = np.clip(np.nan_to_num(shade), 0.0, 255.0).astype(self.dtype) class GeographicHillshade(GeographicGridMixin, Hillshade): + """Hillshade on a latitude/longitude grid.""" + pass class MaxSlope(CalculationMixin, BaseSpatialGrid): - + """Magnitude of the topographic gradient, from centred differences. + + :: + + slope = MaxSlope(elevation=dem) + + Values are dimensionless (rise over run); take ``arctan`` for degrees. + """ + dtype = float64 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation',), '_create_from_elevation')) - + def _create_from_elevation(self, *args, **kwargs): elevation = kwargs['elevation'] - + self._copy_info_from_grid(elevation) self.calcSlope() - + def calcSlope(self): - - Sx, Sy = self._calcFiniteSlopes(self._griddata, self._mean_pixel_dimension(), self._georef_info.nx, self._georef_info.ny) # Calculate slope in X and Y directions + """Replace the grid with the gradient magnitude of its own values.""" + Sx, Sy = self._calcFiniteSlopes(self._griddata, self._mean_pixel_dimension(), + self._georef_info.nx, self._georef_info.ny) self._griddata = np.sqrt(Sx**2 + Sy**2) - + class GeographicMaxSlope(GeographicGridMixin, MaxSlope): - + """Gradient magnitude on a latitude/longitude grid.""" + pass class Laplacian(CalculationMixin, BaseSpatialGrid): - + """Laplacian of elevation -- positive in valleys, negative on ridges. + + :: + + curvature = Laplacian(elevation=dem) + + Uses SciPy's 5-point stencil, scaled by ``1 / dx**2``, so the units are + inverse length. + """ + dtype = float64 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation',), '_create_from_elevation')) - + def _create_from_elevation(self, *args, **kwargs): from scipy.ndimage import laplace elevation = kwargs['elevation'] - + self._copy_info_from_grid(elevation) - self._griddata = laplace(elevation._griddata) / np.power(elevation._georef_info.dx,2) - + # _mean_pixel_dimension(), not _georef_info.dx: on a geographic grid + # dx is in DEGREES, and dividing by it gave a curvature ten orders of + # magnitude too large. The mixin overrides this with the true metric + # cell size; for a projected grid the two are the same number. + cell = self._mean_pixel_dimension(*args, **kwargs) + self._griddata = (laplace(np.asarray(elevation._griddata, dtype=float64)) + / np.power(cell, 2)) + class GeographicLaplacian(GeographicGridMixin, Laplacian): - + """Laplacian of elevation on a latitude/longitude grid.""" + pass class PriorityQueueMixIn(object): - + """Depression filling by Priority-Flood. + + Mix into a grid class to give it :meth:`_flood`, which removes closed + depressions so that every cell has a downhill path to an outlet. + + The algorithms are those of + + Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal + depression-filling and watershed-labeling algorithm for digital + elevation models." *Computers & Geosciences* 62, 117-127. + + and run in :mod:`TopoAnalysis.kernels`, in C++ where the extension is + built and in NumPy otherwise. + + Attributes + ---------- + aggradation_slope : float + Gradient of the increment added while filling. The default of 1e-12 + makes filled surfaces very slightly convergent, which gives D8 routing + something to follow across what would otherwise be a dead flat + plateau. Set it to 0 for the flat fill that TopoToolbox's + ``fillsinks`` produces. + """ + aggradation_slope = 1E-12 class priorityQueue: - #Implements a priority queue using heapq. Python has a priority queue module built in, but it - # is not stably sorted (meaning that two items who are tied in priority are treated arbitrarily, as opposed to being - # returned on a first in first out basis). This circumvents that by keeping a count on the items inserted and using that - # count as a secondary priority - + """Stably-ordered priority queue. + + Retained because external code constructs it directly. The filling + routines no longer use it -- they call the compiled kernel instead -- + so it is kept purely for compatibility. + """ + def __init__(self): - # A counter and the number of items are stored separately to ensure that items remain stably sorted and to - # keep track of the size of the queue (so that we can check if its empty, which will be useful will iterating - # through the queue) + import heapq + self._heapq = heapq self.__pq = [] self.__counter = 0 self.__nItems = 0 - + def get(self): - #Remove an item and its priority from the queue - priority, count, item = heapq.heappop(self.__pq) + """Pop the lowest-priority item; returns ``(priority, item)``.""" + priority, count, item = self._heapq.heappop(self.__pq) self.__nItems -= 1 return priority, item - + def put(self, priority, item): - #Add an item to the priority queue + """Insert ``item``. Equal priorities come back out in FIFO order.""" self.__counter += 1 self.__nItems += 1 - entry = [priority, self.__counter, item] - heapq.heappush(self.__pq, entry) - + self._heapq.heappush(self.__pq, [priority, self.__counter, item]) + def isEmpty(self): return self.__nItems == 0 def randomize_subbasins_with_mask(self, *args, **kwargs): + """Replace elevations inside a mask with random values and re-flood. + + Used to generate synthetic drainage networks with the same outlets as + a real one. + """ mask = args[0] outlets = args[1] - - self.__flood(mask = mask, outlets = outlets, randomize = True) - + + # `self.__flood` inside this class mangles to _PriorityQueueMixIn__flood, + # which does not exist -- the method is _flood. + self._flood(mask=mask, outlets=outlets, randomize=True) + def _flood(self, *args, **kwargs): - # dem is a numpy array of elevations to be flooded, aggInc is the minimum amount to increment elevations by moving upstream - # use priority flood algorithm described in Barnes et al., 2013 - # Priority-Flood: An Optimal Depression-Filling and Watershed-Labeling Algorithm for Digital Elevation Models - # NOTE: They have another algorithm to make this more efficient, but to use that and a slope makes things more - # complicated - - #Create a grid to keep track of which cells have been filled - closed = np.zeros_like(self._griddata) - i = np.where(np.isnan(self._griddata)) - closed[i] = 1 - #using_mask = False - - if kwargs.get('mask') is not None: - closed = (kwargs.get('mask')._griddata != 1).astype(int) - if kwargs.get('outlets') is not None: - v = self._xy_to_rowscols(kwargs.get('outlets')) - #using_mask = True - edgeRows = [a[0] for a in v] - edgeCols = [a[1] for a in v] - else: - #Add all the edge cells to the priority queue, mark those cells as draining (not closed) - edgeRows, edgeCols = self.findDEMedge() - + """Fill depressions in ``self._griddata``, in place. + + Keyword arguments + ----------------- + mask : BaseSpatialGrid, optional + Only cells where the mask equals 1 take part. + outlets : sequence of (x, y), optional + Flood inwards from these points instead of from the DEM edge. + randomize : bool, optional + Randomise elevations inside the mask before flooding. + maximum_pit_depth : float, optional + Leave depressions deeper than this unfilled. + binary_result : bool, optional + Replace the grid with a 0/1 map of the cells the flood reached. + clip_to_fill : bool, optional + Set cells the flood never reached to ``NaN``. + + Returns + ------- + dict + ``{'cells_filled': int, 'max_fill_depth': float, + 'visited': ndarray|None}``. + """ + elevations = np.ascontiguousarray(self._griddata, dtype=float64) + # Kept for the clip_to_fill/maximum_pit_depth comparison below. + original = elevations.copy() if kwargs.get('clip_to_fill') else None + + mask = kwargs.get('mask') + closed = None + if mask is not None: + closed = (np.asarray(mask._griddata) != 1).astype(np.uint8) + if kwargs.get('randomize') is True: - self._randomize_grid_values(mask = kwargs['mask']) - - should_randomize_priority_queue = False - - if kwargs.get('randomize') is not None: - if kwargs.get('randomize') == True: - should_randomize_priority_queue = True - - priority_queue = FilledElevation.priorityQueue() # priority queue to sort filling operation - - for i in range(len(edgeCols)): - row, col = edgeRows[i], edgeCols[i] - - closed[row, col] = True - priority_queue.put(self._griddata[row, col], (row, col)) # store the indices as a vector of row column, in the priority queue prioritized by the dem value - - if kwargs.get('binary_result') is True or kwargs.get('clip_to_fill') is True: - visited = np.zeros_like(self._griddata, dtype = np.float) - - #While there is anything left in the priority queue, continue to fill holes - while not priority_queue.isEmpty(): - - priority, (row, col) = priority_queue.get() - - if kwargs.get('binary_result') is True or kwargs.get('clip_to_fill') is True: - visited[row, col] = 1.0 - - elevation = max(priority, self[row,col]) - - neighborRows, neighborCols, dxMults = self._getNeighborIndices(row, col) - - dxs = self._georef_info.dx * dxMults - - #Look through the upstream neighbors - for i in range(len(neighborCols)): - should_fill = True - - if not closed[neighborRows[i], neighborCols[i]]: - #If this was a hole (lower than the cell downstream), fill it - if self._griddata[neighborRows[i], neighborCols[i]] <= elevation: - - if kwargs.get('maximum_pit_depth'): - fill_depth = elevation - self._griddata[neighborRows[i], neighborCols[i]] - if fill_depth > kwargs.get('maximum_pit_depth'): - should_fill = False - - if should_fill: - self._griddata[neighborRows[i], neighborCols[i]] = elevation + self.aggradation_slope*dxs[i] - - closed[neighborRows[i], neighborCols[i]] = True - if should_fill: - if should_randomize_priority_queue: - # Use elevation as primary priority, random value for tie-breaking - priority = self._griddata[neighborRows[i], neighborCols[i]] + np.random.rand(1)[0] * 1e-10 - else: - priority = self._griddata[neighborRows[i], neighborCols[i]] - priority_queue.put(priority, [neighborRows[i], neighborCols[i]]) - + self._griddata = elevations + self._randomize_grid_values(mask=mask) + elevations = np.ascontiguousarray(self._griddata, dtype=float64) + if original is not None: + original = elevations.copy() + + seeds = None + outlets = kwargs.get('outlets') + if outlets is not None: + flat = [] + for (row, col) in self._xy_to_rowscols(outlets): + if row is None: + continue + flat.append(row * self._georef_info.nx + col) + if not flat: + raise Error.InputError('_flood', 'none of the outlets lie inside the grid') + seeds = np.array(flat, dtype=np.int64) + + want_visited = bool(kwargs.get('binary_result') or kwargs.get('clip_to_fill')) + max_pit_depth = kwargs.get('maximum_pit_depth') or 0.0 + + kernels.warn_if_slow(elevations.size) + result = kernels.priority_flood( + elevations, + closed=closed, + seeds=seeds, + # A non-zero aggradation slope is Barnes et al.'s Algorithm 4 + # (Priority-Flood+epsilon); zero gives the flat fill of + # Algorithm 2, which is what TopoToolbox's fillsinks returns. + mode='epsilon' if self.aggradation_slope > 0 else 'flat', + epsilon=self.aggradation_slope, + cellsize=self._georef_info.dx, + max_pit_depth=max_pit_depth, + track_visited=want_visited, + ) + + self._griddata = elevations.astype(self.dtype, copy=False) + + visited = result['visited'] if kwargs.get('binary_result'): - self._griddata = visited - if kwargs.get('clip_to_fill') is True: - self._griddata[visited == 0] = np.NAN - - - + self._griddata = np.asarray(visited, dtype=self.dtype) + elif kwargs.get('clip_to_fill') is True: + self._griddata = self._griddata.astype(float64, copy=True) + self._griddata[np.asarray(visited) == 0] = np.nan + if max_pit_depth: + # A depth limit leaves whole depressions unfilled that the + # flood nonetheless traverses, so `visited` no longer marks + # them. Compare against an unlimited fill: the cells that + # would have risen but did not are exactly the depressions + # that were declined, which is what the old implementation + # excised here. + unlimited = np.ascontiguousarray( + np.array(original, dtype=float64, copy=True)) + kernels.priority_flood( + unlimited, + closed=None if closed is None else closed.copy(), + seeds=seeds, + mode='epsilon' if self.aggradation_slope > 0 else 'flat', + epsilon=self.aggradation_slope, + cellsize=self._georef_info.dx) + declined = unlimited > self._griddata + 1e-12 + self._griddata[declined] = np.nan + + return result + class PriorityFillGrid(PriorityQueueMixIn, BaseSpatialGrid): - + """Cells of a mask that are connected to a set of outlets. + + Floods outwards from ``outlets`` through the cells where ``mask`` is 1 + and returns a 0/1 grid of everything reached. Used to grow a valley + network from a curvature mask, keeping only the parts that connect to a + real channel. + """ + dtype = uint8 - + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('outlets', 'mask'), '_create_from_outlets_and_mask')) - + def _create_from_outlets_and_mask(self, *args, **kwargs): - + mask = kwargs['mask'] outlets = kwargs['outlets'] + kwargs = dict(kwargs) kwargs['randomize'] = False kwargs['binary_result'] = True - self._copy_info_from_grid(mask,True) - rcs = self._xy_to_rowscols(outlets) - for rc in rcs: - self._griddata[rc[0],rc[1]] = 100 - i = np.where(np.isnan(mask._griddata)) - self._griddata[i] = 0 - i = np.where(mask._griddata != 1) - self._griddata[i] = 0 + + self._copy_info_from_grid(mask, True) + # Flooding needs a surface to descend; give the outlets a high value + # so the flood spreads from them through the rest of the mask. + self._griddata = np.zeros((self._georef_info.ny, self._georef_info.nx), + dtype=float64) + for (row, col) in self._xy_to_rowscols(outlets): + if row is not None: + self._griddata[row, col] = 100.0 + self._griddata[np.asarray(mask._griddata) != 1] = 0.0 + self._flood(*args, **kwargs) - + self._griddata = np.asarray(self._griddata, dtype=self.dtype) + class FilledElevation(PriorityQueueMixIn, Elevation): - - + """A DEM with its closed depressions removed. + + :: + + filled = FilledElevation(elevation=dem) + + Filling uses Priority-Flood (Barnes et al. 2014). By default the filled + surface carries a gradient of + :attr:`PriorityQueueMixIn.aggradation_slope` across former depressions so + that D8 routing has a direction to follow; pass + ``aggradation_slope=0`` -- or set the class attribute -- for the flat fill + that TopoToolbox's ``fillsinks`` produces. + + Other keywords (``mask``, ``outlets``, ``maximum_pit_depth``, + ``clip_to_fill``) are passed to :meth:`PriorityQueueMixIn._flood`. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation',), '_create_from_elevation')) - + def _create_from_elevation(self, *args, **kwargs): elevation = kwargs['elevation'] self._copy_info_from_grid(elevation) - self._flood(*args, **kwargs) - + if 'aggradation_slope' in kwargs: + # Per-instance override, so one call can ask for a flat fill + # without changing the default for every other grid. + self.aggradation_slope = kwargs['aggradation_slope'] + self.fill_report = self._flood(*args, **kwargs) + + class Area(BaseSpatialGrid): - + """D8 contributing (drainage) area. + + :: + + area = Area(flow_direction=d8) + + Values are in squared map units: each cell contributes its own area + (``dx**2``, or the true spherical cell area for a + :class:`GeographicArea`) plus everything upstream of it. + + To count contributing *cells* instead -- which is what TopoToolbox's + ``flowacc`` returns -- divide by the per-cell area, or pass + ``weights=np.ones(...)``. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction',), '_create_from_flow_direction'), (('dx', 'grid'), '_create_from_grid')) def _create_from_flow_direction(self, *args, **kwargs): - + flow_dir = kwargs['flow_direction'] self._copy_info_from_grid(flow_dir, True) - #if flow_dir.__class__ == FlowDirectionD8: - self.__calcD8Area(*args, **kwargs) - - def __calcD8Area(self, *args, **kwargs): - - # I am returning area and flowDirections but NOTE!I might be in danger the way I handle this... - # Flow directions are indices of arrays which may be different sizes, this is not truly giving me flow directions - # Because in the getNeighbor function I don't return values at edges.... might want to think about improving this, - # although it should work in this application - # Calculate the D8 drainage area for the numpy array representing a demMethods in elevGrid, assumes that elevGrid has already been filled - # Assigns BCs to deal with edges - these should be changed to handle flow differently, currently it will not force flow through the edges - + self._calc_d8_area(*args, **kwargs) + + def _calc_d8_area(self, *args, **kwargs): + """Accumulate per-cell weights down the D8 network. + + Keyword arguments + ----------------- + flow_direction : FlowDirectionD8 + weights : ndarray, optional + Per-cell contribution. Defaults to the cell area. + mask, evaluate_at : BaseSpatialGrid, optional + Cells where these are 0 contribute nothing and pass nothing on. + """ flow_dir = kwargs['flow_direction'] + kernels.warn_if_slow(flow_dir._griddata.size) + + weights = kwargs.get('weights') + if weights is None: + weights = self._area_per_pixel(*args, **kwargs) + weights = np.ascontiguousarray(weights, dtype=float64) + + gate = None + for gate_name in ('evaluate_at', 'mask'): + gate_grid = kwargs.get(gate_name) + if gate_grid is None: + continue + # `mask[i, j] is not None` was always true -- __getitem__ returns + # the value, not None, for a masked-out cell -- so a supplied + # mask never actually excluded anything. + this_gate = (np.asarray(gate_grid._griddata) != 0).astype(np.uint8) + gate = this_gate if gate is None else (gate & this_gate) + + # Ordering comes from the flow network itself rather than from an + # elevation sort, so it stays correct for a flow-direction grid + # loaded from disk, where `sort()` would have ordered cells by their + # direction *code*. + receivers = flow_dir.receivers + order = flow_dir.topological_order + self._griddata = np.asarray( + kernels.accumulate(receivers, order, weights, gate)) + + # Preserved under its original name-mangled spelling for any external + # caller that reached in for it. + _Area__calcD8Area = _calc_d8_area - if kwargs.get('sorted_indexes') is not None: - idcs = kwargs.get('sorted_indexes') - else: - idcs = flow_dir.sort() - - area = self._area_per_pixel(*args, **kwargs) # area of a pixel - - [ind_i, ind_j] = np.unravel_index(idcs, flow_dir._griddata.shape) - - has_mask = kwargs.get('mask') is not None - has_evaluate_at = kwargs.get('evaluate_at') is not None - - if has_evaluate_at: - area[kwargs['evaluate_at']._griddata == 0] = 0 - - - for i, j in zip(ind_i, ind_j): # Loop through all the data in sorted order - i_next, j_next, is_good = flow_dir.get_flow_to_cell(i,j) - if is_good and not has_mask: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - area[i_next, j_next] += area[i,j] - elif is_good and kwargs['mask'][i, j] is not None: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - area[i_next, j_next] += area[i,j] - - self._griddata = area # Return non bc version of area - def areas_greater_than(self, min_area): - ij_cols = np.where(self._griddata >= min_area) - ij = zip(ij_cols[0].tolist(), ij_cols[1].tolist()) - return self._rowscols_to_xy(ij) - + """Map coordinates of every cell whose area is at least ``min_area``.""" + rows, cols = np.where(self._griddata >= min_area) + return self._rowscols_to_xy(list(zip(rows.tolist(), cols.tolist()))) + def areas_between(self, fd, min_area, max_area): + """Channel heads: cells in an area band whose donors are all below it. + + These are the points where drainage area first crosses ``min_area``, + so they mark the upstream ends of the channel network. + """ ij_out = [] - ij_cols = np.where((self._griddata >= min_area) & (self._griddata <= max_area)) - for (i,j) in zip(ij_cols[0].tolist(), ij_cols[1].tolist()): - options = fd.get_upstream_cell_indexes(i,j) - max_a = None - for (i_p, j_p) in options: - if max_a is None or (self._griddata[i_p, j_p] > max_a): - max_a = self._griddata[i_p, j_p] + rows, cols = np.where((self._griddata >= min_area) & (self._griddata <= max_area)) + for (i, j) in zip(rows.tolist(), cols.tolist()): + options = fd.get_upstream_cell_indexes(i, j) + # No upstream neighbour at all means this cell *is* a source, so + # it qualifies. The original compared None < min_area and raised. + if not options: + ij_out.append((i, j)) + continue + max_a = max(self._griddata[i_p, j_p] for (i_p, j_p) in options) if max_a < min_area: - ij_out += [(i,j)] + ij_out.append((i, j)) return self._rowscols_to_xy(ij_out) - - + + class GeographicArea(GeographicGridMixin, Area): + """Drainage area on a latitude/longitude grid, using true cell areas.""" + pass -class ValleyArea(Area): - - required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), - (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), - (('flow_direction', 'area', 'laplace', 'valley_laplace_value', 'min_area_value'), '_create_from_flow_direction_and_valley_slope_and_area')) +class _ValleyMaskMixin(object): + """Shared construction of a connected valley-floor mask. + + Cells whose Laplacian exceeds ``valley_laplace_value`` are convergent + (valley-like). Those that are connected to a cell with more than + ``min_area_value`` of drainage area form the valley network; isolated + convergent patches on hillslopes are discarded. + """ + + def _valley_mask(self, *args, **kwargs): + from scipy.ndimage import binary_dilation, binary_erosion - def _create_from_flow_direction_and_valley_slope_and_area(self, *args, **kwargs): - mask = Mask() mask._copy_info_from_grid(kwargs['laplace'], True) - mask._griddata[kwargs['laplace']._griddata >= kwargs['valley_laplace_value'] ] = 1 + mask._griddata[kwargs['laplace']._griddata >= kwargs['valley_laplace_value']] = 1 + outlets = kwargs['area'].areas_greater_than(kwargs['min_area_value']) - pfg = PriorityFillGrid(mask = mask, outlets = outlets) - import scipy.ndimage.morphology as morph - if kwargs.get('iterations') is None or kwargs.get('iterations') == 0: - pass - else: - iterations = kwargs['iterations'] - pfg._griddata = morph.binary_dilation(pfg._griddata, iterations = iterations) - pfg._griddata = morph.binary_erosion(pfg._griddata, iterations = iterations) - - kwargs['evaluate_at'] = pfg + pfg = PriorityFillGrid(mask=mask, outlets=outlets) + + iterations = kwargs.get('iterations') or 0 + if iterations: + # Closing: dilate then erode, to bridge one-cell gaps in the + # valley network without growing it overall. + pfg._griddata = binary_dilation(pfg._griddata, iterations=iterations) + pfg._griddata = binary_erosion( + pfg._griddata, iterations=iterations).astype(pfg.dtype) + return pfg + + def _create_from_flow_direction_and_valley_slope_and_area(self, *args, **kwargs): + kwargs = dict(kwargs) + kwargs['evaluate_at'] = self._valley_mask(*args, **kwargs) self._create_from_flow_direction(*args, **kwargs) - - + + +class ValleyArea(_ValleyMaskMixin, Area): + """Drainage area accumulated only over valley-floor cells. + + :: + + va = ValleyArea(flow_direction=d8, area=area, laplace=laplacian, + valley_laplace_value=0.01, min_area_value=1e6) + """ + + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), + (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), + (('gdal_filename',), '_read_gdal'), + (('flow_direction', 'area', 'laplace', 'valley_laplace_value', 'min_area_value'), '_create_from_flow_direction_and_valley_slope_and_area')) + + class GeographicValleyArea(GeographicGridMixin, ValleyArea): - pass - -class MainstemValleyArea(Area): - + """Valley-floor area on a latitude/longitude grid.""" + + pass + + +class MainstemValleyArea(_ValleyMaskMixin, Area): + """Valley-floor area accumulated along the main stem only. + + Where :class:`ValleyArea` sums every tributary, this follows only the + longest flow path into each cell, so the result is the valley area of the + trunk stream rather than of the whole basin. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('area', 'laplace', 'valley_laplace_value', 'min_area_value', 'flow_direction'), '_create_from_flow_direction_and_valley_slope_and_area')) - def _create_from_flow_direction_and_valley_slope_and_area(self, *args, **kwargs): - - mask = Mask() - mask._copy_info_from_grid(kwargs['laplace'], True) - mask._griddata[kwargs['laplace']._griddata >= kwargs['valley_laplace_value'] ] = 1 - outlets = kwargs['area'].areas_greater_than(kwargs['min_area_value']) - pfg = PriorityFillGrid(mask = mask, outlets = outlets) - import scipy.ndimage.morphology as morph - if kwargs.get('iterations') is None or kwargs.get('iterations') == 0: - pass - else: - iterations = kwargs['iterations'] - pfg._griddata = morph.binary_dilation(pfg._griddata, iterations = iterations) - pfg._griddata = morph.binary_erosion(pfg._griddata, iterations = iterations) - - kwargs['evaluate_at'] = pfg - self._create_from_flow_direction(*args, **kwargs) - - def __calcD8Area(self, *args, **kwargs): - + def _calc_d8_area(self, *args, **kwargs): + """Accumulate down the longest-flow-length path only. + + Notes + ----- + The original version of this method was named ``__calcD8Area``, so + Python mangled it to ``_MainstemValleyArea__calcD8Area`` while the + inherited caller looked for ``_Area__calcD8Area``. It was therefore + never called, and ``MainstemValleyArea`` silently behaved exactly like + :class:`ValleyArea`. It also compared a whole grid object against a + scalar, which would have raised as soon as it did run. + """ flow_dir = kwargs['flow_direction'] + kernels.warn_if_slow(flow_dir._griddata.size) + + dA = np.ascontiguousarray(self._area_per_pixel(*args, **kwargs), dtype=float64) + step = flow_dir.step_lengths(self._mean_pixel_dimension(*args, **kwargs)) + + gate = None + for gate_name in ('evaluate_at', 'mask'): + gate_grid = kwargs.get(gate_name) + if gate_grid is not None: + this_gate = (np.asarray(gate_grid._griddata) != 0).astype(np.uint8) + gate = this_gate if gate is None else (gate & this_gate) + if gate is not None: + dA = np.where(gate != 0, dA, 0.0) + + receivers = flow_dir.receivers + order = flow_dir.topological_order + _, main_stem = kernels.flow_length(receivers, order, + np.ascontiguousarray(step, dtype=float64)) + + self._griddata = np.asarray( + kernels.propagate_along_main_stem(receivers, order, main_stem, gate, dA, 'sum')) + + _Area__calcD8Area = _calc_d8_area + - if kwargs.get('sorted_indexes') is not None: - idcs = kwargs.get('sorted_indexes') - else: - idcs = flow_dir.sort() - - dA = self._area_per_pixel(*args, **kwargs) - dx = self._mean_pixel_dimension(*args, **kwargs) * flow_dir.pixel_scale() - - length = np.zeros_like(dA) - - [ind_i, ind_j] = np.unravel_index(idcs, flow_dir._griddata.shape) - - has_mask = kwargs.get('mask') is not None - has_evaluate_at = kwargs.get('evaluate_at') is not None - - if has_evaluate_at: - dA[kwargs['evaluate_at']._griddata == 0] = 0 - - for i, j in zip(ind_i, ind_j): # Loop through all the data in sorted order - i_next, j_next, is_good = flow_dir.get_flow_to_cell(i,j) - - if is_good: - next_l = length[i,j] + dx[i,j] - if next_l > length[i_next, j_next]: - length[i_next, j_next] = next_l - if not has_mask: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - self._griddata[i_next, j_next] = dA[i_next,j_next] + self._griddata[i,j] - elif kwargs['mask'][i, j] is not None: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - self._griddata[i_next, j_next] = dA[i_next,j_next] + self._griddata[i,j] - elif kwargs['area'] <= kwargs['min_area_value']: - if not has_mask: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - self._griddata[i_next, j_next] += self._griddata[i,j] - elif kwargs['mask'][i, j] is not None: - if not has_evaluate_at or (has_evaluate_at and (kwargs['evaluate_at'][i,j] == 1)): - self._griddata[i_next, j_next] += self._griddata[i,j] - class GeographicMainstemValleyArea(GeographicGridMixin, MainstemValleyArea): + """Main-stem valley area on a latitude/longitude grid.""" + pass -class AlongFlowSmoothing(object): +def _chi_profile_along_path(points, area_grid, elevation_grid, de, theta): + """Chi and elevation sampled along an ordered list of cells. + + ``points`` is a 2 x n integer array of rows and columns running + downstream to upstream. The integral uses the trapezoidal rule over the + along-path distance, with diagonal steps counted as sqrt(2) cell widths. + + Returns + ------- + (ndarray, ndarray) + ``(chi, elevation)``, both length n, with ``chi[0] == 0``. + """ + n = points.shape[1] + adjustment = np.ones(n) + if n > 2: + diagonal = ((points[0, 1:-1] != points[0, 2:]) + & (points[1, 1:-1] != points[1, 2:])) + adjustment[np.flatnonzero(diagonal) + 1] += (SQRT2 - 1.0) + + area_profile = area_grid[points[0], points[1]] + elevation_profile = elevation_grid[points[0], points[1]] + de_profile = de[points[0], points[1]] + + chi_profile = np.zeros_like(elevation_profile, dtype=float64) + if n > 1: + integrand = 0.5 * (np.power(area_profile[1:], -theta) + + np.power(area_profile[:-1], -theta)) + step = 0.5 * (de_profile[1:] + de_profile[:-1]) * adjustment[1:] + chi_profile[1:] = np.cumsum(integrand * step) + return chi_profile, elevation_profile + + +class AlongFlowSmoothing(object): + """Collect the cells within a window along each flow path. + + Mixed into the ``*WithSmoothing`` classes, which fit a model over that + window at every cell. The window is set either by drop + (``vertical_interval``) or by distance (``horizontal_interval``), and + extends both upstream and downstream of the cell being evaluated. + """ + def _find_points_along_path(self, de, **kwargs): area = kwargs['area'] @@ -2572,6 +3618,24 @@ def find_points_along_path(this_i, this_j): return find_points_along_path class KsFromChiWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): + """Channel steepness fitted to chi--elevation in a moving window. + + :: + + ks = KsFromChiWithSmoothing(elevation=dem, area=area, + flow_direction=d8, theta=0.45, + vertical_interval=50.0) + + At every channel cell, chi and elevation are extracted over a window + along the flow path and a line through the origin is fitted; its slope + is k_s. ``save()`` writes seven bands: ks, the number of profiles + crossing each cell, mean squared error, sum of squares, r-squared, + p-value, and the number of points in the regression. + + Needs ``statsmodels``. Pass ``verbose=False`` to silence the progress + ticker. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), @@ -2580,13 +3644,15 @@ class KsFromChiWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): ) def _create_from_elevation_area_flow_direction(self, *args, **kwargs): - + + sm = _require_statsmodels() + elevation = kwargs['elevation'] area = kwargs['area'] theta = kwargs['theta'] area_threshold = kwargs.get('area_threshold', 0) de = area._mean_pixel_dimension() - + self._copy_info_from_grid(elevation) self._griddata = np.zeros_like(elevation._griddata) self._n = np.zeros_like(self._griddata).astype(int) @@ -2599,58 +3665,45 @@ def _create_from_elevation_area_flow_direction(self, *args, **kwargs): find_points_along_path = self._find_points_along_path(de, **kwargs) - def calc_ks(i,j): - points = find_points_along_path(i, j) - if points is not None: - pts = list(zip(*(points))) - points = np.array(pts).astype(int) - adjustment = np.ones((len(points[0]))) - i = np.where((points[0,1:-1] != points[0,2:]) & (points[1,1:-1] != points[1,2:])) - adjustment[i[0]+1] += 0.414 - area_profile = area._griddata[points[0],points[1]] - elevation_profile = elevation._griddata[points[0], points[1]] - de_profile = de[points[0], points[1]] - chi_profile = np.zeros_like(elevation_profile) - chi_profile[1:] = np.cumsum(0.25*(np.power(area_profile[1:], -theta) + np.power(area_profile[0:-1], -theta)) * (de_profile[1:] + de_profile[0:-1])*adjustment[1:]) - X = np.array(chi_profile) - y = np.array(elevation_profile) - elevation_profile[0] - model = sm.OLS(y, X) - res = model.fit() - SS = res.ssr - return res.params[0], SS / float(len(chi_profile)), SS, res.rsquared, points, res.pvalues[0], len(chi_profile) - - else: - - return np.nan, np.nan, np.nan, np.nan, [[],[]], np.nan, 0 - - i = np.where(~np.isnan(area._griddata) & ~np.isnan(elevation._griddata) & (area._griddata > area_threshold)) - ij = list(zip(i[0],i[1])) - totalnumber = len(ij) - counter = 0.0 - next_readout = 0.1 - sys.stdout.write('Percent completion...') - sys.stdout.flush() - for (i,j) in ij: - self._griddata[i,j], self._mse[i,j], self._ss[i,j], self._r2[i,j], pts, self._pval[i,j], self._n_regression[i,j] = calc_ks(i,j) - self._n[pts[0], pts[1]] += 1 - counter += 1.0 / totalnumber - if counter > next_readout: - sys.stdout.write(str(int(next_readout*100)) + "...") - sys.stdout.flush() - next_readout += 0.1 - - sys.stdout.write('Percent completion...') - sys.stdout.flush() - sys.stdout.write('100') - sys.stdout.flush() - + def calc_ks(i, j): + points = find_points_along_path(i, j) + if points is None: + return np.nan, np.nan, np.nan, np.nan, [[], []], np.nan, 0 + + points = np.array(list(zip(*points))).astype(int) + chi_profile, elevation_profile = _chi_profile_along_path( + points, area._griddata, elevation._griddata, de, theta) + + X = np.array(chi_profile) + y = np.array(elevation_profile) - elevation_profile[0] + res = sm.OLS(y, X).fit() + SS = res.ssr + return (res.params[0], SS / float(len(chi_profile)), SS, res.rsquared, + points, res.pvalues[0], len(chi_profile)) + + rows, cols = np.where(~np.isnan(area._griddata) & ~np.isnan(elevation._griddata) + & (area._griddata > area_threshold)) + ij = list(zip(rows, cols)) + progress = _Progress(len(ij), kwargs.get('verbose', True)) + for (i, j) in ij: + (self._griddata[i, j], self._mse[i, j], self._ss[i, j], self._r2[i, j], + pts, self._pval[i, j], self._n_regression[i, j]) = calc_ks(i, j) + # np.add.at, not `+=`: with repeated subscripts a fancy-indexed + # `+=` applies each increment only once, so cells crossed by + # several profiles were undercounted. + if len(pts[0]): + np.add.at(self._n, (pts[0], pts[1]), 1) + progress.tick() + progress.done() + + def save(self, filename): self._create_gdal_representation_from_array(self._georef_info, 'GTiff', [self._griddata, self._n, self._mse, self._ss, self._r2, self._pval, self._n_regression], self.dtype, filename, ['COMPRESS=LZW', 'BIGTIFF=YES'], multiple_bands=True) @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -2659,7 +3712,7 @@ def get_band(gdal_dataset, band_number): nodata_elements = np.where(grid == nodata) from numpy import uint8 if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -2690,6 +3743,16 @@ def get_band(gdal_dataset, band_number): class ThetaFromChiWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): + """Concavity fitted to chi--elevation in a moving window. + + As :class:`KsFromChiWithSmoothing`, but ``theta`` is optimised at each + cell rather than held fixed: the value returned is the one that makes + the chi--elevation relation most nearly linear. The search is bounded + at +/-10. + + Needs ``statsmodels``. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',), '_create'), (('ai_ascii_filename', 'EPSGprojectionCode'), '_read_ai'), (('gdal_filename',), '_read_gdal'), @@ -2701,6 +3764,10 @@ class ThetaFromChiWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): def _create_from_elevation_area_flow_direction(self, *args, **kwargs): + from scipy.optimize import fmin + + sm = _require_statsmodels() + elevation = kwargs['elevation'] area = kwargs['area'] min_area = kwargs['min_area'] @@ -2717,72 +3784,53 @@ def _create_from_elevation_area_flow_direction(self, *args, **kwargs): self._griddata[:] = np.nan find_points_along_path = self._find_points_along_path(de, **kwargs) + empty = (np.nan, np.nan, np.nan, np.nan, [[], []], np.nan, 0) def calc_theta(i, j): points = find_points_along_path(i, j) - if points is not None: - pts = list(zip(*(points))) - points = np.array(pts).astype(int) - adjustment = np.ones((len(points[0]))) - i = np.where((points[0, 1:-1] != points[0, 2:]) & (points[1, 1:-1] != points[1, 2:])) - adjustment[i[0] + 1] += 0.414 - area_profile = area._griddata[points[0], points[1]] - elevation_profile = elevation._griddata[points[0], points[1]] - de_profile = de[points[0], points[1]] - - def r2_for_theta(theta): - if theta > 10 or theta < -10: - return np.inf - chi_profile = np.zeros_like(elevation_profile) - chi_profile[1:] = np.cumsum( - 0.25 * (np.power(area_profile[1:], -theta) + np.power(area_profile[0:-1], -theta)) * ( - de_profile[1:] + de_profile[0:-1]) * adjustment[1:]) - X = np.array(chi_profile) - y = np.array(elevation_profile) - elevation_profile[0] - model = sm.OLS(y, X) - res = model.fit() - return res.ssr - from scipy.optimize import fmin - try: - (theta_bf, funval, iter, funcalls, warnflag) = fmin(r2_for_theta, np.array([0.5]), (), 1E-5, 1E-5, 100, 200, True, False, 0, None) - chi_profile = np.zeros_like(elevation_profile) - chi_profile[1:] = np.cumsum( - 0.25 * (np.power(area_profile[1:], -theta_bf) + np.power(area_profile[0:-1], -theta_bf)) * ( - de_profile[1:] + de_profile[0:-1]) * adjustment[1:]) - X = np.array(chi_profile) - y = np.array(elevation_profile) - elevation_profile[0] - model = sm.OLS(y, X) - res = model.fit() - SS = res.ssr - return theta_bf, SS / float(len(chi_profile)), SS, res.rsquared, points, res.pvalues[0], len(chi_profile) - except: - return np.nan, np.nan, np.nan, np.nan, [[], []], np.nan, 0 - else: + if points is None: + return empty - return np.nan, np.nan, np.nan, np.nan, [[],[]], np.nan, 0 + points = np.array(list(zip(*points))).astype(int) - i = np.where((area._griddata != 0) & ~np.isnan(area._griddata) & ~np.isnan(elevation._griddata) & (area._griddata > min_area)) - ij = list(zip(i[0], i[1])) - totalnumber = len(ij) - counter = 0.0 - next_readout = 0.1 - sys.stdout.write('Percent completion...') - sys.stdout.flush() - for (i, j) in ij: - self._griddata[i, j], self._mse[i, j], self._ss[i, j], self._r2[i, j], pts, self._pval[i, j], \ - self._n_regression[i, j] = calc_theta(i, j) - self._n[pts[0], pts[1]] += 1 - counter += 1.0 / totalnumber - if counter > next_readout: - sys.stdout.write(str(int(next_readout * 100)) + "...") - sys.stdout.flush() - next_readout += 0.1 + def fit_for_theta(theta): + chi_profile, elevation_profile = _chi_profile_along_path( + points, area._griddata, elevation._griddata, de, theta) + X = np.array(chi_profile) + y = np.array(elevation_profile) - elevation_profile[0] + return sm.OLS(y, X).fit(), chi_profile - sys.stdout.write('Percent completion...') - sys.stdout.flush() - sys.stdout.write('100') - sys.stdout.flush() + def ssr_for_theta(theta): + theta = float(np.atleast_1d(theta)[0]) + if theta > 10 or theta < -10: + return np.inf + return fit_for_theta(theta)[0].ssr + + try: + theta_bf, funval, iterations, funcalls, warnflag = fmin( + ssr_for_theta, np.array([0.5]), xtol=1E-5, ftol=1E-5, + maxiter=100, maxfun=200, full_output=True, disp=False) + res, chi_profile = fit_for_theta(float(theta_bf[0])) + SS = res.ssr + return (float(theta_bf[0]), SS / float(len(chi_profile)), SS, + res.rsquared, points, res.pvalues[0], len(chi_profile)) + except (ValueError, np.linalg.LinAlgError, ZeroDivisionError, FloatingPointError): + # A bare `except:` here also swallowed KeyboardInterrupt and + # genuine programming errors. + return empty + + rows, cols = np.where((area._griddata != 0) & ~np.isnan(area._griddata) + & ~np.isnan(elevation._griddata) & (area._griddata > min_area)) + ij = list(zip(rows, cols)) + progress = _Progress(len(ij), kwargs.get('verbose', True)) + for (i, j) in ij: + (self._griddata[i, j], self._mse[i, j], self._ss[i, j], self._r2[i, j], + pts, self._pval[i, j], self._n_regression[i, j]) = calc_theta(i, j) + if len(pts[0]): + np.add.at(self._n, (pts[0], pts[1]), 1) + progress.tick() + progress.done() def save(self, filename): @@ -2793,7 +3841,7 @@ def save(self, filename): @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -2802,7 +3850,7 @@ def get_band(gdal_dataset, band_number): nodata_elements = np.where(grid == nodata) from numpy import uint8 if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -2837,22 +3885,32 @@ def get_band(gdal_dataset, band_number): class ChannelSlopeWithSmoothing(BaseSpatialGrid, AlongFlowSmoothing): + """Channel gradient measured over a window along the flow path. + + Far less noisy than a cell-to-cell slope, which on a real DEM is mostly + vertical quantisation. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',), '_create'), (('ai_ascii_filename', 'EPSGprojectionCode'), '_read_ai'), (('gdal_filename',), '_read_gdal'), (('elevation', 'area', 'flow_direction', 'vertical_interval'), '_create_from_elevation_area_flow_direction'), (('elevation', 'area', 'flow_direction', 'horizontal_interval'), - '_create_from_elevation_flow_direction'), + '_create_from_elevation_area_flow_direction'), ) scale_factor = 1.0 def points_along_path(self, points, i, j): + """Which part of the window to measure over; the whole of it here.""" return points def calc_channel_slope(self, i, j, elevation, de, find_points_along_path): + """Gradient across the window centred on ``(i, j)``. + Returns ``NaN`` where the window runs off the end of the network. + """ points = find_points_along_path(i, j) if points is None or len(points) == 0: @@ -2895,68 +3953,85 @@ def _create_from_elevation_area_flow_direction(self, *args, **kwargs): find_points_along_path = self._find_points_along_path(de, **kwargs) - i = np.where((area._griddata != 0) & ~np.isnan(area._griddata) & ~np.isnan(elevation._griddata) & (area._griddata > min_area)) - ij = list(zip(i[0], i[1])) - totalnumber = len(ij) - counter = 0.0 - next_readout = 0.1 - sys.stdout.write('Percent completion...') - sys.stdout.flush() + rows, cols = np.where((area._griddata != 0) & ~np.isnan(area._griddata) + & ~np.isnan(elevation._griddata) & (area._griddata > min_area)) + ij = list(zip(rows, cols)) + progress = _Progress(len(ij), kwargs.get('verbose', True)) for (i, j) in ij: self._griddata[i, j] = self.calc_channel_slope(i, j, elevation, de, find_points_along_path) - counter += 1.0 / totalnumber - if counter > next_readout: - sys.stdout.write(str(int(next_readout * 100)) + "...") - sys.stdout.flush() - next_readout += 0.1 - sys.stdout.write('Percent completion...') - sys.stdout.flush() - sys.stdout.write('100') - sys.stdout.flush() + progress.tick() + progress.done() class ChannelDownSlopeWithSmoothing(ChannelSlopeWithSmoothing): + """Channel gradient over the downstream half of the window.""" + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',), '_create'), (('ai_ascii_filename', 'EPSGprojectionCode'), '_read_ai'), (('gdal_filename',), '_read_gdal'), (('elevation', 'area', 'flow_direction', 'vertical_interval'), '_create_from_elevation_area_flow_direction'), (('elevation', 'area', 'flow_direction', 'horizontal_interval'), - '_create_from_elevation_flow_direction'), + '_create_from_elevation_area_flow_direction'), ) scale_factor = 2.0 def points_along_path(self, points, i, j): + """Only the downstream half of the window.""" position_of_center = list([ind[0] for ind in zip(range(len(points)),points) if (ind[1][0] == i and ind[1][1] == j)])[0] return points[0:position_of_center] if len(points[0:position_of_center]) > 0 else None class ChannelUpSlopeWithSmoothing(ChannelSlopeWithSmoothing): + """Channel gradient over the upstream half of the window.""" + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',), '_create'), (('ai_ascii_filename', 'EPSGprojectionCode'), '_read_ai'), (('gdal_filename',), '_read_gdal'), (('elevation', 'area', 'flow_direction', 'vertical_interval'), '_create_from_elevation_area_flow_direction'), (('elevation', 'area', 'flow_direction', 'horizontal_interval'), - '_create_from_elevation_flow_direction'), + '_create_from_elevation_area_flow_direction'), ) scale_factor = 2.0 def points_along_path(self, points, i, j): + """Only the upstream half of the window.""" position_of_center = list([ind[0] for ind in zip(range(len(points)),points) if (ind[1][0] == i and ind[1][1] == j)])[0] return points[position_of_center:] if len(points[position_of_center:]) > 0 else None class GeographicKsFromChiWithSmoothing(GeographicGridMixin, KsFromChiWithSmoothing): + """Windowed steepness fit on a latitude/longitude grid.""" + pass -class GeographicThetaFromChiWithSmoothing(GeographicGridMixin, KsFromChiWithSmoothing): +class GeographicThetaFromChiWithSmoothing(GeographicGridMixin, ThetaFromChiWithSmoothing): + """Windowed concavity fit on a latitude/longitude grid.""" + pass class MultiscaleCurvatureValleyWidth(BaseSpatialGrid): + """Valley width from the scale at which curvature is most negative. + + A local quadratic is fitted at a range of window sizes; the width + recorded at each cell is the window that minimises the smaller principal + curvature, i.e. the scale at which the valley is best resolved. The + curvature itself is kept in ``_minC``. + + :: + + width = MultiscaleCurvatureValleyWidth( + elevation=dem, area=area, area_cutoff=1e6, + min_width=30.0, max_width=600.0) + + Pass ``use_dask=True`` to evaluate the scales in parallel. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), (('elevation', 'area', 'area_cutoff', 'max_width', 'min_width'), '_create_from_inputs'),) class Utilities(object): + """The quadratic-fit machinery, as classmethods so dask can pickle it.""" @classmethod def _calc_inv_G_for_kernel(cls, X, Y, N, fix_center=False): @@ -2984,17 +4059,24 @@ def _calc_inv_G_for_kernel(cls, X, Y, N, fix_center=False): @classmethod def _convolve(cls, X, Y, Z, K, fix_center=False): + """Cross-correlate the elevation grid with the five moment kernels. - from numpy.fft import fft2 as fft2 - from numpy.fft import ifft2 as ifft2 - from numpy import flipud as flipud - from numpy import fliplr as fliplr - from numpy.fft import ifftshift + Each returned array is the windowed sum of ``z * basis`` with the + centre cell's own contribution removed, which is what the normal + equations for the local quadratic fit need. + """ + from numpy.fft import fft2, ifft2, fftshift Xt = X Yt = Y - - FZ = fft2(fliplr(fliplr(Z._griddata))) + data = np.asarray(Z._griddata, dtype=float64) + + # The kernels are symmetric about the grid centre, so correlation + # and convolution coincide and the elevation grid can be used as + # is. The original wrote fliplr(fliplr(...)), which is the + # identity -- presumably a typo for a 180-degree rotation, and + # harmless only because of that symmetry. + FZ = fft2(data) FX1 = fft2(np.power(Xt, 2)) FX2 = fft2(np.power(Yt, 2)) FX3 = fft2(Xt * Yt) @@ -3002,13 +4084,20 @@ def _convolve(cls, X, Y, Z, K, fix_center=False): FX5 = fft2(Yt) FX6 = fft2(K) - g = np.real(ifftshift(ifft2(FZ * FX1))) - np.sum(np.power(Xt, 2)) * Z._griddata - h = np.real(ifftshift(ifft2(FZ * FX2))) - np.sum(np.power(Yt, 2)) * Z._griddata - i = np.real(ifftshift(ifft2(FZ * FX3))) - np.sum(Xt * Yt) * Z._griddata - j = np.real(ifftshift(ifft2(FZ * FX4))) - np.sum(Xt) * Z._griddata - k = np.real(ifftshift(ifft2(FZ * FX5))) - np.sum(Yt) * Z._griddata + # fftshift, not ifftshift: the kernels are centred in the array, + # so the convolution output must be rolled forward by half the + # grid. For the odd-sized grids this class constructs the two + # differ by one cell, which displaced every fitted coefficient. + def correlate(kernel_hat): + return np.real(fftshift(ifft2(FZ * kernel_hat))) + + g = correlate(FX1) - np.sum(np.power(Xt, 2)) * data + h = correlate(FX2) - np.sum(np.power(Yt, 2)) * data + i = correlate(FX3) - np.sum(Xt * Yt) * data + j = correlate(FX4) - np.sum(Xt) * data + k = correlate(FX5) - np.sum(Yt) * data if not fix_center: - l = np.real(ifftshift(ifft2(FZ * FX6))) - np.sum(K) * Z._griddata + l = correlate(FX6) - np.sum(K) * data else: l = None @@ -3068,14 +4157,15 @@ def _create_kernel(cls, Z, de): return X * K, Y * K, N, K @classmethod - def _Cmin_for_scale(cls, Z, de, fix_center=False): - + def _Cmin_for_scale(cls, Z, de, fix_center=False, verbose=True): + """Smaller principal curvature of the quadratic fitted at one scale.""" X, Y, N, K = cls._create_kernel(Z, de) H = cls._calc_inv_G_for_kernel(X, Y, N, fix_center) g, h, i, j, k, l = cls._convolve(X, Y, Z, K, fix_center) Cmin = cls._Cmin(H, g, h, i, j, k, l, fix_center) - sys.stdout.write('scale ' + str(de) + '\n') - sys.stdout.flush() + if verbose: + sys.stdout.write('scale ' + str(de) + '\n') + sys.stdout.flush() return Cmin, de @classmethod @@ -3116,7 +4206,10 @@ def _elevation_fit_for_location(cls, x, y, elevation, de, fix_center = False): (nx, ny, dx) = (Z._georef_info.nx, Z._georef_info.ny, Z._georef_info.dx) (xllcenter, yllcenter) = (Z._georef_info.xllcenter, Z._georef_info.yllcenter) - [X,Y] = np.meshgrid(np.arange(xllcenter+dx/2, xllcenter + (nx-0.5)*dx, dx),np.arange(yllcenter+dx/2, yllcenter + (ny-0.5)*dx, dx)) + # arange over a half-open interval of exactly n*dx yields n-1 samples + # as often as n; build the axes from the cell count instead. + [X, Y] = np.meshgrid(xllcenter + dx / 2 + np.arange(nx) * dx, + yllcenter + dx / 2 + np.arange(ny) * dx) X -= xa Y -= ya @@ -3135,7 +4228,11 @@ def _create_from_inputs(self, *args, **kwargs): # Condition inputs to ensure that grids produce square convolution matrices: - (area_cutoff, max_width, min_width, normalize, fix_center, use_dask) = (kwargs['area_cutoff'], kwargs['max_width'], kwargs['min_width'], kwargs.get('normalize', False), kwargs.get('fix_center', False), kwargs.get('use_dask', False)) + (area_cutoff, max_width, min_width, normalize, fix_center, use_dask) = ( + kwargs['area_cutoff'], kwargs['max_width'], kwargs['min_width'], + kwargs.get('normalize', False), kwargs.get('fix_center', False), + kwargs.get('use_dask', False)) + verbose = kwargs.get('verbose', True) if use_dask: from dask import compute, delayed @@ -3175,7 +4272,8 @@ def _create_from_inputs(self, *args, **kwargs): if use_dask: from functools import partial - wrapper = partial(self.Utilities._Cmin_for_scale, Z, fix_center = fix_center) + wrapper = partial(self.Utilities._Cmin_for_scale, Z, + fix_center=fix_center, verbose=verbose) tasks = [delayed(wrapper)(s) for s in scales] results = compute(*tasks) for result in results: @@ -3188,7 +4286,8 @@ def _create_from_inputs(self, *args, **kwargs): ind += 1 else: for scale in scales: - minC, _ = self.Utilities._Cmin_for_scale(Z, scale, fix_center) + minC, _ = self.Utilities._Cmin_for_scale(Z, scale, fix_center, + verbose=verbose) if normalize: minC *= scale i = np.where(minC < g_minC) @@ -3205,11 +4304,13 @@ def _create_from_inputs(self, *args, **kwargs): self._minC = g_minC[start_y_index:, start_x_index:] def remove_padding(self, xpadding = 10, ypadding = 10): - + """Strip tile overlap from both the widths and the curvatures.""" import copy unpadded = copy.deepcopy(self) - unpadded._griddata = unpadded._griddata[ypadding:-ypadding,xpadding:-xpadding] - unpadded._minC = unpadded._minC[ypadding:-ypadding,xpadding:-xpadding] + rows = slice(ypadding, -ypadding if ypadding else None) + cols = slice(xpadding, -xpadding if xpadding else None) + unpadded._griddata = unpadded._griddata[rows, cols] + unpadded._minC = unpadded._minC[rows, cols] unpadded._georef_info.xllcenter += xpadding*unpadded._georef_info.dx unpadded._georef_info.yllcenter += ypadding*unpadded._georef_info.dx unpadded._georef_info.nx -= 2*xpadding @@ -3223,7 +4324,7 @@ def save(self, filename): @classmethod def load(cls, filename): - + """Read back the multi-band grid written by :meth:`save`.""" def get_band(gdal_dataset, band_number): band = gdal_dataset.GetRasterBand(band_number) nodata = band.GetNoDataValue() @@ -3232,7 +4333,7 @@ def get_band(gdal_dataset, band_number): nodata_elements = np.where(grid == nodata) from numpy import uint8 if cls.dtype is not uint8: - grid[nodata_elements] = np.NAN + grid[nodata_elements] = np.nan return grid return_object = cls() @@ -3257,7 +4358,7 @@ def get_band(gdal_dataset, band_number): @classmethod def mosaic(cls, tiles): - + """Reassemble tiles, carrying the curvature band along with the widths.""" xmin = np.nan xmax = np.nan ymin = np.nan @@ -3297,395 +4398,531 @@ def mosaic(cls, tiles): return return_object class DiscreteFlowAccumulation(BaseSpatialGrid): - + """Area drained by a single steepest-descent walk from each outlet. + + Unlike :class:`Area`, which accumulates over the whole D8 network, this + follows one path per outlet, stepping to the lowest unvisited neighbour + each time, and records the running area along it. It is used to trace + individual flow paths across surfaces (debris flows, lava) where the + network abstraction does not apply. + + :: + + dfa = DiscreteFlowAccumulation(elevation=dem, outlets=[(x, y)]) + + Keyword arguments + ----------------- + mask : BaseSpatialGrid + Confine the walk to cells where the mask is 1. + terminations_only : bool + Record the total area only at the end of each path. + display_output : bool + Report progress outlet by outlet. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation', 'outlets'), '_create_from_elevation_outlets'), (('elevation', ), '_create_from_elevation'),) - + def _create_from_elevation(self, *args, **kwargs): + """Start a walk from every valid cell.""" + kwargs = dict(kwargs) elevation = Elevation() elevation._copy_info_from_grid(kwargs['elevation']) if kwargs.get('mask') is not None: - i = np.where(kwargs['mask']._griddata != 1) - elevation._griddata[i] = np.NaN + outside = np.where(kwargs['mask']._griddata != 1) + elevation._griddata[outside] = np.nan kwargs['elevation'] = elevation - i = np.where(~np.isnan(elevation._griddata)) - rcs = zip(i[0].tolist(),i[1].tolist()) - kwargs['outlets'] = elevation._rowscols_to_xy(rcs) + rows, cols = np.where(~np.isnan(elevation._griddata)) + kwargs['outlets'] = elevation._rowscols_to_xy( + list(zip(rows.tolist(), cols.tolist()))) self._create_from_elevation_outlets(*args, **kwargs) - + def _create_from_elevation_outlets(self, *args, **kwargs): - + elevation = kwargs['elevation'] self._copy_info_from_grid(elevation, True) - adjust = [(-1, -1), (0, -1), (1,-1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)] - - counter = 1 - - for outlet in kwargs['outlets']: + adjust = [(-1, -1), (0, -1), (1, -1), (-1, 0), (1, 0), (-1, 1), (0, 1), (1, 1)] + + per_pixel_area = self._area_per_pixel() + terminations_only = kwargs.get('terminations_only') is True + outlets = list(kwargs['outlets']) + + def neighbour_elevations(cell): + """Neighbour elevations, NaN off-grid, as a plain float array.""" + values = np.empty(8, dtype=float64) + for k, (di, dj) in enumerate(adjust): + value = elevation[cell[0] + di, cell[1] + dj] + values[k] = np.nan if value is None else value + return values + + for counter, outlet in enumerate(outlets, start=1): if kwargs.get('display_output') is True: - print('Evaluating outlet ' + str(counter) + ' / ' + str(len(kwargs['outlets']))) - counter += 1 + print('Evaluating outlet {0} / {1}'.format(counter, len(outlets))) + (ij, ) = elevation._xy_to_rowscols((outlet,)) - visited = (ij, ) - ij_a = [(ij[0] + x[0], ij[1] + x[1]) for x in adjust] - e_a = np.array([elevation[i[0], i[1]] if elevation[i[0], i[1]] is not None else np.NaN for i in ij_a]) - new = True - area = 0 - while None not in e_a and np.sum(np.isnan(e_a)) == 0 and new: - area += self._area_per_pixel()[ij[0], ij[1]] - if kwargs.get('terminations_only') is not True: + if ij[0] is None: + continue + + seen = {tuple(ij)} + e_a = neighbour_elevations(ij) + area = 0.0 + + # `None not in numpy_array` never matched, so the original loop + # only ever stopped on NaN. + while not np.any(np.isnan(e_a)): + area += per_pixel_area[ij[0], ij[1]] + if not terminations_only: self._griddata[ij[0], ij[1]] = area - sls = np.argsort(e_a) - new = False - for sl in sls: + moved = False + for sl in np.argsort(e_a): this_ij = (ij[0] + adjust[sl][0], ij[1] + adjust[sl][1]) - if this_ij not in visited: - visited += (this_ij, ) - ij = this_ij - new = True - break - if new: - ij_a = [(ij[0] + x[0], ij[1] + x[1]) for x in adjust] - e_a = np.array([elevation[i[0], i[1]] if elevation[i[0], i[1]] is not None else np.NaN for i in ij_a]) - if kwargs.get('terminations_only') is True: - self._griddata[ij[0], ij[1]] = area + if this_ij in seen: + continue + seen.add(this_ij) + ij = this_ij + moved = True + break + if not moved: + break + e_a = neighbour_elevations(ij) + + if terminations_only: + self._griddata[ij[0], ij[1]] = area def _area_per_pixel(self, *args, **kwargs): return self._georef_info.dx**2 * np.ones((self._georef_info.ny, self._georef_info.nx)) def _mean_pixel_dimension(self, *args, **kwargs): - return self._georef_info.dx * np.ones_like(kwargs['elevation']._griddata, self.dtype) + # Reading the shape from kwargs['elevation'] raised KeyError whenever + # this was called without one; the grid knows its own shape. + return self._georef_info.dx * np.ones( + (self._georef_info.ny, self._georef_info.nx), dtype=float64) + +class GeographicDiscreteFlowAccumulation(GeographicGridMixin, DiscreteFlowAccumulation): + """Discrete flow accumulation on a latitude/longitude grid.""" -class GeographicDiscreteFlowAccumulation(GeographicGridMixin, DiscreteFlowAccumulation): pass class FlowLength(BaseSpatialGrid): - + """Longest upstream flow distance reaching each cell. + + :: + + length = FlowLength(flow_direction=d8) + + Alongside the distances, the grid records which upstream neighbour + supplies each cell's longest path. That "main stem" network is what + :class:`Relief`, :class:`ScaledRelief` and :class:`Ksi` follow, and what + :meth:`locations_along_flow_path_from_outlet` walks. + + .. note:: + The main-stem codes use the same ArcGIS convention as + :class:`FlowDirectionD8`, pointing *from* a cell *at* its upstream + donor. Before version 1.0 they used a row-flipped convention that + was internally consistent but disagreed with every other + flow-direction grid in the library, so ``*_directions`` side-files + written by older versions must be regenerated. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction',), '_create_from_flow_direction_and_sorted_indexes')) - + def _create_from_flow_direction_and_sorted_indexes(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) self.__calculate_flow_length(*args, **kwargs) - + def __calculate_flow_length(self, *args, **kwargs): - - flow_dir = kwargs['flow_direction'] - if kwargs.get('sorted_indexes') is not None: - idcs = kwargs.get('sorted_indexes') - else: - idcs = flow_dir.sort() - - self.__flow_directions = np.zeros_like(flow_dir._griddata, np.uint8) - [ind_i, ind_j] = np.unravel_index(idcs, flow_dir._griddata.shape) - dx = self._mean_pixel_dimension(*args, **kwargs) * flow_dir.pixel_scale() + flow_dir = kwargs['flow_direction'] + kernels.warn_if_slow(flow_dir._griddata.size) + + step = flow_dir.step_lengths(self._mean_pixel_dimension(*args, **kwargs)) + length, main_stem = kernels.flow_length( + flow_dir.receivers, + flow_dir.topological_order, + np.ascontiguousarray(step, dtype=float64)) + + self._griddata = np.asarray(length, dtype=self.dtype) + self.__flow_directions = np.asarray(main_stem, dtype=np.uint8) + + @property + def main_stem_directions(self): + """Codes pointing from each cell at its longest-path donor.""" + return self.__flow_directions + + @staticmethod + def __flow_direction_for_length(from_index, to_index): + """Code stored at ``to_index`` that points back at ``from_index``.""" + d_i = from_index[0] - to_index[0] + d_j = from_index[1] - to_index[1] + for d in range(8): + if int(kernels.D8_DI[d]) == d_i and int(kernels.D8_DJ[d]) == d_j: + return int(kernels.D8_CODES[d]) + return 0 - for i, j in zip(ind_i, ind_j): # Loop through all the data in sorted order - - i_next, j_next, is_good = flow_dir.get_flow_to_cell(i,j) - - if is_good: - next_l = self._griddata[i,j] + dx[i,j] - if next_l > self._griddata[i_next, j_next]: - self._griddata[i_next, j_next] = next_l - self.__flow_directions[i_next, j_next] = self.__flow_direction_for_length((i,j), (i_next,j_next)) - - def __flow_direction_for_length(self, from_index, to_index): - (i_from, j_from) = from_index - (i_to, j_to) = to_index - - d_i = i_to - i_from - d_j = j_to - j_from - - if d_i == 0: - if d_j == 1: - return 16 - if d_j == 0: - return 0 - if d_j == -1: - return 1 - elif d_i == 1: - if d_j == 1: - return 8 - if d_j == 0: - return 4 - if d_j == -1: - return 2 - elif d_i == -1: - if d_j == 1: - return 32 - if d_j == 0: - return 64 - if d_j == -1: - return 128 - - return None - def points_with_length(self, length, fd): - + """Map coordinates where the flow path first exceeds ``length``. + + Used to pick a consistent set of basin outlets: every returned point + has ``length`` of channel upstream of it. + """ tolerance = np.nanmin(self._mean_pixel_dimension()) - max_length = length - min_length = length - tolerance*2.0 - + min_length = length - tolerance * 2.0 + indexes_of_locations = list() - ind = np.where(np.logical_and((self._griddata >= min_length), (self._griddata <= max_length))) - for (this_row,this_col) in zip(ind[0],ind[1]): + ind = np.where((self._griddata >= min_length) & (self._griddata <= length)) + for (this_row, this_col) in zip(ind[0], ind[1]): (next_row, next_col, is_good) = fd.get_flow_to_cell(this_row, this_col) - if next_row is not None and next_col is not None and is_good: - this_value = self[this_row, this_col] - next_value = self[next_row, next_col] - if this_value <= length and next_value >= length: - indexes_of_locations.append((this_row, this_col)) - + if not is_good: + continue + if self[this_row, this_col] <= length <= self[next_row, next_col]: + indexes_of_locations.append((this_row, this_col)) + return self._rowscols_to_xy(indexes_of_locations) - + def indexes_along_flow_path_from_outlet(self, outlet): + """``(row, col)`` along the longest flow path upstream of ``outlet``.""" ij = self._xy_to_rowscols((outlet,))[0] + if ij[0] is None: + return [] return self.__get_upstream_indexes((ij,)) - + def locations_along_flow_path_from_outlet(self, outlet): + """As above, in map coordinates.""" return self._rowscols_to_xy(self.indexes_along_flow_path_from_outlet(outlet)) - + def locations_along_flow_path_from_outlets(self, outlets): + """Concatenated main-stem paths for several outlets.""" points = tuple() for outlet in outlets: points += tuple(self.locations_along_flow_path_from_outlet(outlet)) return points - + + def __step_upstream(self, i, j): + """The single main-stem donor of ``(i, j)``, or ``None``.""" + code = self.__flow_directions[i, j] + if code == 0: + return None + for d in range(8): + if code == kernels.D8_CODES[d]: + up_i = i + int(kernels.D8_DI[d]) + up_j = j + int(kernels.D8_DJ[d]) + if 0 <= up_i < self._georef_info.ny and 0 <= up_j < self._georef_info.nx: + return (up_i, up_j) + return None + return None + def __get_upstream_indexes(self, index): - - (i,j) = index[0] - indexes = list(index) - - - flow_code = self.__flow_directions[i,j] - if flow_code == 1: - indexes += self.__get_upstream_indexes(((i,j+1),)) - elif flow_code == 2: - indexes += self.__get_upstream_indexes(((i-1,j+1),)) - elif flow_code == 4: - indexes += self.__get_upstream_indexes(((i-1,j),)) - elif flow_code == 8: - indexes += self.__get_upstream_indexes(((i-1,j-1),)) - elif flow_code == 16: - indexes += self.__get_upstream_indexes(((i,j-1),)) - elif flow_code == 32: - indexes += self.__get_upstream_indexes(((i+1,j-1),)) - elif flow_code == 64: - indexes += self.__get_upstream_indexes(((i+1,j),)) - elif flow_code == 128: - indexes += self.__get_upstream_indexes(((i+1, j+1),)) - + """Walk the main stem upstream. Iterative, so long rivers are fine.""" + (i, j) = index[0] + indexes = [(i, j)] + seen = {(i, j)} + while True: + nxt = self.__step_upstream(i, j) + if nxt is None or nxt in seen: + break + indexes.append(nxt) + seen.add(nxt) + (i, j) = nxt return indexes - + def __map_flow_from_cell(self, index, **kwargs): - - (i,j) = index[0] - - return_dict = dict() - flow_code = self.__flow_directions[i,j] - - return_dict['index'] = index[0] - return_dict['distance'] = self._griddata[i,j] - for arg in kwargs: - return_dict[arg] = kwargs[arg][i,j] - - return_dict['next'] = [] - return_dict['distance_scale'] = 1.0 - - if flow_code == 1: - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell(((i,j+1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 2: - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell(((i-1,j+1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 4: - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell(((i-1,j),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 8: - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell(((i-1,j-1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 16: - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell(((i,j-1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 32: - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell(((i+1,j-1),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 64: - return_dict['distance_scale'] = 1.0 - child_dict = self.__map_flow_from_cell(((i+1,j),), **kwargs) - return_dict['next'].append(child_dict) - - if flow_code == 128: - return_dict['distance_scale'] = 1.4142135623730951 - child_dict = self.__map_flow_from_cell(((i+1,j+1),), **kwargs) - return_dict['next'].append(child_dict) - - if len(return_dict.get('next', 0)) == 0: - return_dict.pop('next') - - return return_dict + """Nested dictionary along the main stem upstream of ``index``. + + Each node carries ``index``, ``distance`` (the flow length at that + cell), ``distance_scale`` (the step length from its parent, in cell + widths) and one entry per keyword grid. + """ + (i, j) = index[0] + + def make_node(cell, scale): + node = {'index': cell, + 'distance': self._griddata[cell[0], cell[1]], + 'distance_scale': scale} + for name, grid in kwargs.items(): + node[name] = grid[cell[0], cell[1]] + return node + + root = make_node((i, j), 1.0) + node = root + seen = {(i, j)} + while True: + nxt = self.__step_upstream(i, j) + if nxt is None or nxt in seen: + break + diagonal = (nxt[0] != i) and (nxt[1] != j) + child = make_node(nxt, SQRT2 if diagonal else 1.0) + node['next'] = [child] + node = child + seen.add(nxt) + (i, j) = nxt + + return root def is_along_flow_length(self, from_index, to_index): - + """True when ``from_index`` supplies ``to_index``'s longest path.""" (i_to, j_to) = to_index flow_code = self.__flow_directions[i_to, j_to] - + return flow_code == self.__flow_direction_for_length(from_index, to_index) - + def clip_to_extent(self, extent): - + """Clip both the lengths and the main-stem codes to ``extent``.""" return_grid = super(FlowLength, self).clip_to_extent(extent) - - import copy - return_grid.__flow_directions = copy.deepcopy(self.__flow_directions) - lower_left = (extent[0]+self._georef_info.dx, extent[2]+self._georef_info.dx) - upper_right = (extent[1]-self._georef_info.dx, extent[3]-self._georef_info.dx) - idx = self._xy_to_rowscols((lower_left, upper_right)) - return_grid.__flow_directions = return_grid._griddata[idx[1][0]+1:idx[0][0]+1,idx[0][1]:idx[1][1]] + + # Clip the direction grid the same way as the data, through a + # temporary grid -- the old version assigned the clipped *lengths* + # into the direction array. + helper = BaseSpatialGrid() + helper._georef_info = self._georef_info + helper._griddata = self.__flow_directions + return_grid.__flow_directions = helper.clip_to_extent(extent)._griddata.astype(np.uint8) return return_grid def map_values_to_recursive_list(self, outlet, **kwargs): - + """Nested main-stem dictionary upstream of ``outlet``.""" v = (outlet, ) (ij_outlet, ) = self._xy_to_rowscols(v) + if ij_outlet[0] is None: + raise Error.InputError('map_values_to_recursive_list', + 'outlet {0} lies outside the grid'.format(outlet)) return self.__map_flow_from_cell((ij_outlet,), **kwargs) - + def save(self, filename): - + """Write the lengths, plus the main-stem codes to ``filename_directions``.""" super(FlowLength, self).save(filename) flow_dir_name = filename + "_directions" - self._create_gdal_representation_from_array(self._georef_info, 'GTiff', self.__flow_directions, np.uint8, flow_dir_name, ['COMPRESS=LZW']) - + dataset = self._create_gdal_representation_from_array( + self._georef_info, 'GTiff', self.__flow_directions, np.uint8, + flow_dir_name, ['COMPRESS=LZW']) + dataset.FlushCache() + del dataset + @classmethod def load(cls, filename): - + """Read back a grid written by :meth:`save`.""" return_object_bsp = BaseSpatialGrid.load(filename) return_object = cls() return_object._georef_info = return_object_bsp._georef_info return_object._griddata = return_object_bsp._griddata - flow_dir_filename = filename + "_directions" + flow_dir_filename = filename + "_directions" gdal_dataset = gdal.Open(flow_dir_filename) - band = gdal_dataset.GetRasterBand(1) - return_object.__flow_directions = band.ReadAsArray().astype(np.uint8) - - gdal_file = None + try: + band = gdal_dataset.GetRasterBand(1) + return_object.__flow_directions = band.ReadAsArray().astype(np.uint8) + finally: + gdal_dataset = None return return_object - class GeographicFlowLength(GeographicGridMixin, FlowLength): + """Flow length on a latitude/longitude grid.""" + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), (('gdal_filename',), '_read_gdal'), (('flow_direction',), '_create_from_flow_direction_and_sorted_indexes')) class MaxFlowLengthTrackingMixin(object): - + """Propagate a value downstream along longest-flow-length paths only. + + Subclasses set :attr:`_propagate_mode` to ``'carry'`` (the receiver takes + the donor's value) or ``'sum'`` (the receiver adds it) and may define + :meth:`_propagation_gate` to suppress the transfer for some cells. + """ + + #: ``'carry'`` or ``'sum'``. + _propagate_mode = 'carry' + + def _propagation_gate(self, *args, **kwargs): + """0/1 grid of cells allowed to pass their value on; ``None`` = all.""" + return None + def _calculate_by_tracking_down_max_flow_length(self, *args, **kwargs): flow_dir = kwargs['flow_direction'] flow_length = kwargs['flow_length'] - if kwargs.get('sorted_indexes') is not None: - idcs = kwargs.get('sorted_indexes') - else: - idcs = flow_dir.sort() - - [ind_i, ind_j] = np.unravel_index(idcs, flow_dir._griddata.shape) - - for i, j in zip(ind_i, ind_j): # Loop through all the data in sorted order - i_next, j_next, is_good = flow_dir.get_flow_to_cell(i,j) - if is_good and flow_length.is_along_flow_length((i,j), (i_next, j_next)): - self._calculate_grid_value((i,j), (i_next, j_next), *args, **kwargs) - + kernels.warn_if_slow(flow_dir._griddata.size) + + self._griddata = np.asarray(kernels.propagate_along_main_stem( + flow_dir.receivers, + flow_dir.topological_order, + np.ascontiguousarray(flow_length.main_stem_directions, dtype=np.uint8), + self._propagation_gate(*args, **kwargs), + np.ascontiguousarray(self._griddata, dtype=float64), + self._propagate_mode)) + + class Relief(BaseSpatialGrid, MaxFlowLengthTrackingMixin): - + """Height of the head of the longest flow path above each cell. + + :: + + relief = Relief(flow_direction=d8, elevation=dem, flow_length=length) + + Pass ``area`` and ``Ao`` together to stop the relief signal being carried + down from cells with less than ``Ao`` of drainage area, which keeps + hillslope noise out of the channel network. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), - (('flow_direction', 'elevation', 'flow_length'), '_create_from_flow_direction_sorted_indexes_and_elevation')) - + (('gdal_filename',), '_read_gdal'), + (('flow_direction', 'elevation', 'flow_length'), '_create_from_flow_direction_sorted_indexes_and_elevation')) + + _propagate_mode = 'carry' + + def _propagation_gate(self, *args, **kwargs): + area = kwargs.get('area') + Ao = kwargs.get('Ao') + if area is None or Ao is None: + return None + return (np.asarray(area._griddata) > Ao).astype(np.uint8) + def _create_from_flow_direction_sorted_indexes_and_elevation(self, *args, **kwargs): self._copy_info_from_grid(kwargs['elevation']) self._calculate_by_tracking_down_max_flow_length(*args, **kwargs) self._griddata = self._griddata - kwargs['elevation']._griddata - + def _calculate_grid_value(self, pos, next_pos, *args, **kwargs): - (i,j) = pos + """Per-cell transfer rule. + + Retained for subclasses that override it; the bulk path uses the + compiled kernel instead. + """ + (i, j) = pos (i_next, j_next) = next_pos if kwargs.get('area') is not None and kwargs.get('Ao') is not None: - if kwargs.get('area')[i,j] <= kwargs['Ao']: - self._griddata[i_next, j_next] = self._griddata[i_next, j_next] + if kwargs['area'][i, j] <= kwargs['Ao']: return - - self._griddata[i_next, j_next] = self._griddata[i,j] - + + self._griddata[i_next, j_next] = self._griddata[i, j] + + class ScaledRelief(Relief): - + """:class:`Relief` scaled by ``Ao**theta``. + + That scaling puts relief in the same units as :class:`Ksi`, so the ratio + of the two is a channel steepness. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction', 'flooded_dem', 'elevation', 'flow_length', 'Ao', 'theta'), '_create_scaled_from_flow_direction_flooded_dem_and_elevation'), (('flow_direction', 'elevation', 'flow_length', 'Ao', 'theta'), '_create_scaled_from_flow_direction_sorted_indexes_and_elevation')) - + def _create_scaled_from_flow_direction_flooded_dem_and_elevation(self, *args, **kwargs): - self._create_from_flow_direction_flooded_dem_and_elevation(*args, **kwargs) - self._griddata = self._griddata * (np.power(kwargs['Ao'],kwargs['theta'])) - + # The original called a `_create_from_flow_direction_flooded_dem_and_elevation` + # that was never defined, so supplying `flooded_dem` raised + # AttributeError. The flooded DEM plays no part in the calculation + # beyond being an elevation surface, so route it to the same code. + self._create_from_flow_direction_sorted_indexes_and_elevation(*args, **kwargs) + self._griddata = self._griddata * (np.power(kwargs['Ao'], kwargs['theta'])) + def _create_scaled_from_flow_direction_sorted_indexes_and_elevation(self, *args, **kwargs): self._create_from_flow_direction_sorted_indexes_and_elevation(*args, **kwargs) - self._griddata = self._griddata * (np.power(kwargs['Ao'],kwargs['theta'])) + self._griddata = self._griddata * (np.power(kwargs['Ao'], kwargs['theta'])) + class Ksi(BaseSpatialGrid, MaxFlowLengthTrackingMixin): - + r"""The chi coordinate integrated along longest-flow-length paths. + + .. math:: + \chi = \int \left(\frac{A_0}{A}\right)^{\theta} \mathrm{d}x + + :: + + ksi = Ksi(area=area, flow_direction=d8, flow_length=length, + theta=0.45, Ao=1e6) + + Cells with less than ``Ao`` of drainage area are left out of the + integral, so chi is defined only on the channel network. + + .. note:: + Before version 1.0 the integrand was ``(Ao / (A - Ao))**theta``, which + diverges as ``A`` approaches ``Ao`` and does not match the definition + of chi in the literature. Values from older runs are not comparable + with these. + + See also :class:`Chi`, which integrates upstream from chosen outlets over + the whole network rather than along main stems. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('area','flow_direction','theta', 'Ao', 'flow_length'), '_create_from_inputs')) - + + _propagate_mode = 'sum' + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) - area_grid = kwargs['area']._griddata - kwargs['Ao'] - area_grid[area_grid <= 0] = np.nan - self._griddata = np.zeros_like(area_grid) - i = np.where(area_grid > 0) - self._griddata[i] = ( (kwargs['Ao'] / area_grid[i]) ** kwargs['theta']) * self._mean_pixel_dimension(*args, **kwargs)[i] * kwargs['flow_direction'].pixel_scale()[i] + + area = np.asarray(kwargs['area']._griddata, dtype=float64) + Ao = kwargs['Ao'] + theta = kwargs['theta'] + + step = (self._mean_pixel_dimension(*args, **kwargs) + * kwargs['flow_direction'].pixel_scale(float64)) + + self._griddata = np.zeros(area.shape, dtype=float64) + on_network = area > Ao + self._griddata[on_network] = ( + (Ao / area[on_network]) ** theta) * step[on_network] + self._calculate_by_tracking_down_max_flow_length(*args, **kwargs) - + def _calculate_grid_value(self, pos, next_pos, *args, **kwargs): - (i,j) = pos + """Per-cell transfer rule; the bulk path uses the compiled kernel.""" + (i, j) = pos (i_next, j_next) = next_pos - self._griddata[i_next, j_next] += self._griddata[i,j] - + self._griddata[i_next, j_next] += self._griddata[i, j] class GeographicKsi(GeographicGridMixin, Ksi): + """Chi along longest-flow-length paths on a latitude/longitude grid.""" + pass class RestoredElevation(BaseSpatialGrid): - + r"""Topography implied by a uniform channel steepness. + + Starting from the given outlets, elevations are rebuilt upstream from + + .. math:: \frac{\mathrm{d}z}{\mathrm{d}x} = k_s A^{-\theta} + + and the divides are then allowed to migrate towards whichever side ends + up lower. Repeating that (``iterations`` times) converges on the + landscape a steady, spatially uniform ``ks`` would produce, which can be + compared against the real DEM to find where it is out of steady state. + + :: + + restored = RestoredElevation(flow_direction=d8, elevation=dem, + area=area, theta=0.45, ks=50, + outlets=[(x, y)], iterations=5) + + Keyword arguments + ----------------- + randomize : bool + Randomise elevations inside the basin first, then re-route, so the + result does not inherit the real network's planform. + fix_external_outlets : bool + Hold the outer boundary of the basin fixed while divides migrate. + verbose : bool + Report convergence each iteration (default ``True``). + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction','elevation','area','theta','ks','outlets', 'iterations'), '_fill_dem')) - + def _fill_dem(self, *args, **kwargs): import copy self._copy_info_from_grid(kwargs['elevation'], False) @@ -3696,583 +4933,558 @@ def _fill_dem(self, *args, **kwargs): iterations = kwargs['iterations'] ks = kwargs['ks'] theta = kwargs['theta'] - v = self._xy_to_rowscols(outlets) + verbose = kwargs.get('verbose', True) + v = [rc for rc in self._xy_to_rowscols(outlets) if rc[0] is not None] pixel_dimension = self._mean_pixel_dimension(*args, **kwargs) - - - mask = Mask(flow_direction = flow_direction, outlets = outlets) + + mask = Mask(flow_direction=flow_direction, outlets=outlets) external_divides = copy.deepcopy(mask) - - if kwargs.get('fix_external_outlets') == True: + + if kwargs.get('fix_external_outlets') is True: from scipy.ndimage import binary_erosion as erosion - external_divides._griddata = external_divides._griddata - erosion(external_divides._griddata, np.ones((3,3)), border_value = 0).astype(int) + external_divides._griddata = ( + external_divides._griddata + - erosion(external_divides._griddata, np.ones((3, 3)), + border_value=0).astype(int)) else: external_divides._griddata = np.zeros_like(external_divides._griddata, int) + if randomize: - filled = FilledElevation(elevation = self, mask = mask, randomize = randomize, outlets = outlets) + filled = FilledElevation(elevation=self, mask=mask, randomize=randomize, + outlets=outlets) flow_direction.update_flow_codes_in_mask(filled, mask) - i = np.where(mask._griddata == 1) - self._griddata[i] = filled._griddata[i] + inside = np.where(mask._griddata == 1) + self._griddata[inside] = filled._griddata[inside] area = self.__recalculate_area(area, flow_direction, pixel_dimension, outlets) - for i in range(iterations): + + self.convergence = [] + for iteration in range(iterations): last_grid = self._griddata.copy() - print('Iteration {0}'.format(i)) - print('Filling outlets.') divides = self.__fill_outlets(area, flow_direction, pixel_dimension, v, ks, theta) - print('Migrating divides') - flow_direction = self.__migrate_divides(flow_direction, divides, external_divides) + flow_direction = self.__migrate_divides(flow_direction, divides, external_divides, + verbose=verbose) area = self.__recalculate_area(area, flow_direction, pixel_dimension, outlets) - change_in_elevation = np.mean((self._griddata - last_grid)**2) - print('Change: {0}'.format(change_in_elevation)) - + change_in_elevation = float(np.nanmean((self._griddata - last_grid) ** 2)) + self.convergence.append(change_in_elevation) + if verbose: + print('Iteration {0}: mean squared change {1:g}'.format( + iteration, change_in_elevation)) + + self.flow_direction = flow_direction + self.area = area + def __is_unit_area(self, idx, flow_direction): - - (i,j) = idx - - if flow_direction[i,j+1] == 16 or flow_direction[i+1, j+1] == 32 or \ - flow_direction[i+1, j] == 64 or flow_direction[i+1, j-1] == 128 or \ - flow_direction[i, j-1] == 1 or flow_direction[i-1, j-1] == 2 or \ - flow_direction[i-1, j] == 4 or flow_direction[i-1, j+1] == 8: - return False - else: - return True - + """True when no neighbour drains into ``idx`` (it is a divide cell).""" + return not flow_direction.get_upstream_cell_indexes(idx[0], idx[1]) + def __migrate_divides(self, *args, **kwargs): - + """Redirect divide-adjacent cells towards whichever side is lower.""" flow_direction = args[0] divides = args[1] external_divides = args[2] + verbose = kwargs.get('verbose', True) migrated = 0 + ny, nx = self._georef_info.ny, self._georef_info.nx + for (i, j) in divides: - if external_divides[i,j] != 1: - try: - if self[i,j+1] > self[i,j] and external_divides[i,j+1] != 1: - flow_direction[i,j+1] = 16 - migrated += 1 - except: - pass - - try: - if self[i+1, j+1] > self[i,j] and external_divides[i+1,j+1] != 1: - flow_direction[i+1, j+1] = 32 - migrated += 1 - except: - pass - - try: - if self[i+1, j] > self[i,j] and external_divides[i+1,j] != 1: - flow_direction[i+1, j] = 64 - migrated += 1 - except: - pass - - try: - if self[i+1, j-1] > self[i,j] and external_divides[i+1,j-1] != 1: - flow_direction[i+1, j-1] = 128 - migrated += 1 - except: - pass - - try: - if self[i, j-1] > self[i,j] and external_divides[i,j-1] != 1: - flow_direction[i, j-1] = 1 - migrated += 1 - except: - pass - - try: - if self[i-1,j-1] > self[i,j] and external_divides[i-1,j-1] != 1: - flow_direction[i-1, j-1] = 2 - migrated += 1 - except: - pass - - try: - if self[i-1, j] > self[i,j] and external_divides[i-1,j] != 1: - flow_direction[i-1, j] = 4 - migrated += 1 - except: - pass - - try: - if self[i-1, j+1] > self[i,j] and external_divides[i-1,j+1] != 1: - flow_direction[i-1, j+1] = 8 - migrated += 1 - except: - pass - - print("Migrated " + str(migrated) + " divides.") + if external_divides[i, j] == 1: + continue + here = self[i, j] + if here is None: + continue + for d in range(8): + ni = i + int(kernels.D8_DI[d]) + nj = j + int(kernels.D8_DJ[d]) + # Explicit bounds checks instead of eight try/except blocks: + # the exception handlers also swallowed real errors, and + # negative indices wrapped instead of being rejected. + if not (0 <= ni < ny and 0 <= nj < nx): + continue + if external_divides[ni, nj] == 1: + continue + neighbour = self[ni, nj] + if neighbour is None or not neighbour > here: + continue + flow_direction[ni, nj] = int(kernels.D8_CODES[(d + 4) % 8]) + migrated += 1 + + flow_direction._invalidate_network() + if verbose: + print("Migrated " + str(migrated) + " divides.") return flow_direction - + def __fill_outlets(self, area, flow_direction, pixel_dimension, outlet_indexes, ks, theta): - divides = list() for ind in outlet_indexes: - divides += self.__fill_upstream_points(ind, ks, theta, area, flow_direction, pixel_dimension) + divides += self.__fill_upstream_points(ind, ks, theta, area, flow_direction, + pixel_dimension) return divides - + def __fill_upstream_points(self, ind, ks, theta, area, flow_direction, pixel_dimension): - - (i, j) = ind - elevation_for_cell = self[i,j] - is_divide = True + """Integrate dz/dx = ks * A**-theta upstream from ``ind``. + + Returns the divide cells reached. Iterative: the recursive original + needed a million-frame recursion limit and still overflowed on real + basins. + """ + ny, nx = self._georef_info.ny, self._georef_info.nx divides = list() - - try: - if flow_direction[i, j+1] == 16: - self._griddata[i,j+1] = elevation_for_cell + pixel_dimension[i,j]*1.00*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i,j+1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i+1, j+1] == 32: - self._griddata[i+1,j+1] = elevation_for_cell + pixel_dimension[i,j]*1.414*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i+1,j+1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i+1, j] == 64: - self._griddata[i+1,j] = elevation_for_cell + pixel_dimension[i,j]*1.00*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i+1,j), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i+1, j-1] == 128: - self._griddata[i+1,j-1] = elevation_for_cell + pixel_dimension[i,j]*1.414*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i+1,j-1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i, j-1] == 1: - self._griddata[i,j-1] = elevation_for_cell + pixel_dimension[i,j]*1.00*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i,j-1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i-1, j-1] == 2: - self._griddata[i-1,j-1] = elevation_for_cell + pixel_dimension[i,j]*1.414*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i-1,j-1), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i-1, j] == 4: - self._griddata[i-1,j] = elevation_for_cell + pixel_dimension[i,j]*1.00*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i-1,j), ks, theta, area, flow_direction, pixel_dimension) - is_divide = False - except: - pass - - try: - if flow_direction[i-1, j+1] == 8: - self._griddata[i-1,j+1] = elevation_for_cell + pixel_dimension[i,j]*1.414*ks*area[i,j]**(-theta) - divides += self.__fill_upstream_points((i-1,j+1), ks, theta, area, flow_direction, pixel_dimension) + stack = [tuple(ind)] + seen = {tuple(ind)} + + while stack: + (i, j) = stack.pop() + elevation_for_cell = self._griddata[i, j] + a = area[i, j] + is_divide = True + + for d in range(8): + ui = i + int(kernels.D8_DI[d]) + uj = j + int(kernels.D8_DJ[d]) + if not (0 <= ui < ny and 0 <= uj < nx): + continue + if flow_direction[ui, uj] != kernels.D8_CODES[(d + 4) % 8]: + continue is_divide = False - except: - pass - - if is_divide: - return [(i,j)] - else: - return divides - + if (ui, uj) in seen: + continue + seen.add((ui, uj)) + step = pixel_dimension[i, j] * kernels.D8_DIST[d] + self._griddata[ui, uj] = elevation_for_cell + step * ks * a ** (-theta) + stack.append((ui, uj)) + + if is_divide: + divides.append((i, j)) + + return divides + def __recalculate_area(self, area, flow_direction, pixel_dimension, outlets): - outlet_indexes = self._xy_to_rowscols(outlets) - for outlet in outlet_indexes: - self.__recurse_area(outlet, area, flow_direction, pixel_dimension) + """Re-accumulate drainage area after the network has been rewired.""" + weights = np.asarray(pixel_dimension, dtype=float64) ** 2 + accumulated = np.asarray(kernels.accumulate( + flow_direction.receivers, + flow_direction.topological_order, + np.ascontiguousarray(weights))) + + inside = self.__basin_mask(flow_direction, outlets) + area._griddata = np.where(inside, accumulated, area._griddata) return area - - def __recurse_area(self, ind, area, flow_direction, pixel_dimension): - - (i, j) = ind - - this_area = pixel_dimension[i,j]**2 - - try: - if flow_direction[i, j+1] == 16: - ind = (i, j+1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i+1, j+1] == 32: - ind = (i+1,j+1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i+1, j] == 64: - ind = (i+1,j) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i+1, j-1] == 128: - ind = (i+1, j-1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i, j-1] == 1: - ind = (i,j-1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i-1, j-1] == 2: - ind = (i-1, j-1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i-1, j] == 4: - ind = (i-1, j) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - try: - if flow_direction[i-1, j+1] == 8: - ind = (i-1, j+1) - this_area += self.__recurse_area(ind, area, flow_direction, pixel_dimension) - except: - pass - - area[i,j] = this_area - return this_area - + + @staticmethod + def __basin_mask(flow_direction, outlets): + return flow_direction.basin_mask(outlets) + class GeographicRestoredElevation(GeographicGridMixin, RestoredElevation): + """Restored topography on a latitude/longitude grid.""" + pass class ChiScaledRelief(BaseSpatialGrid): + """Height above the basin outlet, scaled by ``Ao**theta``. + + Plotted against :class:`Chi`, the slope of this quantity is the channel + steepness index. + + :: + + relief = ChiScaledRelief(elevation=dem, flow_direction=d8, + theta=0.45, Ao=1e6, outlets=[(x, y)]) + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation', 'flow_direction', 'theta', 'Ao', 'outlets'), '_create_from_inputs'), (('elevation', 'flow_direction', 'flow_length', 'theta', 'Ao', 'basin_length'), '_create_from_basin_length')) - + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['elevation'], True) - outlet_indexes = self._xy_to_rowscols(kwargs['outlets']) elevation = kwargs['elevation'] - scale = np.power(kwargs['Ao'],kwargs['theta']) - outlet_number = 1 - for outlet_index in outlet_indexes: - indexes = kwargs['flow_direction'].get_indexes_of_upstream_cells(outlet_index[0], outlet_index[1]) - elevation_of_outlet = kwargs['elevation'][outlet_index[0],outlet_index[1]] - for index in indexes: - self[index[0],index[1]] = (elevation[index[0], index[1]] - elevation_of_outlet) * scale + flow_direction = kwargs['flow_direction'] + scale = np.power(kwargs['Ao'], kwargs['theta']) + + outlet_indexes = [rc for rc in self._xy_to_rowscols(kwargs['outlets']) + if rc[0] is not None] + for outlet_number, outlet_index in enumerate(outlet_indexes, start=1): + # One vectorised basin extraction per outlet, rather than an + # element-by-element write through __setitem__. + basin = flow_direction.basin_mask( + self._rowscols_to_xy((outlet_index,))) + elevation_of_outlet = elevation[outlet_index[0], outlet_index[1]] + self._griddata[basin] = ( + np.asarray(elevation._griddata)[basin] - elevation_of_outlet) * scale if kwargs.get('output_flag', False): - print('Outlet ' + str(outlet_number) + '/' + str(len(outlet_indexes)) + ' completed.') - outlet_number = outlet_number + 1 - + print('Outlet {0}/{1} completed.'.format(outlet_number, len(outlet_indexes))) + def _create_from_basin_length(self, *args, **kwargs): - kwargs['outlets'] = kwargs['flow_length'].points_with_length(kwargs['basin_length'],kwargs['flow_direction']) + """Use every point with ``basin_length`` of channel upstream as an outlet.""" + kwargs = dict(kwargs) + kwargs['outlets'] = kwargs['flow_length'].points_with_length( + kwargs['basin_length'], kwargs['flow_direction']) kwargs['output_flag'] = True - return self._create_from_inputs(*args, **kwargs) + return self._create_from_inputs(*args, **kwargs) class Chi(BaseSpatialGrid): - + r"""The chi coordinate, integrated upstream from chosen outlets. + + .. math:: + \chi(x) = \int_{x_{\mathrm{outlet}}}^{x} + \left(\frac{A_0}{A(x')}\right)^{\theta}\,\mathrm{d}x' + + :: + + chi = Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e6, + outlets=[(x, y)]) + + Chi is 0 at each outlet and increases upstream. Cells outside the + outlets' drainage areas are left as 0. + + Keyword arguments + ----------------- + trapezoid : bool + Average the integrand across each step instead of taking it at the + upstream cell. This is the rule TopoToolbox's ``chitransform`` uses; + the default (``False``) reproduces the original TopoAnalysis result. + maximum_length : float + Stop integrating beyond this flow distance from the outlet. + mask : BaseSpatialGrid + Restrict the integration to cells where the mask is non-zero. + + .. note:: + Before version 1.0, chi at the outlet itself was one cell-step of the + integrand rather than 0. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('area','flow_direction', 'theta', 'Ao', 'outlets'), '_create_from_inputs'), (('area', 'flow_direction', 'flow_length', 'theta', 'Ao', 'basin_length'), '_create_from_basin_length')) - - + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) self.__calculate_chi(*args, **kwargs) - + def _create_from_basin_length(self, *args, **kwargs): - kwargs['outlets'] = kwargs['flow_length'].points_with_length(kwargs['basin_length'],kwargs['flow_direction']) + """Use every point with ``basin_length`` of channel upstream as an outlet.""" + kwargs = dict(kwargs) + kwargs['outlets'] = kwargs['flow_length'].points_with_length( + kwargs['basin_length'], kwargs['flow_direction']) kwargs['output_flag'] = True return self._create_from_inputs(*args, **kwargs) - + def __calculate_chi(self, *args, **kwargs): - pixel_dimension = self._mean_pixel_dimension(*args, **kwargs) area = kwargs['area'] flow_direction = kwargs['flow_direction'] - Ao = kwargs['Ao'] - theta = kwargs['theta'] - outlet_indexes = self._xy_to_rowscols(kwargs['outlets']) - self.__occupied = np.zeros_like(area._griddata, dtype=int) - outlet_number = 1 - lmax = kwargs.get('maximum_length') - - for outlet in outlet_indexes: - self.__recurse_chi(outlet, area, flow_direction, pixel_dimension, Ao, theta, 0, 1, kwargs.get('mask'), 0.0, lmax = lmax) - if kwargs.get('output_flag', False): - print('Outlet ' + str(outlet_number) + '/' + str(len(outlet_indexes)) + ' completed.') - outlet_number += 1 - self.__occupied = None - try: - self._griddata = self._griddata * kwargs['mask']._griddata - except: - pass - - def __recurse_chi(self, v, area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = None): - - (i,j) = v - - if mask is not None: - if mask[i,j] == 0: - if area[i,j] > 1E4: - return - - if self.__occupied[i,j] == 1: - return - - dl = pixel_dimension[i,j] * scale - l = l + dl - - if lmax is not None and l >= lmax: - return - - self._griddata[i,j] = chi + (Ao / area[i,j])**theta * dl - self.__occupied[i,j] = 1; - chi = self._griddata[i,j] - + kernels.warn_if_slow(flow_direction._griddata.size) + + step = flow_direction.step_lengths(self._mean_pixel_dimension(*args, **kwargs)) + + outlets = [] + for (row, col) in self._xy_to_rowscols(kwargs['outlets']): + if row is not None: + outlets.append(row * self._georef_info.nx + col) + + mask = kwargs.get('mask') + mask_array = None if mask is None else (np.asarray(mask._griddata) != 0).astype(np.uint8) + + chi, distance = kernels.chi( + flow_direction.receivers, + flow_direction.topological_order, + np.ascontiguousarray(area._griddata, dtype=float64), + np.ascontiguousarray(step, dtype=float64), + np.array(outlets, dtype=np.int64), + kwargs['Ao'], + kwargs['theta'], + bool(kwargs.get('trapezoid', False)), + float(kwargs.get('maximum_length') or 0.0), + mask_array) + + chi = np.asarray(chi) + self.distance_from_outlet = np.asarray(distance) + # Cells the integration never reached come back as NaN; the grid's + # documented empty value is 0. + self._griddata = np.where(np.isnan(chi), 0.0, chi) + + if kwargs.get('output_flag', False): + print('{0} outlets completed.'.format(len(outlets))) - try: - if flow_direction[i, j+1] == 16: - scale = 1.0 - self.__recurse_chi((i, j+1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i+1, j+1] == 32: - scale = 1.414 - self.__recurse_chi((i+1, j+1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i+1, j] == 64: - scale = 1.0 - self.__recurse_chi((i+1, j), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i+1, j-1] == 128: - scale = 1.414 - self.__recurse_chi((i+1, j-1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i, j-1] == 1: - scale = 1.0 - self.__recurse_chi((i, j-1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i-1, j-1] == 2: - scale = 1.414 - self.__recurse_chi((i-1, j-1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i-1, j] == 4: - scale = 1.0 - self.__recurse_chi((i-1, j), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass - - try: - if flow_direction[i-1, j+1] == 8: - scale = 1.414 - self.__recurse_chi((i-1, j+1), area, flow_direction, pixel_dimension, Ao, theta, chi, scale, mask, l, lmax = lmax) - except: - pass class GeographicChi(GeographicGridMixin, Chi): + """Chi on a latitude/longitude grid.""" + pass -class CrossDivideDChi(BaseSpatialGrid): - +class _CrossDivideMixin(object): + """Shared machinery for the across-divide chi contrast grids.""" + + def _divide_pairs(self, *args, **kwargs): + """``(from_rows, from_cols, to_rows, to_cols)`` for each divide pair.""" + paired_divides = self._paired_divides(*args, **kwargs) + if not paired_divides: + return None + + # zip() is a one-shot iterator in Python 3 and is not subscriptable, + # so the original `zip(*pairs)[0]` raised immediately. + from_point, to_point = zip(*paired_divides) + + from_idx = self._xy_to_rowscols(from_point) + to_idx = self._xy_to_rowscols(to_point) + + keep = [k for k in range(len(from_idx)) + if from_idx[k][0] is not None and to_idx[k][0] is not None] + if not keep: + return None + + from_rows = np.array([from_idx[k][0] for k in keep], dtype=int) + from_cols = np.array([from_idx[k][1] for k in keep], dtype=int) + to_rows = np.array([to_idx[k][0] for k in keep], dtype=int) + to_cols = np.array([to_idx[k][1] for k in keep], dtype=int) + return from_rows, from_cols, to_rows, to_cols + + +class CrossDivideDChi(_CrossDivideMixin, BaseSpatialGrid): + """Difference in chi across each drainage divide. + + A large value means the two sides of the divide are far from + steady state, so the divide is expected to migrate towards the + higher-chi side (Willett et al. 2014). + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('chi','flow_direction'), '_create_from_inputs')) - + + def _paired_divides(self, *args, **kwargs): + return kwargs['flow_direction'].paired_divides() + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) - paired_divides = kwargs['flow_direction'].paired_divides() - divide_columns = zip(*paired_divides) - from_point = divide_columns[0] - to_point = divide_columns[1] - - from_idx = self._xy_to_rowscols(from_point) - to_idx = self._xy_to_rowscols(to_point) - - from_idx_column = zip(*from_idx) - to_idx_column = zip(*to_idx) - - minchi = np.minimum(kwargs['chi']._griddata[to_idx_column[0],to_idx_column[1]], kwargs['chi']._griddata[from_idx_column[0],from_idx_column[1]]) - maxchi = np.maximum(kwargs['chi']._griddata[to_idx_column[0],to_idx_column[1]], kwargs['chi']._griddata[from_idx_column[0],from_idx_column[1]]) - - self._griddata[from_idx_column[0],from_idx_column[1]] = (maxchi - minchi) * (minchi != 0).astype(float) - -class NormalizedCrossDivideDChi(BaseSpatialGrid): - + pairs = self._divide_pairs(*args, **kwargs) + if pairs is None: + return + from_rows, from_cols, to_rows, to_cols = pairs + + chi = kwargs['chi']._griddata + a = chi[to_rows, to_cols] + b = chi[from_rows, from_cols] + minchi = np.minimum(a, b) + maxchi = np.maximum(a, b) + + self._griddata[from_rows, from_cols] = (maxchi - minchi) * (minchi != 0).astype(float) + + +class NormalizedCrossDivideDChi(_CrossDivideMixin, BaseSpatialGrid): + """Chi contrast across each divide, divided by the mean chi there. + + Normalising makes divides in high- and low-chi parts of a landscape + comparable. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('chi','flow_direction'), '_create_from_inputs')) - + + def _paired_divides(self, *args, **kwargs): + return kwargs['flow_direction'].paired_divides(mask=kwargs['chi']) + def _create_from_inputs(self, *args, **kwargs): self._copy_info_from_grid(kwargs['flow_direction'], True) - paired_divides = kwargs['flow_direction'].paired_divides(mask = kwargs['chi']) - divide_columns = zip(*paired_divides) - from_point = divide_columns[0] - to_point = divide_columns[1] - - from_idx = self._xy_to_rowscols(from_point) - to_idx = self._xy_to_rowscols(to_point) - - from_idx_column = zip(*from_idx) - to_idx_column = zip(*to_idx) - - minchi = np.minimum(kwargs['chi']._griddata[to_idx_column[0],to_idx_column[1]], kwargs['chi']._griddata[from_idx_column[0],from_idx_column[1]]) - maxchi = np.maximum(kwargs['chi']._griddata[to_idx_column[0],to_idx_column[1]], kwargs['chi']._griddata[from_idx_column[0],from_idx_column[1]]) - + pairs = self._divide_pairs(*args, **kwargs) + if pairs is None: + return + from_rows, from_cols, to_rows, to_cols = pairs + + chi = kwargs['chi']._griddata + a = chi[to_rows, to_cols] + b = chi[from_rows, from_cols] + minchi = np.minimum(a, b) + maxchi = np.maximum(a, b) + rangechi = (maxchi - minchi) * (minchi != 0).astype(float) - meanchi = (maxchi + minchi) / 2 - - self._griddata[from_idx_column[0],from_idx_column[1]] = rangechi / meanchi + meanchi = (maxchi + minchi) / 2.0 + with np.errstate(divide='ignore', invalid='ignore'): + self._griddata[from_rows, from_cols] = np.where( + meanchi != 0, rangechi / meanchi, 0.0) - class Deflection(BaseSpatialGrid): - + r"""Flexural deflection of an elastic plate under a topographic load. + + Solves the thin-plate equation in the Fourier domain: + + .. math:: + w(k) = \frac{-\rho_c\,g\,h(k)} + {(\rho_m - \rho_c)\,g + D\,(2\pi k)^4} + + :: + + w = Deflection(elevation=dem, D=1e23, rho_m=3300, rho_c=2700, g=9.81) + + Pass ``restored_elevation`` as well to get the *change* in deflection + between two surfaces, which is the isostatic response to the erosion + between them. + + Because the solution is spectral, the load is implicitly periodic; pad + the DEM if edge effects matter. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('elevation', 'D', 'rho_m', 'rho_c', 'g'), '_deflect')) - + def _deflect(self, *args, **kwargs): if kwargs.get('restored_elevation'): - self._griddata = self.__deflect_elevation(kwargs['elevation'], **kwargs) - self.__deflect_elevation(kwargs['restored_elevation'], **kwargs) + self._griddata = (self.__deflect_elevation(kwargs['elevation'], **kwargs) + - self.__deflect_elevation(kwargs['restored_elevation'], **kwargs)) else: self._griddata = self.__deflect_elevation(kwargs['elevation'], **kwargs) - + def __deflect_elevation(self, *args, **kwargs): elevation = args[0] self._copy_info_from_grid(elevation, False) + kwargs = dict(kwargs) kwargs['flow_direction'] = elevation - dx = np.mean(self._mean_pixel_dimension(*args, **kwargs)) - gr = -self._griddata * kwargs['rho_c'] * kwargs['g'] - grf = np.fft.fft2(gr) - Rx = np.arange(((self._georef_info.nx-1)*dx)/2,0,-dx) - Ry = np.arange(((self._georef_info.ny-1)*dx)/2,0,-dx) - lx = len(Rx) - ly = len(Ry) - wn_x = np.array((1 / dx) * np.arange(0,lx,1) / lx) - wn_y = np.array((1 / dx) * np.arange(0,ly,1) / ly) - wn_x = np.concatenate((wn_x,np.flipud(wn_x))) - wn_y = np.concatenate((wn_y,np.flipud(wn_y))) - [WN_x, WN_y] = np.meshgrid(wn_x, wn_y) - - R = 1 / ((kwargs['rho_m']-kwargs['rho_c'])*kwargs['g'] + (2*3.141)**4 * kwargs['D']*(WN_x**2 + WN_y**2)**2) - - grfR = grf * R - - return np.real(np.fft.ifft2(grfR)) - + dx = float(np.mean(self._mean_pixel_dimension(*args, **kwargs))) + + load = -np.nan_to_num(np.asarray(self._griddata, dtype=float64)) \ + * kwargs['rho_c'] * kwargs['g'] + load_hat = np.fft.fft2(load) + + # np.fft.fftfreq gives exactly one wavenumber per sample, in the + # order fft2 expects. Building the axes by hand from half-width + # ranges produced arrays whose length did not have to match the grid, + # so the multiply below could raise a broadcast error. + wn_x = np.fft.fftfreq(self._georef_info.nx, d=dx) + wn_y = np.fft.fftfreq(self._georef_info.ny, d=dx) + WN_x, WN_y = np.meshgrid(wn_x, wn_y) + + # (2*pi)**4, not (2*3.141)**4 -- the truncated constant was 0.2% low + # in the flexural term. + response = 1.0 / ((kwargs['rho_m'] - kwargs['rho_c']) * kwargs['g'] + + (2 * np.pi) ** 4 * kwargs['D'] + * (WN_x ** 2 + WN_y ** 2) ** 2) + + return np.real(np.fft.ifft2(load_hat * response)) + class GeographicDeflection(GeographicGridMixin, Deflection): + """Flexural deflection on a latitude/longitude grid.""" + pass - + class ChannelSlope(BaseSpatialGrid): - + """Downstream gradient of each cell along its D8 flow direction. + + :: + + slope = ChannelSlope(flow_direction=d8, elevation=dem) + + Positive values are downhill. Cells with no downstream neighbour stay 0. + """ + required_inputs_and_actions = ((('nx', 'ny', 'projection', 'geo_transform',),'_create'), (('ai_ascii_filename','EPSGprojectionCode'),'_read_ai'), - (('gdal_filename',), '_read_gdal'), + (('gdal_filename',), '_read_gdal'), (('flow_direction','elevation'), '_create_from_flow_direction_and_elevation')) - + def _create_from_flow_direction_and_elevation(self, *args, **kwargs): - + flow_dir = kwargs['flow_direction'] elevation = kwargs['elevation'] self._copy_info_from_grid(flow_dir, True) - if flow_dir.__class__ == FlowDirectionD8: - self.__calc_D8_slope(flow_dir, elevation) + # isinstance, not an exact class comparison, so GeographicFlowDirection + # and any other subclass is handled too. + if isinstance(flow_dir, FlowDirectionD8): + self.__calc_D8_slope(flow_dir, elevation, *args, **kwargs) - def __calc_D8_slope(self, fd, dem): + def __calc_D8_slope(self, fd, dem, *args, **kwargs): + + receivers = fd.receivers.reshape(-1) + has_receiver = receivers >= 0 + + z = np.asarray(dem._griddata, dtype=float64).reshape(-1) + step = fd.step_lengths(self._mean_pixel_dimension(*args, **kwargs)).reshape(-1) + + slope = np.zeros(z.shape, dtype=float64) + donors = np.flatnonzero(has_receiver) + targets = receivers[donors] + # Vectorised: the original walked every cell in Python, which cost + # minutes on a large DEM for what is a single subtraction per cell. + slope[donors] = (z[donors] - z[targets]) / step[donors] + + self._griddata = slope.reshape(self._georef_info.ny, self._georef_info.nx) - dx = self._georef_info.dx - idcs = fd.sort() # Get the sorted indices of the array in reverse order (e.g. largest first) - - for idx in idcs: # Loop through all the data in sorted order - [i, j] = np.unravel_index(idx, fd._griddata.shape) # Get the row/column indices - - i_next, j_next, is_good = fd.get_flow_to_cell(i,j) - if is_good: - dist = np.sqrt((i-i_next)**2 + (j-j_next)**2)*dx - self._griddata[i,j] = (dem[i,j] - dem[i_next, j_next]) / dist - def mosaicFolder(folderPath, fileSuffix, outfile): - #This runs the gdal utility gdal_merge on the command line, is mainly here so that I don't have to continue looking - #up how to actually accomplish this - #use os to get files, existing gdal functions to merge them - files = glob.glob1(folderPath, '*'+fileSuffix) # get all the files in the path - ## Need to use existing gdal_merge.py.... need to check on interpolation options. - # could also use gdal_warp i think. Should consider whether I care about the possibility of using the c - #utility... - argument = ['python', 'gdal_merge.py', '-o', outfile] - for file in files: - argument.append(os.path.join(folderPath, file)) - # sys.argv = argv - subprocess.call(argument) + """Merge every ``*fileSuffix`` raster in a folder into ``outfile``. + + Uses GDAL's Python API directly rather than shelling out to + ``gdal_merge.py``, which is not on the path of most installations. + """ + files = sorted(glob.glob1(folderPath, '*' + fileSuffix)) + if not files: + raise Error.InputError( + 'mosaicFolder', + 'no files matching *{0} in {1}'.format(fileSuffix, folderPath)) + + paths = [os.path.join(folderPath, name) for name in files] + vrt = gdal.BuildVRT('', paths) + try: + gdal.Translate(outfile, vrt, creationOptions=['COMPRESS=LZW']) + finally: + vrt = None + def plot(*args, **kwargs): - + """Scatter one grid's values against another's, cell by cell. + + :: + + plot(area, slope, xlabel='Drainage area', ylabel='Slope', + decimation_factor=50) + + Parameters + ---------- + xlabel, ylabel : str + Axis labels. + symbol : str + Matplotlib format string; ``'k.'`` by default. + indexes : tuple of ndarray + Restrict to these cells instead of every cell valid in both grids. + decimation_factor : int + Plot only every n-th point. + interactive : bool + Leave the figure open without blocking. + + Returns + ------- + matplotlib.axes.Axes + """ grid1 = args[0]._griddata grid2 = args[1]._griddata interactive = kwargs.pop('interactive', True) - if kwargs.get('xlabel') is None: - xlabel = 'Grid 1' - - if kwargs.get('ylabel') is None: - ylabel = 'Grid 2' - - if kwargs.get('symbol') is None: - symbol = 'k.' - - if kwargs.get('indexes') is None: - valid_indexes = np.where( ~np.isnan(grid1+grid2) ) - else: - valid_indexes = kwargs.get('indexes') - - if kwargs.get('decimation_factor') is not None: - valid_indexes = (valid_indexes[0][::kwargs.get('decimation_factor')],valid_indexes[1][::kwargs.get('decimation_factor')]) - - plt.plot(grid1[valid_indexes], grid2[valid_indexes], symbol) + # Reading these with .get() into a local only inside the `is None` + # branch meant that actually *passing* xlabel left the name unbound and + # raised NameError two lines later. + xlabel = kwargs.pop('xlabel', None) or 'Grid 1' + ylabel = kwargs.pop('ylabel', None) or 'Grid 2' + symbol = kwargs.pop('symbol', None) or 'k.' + valid_indexes = kwargs.pop('indexes', None) + decimation_factor = kwargs.pop('decimation_factor', None) + + if valid_indexes is None: + valid_indexes = np.where(~np.isnan(grid1 + grid2)) + + if decimation_factor is not None: + valid_indexes = tuple(axis[::decimation_factor] for axis in valid_indexes) + + plt.plot(grid1[valid_indexes], grid2[valid_indexes], symbol, **kwargs) plt.xlabel(xlabel) plt.ylabel(ylabel) if interactive: @@ -4282,8 +5494,4 @@ def plot(*args, **kwargs): plt.ioff() plt.show(block=True) - ax = plt.gca() - - return ax - - + return plt.gca() diff --git a/demMethods.py b/demMethods.py index 5432aea..cd6a327 100644 --- a/demMethods.py +++ b/demMethods.py @@ -1,73 +1,73 @@ -from dem import FilledElevation +"""Batch pipelines that chain the grid classes into a finished analysis.""" + +import numpy as np + +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d +except ImportError: # pragma: no cover + import dem as d + def processAll(prefix_name, Ao, theta, base_name = '.'): - from dem import Elevation, FlowDirectionD8, GeographicArea, Area, GeographicFlowLength, GeographicKsi, ScaledRelief - elevation_name = base_name + "/" + prefix_name + "_dem_15s" area_name = base_name + "/" + prefix_name + "_acc_15s" d8_name = base_name + "/" + prefix_name + "_dir_15s" - elevation = Elevation(gdal_filename = elevation_name) + elevation = d.Elevation(gdal_filename = elevation_name) elevation.save(prefix_name + "_elevation") - area = Area(gdal_filename = area_name) - d8 = FlowDirectionD8(gdal_filename = d8_name) + area = d.Area(gdal_filename = area_name) + d8 = d.FlowDirectionD8(gdal_filename = d8_name) d8.save(prefix_name + "_flow_direction") idx = area.sort(reverse = False) - area = GeographicArea(flow_direction = d8, sorted_indexes = idx) + area = d.GeographicArea(flow_direction = d8, sorted_indexes = idx) area.save(prefix_name + "_area") - flow_length = GeographicFlowLength(flow_direction = d8, sorted_indexes = idx) + flow_length = d.GeographicFlowLength(flow_direction = d8, sorted_indexes = idx) flow_length.save(prefix_name + "_flow_length") - ksi = GeographicKsi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length, sorted_indexes = idx) + ksi = d.GeographicKsi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length, sorted_indexes = idx) ksi.save(prefix_name + "_ksi_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) - relief = ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta, sorted_indexes = idx) + relief = d.ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta, sorted_indexes = idx) relief.save(prefix_name + "_relief_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) def processAllUTM(prefix_name, EPSGprojectionCode, Ao, theta, base_name = '.'): - from dem import Elevation, FlowDirectionD8, Area, FlowLength, Ksi, ScaledRelief - full_path_without_suffix = base_name + "/" + prefix_name elevation_unfilled_ascii_filename = full_path_without_suffix + ".txt" - elevation = Elevation(ai_ascii_filename = elevation_unfilled_ascii_filename, EPSGprojectionCode= EPSGprojectionCode) + elevation = d.Elevation(ai_ascii_filename = elevation_unfilled_ascii_filename, EPSGprojectionCode= EPSGprojectionCode) elevation.save(full_path_without_suffix + "_elevation") - filled = FilledElevation(elevation = elevation) + filled = d.FilledElevation(elevation = elevation) filled.save(full_path_without_suffix + "_filled") - d8 = FlowDirectionD8(flooded_dem = filled) + d8 = d.FlowDirectionD8(flooded_dem = filled) d8.save(full_path_without_suffix + "_flow_direction") - area = Area(flow_direction = d8) + area = d.Area(flow_direction = d8) area.save(full_path_without_suffix + "_area") - flow_length = FlowLength(flow_direction = d8) + flow_length = d.FlowLength(flow_direction = d8) flow_length.save(full_path_without_suffix + "_flow_length") - ksi = Ksi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length) + ksi = d.Ksi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length) ksi.save(full_path_without_suffix + "_ksi_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) - relief = ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta) + relief = d.ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta) relief.save(full_path_without_suffix + "_relief_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) def processForTheta(prefix_name, Ao, theta, base_name = '.'): - from dem import FlowDirectionD8, GeographicArea, GeographicFlowLength, GeographicKsi, ScaledRelief, Elevation - area_name = base_name + "/" + prefix_name + "_area" d8_name = base_name + "/" + prefix_name + "_flow_direction" flow_length_name = base_name + "/" + prefix_name + "_flow_length" elevation_name = base_name + "/" + prefix_name + "_elevation" - area = GeographicArea.load(area_name) - d8 = FlowDirectionD8.load(d8_name) - flow_length = GeographicFlowLength.load(flow_length_name) - elevation = Elevation.load(elevation_name) + area = d.GeographicArea.load(area_name) + d8 = d.FlowDirectionD8.load(d8_name) + flow_length = d.GeographicFlowLength.load(flow_length_name) + elevation = d.Elevation.load(elevation_name) idx = area.sort(reverse = False) - ksi = GeographicKsi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length, sorted_indexes = idx) + ksi = d.GeographicKsi(area = area, flow_direction = d8, theta = theta, Ao = Ao, flow_length = flow_length, sorted_indexes = idx) ksi.save(prefix_name + "_ksi_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) - relief = ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta, sorted_indexes = idx, area = area) + relief = d.ScaledRelief(flow_direction = d8, elevation = elevation, flow_length = flow_length, Ao = Ao, theta = theta, sorted_indexes = idx, area = area) relief.save(prefix_name + "_relief_" + str(Ao).replace('.','_') + "_" + str(theta).replace('.','_')) def plotGrids(x_grid, y_grid, plot_string, **kwargs): - - import numpy as np - + """Scatter one grid against another, keeping only non-negative y.""" x_vec = np.ndarray.flatten(x_grid._griddata) y_vec = np.ndarray.flatten(y_grid._griddata) @@ -82,8 +82,7 @@ def plotGrids(x_grid, y_grid, plot_string, **kwargs): return x_vec, y_vec def extract_values_from_grid(x_grid, y_grid, ignore_zeros=False): - - import numpy as np + """Flatten two grids to matching 1-D vectors.""" x_vec = np.ndarray.flatten(x_grid._griddata) y_vec = np.ndarray.flatten(y_grid._griddata) if ignore_zeros: @@ -94,7 +93,7 @@ def extract_values_from_grid(x_grid, y_grid, ignore_zeros=False): return x_vec, y_vec def create_density(x, y, x_boundaries, y_boundaries): - import numpy as np + """2-D histogram of paired grid values.""" H, xedges, yedges = np.histogram2d(x, y, bins = (x_boundaries, y_boundaries)) return H, xedges, yedges diff --git a/demRecursionTools.py b/demRecursionTools.py index 744c872..e12b308 100644 --- a/demRecursionTools.py +++ b/demRecursionTools.py @@ -1,5 +1,16 @@ +"""Profile extraction and chi/steepness fitting over recursive basin maps. + +These functions consume the nested dictionaries produced by +:meth:`TopoAnalysis.dem.FlowDirectionD8.map_values_to_recursive_list`. +""" + import numpy as np +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d +except ImportError: # pragma: no cover + import dem as d + def extract_chi_elevation_values(ld_list, de, theta, chi_o, elevation, chi, base_elevation, A_mdx = None, xo = 500.0): @@ -113,7 +124,8 @@ def hi_list(ld_list): elevation, dA = extract_dA_elevation_values(ld_list) - if np.nan in elevation: + # `np.nan in array` is always False (NaN != NaN); test explicitly. + if np.isnan(elevation).any(): return 0.0 mean_elevation = np.mean(elevation) @@ -124,7 +136,6 @@ def hi_list(ld_list): def area_elevation_for_mainstem_and_tributaries(outlet, flow_direction, elevation, area, theta = 0.5, minimum_area = 1.0E7): - import dem as d mean_pixel_dimension = d.BaseSpatialGrid() mean_pixel_dimension._copy_info_from_grid(area, True) mean_pixel_dimension._griddata = area._mean_pixel_dimension() @@ -163,7 +174,8 @@ def get_elevations_and_areas(ld_list, area, elevation, de, tributary_ld, minimum for trb_ld in tributary_ld: this_area = [trb_ld['area']] this_elevation = [trb_ld['elevation']] - this_de = [trb_ld['de']+trb_ld['distance_scale']] + # '*' not '+': de is a length and distance_scale is a multiplier. + this_de = [trb_ld['de'] * trb_ld['distance_scale']] (this_area, this_elevation, this_de, next_tributary_ld) = get_elevations_and_areas(trb_ld, this_area, this_elevation, this_de, next_tributary_ld, minimum_area) area.append(this_area) elevation.append(this_elevation) @@ -203,7 +215,7 @@ def chi_for_profile(area, de, theta): def best_ks_with_wrss(chi, elevation): A = np.vstack([chi]).T - sol = np.linalg.lstsq(A, elevation) + sol = np.linalg.lstsq(A, elevation, rcond=None) m = sol[0] WRSS = sol[1] return (m, WRSS) @@ -291,7 +303,7 @@ def chi_for_profile(area, de, theta): def best_ks_with_wrss(chi, elevation): A = np.vstack([chi]).T - sol = np.linalg.lstsq(A, elevation) + sol = np.linalg.lstsq(A, elevation, rcond=None) m = sol[0] WRSS = sol[1] return (m, WRSS) @@ -356,7 +368,6 @@ def map_ld_link(current_chi, ld_list, downstream_sign, minimum_area): return_map[index] = (next_chi, next_elevation) map_ld_link(next_chi, next_list, downstream_sign, minimum_area) - import dem as d mean_pixel_dimension = d.BaseSpatialGrid() mean_pixel_dimension._copy_info_from_grid(area, True) mean_pixel_dimension._griddata = area._mean_pixel_dimension() diff --git a/docs/algorithms.md b/docs/algorithms.md new file mode 100644 index 0000000..0d7ec6a --- /dev/null +++ b/docs/algorithms.md @@ -0,0 +1,397 @@ +# Algorithms + +What each computation actually does, the conventions it assumes, and where +it will disagree with another package. + +--- + +## Depression filling — Priority-Flood + +A DEM straight off a sensor is full of closed depressions: real ones +(playas, craters, karst) and spurious ones (noise, bridges, vegetation +returns). Flow routing needs every cell to have a downhill path to the edge, +so the depressions are filled first. + +TopoAnalysis uses the Priority-Flood family of Barnes, Lehman and Mulla +(2014). The idea is simple: start from the cells water can leave through, +and flood inwards, always processing the lowest cell you have reached so far. + +``` +push every seed cell onto a min-heap, mark it closed +while the heap is not empty: + c = pop the lowest cell + for each of c's eight neighbours n that is not closed: + close n + raise n to at least c's elevation + push n +``` + +Because cells are processed in nondecreasing elevation order, the first time +you reach a cell you have reached it by the lowest possible path, so the +elevation you raise it to is the lowest that lets water out. The filled +surface is the pointwise-minimal surface with no depressions. + +### Seeds + +Water leaves the DEM at the grid perimeter and at no-data holes, so the seed +set is: + +- every cell on the outer ring, and +- every valid cell that touches a `NaN`. + +This is the same seed set TopoToolbox's `fillsinks` uses. A depression that +touches a no-data hole is therefore *not* filled — it drains into the hole. + +You can supply your own seeds with `outlets=[(x, y), ...]`, which floods +inwards from those points instead. + +### The three variants + +`TopoAnalysis.kernels.priority_flood` implements all three algorithms from +the paper: + +| Variant | Paper | When it is used | +|---|---|---| +| Priority-Flood | Algorithm 1 | `reference_algorithm=True`, as a test oracle | +| Improved Priority-Flood | Algorithm 2 | the default for a flat fill | +| Priority-Flood + ε | Algorithm 4 | whenever `aggradation_slope > 0` | + +Algorithm 2 adds a FIFO queue alongside the heap: a cell that needs no +filling sits at the current flood level and cannot be reordered, so it can +skip the heap entirely. That is what makes the practical cost O(n) instead +of O(n log n). The result is identical to Algorithm 1 — the test suite +asserts it on random surfaces. + +The FIFO shortcut is only sound while every cell it holds is at exactly the +current level. Two options break that, and both fall back to Algorithm 1: + +- **ε filling**, where each filled cell is raised by its own increment; and +- **`maximum_pit_depth`**, which re-queues an unfilled cell *below* the + current level. + +### Flat fill or ε fill? + +A flat fill produces exactly level plateaus, which have no downhill +direction — D8 routing has nothing to follow across them. TopoAnalysis +therefore defaults to raising each filled cell by + + parent_elevation + aggradation_slope × step_length + +with `aggradation_slope = 1e-12` per map unit. That is far below any +meaningful elevation difference but strictly monotonic, so every filled cell +has a defined downslope neighbour. + +Set `aggradation_slope=0` for the flat fill TopoToolbox produces. TopoToolbox +then handles the flats separately, by building an auxiliary "geodesic" +surface inside them so cells drain towards the nearest sill. The two +approaches give *different but equally valid* flow directions inside flats; +see [topotoolbox_parity.md](topotoolbox_parity.md). + +### Leaving real basins alone + +`maximum_pit_depth=20.0` leaves any depression deeper than 20 m unfilled. +The flood still traverses it — so the terrain upstream is routed — but the +cells keep their elevations, and the basin becomes a local base level. + +> The original implementation marked such cells closed without queueing +> them, which orphaned everything upstream of a deep pit. + +### Cost + +O(n) time and O(n) memory in practice. About 100 ms per million cells in +C++; 5 s in NumPy. + +--- + +## Flow direction — D8 + +Each cell drains to exactly one of its eight neighbours: the one reached by +the steepest *gradient*, + + slope = (z_centre − z_neighbour) / distance + +with `distance = dx` for the four cardinal neighbours and `√2 · dx` for the +four diagonals. Dividing by distance matters: a diagonal step covers 41% +more ground, so an equal drop over a diagonal is a shallower slope. Choosing +by raw drop biases the network towards diagonals and inflates flow lengths. + +Ties go to the first direction in the order E, SE, S, SW, W, NW, N, NE. Any +rule is arbitrary; what matters is that it is deterministic, so two runs +agree. + +Codes follow the ArcGIS convention, with row 0 at the north edge: + +``` +| 32 64 128 | | (i-1,j-1) (i-1,j) (i-1,j+1) | +| 16 X 1 | == | (i ,j-1) X (i ,j+1) | +| 8 4 2 | | (i+1,j-1) (i+1,j) (i+1,j+1) | +``` + +0 means the cell has no downstream neighbour: a pit, an outlet at the grid +edge, or no-data. On a filled DEM the only zeros are edge outlets. + +For a geographic (lat/lon) grid the distance uses the local cell size, which +varies with latitude. + +--- + +## The flow network + +Once directions are known, the grid is a forest: every cell has one +*receiver* and any number of *donors*, and following receivers always +terminates. + +```python +d8.receivers # flat index of each cell's receiver, or -1 +d8.topological_order # every donor appears before its receiver +``` + +The order comes from Kahn's algorithm on the donor counts, **not** from +sorting by elevation. That distinction matters: a flow-direction grid loaded +from a file carries no elevations, so an elevation sort is unavailable — +and the original code silently sorted such grids by their *direction codes*, +producing wrong accumulations. + +Kahn's algorithm also detects cycles, which a hand-edited or externally +produced flow-direction grid can contain. `d8.cycle_count` reports them and +a warning is raised; a valid network has none. + +Given the order, everything downstream is one linear pass: + +| Quantity | Pass | Recurrence | +|---|---|---| +| accumulation | forward | `A[recv] += A[k]` | +| flow length | forward | `L[recv] = max(L[recv], L[k] + step)` | +| distance to outlet | backward | `D[k] = D[recv] + step` | +| chi | backward | `χ[k] = χ[recv] + integrand · step` | +| basin labels | backward | `label[k] = label[recv]` | +| minima imposition | forward | `z[recv] = min(z[recv], z[k] − sl·step)` | +| Strahler order | forward | the order-increment rule | + +--- + +## Drainage area + +`Area` accumulates the per-cell area downstream, so each cell holds its own +area plus every cell upstream of it, in **square map units**. + +TopoToolbox's `flowacc` accumulates unit weights instead and reports the +**number of cells**. `TopoAnalysis.topotoolbox.flowacc` does the same; the +two differ by exactly `dx²` on a projected grid. + +For a geographic grid, `GeographicArea` uses the true spherical cell area + + A(lat) = R² · Δλ · (sin(lat + Δφ/2) − sin(lat − Δφ/2)) + +with `R = 6 371 000 m`. Cells shrink towards the poles: at 60° latitude they +are half the area of equatorial ones, and using a constant `dx²` would +overestimate drainage area by that factor. + +### Gating + +`evaluate_at` or `mask` restrict accumulation to a subset of cells. A gated- +out cell contributes nothing and passes nothing downstream — the first cell +*outside* the gate still receives what the gate delivers, and the flow +stops there. `ValleyArea` uses this to accumulate only over valley floors. + +--- + +## Flow length and the main stem + +`FlowLength` records, for each cell, the length of the **longest** flow path +reaching it — the standard "flow length" of a basin, and the distance used +to define its relief. + +Alongside the distances it records which upstream neighbour supplies each +cell's longest path. Chaining those donors from an outlet traces the main +stem. Ties are broken towards the donor with the lower flat index, so the +main stem does not depend on which valid topological order was used. + +`Relief`, `ScaledRelief` and `Ksi` all propagate along that main-stem +network, which is why they need a `FlowLength` as well as a flow direction. + +> Before version 1.0 these codes used a row-flipped convention, internally +> consistent but incompatible with `FlowDirectionD8`. `*_directions` files +> written by older versions must be regenerated. + +--- + +## Chi + +For a river in steady state under the stream-power law, elevation is linear +in the integral quantity + + χ(x) = ∫[outlet → x] (A₀ / A(x'))^θ dx' + +(Perron & Royden 2013). The slope of elevation against χ is the steepness +index k_sn, and A₀ makes χ dimensionally a length — usually 1 km² = 10⁶ m². + +Two implementations, because they answer different questions. + +**`Chi`** integrates upstream from outlets you name, over the whole network +above each one. Use it when the basins are the unit of analysis. + +**`Ksi`** integrates along longest-flow-length paths across the whole grid, +without outlets. Use it for a regional map. + +### The integration rule + +Per step from a cell to its receiver: + +- *left endpoint* (the default): `χ[k] = χ[recv] + (A₀/A[k])^θ · dl` +- *trapezoid* (`trapezoid=True`): averages the integrand across the step, + `χ[k] = χ[recv] + ½((A₀/A[k])^θ + (A₀/A[recv])^θ) · dl` + +TopoToolbox's `chitransform` uses the trapezoid rule. Area grows downstream +so the integrand shrinks downstream, which makes the left-endpoint rule the +larger of the two. Use `trapezoid=True` when comparing. + +χ is 0 at each outlet and grows upstream. + +> The `Ksi` integrand used to be `(A₀ / (A − A₀))^θ`, which diverges as area +> approaches the threshold. It is now `(A₀ / A)^θ`. + +--- + +## Terrain derivatives + +### Slope + +`MaxSlope` is the magnitude of the centred-difference gradient, +`√((dz/dx)² + (dz/dy)²)`, with the boundary padded by replicating the edge. + +`topotoolbox.gradient8` is the steepest *downward* gradient over the eight +neighbours, clamped at 0 — a different quantity, always non-negative, 0 in +pits and flats. + +`topotoolbox.arcslope` is the ArcGIS/Horn 3×3 estimator: a 1-2-1 weighted +stencil over an `8·dx` denominator, with no √2 anywhere. It is less +sensitive to single-cell noise than a plain centred difference. + +### Measuring over a length scale + +On a 1 m lidar DEM a 3×3 stencil measures the noise, not the landscape. +`calculate_gradient_over_length_scale(L)` and +`calculate_laplacian_over_length_scale(L)` use a stencil reaching +`N = ⌈L/dx⌉` cells either side, so the sampled cells are `2N·dx` apart. A +band `N` cells wide around the edge comes back as `NaN`. + +### Curvature + +`Laplacian` is `∇²z` from SciPy's 5-point stencil, scaled by `1/dx²`. +Positive in valleys, negative on ridges, and the basis of the +valley-floor masks that `ValleyArea` uses. + +`principal_curvatures()` returns the two principal curvatures of the surface +from the second fundamental form. `topotoolbox.curvature` offers the five +TopoToolbox flavours — profile, plan, tangential, mean and total — from the +Evans 3×3 quadratic fit. + +### Hillshade + +`Hillshade` follows the ESRI definition and returns 0–255: + + H = 255 · (cos(zenith)·cos(slope) + sin(zenith)·sin(slope)·cos(azimuth − aspect)) + +with `zenith = 90° − altitude`, `azimuth_math = 360° − azimuth + 90°`, and +`aspect = atan2(dz/dy, −dz/dx)`. Values below 0 — slopes facing away from +the light — are clamped to 0. + +> The `−dz/dx` is easy to miss and easy to get wrong; without it the +> illumination is mirrored east–west, which is what the original did. + +`topotoolbox.hillshade` returns the raw cosine in [−1, 1] from a +central-difference surface normal, which is what TopoToolbox does (it only +scales to 0–255 when plotting). + +--- + +## Valley extraction + +`ValleyArea` builds a valley-floor mask in three steps: + +1. **Convergence.** Cells whose Laplacian exceeds `valley_laplace_value`. +2. **Connectivity.** `PriorityFillGrid` floods outwards from cells with more + than `min_area_value` of drainage area through the convergent cells, so + only the parts that connect to a real channel survive. Isolated + convergent patches on hillslopes are dropped. +3. **Closing** (optional, `iterations=n`). Dilate then erode, to bridge + one-cell gaps without growing the network. + +Drainage area is then accumulated over that mask only. +`MainstemValleyArea` restricts it further, to the longest flow path. + +`MultiscaleCurvatureValleyWidth` takes a different route: it fits a local +quadratic at a range of window sizes and records, at each cell, the window +that minimises the smaller principal curvature. That scale is the valley +width, because a window matched to the valley resolves it best. + +--- + +## Restored topography + +`RestoredElevation` asks what the landscape would look like if channel +steepness were uniform. Starting from the outlets it integrates + + dz/dx = k_s · A^(−θ) + +upstream, then lets each divide migrate towards whichever side ended up +lower, recomputes drainage area, and repeats. Comparing the result with the +real DEM shows where the landscape is out of steady state. + +`randomize=True` randomises elevations inside the basin first and re-routes, +so the result does not inherit the real network's planform. + +--- + +## Flexural isostasy + +`Deflection` solves the thin elastic plate equation spectrally: + + w(k) = −ρ_c g h(k) / [(ρ_m − ρ_c) g + D (2π|k|)⁴] + +Long wavelengths approach the Airy limit `−h ρ_c/(ρ_m − ρ_c)`; short ones +are supported by plate strength. Because it is spectral the load is +implicitly periodic, so pad the DEM if edge effects matter. + +Pass `restored_elevation` as well to get the *change* in deflection between +two surfaces — the isostatic response to the erosion between them. + +--- + +## Numerical conventions + +| | | +|---|---| +| Row order | row 0 is the **north** edge | +| No-data | `NaN` in floating-point grids; 0 in flow-direction grids | +| Cell registration | coordinates refer to cell **centres** | +| `geoTransform` | GDAL's 6-tuple, origin at the **outer corner** of the NW cell | +| Diagonal distance | `√2 · dx`, never 1.41 | +| Precision | float64 throughout (TopoToolbox often works in float32) | +| Cells | assumed square; `dx` is used for both axes | + +--- + +## References + +Barnes, R., Lehman, C. and Mulla, D. (2014). Priority-flood: An optimal +depression-filling and watershed-labeling algorithm for digital elevation +models. *Computers & Geosciences* 62, 117–127. + +Horn, B.K.P. (1981). Hill shading and the reflectance map. *Proceedings of +the IEEE* 69, 14–47. + +O'Callaghan, J.F. and Mark, D.M. (1984). The extraction of drainage networks +from digital elevation data. *Computer Vision, Graphics and Image +Processing* 28, 323–344. + +Perron, J.T. and Royden, L. (2013). An integral approach to bedrock river +profile analysis. *Earth Surface Processes and Landforms* 38, 570–576. + +Schwanghart, W. and Scherler, D. (2014). TopoToolbox 2 — MATLAB-based +software for topographic analysis and modeling in Earth surface sciences. +*Earth Surface Dynamics* 2, 1–7. + +Willett, S.D., McCoy, S.W., Perron, J.T., Goren, L. and Chen, C.-Y. (2014). +Dynamic reorganization of river basins. *Science* 343, 1248765. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..4fd55ca --- /dev/null +++ b/docs/api.md @@ -0,0 +1,1993 @@ +# API reference + +Generated from the docstrings by `docs/generate_api.py`, so +`help(TopoAnalysis.Area)` in a session shows the same text. For how the +pieces fit together start with [guide.md](guide.md); for what the +computations actually do see [algorithms.md](algorithms.md). + +Attributes prefixed with a single underscore (`_griddata`, +`_georef_info`) are not private in practice -- they are how you reach +the data -- but they are not listed here; see +[guide.md](guide.md#the-grid-model). + +## `TopoAnalysis.dem` -- the grid classes + +Contents: + +- **Core grid types**: `BaseSpatialGrid`, `Georef_info`, `BaseSpatialShape`, `ValueGrid`, `Elevation`, `GeographicElevation` +- **Depression filling**: `PriorityQueueMixIn`, `FilledElevation`, `PriorityFillGrid` +- **Flow routing**: `FlowDirection`, `FlowDirectionD8`, `FlowLength`, `GeographicFlowLength`, `Mask`, `DiscreteFlowAccumulation`, `GeographicDiscreteFlowAccumulation` +- **Drainage area**: `Area`, `GeographicArea`, `LogArea`, `ValleyArea`, `GeographicValleyArea`, `MainstemValleyArea`, `GeographicMainstemValleyArea` +- **Chi and channel steepness**: `Chi`, `GeographicChi`, `Ksi`, `GeographicKsi`, `Relief`, `ScaledRelief`, `ChiScaledRelief`, `ChannelSlope`, `CrossDivideDChi`, `NormalizedCrossDivideDChi`, `RestoredElevation`, `GeographicRestoredElevation` +- **Along-flow smoothing**: `AlongFlowSmoothing`, `KsFromChiWithSmoothing`, `GeographicKsFromChiWithSmoothing`, `ThetaFromChiWithSmoothing`, `GeographicThetaFromChiWithSmoothing`, `ChannelSlopeWithSmoothing`, `ChannelDownSlopeWithSmoothing`, `ChannelUpSlopeWithSmoothing` +- **Terrain attributes**: `CalculationMixin`, `Hillshade`, `GeographicHillshade`, `MaxSlope`, `GeographicMaxSlope`, `Laplacian`, `GeographicLaplacian`, `Gradient`, `LocalRelief`, `MultiscaleCurvatureValleyWidth`, `ScarpWavelet` +- **Isostasy**: `Deflection`, `GeographicDeflection` +- **Mixins**: `GDALMixin`, `GeographicGridMixin`, `MaxFlowLengthTrackingMixin` + +### Core grid types + +#### `BaseSpatialGrid` + +*inherits* `GDALMixin` + +A georeferenced raster, and the base class of every grid here. + +Construct one by keyword; the combination you pass selects how it is +built (see :attr:`required_inputs_and_actions`):: + + BaseSpatialGrid(gdal_filename='grid.tif') + BaseSpatialGrid(dx=30.0, grid=array) + BaseSpatialGrid(nx=100, ny=80, dx=30.0) # random values + BaseSpatialGrid(nx=100, ny=80, projection=wkt, geo_transform=gt) + +The data live in ``_griddata``, a 2-D NumPy array with row 0 at the +**north** edge and ``NaN`` for no-data; the georeferencing lives in +``_georef_info`` (a :class:`Georef_info`). + +Indexing is by ``(row, col)`` and is forgiving: reading outside the grid +returns ``None`` and writing outside it is ignored, which is what the +neighbourhood traversals in this module rely on. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `nx`, `ny`, `dx` +- `dx`, `grid` + +*Stored as* `float64` + +| Member | Signature | Summary | +|---|---|---| +| `apply_moving_window` | `(moving_window)` | Apply a :class:`~TopoAnalysis.MovingWindow.MovingWindow` to the grid. | +| `average_over_distance` | `(distance, grid=None)` | Mean of the grid within ``distance`` of each cell. | +| `calculate_gradient_over_length_scale` | `(length_scale)` | Centred finite-difference gradient measured over ``length_scale``. | +| `calculate_laplacian_over_length_scale` | `(length_scale)` | Laplacian measured over ``length_scale`` with a centred stencil. | +| `clip_to_bounds` | `(bounds)` | Clip to ``((xmin, xmax), (ymin, ymax))``. | +| `clip_to_extent` | `(extent)` | Clip to ``(xmin, xmax, ymin, ymax)``, keeping every cell inside it. | +| `clip_to_mask_grid` | `(mask_grid)` | Reproject and resample this grid onto ``mask_grid``'s frame, in place. | +| `clip_to_shapefile` | `(shapefile)` | Cut the grid to a vector boundary, in place. | +| `coordinate_vectors` | `()` | ``(x, y)`` coordinates of the cell centres, west-to-east and | +| `export_arc_e00_grid` | `(filename)` | Write an ArcInfo E00 interchange grid. | +| `extent` | `()` | ``(xmin, xmax, ymin, ymax)`` of the grid's outer pixel edges. | +| `extent_of_data` | `()` | ``(xmin, xmax, ymin, ymax)`` of the smallest box holding all valid data. | +| `find_nearest_cell_with_greatest_value` | `(index, pixel_radius=5)` | Index of the highest-valued cell within ``pixel_radius``. | +| `find_nearest_cell_with_value` | `(index, value, pixel_radius)` | Index of the cell within ``pixel_radius`` whose value is closest to ``value``. | +| `find_nearest_cell_with_value_greater_than` | `(index, value, pixel_radius)` | Index of the cell within ``pixel_radius`` whose value is the smallest | +| `get_XY_matrices` | `()` | Meshgrid of cell-centre coordinates. | +| `get_XY_matricies` | `()` | Meshgrid of cell-centre coordinates. | +| `load` | `(filename)` | Read a grid previously written by :meth:`save`. *(classmethod)* | +| `location_in_grid` | `(xo)` | True when ``xo = (x, y)`` lands on a cell holding real data. | +| `mosaic` | `(tiles)` | Reassemble tiles produced by :meth:`tile` into a single grid. *(classmethod)* | +| `plot` | `(**kwargs)` | Display the grid with ``imshow``, georeferenced in map coordinates. | +| `principal_curvatures` | `()` | Maximum and minimum principal curvature of the surface. | +| `remove_padding` | `(xpadding=10, ypadding=10)` | Strip the overlap that :meth:`tile` added, ready for :meth:`mosaic`. | +| `resample` | `(de, interpolation='quintic')` | Resample onto a grid of spacing ``de``. | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | +| `set_value_at_rowscols` | `(value, rowscols)` | Set ``value`` at each ``(row, col)`` in ``rowscols``. | +| `snap_locations_to_closest_value` | `(v, value, pixel_radius=5)` | Move each ``(x, y)`` to the nearby cell whose value best matches. | +| `snap_locations_to_greatest_value` | `(v, pixel_radius=5)` | Move each ``(x, y)`` to the highest-valued cell within ``pixel_radius``. | +| `sort` | `(reverse=True, force=False, mask=None)` | Flat indices of the cells, ordered by value. | +| `tile` | `(tile_xdim=400, tile_ydim=400, tile_xpadding=10, tile_ypadding=10)` | Split the grid into overlapping tiles. | +| `vectorize` | `(filename)` | Polygonize the grid into an ESRI shapefile called ``filename.shp``. | +| `write_to_ai` | `(filename, nodata_value=-9999.0)` | Write the grid as an ArcInfo ASCII grid. | + +#### `Georef_info` + +Where a grid sits on the ground. + +Attributes +---------- +geoTransform : tuple + The GDAL 6-tuple ``(x_origin, dx, 0, y_origin, 0, -dy)``, whose + origin is the *outer corner* of the north-west cell. +projection : str or int + WKT, or the integer 0 when no projection is set. +dx : float + Cell size in map units. Cells are assumed square. +xllcenter, yllcenter : float + Centre of the south-west cell. +nx, ny : int + Columns and rows. + +#### `BaseSpatialShape` + +A vector layer, used to cut rasters to a boundary. + +:: + + shape = BaseSpatialShape(shapefile_name='basin.shp') + mask = shape.createMaskFromShape(dem._georef_info, + dem._georef_info.projection, + gdal.GDT_Byte) + +| Member | Signature | Summary | +|---|---|---| +| `createMaskFromShape` | `(geoRefInfo, projection, dtype, noDataValue=0)` | Rasterise the shape onto ``geoRefInfo``'s frame, 1 inside and 0 out. | + +#### `ValueGrid` + +*inherits* `BaseSpatialGrid` + +A plain raster of values, with a helper for scattered writes. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `dx`, `grid` + +| Member | Signature | Summary | +|---|---|---| +| `set_value_at_indexes` | `(indexes, value)` | Set ``value`` at each ``(row, col)`` pair in ``indexes``. | + +#### `Elevation` + +*inherits* `CalculationMixin`, `BaseSpatialGrid` + +A DEM. + +Adds the terrain-derivative helpers of :class:`CalculationMixin` to the +base raster, plus the edge detection that depression filling starts from. + +| Member | Signature | Summary | +|---|---|---| +| `findDEMedge` | `()` | Cells where water can leave the DEM. | +| `outlets_at_coastlines` | `(iterations=3)` | Map coordinates of the cells forming the land/sea boundary. | +| `track_flow_downhill` | `(starting_point, maximum_pit_depth=20)` | Walk downhill from ``starting_point``, hopping over shallow pits. | + +#### `GeographicElevation` + +*inherits* `GeographicGridMixin`, `Elevation` + +A DEM on a latitude/longitude grid. + +### Depression filling + +#### `PriorityQueueMixIn` + +Depression filling by Priority-Flood. + +Mix into a grid class to give it :meth:`_flood`, which removes closed +depressions so that every cell has a downhill path to an outlet. + +The algorithms are those of + + Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal + depression-filling and watershed-labeling algorithm for digital + elevation models." *Computers & Geosciences* 62, 117-127. + +and run in :mod:`TopoAnalysis.kernels`, in C++ where the extension is +built and in NumPy otherwise. + +Attributes +---------- +aggradation_slope : float + Gradient of the increment added while filling. The default of 1e-12 + makes filled surfaces very slightly convergent, which gives D8 routing + something to follow across what would otherwise be a dead flat + plateau. Set it to 0 for the flat fill that TopoToolbox's + ``fillsinks`` produces. + +| Member | Signature | Summary | +|---|---|---| +| `randomize_subbasins_with_mask` | `(*args, **kwargs)` | Replace elevations inside a mask with random values and re-flood. | + +#### `FilledElevation` + +*inherits* `PriorityQueueMixIn`, `Elevation` + +A DEM with its closed depressions removed. + +:: + + filled = FilledElevation(elevation=dem) + +Filling uses Priority-Flood (Barnes et al. 2014). By default the filled +surface carries a gradient of +:attr:`PriorityQueueMixIn.aggradation_slope` across former depressions so +that D8 routing has a direction to follow; pass +``aggradation_slope=0`` -- or set the class attribute -- for the flat fill +that TopoToolbox's ``fillsinks`` produces. + +Other keywords (``mask``, ``outlets``, ``maximum_pit_depth``, +``clip_to_fill``) are passed to :meth:`PriorityQueueMixIn._flood`. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation` + +#### `PriorityFillGrid` + +*inherits* `PriorityQueueMixIn`, `BaseSpatialGrid` + +Cells of a mask that are connected to a set of outlets. + +Floods outwards from ``outlets`` through the cells where ``mask`` is 1 +and returns a 0/1 grid of everything reached. Used to grow a valley +network from a curvature mask, keeping only the parts that connect to a +real channel. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `outlets`, `mask` + +*Stored as* `uint8` + +### Flow routing + +#### `FlowDirection` + +*inherits* `BaseSpatialGrid` + +Base class for flow-direction grids. + +#### `FlowDirectionD8` + +*inherits* `FlowDirection` + +Single-direction (D8) flow directions. + +Values are ArcGIS direction codes; 0 means the cell has no downstream +neighbour (a pit, an outlet at the grid edge, or no-data):: + + | 32 64 128 | + | 16 X 1 | + | 8 4 2 | + +Build one from a filled DEM:: + + d8 = FlowDirectionD8(flooded_dem=FilledElevation(elevation=dem)) + +Direction is chosen by steepest *gradient*, so the drop to a diagonal +neighbour is divided by ``sqrt(2) * dx`` before being compared with the +cardinal ones. Ties go to the first direction in the order E, SE, S, SW, +W, NW, N, NE. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flooded_dem` +- `elevation` + +*Stored as* `uint8` + +| Member | Signature | Summary | +|---|---|---| +| `basin_mask` | `(outlets)` | Boolean array marking every cell draining to any of ``outlets``. | +| `bounds_of_basin_for_outlet` | `(outlet)` | ``((xmin, xmax), (ymin, ymax))`` of the basin draining to ``outlet``. | +| `convert_rivertools_directions_to_arc` | `(no_data=0)` | Convert RiverTools flow codes in place to the ArcGIS convention. | +| `cycle_count` | `` | How many cells lie on circular flow paths (0 for a valid network). | +| `divides` | `()` | Grid marking drainage divides: cells with no upstream neighbour. | +| `divides_for_outlets` | `(outlet1, outlet2)` | Cells on each side of the divide shared by two basins. | +| `get_flow_to_cell` | `(i, j)` | Index of the cell that ``(i, j)`` drains into. | +| `get_indexes_of_upstream_cells` | `(i, j)` | Tuple of ``(row, col)`` for every cell upstream of ``(i, j)``. | +| `get_indexes_of_upstream_cells_for_location` | `(x, y)` | As :meth:`get_indexes_of_upstream_cells`, from a map coordinate. | +| `get_upstream_cell_indexes` | `(i, j)` | Indices of the immediate neighbours that drain into ``(i, j)``. | +| `invalidate_network` | `()` | Drop the cached receivers and ordering. | +| `locations_of_paired_hollows` | `(outlet1, outlet2, area, Ao=100000.0)` | Channel heads reached by flowing down from a shared divide. | +| `map_values_to_recursive_list` | `(outlet, **kwargs)` | Nested dictionary of the basin upstream of ``outlet``. | +| `paired_divides` | `(mask=None)` | Pair each divide cell with the cell directly across the divide. | +| `pixel_scale` | `(dtype=)` | Step-length multiplier for each cell: 1 cardinal, sqrt(2) diagonal. | +| `receivers` | `` | Flat index of the cell each cell drains to, or -1 for none. | +| `search_down_flow_direction` | `(start, search_length=inf)` | The cells along the flow path from ``start``, without distances. | +| `search_down_flow_direction_from_rowscols_location` | `(start, return_rowscols=False, search_length=inf)` | Follow flow downstream from a ``(row, col)`` start. | +| `search_down_flow_direction_from_xy_location` | `(start, return_rowscols=False, search_length=inf)` | Follow flow downstream from an ``(x, y)`` start. | +| `search_down_flow_direction_with_length` | `(start, search_length=inf)` | Follow flow downstream from ``start``, recording distance travelled. | +| `set_value_at_rowscols` | `(value, rowscols)` | Set ``value`` at each ``(row, col)`` in ``rowscols``. | +| `step_lengths` | `(pixel_dimension=None)` | Distance from each cell to its receiver, in map units. | +| `topological_order` | `` | Flat cell indices ordered so every donor precedes its receiver. | +| `update_flow_codes_in_mask` | `(*args, **kwargs)` | Re-derive flow codes inside a mask after the DEM has changed. | + +#### `FlowLength` + +*inherits* `BaseSpatialGrid` + +Longest upstream flow distance reaching each cell. + +:: + + length = FlowLength(flow_direction=d8) + +Alongside the distances, the grid records which upstream neighbour +supplies each cell's longest path. That "main stem" network is what +:class:`Relief`, :class:`ScaledRelief` and :class:`Ksi` follow, and what +:meth:`locations_along_flow_path_from_outlet` walks. + +.. note:: + The main-stem codes use the same ArcGIS convention as + :class:`FlowDirectionD8`, pointing *from* a cell *at* its upstream + donor. Before version 1.0 they used a row-flipped convention that + was internally consistent but disagreed with every other + flow-direction grid in the library, so ``*_directions`` side-files + written by older versions must be regenerated. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction` + +| Member | Signature | Summary | +|---|---|---| +| `clip_to_extent` | `(extent)` | Clip both the lengths and the main-stem codes to ``extent``. | +| `indexes_along_flow_path_from_outlet` | `(outlet)` | ``(row, col)`` along the longest flow path upstream of ``outlet``. | +| `is_along_flow_length` | `(from_index, to_index)` | True when ``from_index`` supplies ``to_index``'s longest path. | +| `load` | `(filename)` | Read back a grid written by :meth:`save`. *(classmethod)* | +| `locations_along_flow_path_from_outlet` | `(outlet)` | As above, in map coordinates. | +| `locations_along_flow_path_from_outlets` | `(outlets)` | Concatenated main-stem paths for several outlets. | +| `main_stem_directions` | `` | Codes pointing from each cell at its longest-path donor. | +| `map_values_to_recursive_list` | `(outlet, **kwargs)` | Nested main-stem dictionary upstream of ``outlet``. | +| `points_with_length` | `(length, fd)` | Map coordinates where the flow path first exceeds ``length``. | +| `save` | `(filename)` | Write the lengths, plus the main-stem codes to ``filename_directions``. | + +#### `GeographicFlowLength` + +*inherits* `GeographicGridMixin`, `FlowLength` + +Flow length on a latitude/longitude grid. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction` + +#### `Mask` + +*inherits* `BaseSpatialGrid` + +0/1 grid marking the cells that drain to a set of outlets. + +:: + + mask = Mask(flow_direction=d8, outlets=[(x, y), ...]) + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `outlets` + +*Stored as* `uint8` + +| Member | Signature | Summary | +|---|---|---| +| `perform_erosion` | `(structure=None, iterations=1)` | Morphological erosion: shrink the mask by ``iterations`` cells. | +| `perform_opening` | `(structure=None, iterations=1)` | Morphological opening: erode then dilate, removing specks. | + +#### `DiscreteFlowAccumulation` + +*inherits* `BaseSpatialGrid` + +Area drained by a single steepest-descent walk from each outlet. + +Unlike :class:`Area`, which accumulates over the whole D8 network, this +follows one path per outlet, stepping to the lowest unvisited neighbour +each time, and records the running area along it. It is used to trace +individual flow paths across surfaces (debris flows, lava) where the +network abstraction does not apply. + +:: + + dfa = DiscreteFlowAccumulation(elevation=dem, outlets=[(x, y)]) + +Keyword arguments +----------------- +mask : BaseSpatialGrid + Confine the walk to cells where the mask is 1. +terminations_only : bool + Record the total area only at the end of each path. +display_output : bool + Report progress outlet by outlet. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `outlets` +- `elevation` + +#### `GeographicDiscreteFlowAccumulation` + +*inherits* `GeographicGridMixin`, `DiscreteFlowAccumulation` + +Discrete flow accumulation on a latitude/longitude grid. + +### Drainage area + +#### `Area` + +*inherits* `BaseSpatialGrid` + +D8 contributing (drainage) area. + +:: + + area = Area(flow_direction=d8) + +Values are in squared map units: each cell contributes its own area +(``dx**2``, or the true spherical cell area for a +:class:`GeographicArea`) plus everything upstream of it. + +To count contributing *cells* instead -- which is what TopoToolbox's +``flowacc`` returns -- divide by the per-cell area, or pass +``weights=np.ones(...)``. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction` +- `dx`, `grid` + +| Member | Signature | Summary | +|---|---|---| +| `areas_between` | `(fd, min_area, max_area)` | Channel heads: cells in an area band whose donors are all below it. | +| `areas_greater_than` | `(min_area)` | Map coordinates of every cell whose area is at least ``min_area``. | + +#### `GeographicArea` + +*inherits* `GeographicGridMixin`, `Area` + +Drainage area on a latitude/longitude grid, using true cell areas. + +#### `LogArea` + +*inherits* `BaseSpatialGrid` + +Base-10 logarithm of drainage area. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `area` + +*Stored as* `float64` + +#### `ValleyArea` + +*inherits* `_ValleyMaskMixin`, `Area` + +Drainage area accumulated only over valley-floor cells. + +:: + + va = ValleyArea(flow_direction=d8, area=area, laplace=laplacian, + valley_laplace_value=0.01, min_area_value=1e6) + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `area`, `laplace`, `valley_laplace_value`, `min_area_value` + +#### `GeographicValleyArea` + +*inherits* `GeographicGridMixin`, `ValleyArea` + +Valley-floor area on a latitude/longitude grid. + +#### `MainstemValleyArea` + +*inherits* `_ValleyMaskMixin`, `Area` + +Valley-floor area accumulated along the main stem only. + +Where :class:`ValleyArea` sums every tributary, this follows only the +longest flow path into each cell, so the result is the valley area of the +trunk stream rather than of the whole basin. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `area`, `laplace`, `valley_laplace_value`, `min_area_value`, `flow_direction` + +#### `GeographicMainstemValleyArea` + +*inherits* `GeographicGridMixin`, `MainstemValleyArea` + +Main-stem valley area on a latitude/longitude grid. + +### Chi and channel steepness + +#### `Chi` + +*inherits* `BaseSpatialGrid` + +The chi coordinate, integrated upstream from chosen outlets. + +.. math:: + \chi(x) = \int_{x_{\mathrm{outlet}}}^{x} + \left(\frac{A_0}{A(x')}\right)^{\theta}\,\mathrm{d}x' + +:: + + chi = Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e6, + outlets=[(x, y)]) + +Chi is 0 at each outlet and increases upstream. Cells outside the +outlets' drainage areas are left as 0. + +Keyword arguments +----------------- +trapezoid : bool + Average the integrand across each step instead of taking it at the + upstream cell. This is the rule TopoToolbox's ``chitransform`` uses; + the default (``False``) reproduces the original TopoAnalysis result. +maximum_length : float + Stop integrating beyond this flow distance from the outlet. +mask : BaseSpatialGrid + Restrict the integration to cells where the mask is non-zero. + +.. note:: + Before version 1.0, chi at the outlet itself was one cell-step of the + integrand rather than 0. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `area`, `flow_direction`, `theta`, `Ao`, `outlets` +- `area`, `flow_direction`, `flow_length`, `theta`, `Ao`, `basin_length` + +#### `GeographicChi` + +*inherits* `GeographicGridMixin`, `Chi` + +Chi on a latitude/longitude grid. + +#### `Ksi` + +*inherits* `BaseSpatialGrid`, `MaxFlowLengthTrackingMixin` + +The chi coordinate integrated along longest-flow-length paths. + +.. math:: + \chi = \int \left(\frac{A_0}{A}\right)^{\theta} \mathrm{d}x + +:: + + ksi = Ksi(area=area, flow_direction=d8, flow_length=length, + theta=0.45, Ao=1e6) + +Cells with less than ``Ao`` of drainage area are left out of the +integral, so chi is defined only on the channel network. + +.. note:: + Before version 1.0 the integrand was ``(Ao / (A - Ao))**theta``, which + diverges as ``A`` approaches ``Ao`` and does not match the definition + of chi in the literature. Values from older runs are not comparable + with these. + +See also :class:`Chi`, which integrates upstream from chosen outlets over +the whole network rather than along main stems. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `area`, `flow_direction`, `theta`, `Ao`, `flow_length` + +#### `GeographicKsi` + +*inherits* `GeographicGridMixin`, `Ksi` + +Chi along longest-flow-length paths on a latitude/longitude grid. + +#### `Relief` + +*inherits* `BaseSpatialGrid`, `MaxFlowLengthTrackingMixin` + +Height of the head of the longest flow path above each cell. + +:: + + relief = Relief(flow_direction=d8, elevation=dem, flow_length=length) + +Pass ``area`` and ``Ao`` together to stop the relief signal being carried +down from cells with less than ``Ao`` of drainage area, which keeps +hillslope noise out of the channel network. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `elevation`, `flow_length` + +#### `ScaledRelief` + +*inherits* `Relief` + +:class:`Relief` scaled by ``Ao**theta``. + +That scaling puts relief in the same units as :class:`Ksi`, so the ratio +of the two is a channel steepness. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `flooded_dem`, `elevation`, `flow_length`, `Ao`, `theta` +- `flow_direction`, `elevation`, `flow_length`, `Ao`, `theta` + +#### `ChiScaledRelief` + +*inherits* `BaseSpatialGrid` + +Height above the basin outlet, scaled by ``Ao**theta``. + +Plotted against :class:`Chi`, the slope of this quantity is the channel +steepness index. + +:: + + relief = ChiScaledRelief(elevation=dem, flow_direction=d8, + theta=0.45, Ao=1e6, outlets=[(x, y)]) + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `flow_direction`, `theta`, `Ao`, `outlets` +- `elevation`, `flow_direction`, `flow_length`, `theta`, `Ao`, `basin_length` + +#### `ChannelSlope` + +*inherits* `BaseSpatialGrid` + +Downstream gradient of each cell along its D8 flow direction. + +:: + + slope = ChannelSlope(flow_direction=d8, elevation=dem) + +Positive values are downhill. Cells with no downstream neighbour stay 0. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `elevation` + +#### `CrossDivideDChi` + +*inherits* `_CrossDivideMixin`, `BaseSpatialGrid` + +Difference in chi across each drainage divide. + +A large value means the two sides of the divide are far from +steady state, so the divide is expected to migrate towards the +higher-chi side (Willett et al. 2014). + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `chi`, `flow_direction` + +#### `NormalizedCrossDivideDChi` + +*inherits* `_CrossDivideMixin`, `BaseSpatialGrid` + +Chi contrast across each divide, divided by the mean chi there. + +Normalising makes divides in high- and low-chi parts of a landscape +comparable. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `chi`, `flow_direction` + +#### `RestoredElevation` + +*inherits* `BaseSpatialGrid` + +Topography implied by a uniform channel steepness. + +Starting from the given outlets, elevations are rebuilt upstream from + +.. math:: \frac{\mathrm{d}z}{\mathrm{d}x} = k_s A^{-\theta} + +and the divides are then allowed to migrate towards whichever side ends +up lower. Repeating that (``iterations`` times) converges on the +landscape a steady, spatially uniform ``ks`` would produce, which can be +compared against the real DEM to find where it is out of steady state. + +:: + + restored = RestoredElevation(flow_direction=d8, elevation=dem, + area=area, theta=0.45, ks=50, + outlets=[(x, y)], iterations=5) + +Keyword arguments +----------------- +randomize : bool + Randomise elevations inside the basin first, then re-route, so the + result does not inherit the real network's planform. +fix_external_outlets : bool + Hold the outer boundary of the basin fixed while divides migrate. +verbose : bool + Report convergence each iteration (default ``True``). + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `flow_direction`, `elevation`, `area`, `theta`, `ks`, `outlets`, `iterations` + +#### `GeographicRestoredElevation` + +*inherits* `GeographicGridMixin`, `RestoredElevation` + +Restored topography on a latitude/longitude grid. + +### Along-flow smoothing + +#### `AlongFlowSmoothing` + +Collect the cells within a window along each flow path. + +Mixed into the ``*WithSmoothing`` classes, which fit a model over that +window at every cell. The window is set either by drop +(``vertical_interval``) or by distance (``horizontal_interval``), and +extends both upstream and downstream of the cell being evaluated. + +#### `KsFromChiWithSmoothing` + +*inherits* `BaseSpatialGrid`, `AlongFlowSmoothing` + +Channel steepness fitted to chi--elevation in a moving window. + +:: + + ks = KsFromChiWithSmoothing(elevation=dem, area=area, + flow_direction=d8, theta=0.45, + vertical_interval=50.0) + +At every channel cell, chi and elevation are extracted over a window +along the flow path and a line through the origin is fitted; its slope +is k_s. ``save()`` writes seven bands: ks, the number of profiles +crossing each cell, mean squared error, sum of squares, r-squared, +p-value, and the number of points in the regression. + +Needs ``statsmodels``. Pass ``verbose=False`` to silence the progress +ticker. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `theta`, `vertical_interval` +- `elevation`, `area`, `flow_direction`, `theta`, `horizontal_interval` + +| Member | Signature | Summary | +|---|---|---| +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | + +#### `GeographicKsFromChiWithSmoothing` + +*inherits* `GeographicGridMixin`, `KsFromChiWithSmoothing` + +Windowed steepness fit on a latitude/longitude grid. + +#### `ThetaFromChiWithSmoothing` + +*inherits* `BaseSpatialGrid`, `AlongFlowSmoothing` + +Concavity fitted to chi--elevation in a moving window. + +As :class:`KsFromChiWithSmoothing`, but ``theta`` is optimised at each +cell rather than held fixed: the value returned is the one that makes +the chi--elevation relation most nearly linear. The search is bounded +at +/-10. + +Needs ``statsmodels``. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `vertical_interval`, `min_area` +- `elevation`, `area`, `flow_direction`, `horizontal_interval`, `min_area` + +| Member | Signature | Summary | +|---|---|---| +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | + +#### `GeographicThetaFromChiWithSmoothing` + +*inherits* `GeographicGridMixin`, `ThetaFromChiWithSmoothing` + +Windowed concavity fit on a latitude/longitude grid. + +#### `ChannelSlopeWithSmoothing` + +*inherits* `BaseSpatialGrid`, `AlongFlowSmoothing` + +Channel gradient measured over a window along the flow path. + +Far less noisy than a cell-to-cell slope, which on a real DEM is mostly +vertical quantisation. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `vertical_interval` +- `elevation`, `area`, `flow_direction`, `horizontal_interval` + +| Member | Signature | Summary | +|---|---|---| +| `calc_channel_slope` | `(i, j, elevation, de, find_points_along_path)` | Gradient across the window centred on ``(i, j)``. | +| `points_along_path` | `(points, i, j)` | Which part of the window to measure over; the whole of it here. | + +#### `ChannelDownSlopeWithSmoothing` + +*inherits* `ChannelSlopeWithSmoothing` + +Channel gradient over the downstream half of the window. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `vertical_interval` +- `elevation`, `area`, `flow_direction`, `horizontal_interval` + +| Member | Signature | Summary | +|---|---|---| +| `points_along_path` | `(points, i, j)` | Only the downstream half of the window. | + +#### `ChannelUpSlopeWithSmoothing` + +*inherits* `ChannelSlopeWithSmoothing` + +Channel gradient over the upstream half of the window. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `flow_direction`, `vertical_interval` +- `elevation`, `area`, `flow_direction`, `horizontal_interval` + +| Member | Signature | Summary | +|---|---|---| +| `points_along_path` | `(points, i, j)` | Only the upstream half of the window. | + +### Terrain attributes + +#### `CalculationMixin` + +Finite-difference terrain derivatives. + +Supplies the slope, curvature and boundary-padding helpers that +:class:`Elevation`, :class:`Hillshade` and :class:`MaxSlope` share. + +| Member | Signature | Summary | +|---|---|---| +| `assignBCs` | `(grid, nx, ny)` | Pad a grid by one cell, replicating the edge values. | +| `calcAverageSlopeOfGridSubset` | `(gridSubset, dx)` | Least-squares plane through a block of cells; returns its slopes. | +| `calcContourCurvature` | `(grid, dx)` | Contour (plan) curvature: the curvature of the contour lines. | +| `calcFiniteCurv` | `(grid, dx)` | Laplacian from a 5-point centred stencil, same shape as the input. | + +#### `Hillshade` + +*inherits* `CalculationMixin`, `BaseSpatialGrid` + +Shaded relief, following the ESRI hillshade definition. + +:: + + hs = Hillshade(elevation=dem, azimuth=315, inclination=45) + +``azimuth`` is the compass bearing of the light source in degrees +(0 = north, 90 = east) and ``inclination`` its height above the horizon. +Values run 0-255. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `azimuth`, `inclination` + +*Stored as* `uint8` + +| Member | Signature | Summary | +|---|---|---| +| `calcHillshade` | `(az, elev, z_factor=1.0)` | Fill the grid with an ESRI-style hillshade. | + +#### `GeographicHillshade` + +*inherits* `GeographicGridMixin`, `Hillshade` + +Hillshade on a latitude/longitude grid. + +#### `MaxSlope` + +*inherits* `CalculationMixin`, `BaseSpatialGrid` + +Magnitude of the topographic gradient, from centred differences. + +:: + + slope = MaxSlope(elevation=dem) + +Values are dimensionless (rise over run); take ``arctan`` for degrees. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation` + +*Stored as* `float64` + +| Member | Signature | Summary | +|---|---|---| +| `calcSlope` | `()` | Replace the grid with the gradient magnitude of its own values. | + +#### `GeographicMaxSlope` + +*inherits* `GeographicGridMixin`, `MaxSlope` + +Gradient magnitude on a latitude/longitude grid. + +#### `Laplacian` + +*inherits* `CalculationMixin`, `BaseSpatialGrid` + +Laplacian of elevation -- positive in valleys, negative on ridges. + +:: + + curvature = Laplacian(elevation=dem) + +Uses SciPy's 5-point stencil, scaled by ``1 / dx**2``, so the units are +inverse length. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation` + +*Stored as* `float64` + +#### `GeographicLaplacian` + +*inherits* `GeographicGridMixin`, `Laplacian` + +Laplacian of elevation on a latitude/longitude grid. + +#### `Gradient` + +*inherits* `BaseSpatialGrid` + +The two components of the topographic gradient. + +:: + + gradient = Gradient(elevation=dem) + gradient._gx, gradient._gy + +Stored as two bands rather than one, so direction survives ``save()``. +:meth:`average_gradient` smooths both components over a length scale, +which is how a regional aspect is obtained. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation` + +| Member | Signature | Summary | +|---|---|---| +| `average_gradient` | `(distance)` | Both gradient components averaged over a disc of radius ``distance``. | +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `plot` | `(**kwargs)` | Display the grid with ``imshow``, georeferenced in map coordinates. | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | + +#### `LocalRelief` + +*inherits* `BaseSpatialGrid` + +Elevation range within a circular window. + +:: + + relief = LocalRelief(elevation=dem, pixel_radius=25) + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `pixel_radius` + +#### `MultiscaleCurvatureValleyWidth` + +*inherits* `BaseSpatialGrid` + +Valley width from the scale at which curvature is most negative. + +A local quadratic is fitted at a range of window sizes; the width +recorded at each cell is the window that minimises the smaller principal +curvature, i.e. the scale at which the valley is best resolved. The +curvature itself is kept in ``_minC``. + +:: + + width = MultiscaleCurvatureValleyWidth( + elevation=dem, area=area, area_cutoff=1e6, + min_width=30.0, max_width=600.0) + +Pass ``use_dask=True`` to evaluate the scales in parallel. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `area`, `area_cutoff`, `max_width`, `min_width` + +| Member | Signature | Summary | +|---|---|---| +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `mosaic` | `(tiles)` | Reassemble tiles, carrying the curvature band along with the widths. *(classmethod)* | +| `remove_padding` | `(xpadding=10, ypadding=10)` | Strip tile overlap from both the widths and the curvatures. | +| `save` | `(filename)` | Write the grid to an LZW-compressed GeoTIFF. | + +#### `ScarpWavelet` + +*inherits* `BaseSpatialGrid` + +Template-matched scarp amplitude, age, orientation and signal-to-noise. + +Holds the four-band output of a scarp template search. Load one with +:meth:`load`; the bands become ``_A``, ``_kt``, ``_orientation`` and +``_SNR``. :meth:`template_window` needs the separate ``scarplet`` +package. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` + +| Member | Signature | Summary | +|---|---|---| +| `calculate_valid_orientations` | `(window_size, age, num=46, discard_max_min=True)` | Range of template orientations that stay inside the valid data. | +| `load` | `(filename)` | Read back the multi-band grid written by :meth:`save`. *(classmethod)* | +| `load_elevation` | `(filename)` | Attach the DEM the templates were matched against. | +| `plot_orientations` | `(*args, **kwargs)` | Draw scarp orientations over a hillshade, faded by signal-to-noise. | +| `template_window` | `(window_size, age, orientation, use_pixels=False)` | The scarp template mask at one size, age and orientation. | +| `valid_data` | `()` | 1 where the attached DEM has usable, above-sea-level data; NaN elsewhere. | + +### Isostasy + +#### `Deflection` + +*inherits* `BaseSpatialGrid` + +Flexural deflection of an elastic plate under a topographic load. + +Solves the thin-plate equation in the Fourier domain: + +.. math:: + w(k) = \frac{-\rho_c\,g\,h(k)} + {(\rho_m - \rho_c)\,g + D\,(2\pi k)^4} + +:: + + w = Deflection(elevation=dem, D=1e23, rho_m=3300, rho_c=2700, g=9.81) + +Pass ``restored_elevation`` as well to get the *change* in deflection +between two surfaces, which is the isostatic response to the erosion +between them. + +Because the solution is spectral, the load is implicitly periodic; pad +the DEM if edge effects matter. + +*Constructor keywords* (any one combination) + +- `nx`, `ny`, `projection`, `geo_transform` +- `ai_ascii_filename`, `EPSGprojectionCode` +- `gdal_filename` +- `elevation`, `D`, `rho_m`, `rho_c`, `g` + +#### `GeographicDeflection` + +*inherits* `GeographicGridMixin`, `Deflection` + +Flexural deflection on a latitude/longitude grid. + +### Mixins + +#### `GDALMixin` + +Reading and writing rasters through GDAL. + +Mixed into :class:`BaseSpatialGrid`; nothing here is meant to be called +directly. Every method reaches GDAL through the module-level lazy +proxies, so the mixin can be present without GDAL being installed. + +| Member | Signature | Summary | +|---|---|---| +| `getDEMcoords` | `(GdalData, dx)` | Cell-centre x and y coordinate lists for an open GDAL dataset. | + +#### `GeographicGridMixin` + +Spherical geometry for grids in degrees of latitude and longitude. + +Mix it in *before* the grid class to override the cell area and cell +dimension with their true, latitude-dependent values:: + + class GeographicArea(GeographicGridMixin, Area): + pass + +Every ``Geographic*`` class in this module is exactly that. + +#### `MaxFlowLengthTrackingMixin` + +Propagate a value downstream along longest-flow-length paths only. + +Subclasses set :attr:`_propagate_mode` to ``'carry'`` (the receiver takes +the donor's value) or ``'sum'`` (the receiver adds it) and may define +:meth:`_propagation_gate` to suppress the transfer for some cells. + +### Module-level functions in `TopoAnalysis.dem` + +| Function | Summary | +|---|---| +| `gdal_is_available()` | True when the GDAL Python bindings can be imported. | +| `mosaicFolder(folderPath, fileSuffix, outfile)` | Merge every ``*fileSuffix`` raster in a folder into ``outfile``. | +| `plot(*args, **kwargs)` | Scatter one grid's values against another's, cell by cell. | + +## `TopoAnalysis.topotoolbox` -- TopoToolbox-equivalent functions + +TopoToolbox-equivalent terrain analysis. + +Every function here reproduces the numerical result of the identically named +function in TopoToolbox 2 (Schwanghart & Scherler 2014), the MATLAB package, +so that a result computed in TopoAnalysis can be checked against one computed +in MATLAB. + +The rest of TopoAnalysis keeps its own idioms -- drainage area in square +metres, ESRI-style hillshade, chi integrated with a left-endpoint rule. This +module exists for the cases where you need the *same number*, not just the +same quantity. :doc:`docs/topotoolbox_parity` tabulates the correspondence +and lists the places where exact agreement is not achievable. + +See [topotoolbox_parity.md](topotoolbox_parity.md) for the full +correspondence table and the places where exact agreement is not +achievable. + +| Function | Summary | +|---|---| +| `arcslope(elevation, unit='tangent')` | ArcGIS/Horn 3x3 slope magnitude. | +| `aspect(elevation, classify=False)` | Down-slope direction in degrees clockwise from north. | +| `cellarea(grid, unit='m')` | Area of each cell of a geographic (lat/lon) grid. | +| `chitransform(flow_direction, area, mn=0.45, a0=1000000.0, outlets=None, correctcellsize=True, stream_mask=None)` | The chi integral transform. | +| `coord2sub(grid, x, y)` | Map coordinates to 0-based ``(row, col)`` subscripts. | +| `curvature(elevation, ctype='profc')` | Surface curvature from the Evans 3x3 quadratic fit. | +| `drainagebasins(flow_direction, outlets=None, valid=None)` | Label drainage basins. | +| `fillsinks(elevation, maxdepth=None)` | Fill topographic depressions to a flat surface. | +| `flowacc(flow_direction, weights=None)` | Flow accumulation in **number of cells**. | +| `flowdistance(flow_direction, direction='upstream')` | Along-flow distance in map units. | +| `flowobj(elevation, preprocess='fill')` | Build a D8 flow-direction grid the way ``FLOWobj`` does. | +| `getcoordinates(grid)` | ``(x, y)`` cell-centre coordinate vectors. | +| `getextent(grid)` | ``(xmin, xmax, ymin, ymax)`` of the outermost **cell centres**. | +| `gradient8(elevation, unit='tangent')` | Steepest downward gradient over the 8 neighbours. | +| `hillshade(elevation, azimuth=315.0, altitude=60.0, exaggerate=1.0)` | Shaded relief as the cosine of the illumination incidence angle. | +| `identifyflats(elevation, output='both')` | Locate flat areas, their sills and the closed basins. | +| `imposemin(flow_direction, elevation, sl=0.0)` | Carve elevations so every flow path descends at gradient >= ``sl``. | +| `ksn(flow_direction, elevation, area, theta=0.45, correctcellsize=True, min_gradient=1e-05)` | Normalized channel steepness index, :math:`k_{sn} = S A^{\theta}`. | +| `localtopography(elevation, radius=5000.0, type='range')` | Statistic of elevation within a circular window. | +| `mchi(flow_direction, elevation, chi)` | Gradient of elevation in chi space, :math:`M_{\chi}`. | +| `streamorder(flow_direction, stream_mask, order_type='strahler')` | Strahler or Shreve stream order. | +| `sub2coord(grid, row, col)` | 0-based ``(row, col)`` subscripts to cell-centre map coordinates. | + +### `fillsinks(elevation, maxdepth=None)` + +Fill topographic depressions to a flat surface. + +Equivalent to ``GRIDobj/fillsinks(DEM)`` and, with ``maxdepth``, to +``fillsinks(DEM, maxdepth)``. + +Unlike :class:`TopoAnalysis.dem.FilledElevation`, whose default adds a +tiny gradient across filled areas so that D8 routing has a direction to +follow, this fills to an exactly flat plateau -- which is what +TopoToolbox does, and what ``identifyflats`` then relies on. + +Parameters +---------- +elevation : BaseSpatialGrid + The DEM. ``NaN`` is no-data and, as in TopoToolbox, acts as an + outlet: water reaching a NaN cell leaves the grid. +maxdepth : float, optional + Leave depressions deeper than this unfilled. + +Returns +------- +Elevation + A new grid; the input is not modified. + +### `identifyflats(elevation, output='both')` + +Locate flat areas, their sills and the closed basins. + +Equivalent to ``GRIDobj/identifyflats``. + +* A **flat** is a cell with no strictly lower neighbour -- the MATLAB test + is ``imerode(dem, ones(3)) == dem``. An isolated pit therefore counts + as a flat. The grid's outer ring is cleared, as is the one-cell halo + around any no-data. +* A **sill** is a non-flat cell at the same elevation as an adjacent + flat: the point a flat drains through. +* A **closed basin** is a regional minimum that does not touch the grid + edge. + +Parameters +---------- +elevation : BaseSpatialGrid + Normally the output of :func:`fillsinks`. +output : {'both', 'flats', 'sills', 'closedbasins', 'all'} + ``'both'`` returns ``(flats, sills)``; ``'all'`` adds the closed + basins. + +Returns +------- +BaseSpatialGrid or tuple + Boolean grids. + +### `imposemin(flow_direction, elevation, sl=0.0)` + +Carve elevations so every flow path descends at gradient >= ``sl``. + +Equivalent to ``FLOWobj/imposemin(FD, DEM, sl)``. Cells are only ever +lowered, so channels are cut through obstructions instead of being +drowned by them. + +Returns +------- +Elevation + A new grid. + +### `flowobj(elevation, preprocess='fill')` + +Build a D8 flow-direction grid the way ``FLOWobj`` does. + +Parameters +---------- +elevation : BaseSpatialGrid +preprocess : {'fill', 'none'} + ``'fill'`` removes depressions first (TopoToolbox's + ``'preprocess','fill'``). ``'carve'`` is not implemented; see the + note below. + +Returns +------- +FlowDirectionD8 + +Notes +----- +TopoToolbox routes flow across the flat areas left by filling using an +auxiliary "geodesic" topography, so that flat cells drain towards the +nearest sill. TopoAnalysis instead adds a small gradient while filling +(:attr:`~TopoAnalysis.dem.PriorityQueueMixIn.aggradation_slope`). Both +produce a valid, connected drainage network with identical basin +boundaries wherever the terrain is not flat, but **inside flats the two +networks differ**. TopoToolbox's ``'carve'`` preprocessing lowers the +DEM instead of raising it and is a different surface again. + +### `flowacc(flow_direction, weights=None)` + +Flow accumulation in **number of cells**. + +Equivalent to ``FLOWobj/flowacc(FD)``: every cell counts itself, so the +minimum value is 1. + +Parameters +---------- +flow_direction : FlowDirectionD8 +weights : ndarray or BaseSpatialGrid, optional + Per-cell weight; defaults to 1. + +Returns +------- +BaseSpatialGrid + +See Also +-------- +TopoAnalysis.dem.Area : the same accumulation weighted by cell area. + +### `flowdistance(flow_direction, direction='upstream')` + +Along-flow distance in map units. + +Equivalent to ``FLOWobj/flowdistance(FD, direction)`` with no seed +points. + +Parameters +---------- +direction : {'upstream', 'downstream', 'maxdownstream'} + ``'upstream'`` gives the distance from each cell down to its outlet. + ``'downstream'`` (a synonym for ``'maxdownstream'``) gives the length + of the longest flow path reaching each cell -- the quantity + :class:`TopoAnalysis.dem.FlowLength` computes. + +### `drainagebasins(flow_direction, outlets=None, valid=None)` + +Label drainage basins. + +Equivalent to ``FLOWobj/drainagebasins``. Without ``outlets`` every +terminal cell that something drains into seeds its own basin; with them, +each cell is labelled by the 1-based position of the outlet it drains to. + +Parameters +---------- +outlets : sequence of (x, y), optional +valid : ndarray or BaseSpatialGrid, optional + Restrict labelling to these cells (e.g. ``~np.isnan(dem._griddata)``). + +Returns +------- +BaseSpatialGrid + Integer labels; 0 means "reaches none of the outlets". + +Notes +----- +TopoToolbox numbers basins in the order its edge list happens to meet +them, so the *partition* matches but the integer labels generally do +not. Compare partitions, not label values. + +### `gradient8(elevation, unit='tangent')` + +Steepest downward gradient over the 8 neighbours. + +Equivalent to ``GRIDobj/gradient8``. The cardinal and diagonal drops are +divided by ``cellsize`` and ``sqrt(2)*cellsize`` respectively, and the +result is clamped at 0, so pits and flats give exactly 0 rather than a +negative gradient. + +Parameters +---------- +unit : {'tangent', 'degree', 'radian', 'sine', 'percent'} + +### `arcslope(elevation, unit='tangent')` + +ArcGIS/Horn 3x3 slope magnitude. + +Equivalent to ``GRIDobj/arcslope``. Uses the 1-2-1 weighted 3x3 stencil +with an ``8 * cellsize`` denominator -- there is no ``sqrt(2)`` anywhere; +the diagonals enter through the weights. + +``NaN`` holes are filled from their nearest valid neighbour before +differencing, then restored, which is how TopoToolbox handles them here +(and is the opposite of what :func:`gradient8` does). + +### `hillshade(elevation, azimuth=315.0, altitude=60.0, exaggerate=1.0)` + +Shaded relief as the cosine of the illumination incidence angle. + +Equivalent to ``GRIDobj/hillshade`` with its default ``'surfnorm'`` +method, including its defaults of ``azimuth=315`` and ``altitude=60``. + +Returns +------- +BaseSpatialGrid + Values in ``[-1, 1]``. TopoToolbox only scales to 0-255 when it is + plotting, so neither does this. + +See Also +-------- +TopoAnalysis.dem.Hillshade : the ESRI formulation, returning 0-255. + +### `aspect(elevation, classify=False)` + +Down-slope direction in degrees clockwise from north. + +Equivalent to ``GRIDobj/aspect``. 0 = north, 90 = east, 180 = south, +270 = west. Perfectly flat cells come out as 90, exactly as they do in +TopoToolbox. + +Parameters +---------- +classify : bool + Return the 8-class Gomez-Plaza wetness ranking (1 = N, 3 = NE, + 5 = E, 7 = SE, 8 = S, 6 = SW, 4 = W, 2 = NW) instead of degrees. + +### `curvature(elevation, ctype='profc')` + +Surface curvature from the Evans 3x3 quadratic fit. + +Equivalent to ``GRIDobj/curvature``. + +Parameters +---------- +ctype : {'profc', 'planc', 'tangc', 'meanc', 'total'} + Profile, plan, tangential, mean, or total curvature. ``'profc'`` + (the default) is positive where a slope steepens downhill. + +Returns +------- +BaseSpatialGrid + Units are 1/length, except ``'total'`` which is 1/length^2. + +Notes +----- +Cells where the gradient vanishes give 0/0; TopoToolbox sets those to 0, +and so does this. Cells adjacent to no-data therefore come out as 0 +rather than ``NaN`` -- again matching TopoToolbox. + +### `cellarea(grid, unit='m')` + +Area of each cell of a geographic (lat/lon) grid. + +Equivalent to ``GRIDobj/cellarea``. Cells are treated as spherical +quadrangles on a sphere whose total area is +:data:`EARTH_SURFACE_AREA_KM2`. + +Parameters +---------- +unit : {'m', 'km'} + Square metres (the default) or square kilometres. + +Returns +------- +BaseSpatialGrid + Constant along each row, since area depends only on latitude. + +### `localtopography(elevation, radius=5000.0, type='range')` + +Statistic of elevation within a circular window. + +Equivalent to ``GRIDobj/localtopography``. The window radius is given in +map units and rounded up to a whole number of cells. + +Parameters +---------- +type : {'range', 'max', 'min', 'mean', 'median', 'std'} + +Notes +----- +TopoToolbox's default disk (``strel('disk', r)`` with ``N=8``) is a +periodic-line approximation whose exact shape is an internal MATLAB +detail. This uses an exact disc -- TopoToolbox's ``N=0`` -- so values +can differ slightly near the window scale. + +### `streamorder(flow_direction, stream_mask, order_type='strahler')` + +Strahler or Shreve stream order. + +Equivalent to ``FLOWobj/streamorder`` / ``STREAMobj/streamorder``. + +Parameters +---------- +stream_mask : ndarray or BaseSpatialGrid + Non-zero on the channel network, e.g. ``flowacc(fd)._griddata > 1000``. +order_type : {'strahler', 'shreve'} + +Returns +------- +BaseSpatialGrid + Integer orders; 0 off the network. + +### `chitransform(flow_direction, area, mn=0.45, a0=1000000.0, outlets=None, correctcellsize=True, stream_mask=None)` + +The chi integral transform. + +Equivalent to ``STREAMobj/chitransform``: + +.. math:: + \chi = \int_{\mathrm{outlet}}^{x} + \left(\frac{A_0}{A(x')}\right)^{m/n}\,\mathrm{d}x' + +integrated upstream with the trapezoidal rule, so chi is 0 at each outlet +and increases upstream. + +Parameters +---------- +area : BaseSpatialGrid + Flow accumulation. With ``correctcellsize=True`` (the default, as in + TopoToolbox) it is taken to be in **number of cells** and multiplied + by ``cellsize**2``; pass ``False`` if it is already in m^2, e.g. if + it came from :class:`TopoAnalysis.dem.Area`. +mn : float + The m/n concavity ratio. TopoToolbox's default is 0.45. +a0 : float + Reference area in m^2. TopoToolbox's default is 1e6 (1 km^2). +outlets : sequence of (x, y), optional + Integration starts here. Defaults to every terminal cell of the + network, which is what a STREAMobj built over the whole grid gives. +stream_mask : ndarray or BaseSpatialGrid, optional + Restrict the integration to the channel network. + +Returns +------- +BaseSpatialGrid + Chi in map units; ``NaN`` away from the integrated network. + +### `ksn(flow_direction, elevation, area, theta=0.45, correctcellsize=True, min_gradient=1e-05)` + +Normalized channel steepness index, :math:`k_{sn} = S A^{\theta}`. + +Equivalent to ``STREAMobj/ksn``, including its unconditional minima +imposition at a gradient of 1e-5 before the slope is measured. + +Parameters +---------- +area : BaseSpatialGrid + Flow accumulation in cells (see ``correctcellsize``). +theta : float + Reference concavity; TopoToolbox's default is 0.45. +min_gradient : float + Gradient imposed by the carving step. TopoToolbox hard-codes 1e-5. + +Returns +------- +BaseSpatialGrid + ``ksn`` with a reference area of 1 m^2 -- TopoToolbox's ``ksn`` has + no ``a0``. Multiply by ``a0**theta`` to compare against + ``chiplot``'s ``ks``. + +### `mchi(flow_direction, elevation, chi)` + +Gradient of elevation in chi space, :math:`M_{\chi}`. + +Equivalent to ``STREAMobj/mchi``: a forward difference assigned to the +upstream cell, with no carving and no smoothing. + +``M_chi`` scales with the ``a0`` baked into ``chi``: computing chi with +``a0=1`` puts it in the same units as :func:`ksn`, and +``mchi(a0) * a0**mn`` is invariant. + +It is *close to* but not equal to :func:`ksn`, despite what TopoToolbox's +own docstring says: ``ksn`` carves the DEM first and evaluates the +integrand at the upstream cell, while ``mchi`` uses the raw elevations +and inherits chi's trapezoidal averaging. + +### `getcoordinates(grid)` + +``(x, y)`` cell-centre coordinate vectors. + +Equivalent to ``GRIDobj/getcoordinates``. ``x`` runs west to east; +``y`` runs **north to south**, matching the row order of the grid. + +### `getextent(grid)` + +``(xmin, xmax, ymin, ymax)`` of the outermost **cell centres**. + +Equivalent to ``GRIDobj/getextent``. Note this is half a cell inside +:meth:`TopoAnalysis.dem.BaseSpatialGrid.extent`, which returns the pixel +edges that matplotlib wants. + +### `coord2sub(grid, x, y)` + +Map coordinates to 0-based ``(row, col)`` subscripts. + +Equivalent to ``GRIDobj/coord2sub`` apart from the index base. Points +outside the grid give ``NaN``. + +Notes +----- +Halves are rounded away from zero, as MATLAB does. Python's built-in +``round`` -- which +:meth:`TopoAnalysis.dem.BaseSpatialGrid._xy_to_rowscols` uses -- rounds +halves to even, so the two disagree for points landing exactly on a cell +boundary. + +### `sub2coord(grid, row, col)` + +0-based ``(row, col)`` subscripts to cell-centre map coordinates. + +Equivalent to ``GRIDobj/sub2coord``. Subscripts outside the grid are +extrapolated rather than rejected, as in TopoToolbox. + +## `TopoAnalysis.kernels` -- backend dispatch + +Backend dispatch for the TopoAnalysis grid kernels. + +Import the kernels from here rather than from :mod:`TopoAnalysis.fastops` or +the compiled extension directly:: + + from TopoAnalysis import kernels + kernels.priority_flood(z, mode="flat") + +If the C++ extension built successfully it is used; otherwise the pure-NumPy +implementations in :mod:`TopoAnalysis.fastops` take over. Both produce the +same numbers -- the test suite asserts it -- so the choice is purely about +speed. + +Set the environment variable ``TOPOANALYSIS_PURE_PYTHON=1`` to force the +NumPy path, which is useful when debugging or comparing the two. + +| Function | Summary | +|---|---| +| `accumulate(recv, order, weights, gate=None)` | Downstream accumulation; see :func:`TopoAnalysis.fastops.accumulate`. | +| `backend() -> 'str'` | ``'c++'`` or ``'python'``. | +| `chi(recv, order, area, step_length, outlets, A0, theta, trapezoid=False, max_length=0.0, mask=None)` | ``(chi, distance)``; see :func:`TopoAnalysis.fastops.chi`. | +| `default_seeds(elevations)` | Flat indices of the cells a flood should start from. | +| `downstream_distance(recv, order, step_length)` | See :func:`TopoAnalysis.fastops.downstream_distance`. | +| `drainage_basins(recv, order, outlets=None, valid=None)` | See :func:`TopoAnalysis.fastops.drainage_basins`. | +| `flow_directions(elevations, cellsize_grid=None, cellsize=1.0)` | Steepest-descent D8 codes; see :func:`TopoAnalysis.fastops.flow_directions`. | +| `flow_length(recv, order, step_length)` | ``(length, from_codes)``; see :func:`TopoAnalysis.fastops.flow_length`. | +| `have_extension() -> 'bool'` | True when the compiled kernels are in use. | +| `imposemin(recv, order, step_length, sl, elevations)` | Carve in place; see :func:`TopoAnalysis.fastops.imposemin`. | +| `priority_flood(elevations, closed=None, seeds=None, mode='flat', epsilon=0.0, cellsize=1.0, max_pit_depth=0.0, track_visited=False, reference_algorithm=False)` | Fill depressions in place; see :func:`TopoAnalysis.fastops.priority_flood`. | +| `propagate_along_main_stem(recv, order, main_stem_from_codes, gate, values, mode)` | See :func:`TopoAnalysis.fastops.propagate_along_main_stem`. | +| `receivers(codes)` | Receiver index per cell; see :func:`TopoAnalysis.fastops.receivers`. | +| `stream_order(recv, order, is_stream, kind='strahler')` | See :func:`TopoAnalysis.fastops.stream_order`. | +| `topological_order(recv)` | ``(order, n_cells_on_cycles)``; see :func:`TopoAnalysis.fastops.topological_order`. | +| `upstream_mask(recv, outlets)` | See :func:`TopoAnalysis.fastops.upstream_mask`. | + +## `TopoAnalysis.fastops` -- the reference NumPy kernels + +Pure-NumPy implementations of the TopoAnalysis grid kernels. + +Every function here mirrors, name for name and result for result, one of the +functions in the compiled ``TopoAnalysis._topoanalysis`` extension. They are +the reference semantics: the C++ code is validated against them in the test +suite, and they are what :mod:`TopoAnalysis.kernels` falls back to when the +extension is unavailable. + +The depression-filling routine implements the Priority-Flood algorithms of + + Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal + depression-filling and watershed-labeling algorithm for digital elevation + models." *Computers & Geosciences* 62, 117-127. + +These implementations are correct but roughly one to two orders of magnitude +slower than the compiled kernels on real DEMs. If you are processing +anything larger than a few hundred thousand cells, build the extension:: + + pip install -e . + +### `default_seeds(elevations)` + +Flat indices of the cells a flood should start from. + +These are the grid perimeter plus every valid cell touching a no-data +cell -- the same set that TopoToolbox's ``fillsinks`` keeps in its marker +image. + +Parameters +---------- +elevations : ndarray + 2-D elevation grid; ``NaN`` marks no-data. + +Returns +------- +ndarray of int64 + Flat (C-order) indices of the seed cells. + +### `priority_flood(elevations, closed=None, seeds=None, mode='flat', epsilon=0.0, cellsize=1.0, max_pit_depth=0.0, track_visited=False, reference_algorithm=False)` + +Fill depressions in ``elevations`` in place (Barnes et al. 2014). + +Parameters +---------- +elevations : ndarray + 2-D float64 grid, modified in place. ``NaN`` is no-data. +closed : ndarray of uint8, optional + Cells with a non-zero entry are never processed. Use this to restrict + the flood to a mask. +seeds : ndarray of int64, optional + Flat indices to start from. Defaults to :func:`default_seeds`. +mode : {'flat', 'epsilon'} + ``'flat'`` fills depressions to a level surface (Algorithm 2), which + is what TopoToolbox's ``fillsinks`` does. ``'epsilon'`` adds a small + upslope increment so the result has no flats (Algorithm 4). +epsilon : float + Gradient of the increment used by ``mode='epsilon'``. Each filled + cell is raised to ``parent + epsilon * cellsize * step``. Zero means + "use the smallest representable increment". +cellsize : float + Grid spacing, used only to scale ``epsilon``. +max_pit_depth : float + Depressions deeper than this are left unfilled. Zero disables the + check. Unlike the original TopoAnalysis implementation, the flood + still traverses such depressions, so terrain beyond them is reached. +track_visited : bool + Also return a mask of the cells the flood reached. +reference_algorithm : bool + Use Algorithm 1 (everything through the priority queue) instead of the + faster Algorithm 2. The filled surface is identical; this exists for + testing. + +Returns +------- +dict + ``{'visited': ndarray|None, 'cells_filled': int, 'max_fill_depth': float}`` + +### `flow_directions(elevations, cellsize_grid=None, cellsize=1.0)` + +Steepest-descent D8 flow directions as ArcGIS codes. + +The drop to each neighbour is divided by the centre-to-centre distance, so +a diagonal neighbour only wins if its *gradient* is steeper. Ties go to +the first direction in the order E, SE, S, SW, W, NW, N, NE. + +Cells with no lower neighbour -- pits, and no-data cells -- get code 0. + +### `receivers(codes)` + +Flat index of the cell each cell drains to, or -1 if there is none. + +### `topological_order(recv)` + +Order the cells donors-first. + +Returns ``(order, n_cells_on_cycles)``. Cells that lie on a cycle -- only +possible in a hand-edited or externally supplied flow-direction grid -- +cannot be ordered and are appended at the end in index order. + +### `accumulate(recv, order, weights, gate=None)` + +Accumulate ``weights`` downstream along the D8 network. + +``order`` is accepted for signature compatibility with the compiled +kernel; the result does not depend on which valid topological order is +used, so this implementation derives its own level structure -- which +lets it run vectorised rather than one cell at a time. + +### `flow_length(recv, order, step_length)` + +Longest upstream flow distance, and the main-stem donor direction. + +Returns ``(length, from_codes)``. ``from_codes[k]`` is the direction code +pointing from ``k`` back at the donor that supplies its longest path. +Ties are settled in favour of the donor with the lower flat index, so the +main stem does not depend on which topological order was used. + +### `propagate_along_main_stem(recv, order, main_stem_from_codes, gate, values, mode)` + +Carry or sum ``values`` downstream, but only along longest-flow paths. + +``mode='carry'`` overwrites the receiver with the donor's value (used by +:class:`~TopoAnalysis.dem.Relief`); ``mode='sum'`` adds it (used by +:class:`~TopoAnalysis.dem.Ksi`). + +### `upstream_mask(recv, outlets)` + +Mask of every cell draining to any of ``outlets`` (flat indices). + +### `chi(recv, order, area, step_length, outlets, A0, theta, trapezoid=False, max_length=0.0, mask=None)` + +Integrate the chi coordinate upstream from ``outlets``. + +``chi(cell) = chi(receiver) + (A0 / A)^theta * step_length`` + +With ``trapezoid=True`` the integrand is averaged between the cell and its +receiver, which is the rule TopoToolbox's ``chitransform`` uses. Cells +outside the outlets' drainage areas stay ``NaN``. + +Returns ``(chi, distance_from_outlet)``. + +### `drainage_basins(recv, order, outlets=None, valid=None)` + +Label drainage basins. + +With ``outlets`` given, each cell is labelled with the 1-based position of +the outlet it drains to (0 if it reaches none). Without, every terminal +cell -- pit or edge outlet -- seeds its own basin, matching TopoToolbox's +``drainagebasins(FD)``. + +## `TopoAnalysis.demRecursionTools` + +Profile extraction and chi/steepness fitting over recursive basin maps. + +These functions consume the nested dictionaries produced by +:meth:`TopoAnalysis.dem.FlowDirectionD8.map_values_to_recursive_list`. + +| Function | Summary | +|---|---| +| `area_elevation_for_mainstem_and_tributaries(outlet, flow_direction, elevation, area, theta=0.5, minimum_area=10000000.0)` | | +| `best_ks_and_theta_with_wrss(elevation, flow_direction_or_length, area, outlet, xo=500)` | | +| `best_ks_and_theta_with_wrss_list(ld_list, de, xo=500, maxiter=100, maxfun=200)` | | +| `best_ks_theta(outlet, flow_direction, elevation, area, minimum_area)` | | +| `best_ks_theta_wrss_for_outlet(outlet, flow_direction, elevation, area, minimum_area=10000000.0)` | | +| `best_ks_with_r2_list(ld_list, de, theta, xo=500)` | | +| `best_ks_with_wrss_list(ld_list, de, theta, xo=500)` | | +| `chi_elevation(ld_list, de, theta, xo=500.0)` | | +| `extract_chi_elevation_values(ld_list, de, theta, chi_o, elevation, chi, base_elevation, A_mdx=None, xo=500.0)` | | +| `extract_dA_elevation_values(ld_list)` | | +| `extract_profile_values(ld_list, xo=500.0, items=())` | | +| `hi(elevation, flow_direction, dA, outlet)` | | +| `hi_list(ld_list)` | | +| `map_chi_profiles(elevation, flow_direction, area, outlet, minimum_area=1000000.0, theta=0.5, start_at=0.0, downstream=True, Ao=1000000.0)` | | +| `uninformative_SS_list(ld_list, de, xo=500)` | | + +## `TopoAnalysis.plotting` + +Longitudinal- and chi-profile plotting helpers. + +| Function | Summary | +|---|---| +| `interactive_chi_profiles_and_map_view(prefix, code, plot_code, dem, fd, area, hillshade, minimum_area=10000000.0, theta=0.5, Ao=1000000.0)` | | +| `plot_chi_profiles(elevation, flow_direction, area, outlet, plot_code, minimum_area=1000000.0, figure=None, theta=0.5, start_at=0.0, downstream=True, Ao=1000000.0)` | | +| `plot_chi_profiles_with_outlet_code(prefix, code, plot_code, dem, fd, area, minimum_area=10000000.0, theta=0.5)` | | +| `plot_downstream_profile(elevation, flow_direction, outlet, plot_code, downstream=True, start_at=0.0, mean_pixel_dimension=None, figure=None)` | | +| `plot_profiles(elevation, flow_direction, area, outlet, plot_code, minimum_area=1000000.0, figure=None)` | | +| `plot_profiles_with_outlet_code(prefix, code, plot_code, dem, fd, area, minimum_area=10000000.0)` | | +| `plot_recursive_upstream_profiles(elevation, flow_direction, area, outlet, plot_code, downstream=False, start_at=0.0, figure=None, minimum_area=1000000.0)` | | + +## `TopoAnalysis.datasets` + +Utilities for generating synthetic data and loading examples + +| Function | Summary | +|---|---| +| `sinusoid_grid(ny, nx, width, amp=1, sig=0, slope_y=None)` | Returns a synthetic landscape with sinusoidal ridges and valleys | +| `triangle_grid(ny, nx, width, amp=1, sig=0, slope_y=None)` | Returns a synthetic landscape with triangular ridges and valleys | + +## `TopoAnalysis.utils` + +Convenience wrappers for fitting channel steepness at a single outlet. + +| Function | Summary | +|---|---| +| `calc_ks_for_outlet(outlet, theta, **kwargs)` | Fit channel steepness to the basin draining to ``outlet``. | + +## `TopoAnalysis.analysis` + +Quadrat sampling of a raster. + +### `Quadrats` + + + +| Method | Summary | +|---|---| +| `load_data(filename, band=1)` | Read a raster band with rasterio. | +| `make_quadrats(dx, dy=None)` | | +| `map_quadrats(func, **kwargs)` | | +| `plot(values, ax=None, **kwargs)` | | +| `quiver(u, v, ax=None, **kwargs)` | | + +## `TopoAnalysis.cli` + +Command-line entry point: ``topoanalysis-process``. + +Runs the standard pipeline over a DEM and writes the derived grids as +GeoTIFFs beside it:: + + topoanalysis-process srtm.tif --outdir results --theta 0.45 --a0 1e6 + +Use ``topoanalysis-process --info`` to report which compute backend is +active and whether GDAL is available. + +| Function | Summary | +|---|---| +| `main(argv=None)` | Run the CLI. Returns a process exit status. | + +## `TopoAnalysis.MovingWindow` + +Moving-window filters over a grid. + +A :class:`MovingWindow` applies a reducing function to the cells inside a +window centred on every cell. Subclasses choose the window shape; a +concrete class supplies the reduction as :attr:`~MovingWindow.function`:: + + class WindowMean(CircularMovingWindow): + function = staticmethod(np.mean) + + grid.apply_moving_window(WindowMean(window_radius=200.0)) + +Cells outside the grid, and no-data cells, are dropped from the window +rather than filled, so the function always sees real data. + +For the common reductions (min, max, mean, median, standard deviation) +:func:`TopoAnalysis.topotoolbox.localtopography` is far faster, because it +uses SciPy's separable filters instead of a Python loop. + +### `CircularMovingWindow` + +A disc-shaped window of radius ``round(window_radius / dx)`` cells. + +### `MovingWindow` + +Abstract base for moving-window filters. + +Parameters +---------- +window_radius : float + Radius of the window in map units. ``window_dimension`` is accepted + as a synonym for backward compatibility. + +| Method | Summary | +|---|---| +| `apply_moving_window(grid, dx, dtype)` | Run the window over every cell of ``grid``. | + +### `RectangularMovingWindow` + +A square window of side ``2 * round(window_radius / dx) + 1`` cells. + +## `TopoAnalysis.error` + +### `Error` + +Base class for exceptions in this module. + +### `InputError` + +Exception raised for errors in the input. + +Attributes: + expr -- input expression in which the error occurred + msg -- explanation of the error + +### `TransitionError` + +Raised when an operation attempts a state transition that's not +allowed. + +Attributes: + prev -- state at beginning of transition + next -- attempted new state + msg -- explanation of why the specific transition is not allowed + diff --git a/docs/generate_api.py b/docs/generate_api.py new file mode 100644 index 0000000..1f2badb --- /dev/null +++ b/docs/generate_api.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Regenerate ``docs/api.md`` from the live docstrings. + + python docs/generate_api.py + +Run it after adding or renaming anything public. The output is checked in +so the reference is readable on the repository page without building Sphinx. +""" + +from __future__ import annotations + +import inspect +import os +import pathlib +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) + +from TopoAnalysis import (MovingWindow, analysis, cli, datasets, dem, # noqa: E402 + demRecursionTools, error, fastops, kernels, + plotting, topotoolbox, utils) + +#: Grid classes, grouped the way the guide introduces them. +GROUPS = [ + ('Core grid types', ['BaseSpatialGrid', 'Georef_info', 'BaseSpatialShape', + 'ValueGrid', 'Elevation', 'GeographicElevation']), + ('Depression filling', ['PriorityQueueMixIn', 'FilledElevation', 'PriorityFillGrid']), + ('Flow routing', ['FlowDirection', 'FlowDirectionD8', 'FlowLength', + 'GeographicFlowLength', 'Mask', 'DiscreteFlowAccumulation', + 'GeographicDiscreteFlowAccumulation']), + ('Drainage area', ['Area', 'GeographicArea', 'LogArea', 'ValleyArea', + 'GeographicValleyArea', 'MainstemValleyArea', + 'GeographicMainstemValleyArea']), + ('Chi and channel steepness', ['Chi', 'GeographicChi', 'Ksi', 'GeographicKsi', + 'Relief', 'ScaledRelief', 'ChiScaledRelief', + 'ChannelSlope', 'CrossDivideDChi', + 'NormalizedCrossDivideDChi', 'RestoredElevation', + 'GeographicRestoredElevation']), + ('Along-flow smoothing', ['AlongFlowSmoothing', 'KsFromChiWithSmoothing', + 'GeographicKsFromChiWithSmoothing', + 'ThetaFromChiWithSmoothing', + 'GeographicThetaFromChiWithSmoothing', + 'ChannelSlopeWithSmoothing', + 'ChannelDownSlopeWithSmoothing', + 'ChannelUpSlopeWithSmoothing']), + ('Terrain attributes', ['CalculationMixin', 'Hillshade', 'GeographicHillshade', + 'MaxSlope', 'GeographicMaxSlope', 'Laplacian', + 'GeographicLaplacian', 'Gradient', 'LocalRelief', + 'MultiscaleCurvatureValleyWidth', 'ScarpWavelet']), + ('Isostasy', ['Deflection', 'GeographicDeflection']), + ('Mixins', ['GDALMixin', 'GeographicGridMixin', 'MaxFlowLengthTrackingMixin']), +] + + +def first_line(obj): + doc = inspect.getdoc(obj) or '' + return doc.strip().split('\n')[0] if doc else '' + + +def signature(obj): + """A call signature with the bound first argument removed.""" + try: + sig = str(inspect.signature(obj)) + except (TypeError, ValueError): + return '(...)' + return (sig.replace('self, ', '').replace('(self)', '()') + .replace('cls, ', '').replace('(cls)', '()')) + + +def esc(text): + """Escape the one character that breaks a Markdown table.""" + return text.replace('|', '\\|') + + +def function_table(module, names=None): + rows = [] + for name in sorted(names or getattr(module, '__all__', [])): + obj = getattr(module, name, None) + if not inspect.isfunction(obj): + continue + rows.append('| `{0}{1}` | {2} |'.format( + name, esc(signature(obj)), esc(first_line(obj)))) + return (['| Function | Summary |', '|---|---|'] + rows + ['']) if rows else [] + + +def class_section(out, cls, name): + w = out.append + bases = [b.__name__ for b in cls.__bases__ if b is not object] + w('#### `{0}`\n'.format(name)) + if bases: + w('*inherits* `{0}`\n'.format('`, `'.join(bases))) + doc = inspect.getdoc(cls) + if doc: + w(doc + '\n') + + combos = getattr(cls, 'required_inputs_and_actions', None) + if combos and 'required_inputs_and_actions' in vars(cls): + w('*Constructor keywords* (any one combination)\n') + for keys, _action in combos: + w('- `{0}`'.format('`, `'.join(keys))) + w('') + + dtype = vars(cls).get('dtype') + if dtype is not None: + w('*Stored as* `{0}`\n'.format(np.dtype(dtype).name)) + + rows = [] + for mname, member in sorted(vars(cls).items()): + if mname.startswith('_'): + continue + if isinstance(member, property): + rows.append((mname, '', first_line(member.fget))) + continue + target = member.__func__ if isinstance(member, (classmethod, staticmethod)) else member + if not inspect.isfunction(target): + continue # class constants such as dtype, reported above + kind = '' + if isinstance(member, classmethod): + kind = ' *(classmethod)*' + elif isinstance(member, staticmethod): + kind = ' *(staticmethod)*' + rows.append((mname, signature(target), first_line(target) + kind)) + + if rows: + w('| Member | Signature | Summary |') + w('|---|---|---|') + for mname, sig, summary in rows: + w('| `{0}` | `{1}` | {2} |'.format(mname, esc(sig), esc(summary))) + w('') + + +def build(): + out = [] + w = out.append + + w('# API reference\n') + w('Generated from the docstrings by `docs/generate_api.py`, so') + w('`help(TopoAnalysis.Area)` in a session shows the same text. For how the') + w('pieces fit together start with [guide.md](guide.md); for what the') + w('computations actually do see [algorithms.md](algorithms.md).\n') + w('Attributes prefixed with a single underscore (`_griddata`,') + w('`_georef_info`) are not private in practice -- they are how you reach') + w('the data -- but they are not listed here; see') + w('[guide.md](guide.md#the-grid-model).\n') + + w('## `TopoAnalysis.dem` -- the grid classes\n') + w('Contents:\n') + for title, names in GROUPS: + w('- **{0}**: {1}'.format(title, ', '.join('`{0}`'.format(n) for n in names))) + w('') + + for title, names in GROUPS: + w('### {0}\n'.format(title)) + for name in names: + cls = getattr(dem, name, None) + if cls is not None: + class_section(out, cls, name) + + w('### Module-level functions in `TopoAnalysis.dem`\n') + out.extend(function_table(dem, ['mosaicFolder', 'plot', 'gdal_is_available'])) + + w('## `TopoAnalysis.topotoolbox` -- TopoToolbox-equivalent functions\n') + w(inspect.getdoc(topotoolbox).split('Example')[0].strip() + '\n') + w('See [topotoolbox_parity.md](topotoolbox_parity.md) for the full') + w('correspondence table and the places where exact agreement is not') + w('achievable.\n') + out.extend(function_table(topotoolbox)) + for name in topotoolbox.__all__: + obj = getattr(topotoolbox, name) + if inspect.isfunction(obj): + w('### `{0}{1}`\n'.format(name, signature(obj))) + w((inspect.getdoc(obj) or '') + '\n') + + w('## `TopoAnalysis.kernels` -- backend dispatch\n') + w(inspect.getdoc(kernels) + '\n') + out.extend(function_table(kernels)) + + w('## `TopoAnalysis.fastops` -- the reference NumPy kernels\n') + w(inspect.getdoc(fastops).split('Conventions')[0].strip() + '\n') + for name in fastops.__all__: + obj = getattr(fastops, name) + if inspect.isfunction(obj): + w('### `{0}{1}`\n'.format(name, signature(obj))) + w((inspect.getdoc(obj) or '') + '\n') + + for mod, title in ((demRecursionTools, 'TopoAnalysis.demRecursionTools'), + (plotting, 'TopoAnalysis.plotting'), + (datasets, 'TopoAnalysis.datasets'), + (utils, 'TopoAnalysis.utils'), + (analysis, 'TopoAnalysis.analysis'), + (cli, 'TopoAnalysis.cli'), + (MovingWindow, 'TopoAnalysis.MovingWindow')): + w('## `{0}`\n'.format(title)) + doc = inspect.getdoc(mod) + if doc: + w(doc + '\n') + names = sorted(n for n, o in vars(mod).items() + if not n.startswith('_') and inspect.isfunction(o) + and o.__module__ == mod.__name__) + out.extend(function_table(mod, names)) + for cname in sorted(n for n, o in vars(mod).items() + if not n.startswith('_') and inspect.isclass(o) + and o.__module__ == mod.__name__): + cls = getattr(mod, cname) + w('### `{0}`\n'.format(cname)) + w((inspect.getdoc(cls) or '') + '\n') + rows = ['| `{0}{1}` | {2} |'.format(mname, esc(signature(member)), + esc(first_line(member))) + for mname, member in sorted(vars(cls).items()) + if not mname.startswith('_') and inspect.isfunction(member)] + if rows: + w('| Method | Summary |') + w('|---|---|') + out.extend(rows) + w('') + + w('## `TopoAnalysis.error`\n') + for cname in ('Error', 'InputError', 'TransitionError'): + cls = getattr(error, cname) + w('### `{0}`\n'.format(cname)) + w((inspect.getdoc(cls) or '') + '\n') + + return '\n'.join(out) + '\n' + + +if __name__ == '__main__': + target = pathlib.Path(__file__).parent / 'api.md' + target.write_text(build()) + print('wrote {0} ({1} lines)'.format(target, build().count('\n'))) diff --git a/docs/guide.md b/docs/guide.md new file mode 100644 index 0000000..fefdf18 --- /dev/null +++ b/docs/guide.md @@ -0,0 +1,282 @@ +# User guide + +## The grid model + +Everything in TopoAnalysis is a grid. A grid holds a 2-D NumPy array and the +georeferencing that places it on the ground: + +```python +grid._griddata # the array; row 0 is the NORTH edge +grid._georef_info.dx # cell size, in map units +grid._georef_info.nx, .ny # columns, rows +grid._georef_info.xllcenter, .yllcenter # centre of the SOUTH-WEST cell +grid._georef_info.geoTransform # the GDAL 6-tuple +grid._georef_info.projection # WKT, or 0 if unset +``` + +`NaN` is no-data in floating-point grids. + +Subclasses add meaning rather than structure. `Elevation` is a grid of +heights, `FlowDirectionD8` a grid of direction codes, `Area` a grid of +contributing areas — all of them are `BaseSpatialGrid` underneath and share +its clipping, tiling, resampling and plotting. + +## Constructing grids + +Grids are built by keyword, and the combination you pass selects the +constructor: + +```python +from TopoAnalysis import Elevation + +Elevation(gdal_filename='dem.tif') # from a file +Elevation(ai_ascii_filename='dem.asc', EPSGprojectionCode=32611) +Elevation(dx=30.0, grid=array) # from an array +Elevation(nx=100, ny=80, projection=wkt, geo_transform=gt) # empty +Elevation(nx=100, ny=80, dx=30.0) # random values +``` + +Derived grids name the grid they come from: + +```python +FilledElevation(elevation=dem) +FlowDirectionD8(flooded_dem=filled) +Area(flow_direction=d8) +Ksi(area=area, flow_direction=d8, flow_length=length, theta=0.45, Ao=1e6) +``` + +Each class lists what it accepts in `required_inputs_and_actions`, and an +unrecognised combination raises an error that prints the list. + +## The analysis chain + +```python +from TopoAnalysis import (Elevation, FilledElevation, FlowDirectionD8, + Area, FlowLength, Chi, Ksi, ScaledRelief) + +dem = Elevation(gdal_filename='srtm.tif') +filled = FilledElevation(elevation=dem) +d8 = FlowDirectionD8(flooded_dem=filled) +area = Area(flow_direction=d8) +length = FlowLength(flow_direction=d8) +``` + +Each step depends on the one before. Filling removes closed depressions so +every cell has a downhill path out; routing turns that surface into a +network; accumulation and length are properties of the network. + +### Filling + +```python +filled = FilledElevation(elevation=dem) # default +filled = FilledElevation(elevation=dem, aggradation_slope=0.0) # flat +filled = FilledElevation(elevation=dem, maximum_pit_depth=20.0) +filled = FilledElevation(elevation=dem, outlets=[(x, y)]) +print(filled.fill_report) # {'cells_filled': ..., 'max_fill_depth': ...} +``` + +By default the filled surface carries a very slight gradient +(`aggradation_slope = 1e-12` per map unit) across what were depressions, so +D8 routing has a direction to follow. `aggradation_slope=0` gives a flat +fill, which is what TopoToolbox's `fillsinks` returns — but then the flats +have no defined flow direction. + +`maximum_pit_depth` leaves genuinely closed basins — playas, craters — +unfilled, while still routing the terrain beyond them. + +### Flow direction + +D8 codes follow the ArcGIS convention, with row 0 at the north: + +``` +| 32 64 128 | +| 16 X 1 | +| 8 4 2 | +``` + +0 means the cell has no downstream neighbour: a pit, an outlet at the grid +edge, or no-data. + +The grid also exposes the network directly, which is what makes the +downstream computations linear-time: + +```python +d8.receivers # flat index of each cell's receiver, or -1 +d8.topological_order # donors before receivers +d8.cycle_count # 0 for a valid network +d8.step_lengths() # distance from each cell to its receiver +d8.basin_mask([(x, y)]) # boolean mask of everything draining to a point +``` + +### Drainage area + +```python +area = Area(flow_direction=d8) # square map units +``` + +Each cell contributes its own area plus everything upstream. For a +geographic (lat/lon) grid use `GeographicArea`, which uses the true spherical +cell area — cells shrink towards the poles. + +To count *cells* instead, which is what TopoToolbox reports: + +```python +from TopoAnalysis import topotoolbox as tt +cells = tt.flowacc(d8) +``` + +## Chi and channel steepness + +Chi is the integral + + chi(x) = integral from the outlet to x of (A0 / A)^theta dx' + +and a channel in steady state plots as a straight line in chi–elevation +space, with slope k_sn. There are two ways to compute it here. + +**`Chi`** integrates upstream from outlets you name, over the whole network +above each one: + +```python +outlets = area.areas_greater_than(1e8) +chi = Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e6, outlets=outlets) +``` + +**`Ksi`** integrates along longest-flow-length paths across the whole grid, +with no outlets needed: + +```python +ksi = Ksi(area=area, flow_direction=d8, flow_length=length, + theta=0.45, Ao=1e6) +``` + +Pair either with the matching relief grid and the slope of the resulting +plot is the steepness index: + +```python +relief = ScaledRelief(flow_direction=d8, elevation=dem, flow_length=length, + Ao=1e6, theta=0.45, area=area) + +from TopoAnalysis import plot +plot(ksi, relief, xlabel='chi (m)', ylabel='scaled relief (m)') +``` + +`Chi` takes `trapezoid=True` to average the integrand across each step +instead of taking it at the upstream cell. That is the rule TopoToolbox uses, +and the one to pick when comparing against it. + +### Fitting steepness cell by cell + +`KsFromChiWithSmoothing` fits chi–elevation over a moving window along each +flow path, giving a map of k_sn rather than one number per basin: + +```python +ks = KsFromChiWithSmoothing(elevation=dem, area=area, flow_direction=d8, + theta=0.45, vertical_interval=50.0, + area_threshold=1e6) +ks.save('ksn.tif') # 7 bands: ks, n, mse, ss, r2, p, n_regression +``` + +Use `horizontal_interval` instead of `vertical_interval` to set the window +by distance rather than by drop. `ThetaFromChiWithSmoothing` fits the +concavity instead of holding it fixed. Both need `statsmodels`. + +### Profiles + +For individual profiles rather than maps, extract the basin as a nested +dictionary and work on that: + +```python +from TopoAnalysis import demRecursionTools, plotting + +tree = d8.map_values_to_recursive_list(outlet, elevation=dem, area=area) +e, c = demRecursionTools.chi_elevation(tree, area._mean_pixel_dimension(), + [0.45], xo=1000.0) +ks, r2 = demRecursionTools.best_ks_with_r2_list(tree, de, [0.45]) + +plotting.plot_profiles(dem, d8, area, outlet, 'k-') +plotting.plot_chi_profiles(dem, d8, area, outlet, 'k-', theta=0.45) +``` + +Each node of the tree carries `index`, `distance_scale` (the step length +from its parent, 1 or sqrt(2)), `next` (its upstream children), and one entry +per grid you passed as a keyword. + +## Terrain attributes + +```python +from TopoAnalysis import Hillshade, MaxSlope, Laplacian, LocalRelief + +Hillshade(elevation=dem, azimuth=315, inclination=45) # ESRI, 0-255 +MaxSlope(elevation=dem) # gradient magnitude +Laplacian(elevation=dem) # + in valleys +LocalRelief(elevation=dem, pixel_radius=25) # range in a disc + +dem.principal_curvatures() # (k1, k2) +dem.calculate_gradient_over_length_scale(500.0) # (dz/dx, dz/dy) +dem.calculate_laplacian_over_length_scale(500.0) +``` + +Measuring over an explicit length scale is usually what you want on a noisy +DEM: a 3×3 stencil on 1 m lidar measures the noise, not the landscape. + +## Working with large DEMs + +The compiled kernels handle a few million cells comfortably. Beyond that, +process in tiles: + +```python +tiles = dem.tile(tile_xdim=2000, tile_ydim=2000, + tile_xpadding=100, tile_ypadding=100) + +results = [] +for tile in tiles: + filled = FilledElevation(elevation=tile) + d8 = FlowDirectionD8(flooded_dem=filled) + results.append(Area(flow_direction=d8).remove_padding(100, 100)) + +mosaic = Area.mosaic(results) +``` + +The padding gives each tile enough context that the interior is correct; it +is stripped before the tiles are reassembled. Pick padding larger than the +longest flow path you need to resolve within a tile — for drainage area, +which integrates over the whole basin, tiling is only valid when basins fit +inside a tile. + +## Choosing a backend + +```python +import TopoAnalysis +TopoAnalysis.backend() # 'c++' or 'python' +``` + +Set `TOPOANALYSIS_PURE_PYTHON=1` to force the NumPy kernels. The two produce +identical results — the test suite asserts it — so this is for debugging and +for comparing, not for changing answers. + +## Reading and writing + +```python +grid.save('out.tif') # LZW-compressed GeoTIFF, NaN as nodata +grid.write_to_ai('out.asc') # ArcInfo ASCII, -9999 as nodata +grid.vectorize('polygons') # -> polygons.shp +Elevation.load('out.tif') +``` + +`FlowLength.save` also writes a `*_directions` side-file holding its +main-stem direction codes; `FlowLength.load` expects to find it. + +## Cross-checking against TopoToolbox + +See [topotoolbox_parity.md](topotoolbox_parity.md). The short version: + +```python +from TopoAnalysis import topotoolbox as tt + +tt.fillsinks(dem) # flat fill +tt.flowobj(dem) # FLOWobj(DEM,'preprocess','fill') +tt.flowacc(fd) # in CELLS, not m^2 +tt.chitransform(fd, a) # trapezoidal, mn=0.45, a0=1e6 +tt.ksn(fd, dem, a) # carves at 1e-5 first, no a0 +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..4dcd0be --- /dev/null +++ b/docs/index.md @@ -0,0 +1,82 @@ +# TopoAnalysis documentation + +Digital elevation model analysis for tectonic geomorphology. + +```bash +git clone +cd TopoAnalysis +python3 install.py +``` + +That installs everything, GDAL included — both its C library and its Python +bindings, which pip cannot supply on its own. + +## Start here + +- **[Installation](installation.md)** — the one-command install, how GDAL is + obtained on each platform, and the offline and no-compiler cases. +- **[User guide](guide.md)** — the grid model, the analysis chain from DEM + to channel steepness, and worked examples. +- **[Algorithms](algorithms.md)** — what each computation actually does, + the conventions it assumes, and where it will disagree with another + package. +- **[API reference](api.md)** — every public class and function, generated + from the docstrings. +- **[TopoToolbox parity](topotoolbox_parity.md)** — the function-by-function + correspondence with the MATLAB package, and the places where exact + agreement is not achievable. +- **[Changelog](../CHANGELOG.md)** — the defects repaired in 1.0, and the + handful of results that change because they were wrong. + +## Thirty seconds + +```python +from TopoAnalysis import Elevation, FilledElevation, FlowDirectionD8, Area + +dem = Elevation(gdal_filename='srtm.tif') +filled = FilledElevation(elevation=dem) # Priority-Flood, Barnes et al. 2014 +d8 = FlowDirectionD8(flooded_dem=filled) # steepest-descent D8 +area = Area(flow_direction=d8) # contributing area, m^2 +area.save('area.tif') +``` + +## Two things to know before you start + +**Row 0 is the north edge.** Grids are stored the way a GeoTIFF is, so the +first row of `_griddata` is the top of the map and row index increases +southwards. Every convention in the library follows from that: the D8 code +for "south" is 4, `y` decreases as the row index grows, and +`geoTransform[5]` is negative. + +**Drainage area is in square metres, not cells.** `Area` accumulates +`dx**2` per cell. TopoToolbox's `flowacc` counts cells. When comparing the +two, or reading a paper that reports one, check which is meant — +`TopoAnalysis.topotoolbox.flowacc` gives the cell count. + +## Which backend am I on? + +```python +import TopoAnalysis +TopoAnalysis.backend() # 'c++' or 'python' +``` + +or from a shell: + +```bash +topoanalysis-process --info +``` + +Both backends produce identical numbers; the C++ one is about 50× faster. +`TOPOANALYSIS_PURE_PYTHON=1` forces the NumPy path. + +## Regenerating these pages + +`api.md` is generated from the docstrings: + +```bash +python docs/generate_api.py # or: make docs +``` + +The rest are written by hand. They are plain Markdown and are meant to be +read as they are, on the repository page or in an editor -- there is no +Sphinx build. The `[docs]` extra exists for anyone who wants to wire one up. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..42d2bb3 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,392 @@ +# Installation + +```bash +git clone +cd TopoAnalysis +python3 install.py +``` + +That is the whole procedure. `install.py` creates a virtual environment, +installs the dependencies, compiles the Priority-Flood kernels, installs GDAL +— the C library *and* the Python bindings — and then proves the result works +by writing a georeferenced GeoTIFF and reading it back. + +It uses nothing but the standard library, so it runs on a machine where +nothing has been set up yet. When it finishes it tells you how to activate +the environment. + +``` +======================================================================== +6. Checking the installation +======================================================================== + TopoAnalysis 1.0.0 + compute backend : c++ + in-memory pipeline : ok (max area 8100 m^2) + GDAL available : True + GDAL version : 3.13.3 + GeoTIFF round trip : ok (EPSG:32611, 5x5) +``` + +## What it installs, and what is optional + +| Component | Needed for | If it is missing | +|---|---|---| +| NumPy, SciPy, matplotlib | everything | hard requirement | +| a C++17 compiler | the fast kernels | falls back to NumPy, same numbers, ~50× slower | +| GDAL | reading and writing raster files | everything except file I/O still works | + +Neither the compiler nor GDAL is required, and `install.py` will finish +without them rather than failing. + +## Options + +```bash +python3 install.py --no-gdal # skip GDAL entirely +python3 install.py --venv ~/envs/topo # a virtualenv somewhere specific +python3 install.py --system # no virtualenv; use this interpreter +python3 install.py --gdal-strategy source # force a from-source GDAL build +python3 install.py --allow-root # let apt/dnf use sudo +python3 install.py --jobs 8 # parallel compile jobs +python3 install.py --dry-run # show the plan, change nothing +``` + +`--dry-run` is worth running first on an unfamiliar machine. It reports what +GDAL is present, what is wrong with it, and exactly which commands an install +would run. + +`--venv` forces a virtual environment even from inside a conda environment, +which is otherwise used as it stands. + +**Exit status.** `0` means everything asked for is installed and verified. +`1` means something is missing — including the case where TopoAnalysis +installed correctly but GDAL did not, which the output states explicitly. Use +`--no-gdal` if you do not want GDAL and do not want its absence to be an +error. + +**`--system` and PEP 668.** Debian, Ubuntu, Fedora and Homebrew mark their +Python installations as externally managed, and pip refuses to install into +them. `install.py --system` stops with an explanation rather than letting pip +fail obscurely. Inside a container, where there is no host environment to +protect, add `--break-system-packages`. + +### Why it makes a virtualenv by default + +Because installing into a system Python usually fails, and when it does not +fail it is a bad idea. Debian, Ubuntu, Fedora and Homebrew all mark their +Python installations "externally managed" (PEP 668), so `pip install` refuses +to touch them. Creating `.venv` sidesteps that and gives a from-source GDAL +build somewhere self-contained to live. If you are already inside a +virtualenv or a conda environment, that one is used and nothing new is +created. + +--- + +# GDAL + +GDAL is the one dependency pip cannot supply on its own, so it gets its own +tool. If you only want the summary: run `topoanalysis-gdal install` and it +will work out what to do. + +## Why it is difficult + +There is no GDAL wheel on PyPI. Not for your platform — for *any* platform, +at *any* version. Every release is an sdist whose `setup.py` shells out to +`gdal-config` and compiles the SWIG bindings against a GDAL C library that +must already be on the machine. So installing GDAL is two jobs that have to +agree with each other: + +1. the **C library**, `libgdal`, which pip knows nothing about, and +2. the **Python bindings**, `osgeo`, whose version must match it. + +`pip install GDAL` does only the second half, and fails confusingly when the +first half is absent or a different version. + +## The tool + +```bash +topoanalysis-gdal # what is installed, and what is wrong with it +topoanalysis-gdal plan # what an install would do, step by step +topoanalysis-gdal install # do it +``` + +`doctor` is the one to reach for when something is wrong: + +``` +python : 3.11.9 (/home/you/TopoAnalysis/.venv/bin/python) +numpy : 2.1.0 +GDAL bindings : 3.4.1 +GDAL C library : 3.8.4 +gdal-config : /usr/bin/gdal-config +numpy bridge : MISSING (ReadAsArray will fail) +projection lookup : works + +problems: + [version-mismatch] the bindings are 3.4.1 but the C library is 3.8.4; they + must agree to at least major.minor + [no-gdal-array] osgeo.gdal_array is missing, so Band.ReadAsArray() will + fail. The bindings were built without numpy present. +``` + +## The routes it can take + +`topoanalysis-gdal install` picks the cheapest one that stands a chance. +`--strategy` forces a particular choice. + +| Strategy | When it is chosen | Cost | +|---|---|---| +| `bindings` | the C library is already here; just build `osgeo` against it | seconds | +| `conda` | a conda environment is active | a minute or two | +| `wheel` | Windows, where prebuilt wheels bundling the C library are the only practical option | a minute | +| `system` | apt/dnf/pacman/zypper/apk/brew can supply the C library | a few minutes, needs root except under Homebrew | +| `source` | nothing else is available | about half an hour | + +### `source` — building GDAL without root + +This is the route that makes an unattended install possible on a machine with +no conda, no package manager and no administrator rights. It downloads, +verifies and builds: + +``` +sqlite3 only when the system copy is unusable. PROJ needs the library, the + header, and the sqlite3 command, which it runs to assemble proj.db. +PROJ mandatory for GDAL 3.x. Built without TIFF and curl, which are only + needed for the optional high-accuracy datum grids. +GDAL built against its own bundled libtiff, libgeotiff, libpng, libjpeg + and zlib, so the only external dependency left is PROJ. +``` + +Using GDAL's bundled libraries is what makes this realistic: there are no +system `-dev` packages to hunt for. The result reads GeoTIFF, ArcInfo ASCII, +ENVI, HFA and the other built-in drivers — everything TopoAnalysis uses. + +Everything installs into one prefix, by default the active virtualenv, so +nothing outside it is touched and deleting the environment removes GDAL with +it. The only prerequisites are a C/C++ compiler and CMake. + +Every tarball is pinned by SHA-256, checked against what the projects +themselves publish. A download that does not match the pin is not built. To +move a pin — a newer GDAL, or a local mirror — set all three variables +together, so the integrity check cannot be disabled by accident: + +```bash +export TOPOANALYSIS_GDAL_SOURCE_GDAL_VERSION=3.12.4 +export TOPOANALYSIS_GDAL_SOURCE_GDAL_URL=https://example.org/gdal-3.12.4.tar.gz +export TOPOANALYSIS_GDAL_SOURCE_GDAL_SHA256=<64 hex digits> +``` + +The pinned GDAL is deliberately recent. GDAL bundles its own copies of zlib +and libpng, and the copies shipped up to about 3.9 do not compile against a +current Apple SDK: both take their "Mac OS classic" branch, because that is +selected on `TARGET_OS_MAC`, which modern SDKs define on every Apple +platform. zlib then redefines `fdopen` to `NULL` and the system `_stdio.h` +fails to parse. + +## Platform notes + +**conda (any platform)** — the most reliable route, and the only one that is +equally good on Windows: + +```bash +conda env create -f environment.yml +conda activate topoanalysis +python -m pip install -e . +``` + +`environment.yml` puts `conda-forge` first, because mixing GDAL from +`defaults` with compilers from `conda-forge` produces a C++ ABI mismatch that +only appears at import time. + +**Ubuntu / Debian** — `sudo apt install gdal-bin libgdal-dev`, then +`topoanalysis-gdal install`. Note that 22.04 ships GDAL 3.4 and 24.04 ships +3.8; both are below the version where the sdist started declaring numpy as a +build dependency, so install the bindings with `topoanalysis-gdal` rather than +plain pip — see the trap below. + +**macOS** — Homebrew's GDAL is built for the Mac's native architecture. If +your Python is x86_64 running under Rosetta on an Apple Silicon machine, the +two cannot link. `topoanalysis-gdal doctor` detects this and will not try to +"fix" it by reinstalling the same thing; use conda, a matching Python, or +`--strategy source`, which builds for whichever architecture the interpreter +is. + +**Windows** — use conda, or let `topoanalysis-gdal install` fetch a prebuilt +wheel that bundles the C library. Building GDAL from source on Windows is not +attempted. + +**Docker** — the `osgeo/gdal` images already have both halves installed and +matched; `topoanalysis-gdal doctor` will report nothing to do and +`pip install -e .` is enough. + +## Two traps worth knowing about + +**The silent `gdal_array` trap.** GDAL sdists only grew a `pyproject.toml` in +version 3.9. Before that they declare no build dependencies at all, so under +pip's default build isolation numpy is absent when the bindings compile — and +the numpy bridge, `osgeo.gdal_array`, is quietly skipped. Everything imports +cleanly; `Band.ReadAsArray()` then fails at run time. Ubuntu 22.04 and 24.04 +both ship GDAL versions on the wrong side of that line, so this is the common +case, not an exotic one. `topoanalysis-gdal` installs the bindings with +`--no-build-isolation` against an interpreter that already has numpy, which +both avoids the trap and builds the bridge against the numpy you will +actually run. + +**The architecture trap.** Described under macOS above. The symptom is a +missing-symbol error at link or import time that says nothing about +architectures. + +--- + +## Checking the installation + +```bash +topoanalysis-process --info +topoanalysis-gdal doctor +``` + +``` +TopoAnalysis 1.0.0 + compute backend : c++ + GDAL available : True + fill algorithm : Barnes, R., Lehman, C., Mulla, D. (2014). ... +``` + +Then run the tests: + +```bash +python -m pytest -q # a few seconds +TOPOANALYSIS_PURE_PYTHON=1 python -m pytest -q # same tests, NumPy kernels +``` + +The second run is worth doing after any change to the C++. It re-runs every +test against the NumPy kernels, and the suite contains explicit +backend-comparison tests (`test_kernels.py::test_fill_backends_agree` and +`test_routing_backends_agree`) that call both implementations on the same +inputs and require identical output — so a divergence shows up as a failure +rather than as a quietly different answer. + +## When the compiler is missing + +The extension is declared `optional`, so a failed compile is a warning, not an +installation failure: + +``` +TopoAnalysis: pybind11 not found; building without the compiled kernels. +``` + +The library still installs and works. To get the speed back: + +| Platform | What to install | +|---|---| +| Ubuntu / Debian | `sudo apt install build-essential python3-dev` | +| RHEL / Fedora | `sudo dnf install gcc-c++ python3-devel` | +| macOS | `xcode-select --install` | +| Windows | [Build Tools for Visual Studio](https://visualstudio.microsoft.com/visual-cpp-build-tools/), "Desktop development with C++" | +| conda (any) | `conda install -c conda-forge compilers pybind11` | + +Then reinstall: + +```bash +python -m pip install --force-reinstall --no-deps -e . +python -c "import TopoAnalysis; print(TopoAnalysis.backend())" # -> c++ +``` + +Editing a header under `_cpp/` does trigger a rebuild — the extension declares +them in `depends` — but a stale `build/` directory can still confuse +setuptools. `make distclean && make build` clears it. + +## Installing without a network + +`pip install` needs `pybind11` at build time. If the machine is offline, fetch +the wheel somewhere with a network and carry it over: + +```bash +# on a connected machine +pip download pybind11 setuptools wheel -d wheelhouse + +# on the target +python -m pip install --no-index --find-links wheelhouse pybind11 setuptools wheel +python -m pip install --no-build-isolation --no-index -e . +``` + +`--no-build-isolation` is the important flag: without it pip builds in a fresh +environment and tries to download `pybind11` again. + +A from-source GDAL build also needs a network, but only to fetch its three +tarballs. They are cached under `~/.cache/TopoAnalysis/downloads`, so copying +that directory across is enough to make the build work offline. + +## Building from an sdist + +```bash +python -m build --sdist +python -m pip install dist/TopoAnalysis-1.0.0.tar.gz +``` + +The sdist carries `_cpp/*.hpp` and `_cpp/*.cpp`, so the kernels are rebuilt on +the target machine for its own Python and architecture. + +## Both import styles + +The repository directory *is* the package, so which style works depends on +what is on `sys.path`: + +```python +from TopoAnalysis import dem # parent directory on the path (installed) +import dem # the package directory itself on the path +``` + +Every module supports both. The first is preferred; the second is how the +original scripts in this repository loaded it, and is kept working +deliberately. + +## Optional extras + +```bash +pip install -e ".[stats]" # statsmodels, for the steepness regressions +pip install -e ".[parallel]" # dask, for multiscale curvature +pip install -e ".[quadrats]" # rasterio, for analysis.Quadrats +pip install -e ".[all]" # all of the above +pip install -e ".[docs]" # sphinx, if you want to build a site +``` + +`[all]` deliberately excludes GDAL. Listing it there would make +`pip install .[all]` fail on any machine without a GDAL C library already +present, taking the rest of the package down with it. + +Each is imported only when the code that needs it runs, and the error names +the missing package and how to get it. + +There is a `[gdal]` extra, but it will not work on its own: it installs the +bindings, which cannot build without the C library. Use `topoanalysis-gdal +install` instead, which installs both. + +## Troubleshooting + +**`ImportError: TopoAnalysis needs the GDAL Python bindings ...`** — you called +something that reads or writes a file. Run `topoanalysis-gdal install`, or +build grids in memory with `dx=..., grid=...`. + +**`AttributeError` or a numpy error from `ReadAsArray`** — the `gdal_array` +trap above. `topoanalysis-gdal doctor` confirms it; `topoanalysis-gdal +install` rebuilds the bindings correctly. + +**`symbol not found` or a segfault when importing `osgeo`** — either a C++ ABI +mismatch from mixing conda channels, or the architecture trap. Rebuild the +environment from `environment.yml` with `conda-forge` first, or run +`topoanalysis-gdal doctor`, which checks the architecture explicitly. + +**`ERROR 1: PROJ: proj_create_from_database: Cannot find proj.db`** — PROJ's +data files are missing or `PROJ_LIB` points somewhere wrong. +`topoanalysis-gdal doctor` reports this as `projection lookup : FAILS`. + +**`backend()` says `python` after installing a compiler** — the wheel was +cached. `pip install --force-reinstall --no-deps --no-cache-dir -e .` + +**`RuntimeWarning: TopoAnalysis is using its pure-Python kernels on a grid of N +cells`** — exactly what it says; see the compiler table above. + +**Tests fail with `ModuleNotFoundError: No module named 'TopoAnalysis'`** — run +them from the package directory (`cd TopoAnalysis && pytest`) or from its +parent (`pytest TopoAnalysis/tests`). Both work, and so does a checkout under +any other name (a GitHub zip unpacks to `TopoAnalysis-main`); +`tests/conftest.py` sorts out the import either way. diff --git a/docs/topotoolbox_parity.md b/docs/topotoolbox_parity.md new file mode 100644 index 0000000..d321dc4 --- /dev/null +++ b/docs/topotoolbox_parity.md @@ -0,0 +1,249 @@ +# Parity with TopoToolbox + +[TopoToolbox 2](https://topotoolbox.wordpress.com/) (Schwanghart & Scherler +2014) is the MATLAB package that covers much of the same ground as +TopoAnalysis. This page says, function by function, where the two agree, +where they differ, and what to do about it. + +`TopoAnalysis.topotoolbox` holds functions written to reproduce +TopoToolbox's numbers exactly. The rest of TopoAnalysis keeps its own +conventions — area in square metres, ESRI hillshade, left-endpoint chi — +because those are what the rest of the library is built on. + +```python +from TopoAnalysis import dem, topotoolbox as tt + +z = dem.Elevation(gdal_filename='srtm.tif') +fd = tt.flowobj(z) # == FLOWobj(DEM,'preprocess','fill') +a = tt.flowacc(fd) # == flowacc(FD) +c = tt.chitransform(fd, a) # == chitransform(S,A) +``` + +--- + +## Correspondence table + +**Agreement** is one of: + +- **exact** — the same arithmetic on the same inputs; +- **exact¹** — exact up to float64-vs-float32 rounding (TopoToolbox works in + single precision when the DEM is single); +- **equivalent** — the same quantity, related by a stated conversion; +- **differs** — genuinely different results; the reason is given. + +### Depressions and flats + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `fillsinks(DEM)` | `tt.fillsinks(dem)` | exact | Flat fill; seeds are the perimeter plus every cell touching NaN; 8-connected. | +| | `FilledElevation(elevation=dem)` | differs | Adds `aggradation_slope` (default 1e-12/m) across filled areas. Pass `aggradation_slope=0` for the flat fill. | +| `fillsinks(DEM, maxdepth)` | `tt.fillsinks(dem, maxdepth=...)` | exact | All-or-nothing per connected depression, iterated as MATLAB does — not a per-cell depth test. | +| `identifyflats(DEM)` | `tt.identifyflats(dem)` | exact | Returns `(flats, sills)`; `output='all'` adds the closed basins. Includes MATLAB's cleared rim and no-data halo, and its treatment of an isolated pit as a flat. | +| `imposemin(FD, DEM, sl)` | `tt.imposemin(fd, dem, sl)` | exact | Carves rather than fills: only lowers cells. | +| `elevateminima` | — | — | Not implemented. | + +### Flow + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `FLOWobj(DEM,'preprocess','fill')` | `tt.flowobj(dem)` | **differs inside flats** — see below | Identical elsewhere. | +| `FLOWobj(DEM,'preprocess','carve')` | — | — | Not implemented; `tt.flowobj` raises rather than guessing. | +| `FLOWobj2GRIDobj(FD)` | `d8._griddata` | exact | Same ArcGIS codes, same row-0-is-north orientation. | +| `flowacc(FD)` | `tt.flowacc(fd)` | exact | **Number of cells**, minimum 1. | +| | `Area(flow_direction=d8)` | equivalent | `Area = flowacc × dx²`. | +| `flowacc(FD, W0)` | `tt.flowacc(fd, weights)` | exact | | +| `flowdistance(FD,'upstream')` | `tt.flowdistance(fd,'upstream')` | exact¹ | Distance to the outlet. TopoToolbox accumulates in float32. | +| `flowdistance(FD,'downstream')` | `tt.flowdistance(fd,'downstream')` | exact¹ | Longest path reaching each cell. | +| | `FlowLength(flow_direction=d8)` | exact¹ | The same quantity as `'downstream'`. | +| `drainagebasins(FD)` | `tt.drainagebasins(fd)` | **partition** exact, labels differ | Both number basins in edge-list order; the orders differ. Compare partitions. | +| `drainagebasins(FD,IX)` | `tt.drainagebasins(fd, outlets=...)` | exact | Labels are positions in the outlet list. | +| `streamorder(S,'strahler')` | `tt.streamorder(fd, mask)` | exact | | +| `streamorder(S,'shreve')` | `tt.streamorder(fd, mask, 'shreve')` | exact | | + +### Terrain attributes + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `gradient8(DEM)` | `tt.gradient8(dem)` | exact | Steepest descent over 8 neighbours, clamped at 0. Diagonals use `√2·dx`. | +| `gradient8(DEM,unit)` | `tt.gradient8(dem, unit)` | exact | `tangent`, `degree`, `radian`, `sine`, `percent`. | +| `arcslope(DEM)` | `tt.arcslope(dem)` | exact² | Horn 3×3, `8·dx` denominator. ²NaN infill uses SciPy's distance transform; ties next to a NaN hole can break differently from MATLAB's `bwdist`. | +| | `MaxSlope(elevation=dem)` | differs | Centred-difference gradient magnitude — a different estimator. | +| `hillshade(DEM)` | `tt.hillshade(dem)` | exact | Defaults azimuth 315, altitude 60 (the *code's* defaults; the MATLAB docstring disagrees with itself). Returns a cosine in [−1, 1]. | +| | `Hillshade(elevation=dem, azimuth=, inclination=)` | differs | ESRI formulation: 0–255, and an aspect of `atan2(dz/dy, −dz/dx)` from centred differences. Strongly correlated with the TopoToolbox version but not equal. | +| `hillshade(...,'method','mdow')` | — | — | Not implemented. | +| `aspect(DEM)` | `tt.aspect(dem)` | exact | Degrees clockwise from north; flat cells report 90. | +| `aspect(DEM,true)` | `tt.aspect(dem, classify=True)` | exact | The 8-class Gómez-Plaza ranking. | +| `curvature(DEM,type)` | `tt.curvature(dem, ctype)` | exact | `profc`, `planc`, `tangc`, `meanc`, `total`. | +| | `Laplacian(elevation=dem)` | differs | `∇²z` from a 5-point stencil — a different quantity from any of the five. | +| `cellarea(DEM)` | `tt.cellarea(grid)` | exact³ | ³Assumes a sphere of total area 510 072 000 km². `GeographicArea` uses R = 6 371 000 m, which is 1.5 × 10⁻⁵ smaller. | +| `localtopography(DEM,r,'range')` | `tt.localtopography(dem, r, 'range')` | **≈** | TopoToolbox's default disc is a periodic-line approximation; this uses an exact disc (TopoToolbox's `N=0`). `'mean'` averages over that binary disc where MATLAB's `fspecial('disk')` weights the rim fractionally; `'std'` uses the N−1 sample deviation, as `stdfilt` does. | +| | `LocalRelief(elevation=dem, pixel_radius=)` | exact⁵ | Identical to `tt.localtopography(..., 'range')` with the matching radius, on grids with no no-data. ⁵`LocalRelief` does not mask `NaN`, so around a hole the two differ; use `tt.localtopography` there. | +| `roughness`, `evansslope` | — | — | Not implemented. | + +### Chi and steepness + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `chitransform(S,A)` | `tt.chitransform(fd, a)` | exact | `mn=0.45`, `a0=1e6`, trapezoidal, area in **cells**. | +| | `Chi(..., trapezoid=True)` | exact | Given the same outlets and area in m². | +| | `Chi(...)` (default) | differs | Left-endpoint rule instead of trapezoidal. | +| | `Ksi(...)` | differs | Integrates along longest-flow-length paths, not upstream from outlets. | +| `cumtrapz(S,g)` | — | equivalent | The integration inside `tt.chitransform`. | +| `ksn(S,DEM,A)` | `tt.ksn(fd, dem, a)` | exact | Carves at `sl=1e-5` first; no `a0` (reference area 1 m²). | +| `mchi(S,DEM,chi)` | `tt.mchi(fd, dem, chi)` | exact | `dz/dχ`, no carving. | +| `chiplot`, `slopearea` | — | — | Not implemented; `KsFromChiWithSmoothing` is the nearest equivalent. | + +### Referencing + +| TopoToolbox | TopoAnalysis | Agreement | Notes | +|---|---|---|---| +| `getcoordinates(DEM)` | `tt.getcoordinates(grid)` | exact | `y` runs **north to south**, matching row order. | +| | `grid.coordinate_vectors()` | differs in order | `y` runs south to north. | +| `getextent(DEM)` | `tt.getextent(grid)` | exact | Outermost **cell centres**. | +| | `grid.extent()` | differs | Outer pixel **edges** — what `imshow` wants; half a cell larger each way. | +| `coord2sub`, `coord2ind` | `tt.coord2sub(grid, x, y)` | exact⁴ | ⁴0-based here, 1-based in MATLAB. | +| | `grid._xy_to_rowscols(v)` | **≈** | Rounds halves to even (Python) rather than away from zero (MATLAB); differs only for points landing exactly on a cell boundary. | +| `sub2coord`, `ind2coord` | `tt.sub2coord(grid, r, c)` | exact⁴ | | +| `GRIDobj2ascii` | `grid.write_to_ai(path)` | equivalent | Both centre-registered; TopoAnalysis writes `-9999` for no-data. | +| `GRIDobj2geotiff` | `grid.save(path)` | equivalent | TopoAnalysis adds LZW compression and writes NaN as the no-data value. | + +--- + +## Where exact agreement is not achievable + +### Flow directions inside flats + +This is the one difference that changes a drainage network. + +After filling, a depression becomes a plateau with no downhill direction. +The two packages resolve it differently: + +- **TopoToolbox** fills flat and then builds an *auxiliary topography* — a + grey-weighted geodesic distance transform inside each flat — so cells + drain towards the nearest sill. Flow directions inside a flat come from + the ranking that surface induces. +- **TopoAnalysis** adds a tiny gradient while filling + (`aggradation_slope`), so each cell's direction points back along the + path the flood arrived by. + +Both give a valid, connected network, and both drain each flat to the same +sill. **Which cells they route through inside the flat is different.** +Drainage area therefore differs inside flats and immediately downstream of +them. Everything outside flats is identical. + +There is no way to reconcile this short of implementing the geodesic +transform. If exact agreement inside flats matters, compute the flow +directions in MATLAB and load them: + +```python +d8 = FlowDirectionD8(gdal_filename='flowdir_from_matlab.tif') +``` + +The code convention is identical, so the grid transfers without conversion. + +TopoToolbox's default is actually `'carve'`, not `'fill'` — it *lowers* the +DEM through depressions rather than raising it. That produces a different +surface again, and is not implemented here. + +### Tie-breaking on equal gradients + +Where two neighbours are equally steep, TopoToolbox prefers the diagonal +(its test is `G1 <= G2`) and then the larger linear index; TopoAnalysis +takes the first in the order E, SE, S, SW, W, NW, N, NE. Exact ties are rare +on a real DEM with an ε-filled surface, but common on synthetic or +integer-valued ones. + +### Border cells of `hillshade` and `aspect` + +None — these agree exactly. MATLAB's `surfnorm` expands the grid with a +*quadratic* ghost cell (`3·z₁ − 3·z₂ + z₃`), and a centred difference across +that ghost is algebraically the second-order one-sided difference, which is +what `numpy.gradient(..., edge_order=2)` computes. +`test_surfnorm_matches_matlab_including_the_border` checks it against a +literal port of `surfnorm.m`. + +### Precision + +TopoToolbox works in whatever class the DEM is, usually float32 from a +GeoTIFF; TopoAnalysis works in float64 throughout. Accumulated quantities — +flow length especially, which sums thousands of terms — drift apart in the +last few digits. Compare with a relative tolerance of about 1e-6, not +exactly. + +### Basin labels + +Both packages number basins in the order their edge list happens to meet +them, and the two orders differ. The *partition* is identical. Compare +partitions: + +```python +import numpy as np + +def same_partition(a, b): + """True when two labellings group the cells identically.""" + pairs = np.unique(np.stack([a.ravel(), b.ravel()]), axis=1) + return (len(np.unique(pairs[0])) == pairs.shape[1] + and len(np.unique(pairs[1])) == pairs.shape[1]) +``` + +### Earth radius + +`tt.cellarea` uses TopoToolbox's sphere (total area 510 072 000 km², so +R = 6 371 047 m); `GeographicArea` uses R = 6 371 000 m. Areas differ by +1.5 × 10⁻⁵ — negligible for most work, but it will show up in an exact +comparison. Setting the radius brings them to within 3 × 10⁻¹²: + +```python +grid.earth_radius = (tt.EARTH_SURFACE_AREA_KM2 / (4 * np.pi)) ** 0.5 * 1000.0 +grid._GeographicGridMixin__app = None # the area grid is cached +``` + +The residue is because `GeographicArea` evaluates the longitude difference +per cell, so its areas vary in the last few bits along a row, whereas +`cellarea` computes one value per row and broadcasts it. + +--- + +## Cross-checking in practice + +A recipe for comparing a result against MATLAB: + +```python +import numpy as np +from TopoAnalysis import dem, topotoolbox as tt + +z = dem.Elevation(gdal_filename='srtm.tif') +fd = tt.flowobj(z) +a = tt.flowacc(fd) # cells, like flowacc(FD) +c = tt.chitransform(fd, a, mn=0.45, a0=1e6) + +matlab = dem.BaseSpatialGrid(gdal_filename='chi_from_matlab.tif') +difference = c._griddata - matlab._griddata +print(np.nanmax(np.abs(difference)), + np.nanpercentile(np.abs(difference), 99)) +``` + +Expect agreement to ~1e-6 relative away from flats, and visible differences +inside them. If the difference is a constant factor, check the units: area +in cells versus m², or `a0` versus `a0^mn`. + +## How this was verified + +Without MATLAB available, parity was established three ways: + +1. **From the source.** Every function above was transcribed from the + TopoToolbox MATLAB source in this repository — the exact stencils, + constants, default values, rounding rules and NaN handling — and each + transcription was independently re-checked against the source. +2. **Against closed forms.** Planes, quadratics and spherical cells have + answers that can be written down: a 1:1 east-facing plane has gradient + exactly 1, aspect exactly 90°, curvature exactly 0. `test_topotoolbox.py` + checks those. +3. **Against each other.** Where TopoAnalysis and the parity module compute + the same quantity by different routes, the tests assert the documented + conversion holds — `flowacc × dx² == Area`, `chitransform == + Chi(trapezoid=True)`, `localtopography('range') == LocalRelief`. + +What this does **not** establish is bit-for-bit agreement with a specific +MATLAB release. If you have MATLAB, running the same DEM through both and +comparing is worth doing, and the differences above are what to expect. diff --git a/environment.yml b/environment.yml index 093c386..67ce615 100644 --- a/environment.yml +++ b/environment.yml @@ -1,9 +1,24 @@ -name: TopoAnalysis +# Conda environment for TopoAnalysis. +# +# conda env create -f environment.yml +# conda activate topoanalysis +# pip install -e . +# +# conda-forge is listed first because it is the only channel that reliably +# provides a working GDAL together with a matching libstdc++. +name: topoanalysis channels: - - defaults - conda-forge + - defaults dependencies: - - numpy - - matplotlib - - gdal - - scipy + - python>=3.8 + - numpy>=1.20 + - scipy>=1.7 + - matplotlib-base>=3.3 + - gdal>=3.0 # file I/O only; the rest of the library runs without it + - pybind11>=2.6 # builds the Priority-Flood kernels + - compilers # a C++17 toolchain on every platform + - statsmodels>=0.12 # channel-steepness regressions + - dask>=2021.0 # optional parallel multiscale curvature + - pytest>=7.0 + - pip diff --git a/fastops.py b/fastops.py new file mode 100644 index 0000000..4f262b5 --- /dev/null +++ b/fastops.py @@ -0,0 +1,811 @@ +"""Pure-NumPy implementations of the TopoAnalysis grid kernels. + +Every function here mirrors, name for name and result for result, one of the +functions in the compiled ``TopoAnalysis._topoanalysis`` extension. They are +the reference semantics: the C++ code is validated against them in the test +suite, and they are what :mod:`TopoAnalysis.kernels` falls back to when the +extension is unavailable. + +The depression-filling routine implements the Priority-Flood algorithms of + + Barnes, R., Lehman, C., Mulla, D. (2014). "Priority-flood: An optimal + depression-filling and watershed-labeling algorithm for digital elevation + models." *Computers & Geosciences* 62, 117-127. + +These implementations are correct but roughly one to two orders of magnitude +slower than the compiled kernels on real DEMs. If you are processing +anything larger than a few hundred thousand cells, build the extension:: + + pip install -e . + +Conventions +----------- +Grids are 2-D, row-major, with row 0 at the north edge. ``NaN`` is no-data. +Flow-direction codes follow the ArcGIS convention:: + + | 32 64 128 | + | 16 X 1 | + | 8 4 2 | +""" + +from __future__ import annotations + +import heapq +import math + +import numpy as np + +__all__ = [ + "D8_DI", + "D8_DJ", + "D8_DIST", + "D8_CODES", + "default_seeds", + "priority_flood", + "flow_directions", + "receivers", + "topological_order", + "accumulate", + "flow_length", + "propagate_along_main_stem", + "upstream_mask", + "chi", + "drainage_basins", +] + +# Neighbour order E, SE, S, SW, W, NW, N, NE -- so that the ArcGIS code for +# direction ``d`` is simply ``1 << d``. +D8_DI = np.array([0, 1, 1, 1, 0, -1, -1, -1], dtype=np.int64) +D8_DJ = np.array([1, 1, 0, -1, -1, -1, 0, 1], dtype=np.int64) +D8_DIST = np.array([1.0, math.sqrt(2.0)] * 4, dtype=np.float64) +D8_CODES = np.array([1, 2, 4, 8, 16, 32, 64, 128], dtype=np.uint8) + +_SQRT2 = math.sqrt(2.0) + + +def _as_2d_float(grid, name): + arr = np.asarray(grid, dtype=np.float64) + if arr.ndim != 2: + raise ValueError("{0} must be a 2-D array".format(name)) + return arr + + +def _shifted(grid, di, dj, fill): + """``grid`` translated so that element (i, j) holds grid[i+di, j+dj].""" + out = np.full_like(grid, fill) + ny, nx = grid.shape + src_i = slice(max(di, 0), ny + min(di, 0)) + src_j = slice(max(dj, 0), nx + min(dj, 0)) + dst_i = slice(max(-di, 0), ny + min(-di, 0)) + dst_j = slice(max(-dj, 0), nx + min(-dj, 0)) + out[dst_i, dst_j] = grid[src_i, src_j] + return out + + +# --------------------------------------------------------------------------- +# Depression filling +# --------------------------------------------------------------------------- + + +def default_seeds(elevations): + """Flat indices of the cells a flood should start from. + + These are the grid perimeter plus every valid cell touching a no-data + cell -- the same set that TopoToolbox's ``fillsinks`` keeps in its marker + image. + + Parameters + ---------- + elevations : ndarray + 2-D elevation grid; ``NaN`` marks no-data. + + Returns + ------- + ndarray of int64 + Flat (C-order) indices of the seed cells. + """ + z = _as_2d_float(elevations, "elevations") + ny, nx = z.shape + if ny == 0 or nx == 0: + return np.empty(0, dtype=np.int64) + nodata = np.isnan(z) + + seed = np.zeros((ny, nx), dtype=bool) + seed[0, :] = True + seed[-1, :] = True + seed[:, 0] = True + seed[:, -1] = True + for di, dj in zip(D8_DI, D8_DJ): + seed |= _shifted(nodata.astype(np.float64), int(di), int(dj), 0.0) > 0.5 + seed &= ~nodata + return np.flatnonzero(seed).astype(np.int64) + + +def priority_flood( + elevations, + closed=None, + seeds=None, + mode="flat", + epsilon=0.0, + cellsize=1.0, + max_pit_depth=0.0, + track_visited=False, + reference_algorithm=False, +): + """Fill depressions in ``elevations`` in place (Barnes et al. 2014). + + Parameters + ---------- + elevations : ndarray + 2-D float64 grid, modified in place. ``NaN`` is no-data. + closed : ndarray of uint8, optional + Cells with a non-zero entry are never processed. Use this to restrict + the flood to a mask. + seeds : ndarray of int64, optional + Flat indices to start from. Defaults to :func:`default_seeds`. + mode : {'flat', 'epsilon'} + ``'flat'`` fills depressions to a level surface (Algorithm 2), which + is what TopoToolbox's ``fillsinks`` does. ``'epsilon'`` adds a small + upslope increment so the result has no flats (Algorithm 4). + epsilon : float + Gradient of the increment used by ``mode='epsilon'``. Each filled + cell is raised to ``parent + epsilon * cellsize * step``. Zero means + "use the smallest representable increment". + cellsize : float + Grid spacing, used only to scale ``epsilon``. + max_pit_depth : float + Depressions deeper than this are left unfilled. Zero disables the + check. Unlike the original TopoAnalysis implementation, the flood + still traverses such depressions, so terrain beyond them is reached. + track_visited : bool + Also return a mask of the cells the flood reached. + reference_algorithm : bool + Use Algorithm 1 (everything through the priority queue) instead of the + faster Algorithm 2. The filled surface is identical; this exists for + testing. + + Returns + ------- + dict + ``{'visited': ndarray|None, 'cells_filled': int, 'max_fill_depth': float}`` + """ + if mode not in ("flat", "epsilon"): + raise ValueError("mode must be 'flat' or 'epsilon'") + + z = elevations + if not isinstance(z, np.ndarray) or z.dtype != np.float64 or z.ndim != 2: + raise TypeError("elevations must be a 2-D float64 ndarray (modified in place)") + if not z.flags['C_CONTIGUOUS']: + # reshape(-1) would return a *copy* of a non-contiguous array, so the + # fill would land in the copy and the caller's grid would come back + # untouched while the report claimed success. The compiled kernel + # rejects the same input, so both backends agree. + raise TypeError( + "elevations must be C-contiguous to be filled in place; " + "pass np.ascontiguousarray(elevations)") + ny, nx = z.shape + n = ny * nx + flat = z.reshape(-1) + + if closed is None: + closed_flat = np.zeros(n, dtype=bool) + else: + closed_flat = np.asarray(closed, dtype=bool).reshape(-1).copy() + if closed_flat.size != n: + raise ValueError("closed must have the same shape as elevations") + closed_flat |= np.isnan(flat) + + if seeds is None: + seed_idx = default_seeds(z) + keep = seed_idx[~closed_flat[seed_idx]] + if closed is not None and keep.size == 0: + keep = _mask_boundary_seeds(closed_flat.reshape(ny, nx)) + seed_idx = keep + else: + seed_idx = np.asarray(seeds, dtype=np.int64).reshape(-1) + + use_epsilon = mode == "epsilon" + limit_depth = max_pit_depth > 0.0 + # The FIFO shortcut is only sound when every cell it holds sits at exactly + # the current flood level. Two options break that invariant: an epsilon + # fill raises each cell by its own increment, and a depth limit re-queues + # unfilled cells *below* the current level. In either case everything goes + # through the priority queue, i.e. the run degrades to Algorithm 1. + use_pit = not use_epsilon and not limit_depth and not reference_algorithm + + visited = np.zeros(n, dtype=np.uint8) if track_visited else None + unfilled = np.zeros(n, dtype=np.uint8) if limit_depth else None + cells_filled = 0 + max_fill_depth = 0.0 + + # ``order`` makes the heap ordering total so ties resolve first-in + # first-out, exactly as the compiled kernel does. + order = 0 + open_heap = [] + for k in seed_idx: + k = int(k) + if k < 0 or k >= n or closed_flat[k]: + continue + closed_flat[k] = True + heapq.heappush(open_heap, (flat[k], order, k)) + order += 1 + + from collections import deque + + pit = deque() + + while open_heap or pit: + if pit and open_heap and open_heap[0][0] <= flat[pit[0]]: + cur_z, _, cur = heapq.heappop(open_heap) + elif pit: + cur = pit.popleft() + cur_z = flat[cur] + else: + cur_z, _, cur = heapq.heappop(open_heap) + + if visited is not None: + visited[cur] = 1 + + i, j = divmod(cur, nx) + for d in range(8): + ni = i + int(D8_DI[d]) + nj = j + int(D8_DJ[d]) + if ni < 0 or nj < 0 or ni >= ny or nj >= nx: + continue + nk = ni * nx + nj + if closed_flat[nk]: + continue + closed_flat[nk] = True + if np.isnan(flat[nk]): + continue + + if use_epsilon: + if epsilon > 0.0: + target = cur_z + epsilon * cellsize * D8_DIST[d] + if target <= cur_z: + target = np.nextafter(cur_z, np.inf) + else: + target = np.nextafter(cur_z, np.inf) + else: + target = cur_z + + if flat[nk] < target: + depth = float(target) - float(flat[nk]) + if limit_depth and depth > max_pit_depth: + unfilled[nk] = 1 + heapq.heappush(open_heap, (flat[nk], order, nk)) + order += 1 + continue + flat[nk] = target + cells_filled += 1 + if depth > max_fill_depth: + max_fill_depth = depth + if use_pit: + pit.append(nk) + else: + heapq.heappush(open_heap, (flat[nk], order, nk)) + order += 1 + else: + heapq.heappush(open_heap, (flat[nk], order, nk)) + order += 1 + + return { + "visited": None if visited is None else visited.reshape(ny, nx), + "unfilled": None if unfilled is None else unfilled.reshape(ny, nx), + "cells_filled": cells_filled, + "max_fill_depth": max_fill_depth, + } + + +def _mask_boundary_seeds(closed_2d): + """Seeds on the edge of the open region when the mask excludes the rim.""" + open_region = ~closed_2d + ny, nx = closed_2d.shape + boundary = np.zeros((ny, nx), dtype=bool) + boundary[0, :] = True + boundary[-1, :] = True + boundary[:, 0] = True + boundary[:, -1] = True + for di, dj in zip(D8_DI, D8_DJ): + boundary |= _shifted(closed_2d.astype(np.float64), int(di), int(dj), 0.0) > 0.5 + return np.flatnonzero((boundary & open_region).reshape(-1)).astype(np.int64) + + +# --------------------------------------------------------------------------- +# Flow directions +# --------------------------------------------------------------------------- + + +def flow_directions(elevations, cellsize_grid=None, cellsize=1.0): + """Steepest-descent D8 flow directions as ArcGIS codes. + + The drop to each neighbour is divided by the centre-to-centre distance, so + a diagonal neighbour only wins if its *gradient* is steeper. Ties go to + the first direction in the order E, SE, S, SW, W, NW, N, NE. + + Cells with no lower neighbour -- pits, and no-data cells -- get code 0. + """ + z = _as_2d_float(elevations, "elevations") + if cellsize_grid is None: + de = np.full(z.shape, float(cellsize)) + else: + de = _as_2d_float(cellsize_grid, "cellsize_grid") + if de.shape != z.shape: + raise ValueError("cellsize_grid must match the elevation grid") + + best_slope = np.zeros(z.shape) + codes = np.zeros(z.shape, dtype=np.uint8) + + for d in range(8): + neighbour = _shifted(z, int(D8_DI[d]), int(D8_DJ[d]), np.nan) + slope = (z - neighbour) / (de * D8_DIST[d]) + better = slope > best_slope + # NaN comparisons are False, so no-data neighbours never win. + best_slope = np.where(better, slope, best_slope) + codes = np.where(better, D8_CODES[d], codes).astype(np.uint8) + + codes[np.isnan(z)] = 0 + return codes + + +def receivers(codes): + """Flat index of the cell each cell drains to, or -1 if there is none.""" + codes = np.asarray(codes, dtype=np.uint8) + if codes.ndim != 2: + raise ValueError("codes must be a 2-D array") + ny, nx = codes.shape + ii, jj = np.mgrid[0:ny, 0:nx] + + out = np.full(codes.shape, -1, dtype=np.int64) + for d in range(8): + sel = codes == D8_CODES[d] + if not sel.any(): + continue + ni = ii + int(D8_DI[d]) + nj = jj + int(D8_DJ[d]) + inside = sel & (ni >= 0) & (nj >= 0) & (ni < ny) & (nj < nx) + out[inside] = (ni * nx + nj)[inside] + return out + + +# --------------------------------------------------------------------------- +# Topological ordering and accumulation +# --------------------------------------------------------------------------- + + +def _levels(recv_flat, n): + """Kahn's algorithm, one frontier at a time. + + Returns ``(chunks, n_on_cycles)`` where each chunk is an array of cell + indices that can be processed simultaneously: no cell in a chunk drains + into another cell of the same chunk. + """ + has_recv = (recv_flat >= 0) & (recv_flat < n) & (recv_flat != np.arange(n)) + indegree = np.bincount(recv_flat[has_recv], minlength=n) + + frontier = np.flatnonzero(indegree == 0) + chunks = [] + seen = 0 + while frontier.size: + chunks.append(frontier) + seen += frontier.size + moving = frontier[has_recv[frontier]] + if moving.size == 0: + break + targets = recv_flat[moving] + indegree -= np.bincount(targets, minlength=n) + candidates = np.unique(targets) + frontier = candidates[indegree[candidates] == 0] + + return chunks, n - seen + + +def topological_order(recv): + """Order the cells donors-first. + + Returns ``(order, n_cells_on_cycles)``. Cells that lie on a cycle -- only + possible in a hand-edited or externally supplied flow-direction grid -- + cannot be ordered and are appended at the end in index order. + """ + recv_flat = np.asarray(recv, dtype=np.int64).reshape(-1) + n = recv_flat.size + chunks, n_cycles = _levels(recv_flat, n) + if chunks: + order = np.concatenate(chunks) + else: + order = np.empty(0, dtype=np.int64) + if n_cycles: + emitted = np.zeros(n, dtype=bool) + emitted[order] = True + order = np.concatenate([order, np.flatnonzero(~emitted)]) + return order.astype(np.int64), int(n_cycles) + + +def accumulate(recv, order, weights, gate=None): + """Accumulate ``weights`` downstream along the D8 network. + + ``order`` is accepted for signature compatibility with the compiled + kernel; the result does not depend on which valid topological order is + used, so this implementation derives its own level structure -- which + lets it run vectorised rather than one cell at a time. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + shape = np.asarray(weights).shape + recv_flat = recv_arr.reshape(-1) + n = recv_flat.size + + acc = np.asarray(weights, dtype=np.float64).reshape(-1).copy() + if gate is not None: + gate_flat = np.asarray(gate).reshape(-1).astype(bool) + acc[~gate_flat] = 0.0 + else: + gate_flat = None + + chunks, _ = _levels(recv_flat, n) + for chunk in chunks: + if gate_flat is not None: + chunk = chunk[gate_flat[chunk]] + if chunk.size == 0: + continue + r = recv_flat[chunk] + valid = (r >= 0) & (r < n) & (r != chunk) + if not valid.any(): + continue + np.add.at(acc, r[valid], acc[chunk[valid]]) + + return acc.reshape(shape) + + +def flow_length(recv, order, step_length): + """Longest upstream flow distance, and the main-stem donor direction. + + Returns ``(length, from_codes)``. ``from_codes[k]`` is the direction code + pointing from ``k`` back at the donor that supplies its longest path. + Ties are settled in favour of the donor with the lower flat index, so the + main stem does not depend on which topological order was used. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + step = np.asarray(step_length, dtype=np.float64).reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + + length = np.zeros(n, dtype=np.float64) + from_index = np.full(n, -1, dtype=np.int64) + + for k in order: + k = int(k) + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + candidate = length[k] + step[k] + incumbent = int(from_index[r]) + if candidate > length[r] or ( + candidate == length[r] and (incumbent < 0 or k < incumbent) + ): + length[r] = candidate + from_index[r] = k + + code_for_offset = { + (int(D8_DI[d]), int(D8_DJ[d])): int(D8_CODES[d]) for d in range(8) + } + from_codes = np.zeros(n, dtype=np.uint8) + contributing = np.flatnonzero(from_index >= 0) + for r in contributing: + r = int(r) + k = int(from_index[r]) + offset = (k // nx - r // nx, k % nx - r % nx) + from_codes[r] = code_for_offset.get(offset, 0) + + return length.reshape(ny, nx), from_codes.reshape(ny, nx) + + +def propagate_along_main_stem(recv, order, main_stem_from_codes, gate, values, mode): + """Carry or sum ``values`` downstream, but only along longest-flow paths. + + ``mode='carry'`` overwrites the receiver with the donor's value (used by + :class:`~TopoAnalysis.dem.Relief`); ``mode='sum'`` adds it (used by + :class:`~TopoAnalysis.dem.Ksi`). + """ + if mode not in ("carry", "sum"): + raise ValueError("mode must be 'carry' or 'sum'") + + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + codes_flat = np.asarray(main_stem_from_codes, dtype=np.uint8).reshape(-1) + out = np.asarray(values, dtype=np.float64).reshape(-1).copy() + gate_flat = None if gate is None else np.asarray(gate).reshape(-1).astype(bool) + order = np.asarray(order, dtype=np.int64).reshape(-1) + + code_for_offset = { + (int(D8_DI[d]), int(D8_DJ[d])): int(D8_CODES[d]) for d in range(8) + } + + for k in order: + k = int(k) + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + offset = (k // nx - r // nx, k % nx - r % nx) + code = code_for_offset.get(offset, 0) + if code == 0 or codes_flat[r] != code: + continue + if gate_flat is not None and not gate_flat[k]: + continue + if mode == "sum": + out[r] += out[k] + else: + out[r] = out[k] + + return out.reshape(ny, nx) + + +# --------------------------------------------------------------------------- +# Traversal +# --------------------------------------------------------------------------- + + +def upstream_mask(recv, outlets): + """Mask of every cell draining to any of ``outlets`` (flat indices).""" + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + + valid = (recv_flat >= 0) & (recv_flat < n) & (recv_flat != np.arange(n)) + donor_of = np.flatnonzero(valid) + target_of = recv_flat[valid] + sort = np.argsort(target_of, kind="stable") + donors_sorted = donor_of[sort] + starts = np.searchsorted(target_of[sort], np.arange(n), side="left") + ends = np.searchsorted(target_of[sort], np.arange(n), side="right") + + mask = np.zeros(n, dtype=np.uint8) + stack = [] + for k in np.asarray(outlets, dtype=np.int64).reshape(-1): + k = int(k) + if k < 0 or k >= n or mask[k]: + continue + mask[k] = 1 + stack.append(k) + while stack: + k = stack.pop() + for donor in donors_sorted[starts[k]:ends[k]]: + donor = int(donor) + if mask[donor]: + continue + mask[donor] = 1 + stack.append(donor) + return mask.reshape(ny, nx) + + +def chi( + recv, + order, + area, + step_length, + outlets, + A0, + theta, + trapezoid=False, + max_length=0.0, + mask=None, +): + """Integrate the chi coordinate upstream from ``outlets``. + + ``chi(cell) = chi(receiver) + (A0 / A)^theta * step_length`` + + With ``trapezoid=True`` the integrand is averaged between the cell and its + receiver, which is the rule TopoToolbox's ``chitransform`` uses. Cells + outside the outlets' drainage areas stay ``NaN``. + + Returns ``(chi, distance_from_outlet)``. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + area_flat = np.asarray(area, dtype=np.float64).reshape(-1) + step = np.asarray(step_length, dtype=np.float64).reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + mask_flat = None if mask is None else np.asarray(mask).reshape(-1).astype(bool) + + chi_flat = np.full(n, np.nan) + dist = np.full(n, np.nan) + for k in np.asarray(outlets, dtype=np.int64).reshape(-1): + k = int(k) + if k < 0 or k >= n: + continue + if mask_flat is not None and not mask_flat[k]: + continue + chi_flat[k] = 0.0 + dist[k] = 0.0 + + # Receivers before donors: walk the topological order backwards. + for k in order[::-1]: + k = int(k) + if not np.isnan(chi_flat[k]): + continue + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k or np.isnan(chi_flat[r]): + continue + if mask_flat is not None and not mask_flat[k]: + continue + a = area_flat[k] + if not a > 0.0: + continue + dl = step[k] + if max_length > 0.0 and dist[r] + dl > max_length: + continue + integrand = (A0 / a) ** theta + if trapezoid: + ar = area_flat[r] + integrand = 0.5 * (integrand + ((A0 / ar) ** theta if ar > 0.0 else integrand)) + chi_flat[k] = chi_flat[r] + integrand * dl + dist[k] = dist[r] + dl + + return chi_flat.reshape(ny, nx), dist.reshape(ny, nx) + + +def downstream_distance(recv, order, step_length): + """Along-flow distance from each cell to the end of its flow path. + + Zero at outlets and at cells with no receiver, increasing upstream -- + the same quantity as TopoToolbox's ``flowdistance(FD, 'upstream')``. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + shape = recv_arr.shape + n = recv_arr.size + recv_flat = recv_arr.reshape(-1) + step = np.asarray(step_length, dtype=np.float64).reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + + distance = np.zeros(n, dtype=np.float64) + for k in order[::-1]: + k = int(k) + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + distance[k] = distance[r] + step[k] + return distance.reshape(shape) + + +def imposemin(recv, order, step_length, sl, elevations): + """Carve ``elevations`` in place so flow paths descend at gradient >= ``sl``. + + Only lowers cells, never raises them -- the complement of depression + filling, and what TopoToolbox's ``imposemin`` does. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + n = recv_arr.size + recv_flat = recv_arr.reshape(-1) + step = np.asarray(step_length, dtype=np.float64).reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + if not isinstance(elevations, np.ndarray) or not elevations.flags['C_CONTIGUOUS']: + # As in priority_flood: reshape(-1) copies a non-contiguous array, so + # the carving would be thrown away. + raise TypeError( + "elevations must be a C-contiguous ndarray to be carved in place; " + "pass np.ascontiguousarray(elevations)") + z = elevations.reshape(-1) + + for k in order: + k = int(k) + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + candidate = z[k] - sl * step[k] + if candidate < z[r]: + z[r] = candidate + + +def stream_order(recv, order, is_stream, kind="strahler"): + """Strahler or Shreve stream order over the cells marked in ``is_stream``.""" + recv_arr = np.asarray(recv, dtype=np.int64) + shape = recv_arr.shape + n = recv_arr.size + recv_flat = recv_arr.reshape(-1) + stream = np.asarray(is_stream).reshape(-1).astype(bool) + order = np.asarray(order, dtype=np.int64).reshape(-1) + + result = np.zeros(n, dtype=np.int32) + + if kind == "shreve": + has_donor = np.zeros(n, dtype=bool) + for k in range(n): + if not stream[k]: + continue + r = int(recv_flat[k]) + if 0 <= r < n and r != k and stream[r]: + has_donor[r] = True + result[stream & ~has_donor] = 1 + for k in order: + k = int(k) + if not stream[k]: + continue + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k or not stream[r]: + continue + result[r] += result[k] + return result.reshape(shape) + + # Each junction is resolved as a whole -- the largest incoming order and + # how many donors carry it -- rather than donor by donor. The + # incremental form depends on the sequence the donors arrive in, which + # would make the answer depend on which valid topological order was used. + best = np.zeros(n, dtype=np.int32) + ties = np.zeros(n, dtype=np.int32) + + for k in order: + k = int(k) + if not stream[k]: + continue + result[k] = best[k] + 1 if ties[k] >= 2 else max(int(best[k]), 1) + + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k or not stream[r]: + continue + if result[k] > best[r]: + best[r] = result[k] + ties[r] = 1 + elif result[k] == best[r]: + ties[r] += 1 + + return result.reshape(shape) + + +def drainage_basins(recv, order, outlets=None, valid=None): + """Label drainage basins. + + With ``outlets`` given, each cell is labelled with the 1-based position of + the outlet it drains to (0 if it reaches none). Without, every terminal + cell -- pit or edge outlet -- seeds its own basin, matching TopoToolbox's + ``drainagebasins(FD)``. + """ + recv_arr = np.asarray(recv, dtype=np.int64) + ny, nx = recv_arr.shape + n = ny * nx + recv_flat = recv_arr.reshape(-1) + order = np.asarray(order, dtype=np.int64).reshape(-1) + valid_flat = None if valid is None else np.asarray(valid).reshape(-1).astype(bool) + + labels = np.zeros(n, dtype=np.int32) + if outlets is None: + # Only a terminal cell that something actually drains into seeds a + # basin; an isolated cell stays 0, as in TopoToolbox. + has_donor = np.zeros(n, dtype=bool) + for k in range(n): + if valid_flat is not None and not valid_flat[k]: + continue + r = int(recv_flat[k]) + if 0 <= r < n and r != k: + has_donor[r] = True + next_label = 0 + for k in range(n): + if valid_flat is not None and not valid_flat[k]: + continue + r = recv_flat[k] + if (r < 0 or r >= n or r == k) and has_donor[k]: + next_label += 1 + labels[k] = next_label + else: + for position, k in enumerate(np.asarray(outlets, dtype=np.int64).reshape(-1), start=1): + k = int(k) + if 0 <= k < n and (valid_flat is None or valid_flat[k]): + labels[k] = position + + for k in order[::-1]: + k = int(k) + if labels[k] != 0: + continue + if valid_flat is not None and not valid_flat[k]: + continue + r = int(recv_flat[k]) + if r < 0 or r >= n or r == k: + continue + labels[k] = labels[r] + + return labels.reshape(ny, nx) diff --git a/gdal_setup.py b/gdal_setup.py new file mode 100644 index 0000000..5f403c2 --- /dev/null +++ b/gdal_setup.py @@ -0,0 +1,1561 @@ +"""Find, diagnose and install GDAL -- both the C library and the Python bindings. + +Why this module exists +---------------------- +GDAL is the one dependency ``pip install`` cannot supply by itself. There is +no GDAL wheel on PyPI: at *every* version, for *every* platform, the only +artifact published is an sdist whose ``setup.py`` shells out to +``gdal-config`` and compiles the SWIG bindings against a GDAL C library that +has to be on the machine already. So "install GDAL" is really two jobs that +must agree with each other: + + 1. the C library, ``libgdal``, which pip knows nothing about, and + 2. the Python bindings, ``osgeo``, whose version must match it. + +This module does both. It probes what is present, works out the cheapest +route to a working install, and takes it. In rough order of preference: + + ``bindings`` the C library is already here -- just build ``osgeo`` + against it. Seconds. + ``conda`` a conda environment is active; conda-forge ships both + halves, already matched. A minute or two. + ``wheel`` Windows only, where prebuilt wheels that bundle the C + library are the only practical option. + ``system`` apt/dnf/pacman/zypper/apk/brew supply the C library, then + fall through to ``bindings``. Needs administrator rights + except under Homebrew. + ``source`` download and build PROJ and GDAL from pinned, checksummed + tarballs into a prefix of our own, then fall through to + ``bindings``. Half an hour, but needs nothing but a + compiler and CMake -- no root, no package manager. + +Two traps this module exists to avoid +------------------------------------- +**The silent ``gdal_array`` trap.** GDAL sdists only grew a +``pyproject.toml`` in 3.9. Before that they declare no build dependencies at +all, so under pip's default build isolation numpy is absent when the bindings +compile -- and the numpy bridge, ``osgeo.gdal_array``, is quietly skipped. +Everything imports; ``Band.ReadAsArray()`` then fails at run time with an +unhelpful error. Ubuntu 22.04 ships GDAL 3.4 and 24.04 ships 3.8, so this is +the common case, not the exotic one. We install the bindings with +``--no-build-isolation`` against an interpreter that already has numpy, which +both fixes the trap and builds the bridge against the numpy the user will +actually run. + +**The architecture trap.** A Homebrew GDAL on Apple Silicon is arm64; a +Python installed under Rosetta is x86_64. The bindings cannot link the two, +and the error names a missing symbol rather than the real cause. The probe +checks for this before anything is built. + +Command line +------------ +:: + + topoanalysis-gdal # what is installed, and what is wrong + topoanalysis-gdal plan # what an install would do, step by step + topoanalysis-gdal install # do it +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import shutil +import subprocess +import sys + +try: # pragma: no cover - exercised by whichever import style is in use + from . import _gdal_source +except ImportError: # pragma: no cover + import _gdal_source + +__all__ = [ + "Plan", + "Probe", + "Step", + "activate_environment", + "apply_plan", + "ensure", + "main", + "make_plan", + "probe", +] + + +#: The oldest GDAL whose C API TopoAnalysis is known to work against. +MIN_LIBRARY_VERSION = (3, 0) + +#: GDAL sdists from this version on declare numpy in ``[build-system] +#: requires``. Below it, build isolation leaves numpy out and the +#: ``gdal_array`` bridge is silently dropped. See the module docstring. +NUMPY_BUILD_REQUIRES_FROM = (3, 9) + +#: Fallback when the GitHub API cannot be reached; see :func:`_windows_find_links`. +CGOHLKE_FALLBACK_TAG = "v2026.8.20" + + +# --------------------------------------------------------------------------- +# Probing +# --------------------------------------------------------------------------- + +# Run in a subprocess, never in-process. A bindings/library version mismatch +# or an architecture mismatch can abort the interpreter outright rather than +# raising ImportError, and a diagnostic tool that dies while diagnosing is +# worse than useless. +_PROBE_SCRIPT = r""" +import json, os, platform, sys +out = {"executable": sys.executable, "python": sys.version.split()[0]} +# The *target's* architecture, not ours. An x86_64 Python under Rosetta on +# an arm64 machine is the whole reason the architecture check exists, and +# asking the wrong process gives exactly the wrong answer. +out["machine"] = platform.machine() +# Asked here rather than inferred from the parent process: --python can point +# at a different interpreter, and it is *its* environment we install into. +out["sys_prefix"] = sys.prefix +out["base_prefix"] = getattr(sys, "base_prefix", sys.prefix) + +# The same data-file fixup TopoAnalysis applies at import time, repeated here +# rather than imported: the interpreter being probed very often does not have +# TopoAnalysis installed yet. Without it the probe would report a broken +# GDAL that works perfectly well in use. +def _activate(): + if os.environ.get("TOPOANALYSIS_NO_GDAL_ENV"): + return + root = os.environ.get("XDG_CACHE_HOME") or os.path.join( + os.path.expanduser("~"), ".cache") + for path in (os.path.join(sys.prefix, "share", "topoanalysis", "gdal-prefix.json"), + os.path.join(root, "TopoAnalysis", "gdal-prefix.json")): + try: + with open(path) as handle: + manifest = json.load(handle) + except Exception: + continue + recorded = manifest.get("python_prefix") + if recorded is not None and recorded != sys.prefix: + return + proj_lib = manifest.get("proj_lib") + if proj_lib and os.path.isfile(os.path.join(proj_lib, "proj.db")): + os.environ["PROJ_LIB"] = proj_lib + os.environ["PROJ_DATA"] = proj_lib + gdal_data = manifest.get("gdal_data") + if gdal_data and os.path.isdir(gdal_data): + os.environ["GDAL_DATA"] = gdal_data + return + +try: + _activate() +except Exception: + pass +try: + import numpy + out["numpy"] = numpy.__version__ +except Exception: + out["numpy"] = None +try: + from osgeo import gdal + out["bindings_version"] = gdal.__version__ + out["osgeo_file"] = getattr(gdal, "__file__", None) +except Exception as exc: + out["bindings_error"] = "%s: %s" % (type(exc).__name__, exc) +try: + from osgeo import gdal_array # noqa: F401 + out["has_gdal_array"] = True +except Exception as exc: + out["has_gdal_array"] = False + out["gdal_array_error"] = "%s: %s" % (type(exc).__name__, exc) +out["proj_env"] = { + name: os.environ.get(name) + for name in ("PROJ_LIB", "PROJ_DATA", "GDAL_DATA") + if os.environ.get(name) +} +try: + from osgeo import osr + try: + out["proj_search_paths"] = list(osr.GetPROJSearchPaths() or []) + except Exception: + pass + # Without this a failed lookup returns an error code rather than raising, + # and the diagnosis loses the one message that explains it -- PROJ names + # the offending proj.db and why it was rejected. + osr.UseExceptions() + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + out["projection_works"] = bool(srs.ExportToWkt()) + if not out["projection_works"]: + out["projection_error"] = "EPSG:4326 produced an empty definition" +except Exception as exc: + out["projection_works"] = False + out["projection_error"] = "%s: %s" % (type(exc).__name__, exc) +sys.stdout.write(json.dumps(out)) +""" + + +class Probe(object): + """A snapshot of the GDAL situation on one interpreter.""" + + def __init__(self, **fields): + self.python = fields.get("python_executable") or sys.executable + self.python_version = fields.get("python") + self.numpy_version = fields.get("numpy") + self.bindings_version = fields.get("bindings_version") + self.bindings_error = fields.get("bindings_error") + self.osgeo_file = fields.get("osgeo_file") + self.has_gdal_array = bool(fields.get("has_gdal_array")) + self.gdal_array_error = fields.get("gdal_array_error") + self.projection_works = bool(fields.get("projection_works")) + self.projection_error = fields.get("projection_error") + self.library_version = fields.get("library_version") + self.gdal_config = fields.get("gdal_config") + self.prefix = fields.get("prefix") + self.crashed = bool(fields.get("crashed")) + self.problems = tuple(fields.get("problems", ())) + self.sys_prefix = fields.get("sys_prefix") + self.base_prefix = fields.get("base_prefix") + self.machine = fields.get("machine") or platform.machine() + self.proj_env = dict(fields.get("proj_env") or {}) + self.proj_search_paths = tuple(fields.get("proj_search_paths") or ()) + + # -- derived ----------------------------------------------------------- + + @property + def is_virtualenv(self): + return bool(self.sys_prefix) and self.sys_prefix != self.base_prefix + + @property + def is_conda(self): + return bool(self.sys_prefix) and os.path.isdir( + os.path.join(self.sys_prefix, "conda-meta") + ) + + @property + def target_prefix(self): + """The environment an install should go into. + + A virtualenv or conda environment is self-contained, so a C library + built into it is removed with it and its ``bin`` is already on PATH + when the environment is active -- which is exactly what building the + bindings needs. Anything else falls back to a per-user directory + rather than writing into a system prefix. + """ + if self.is_virtualenv or self.is_conda: + return self.sys_prefix + return default_source_prefix() + + @property + def ok(self): + """True when nothing needs doing.""" + return not self.problems + + @property + def codes(self): + return tuple(code for code, _ in self.problems) + + def has(self, code): + return code in self.codes + + def describe(self): + """A human-readable report, as a list of lines.""" + lines = [ + "python : %s (%s)" % (self.python_version or "?", self.python), + "numpy : %s" % (self.numpy_version or "not installed"), + "GDAL bindings : %s" % (self.bindings_version or "not installed"), + "GDAL C library : %s" % (self.library_version or "not found"), + ] + if self.gdal_config: + lines.append("gdal-config : %s" % self.gdal_config) + lines.append( + "numpy bridge : %s" + % ("present" if self.has_gdal_array else "MISSING (ReadAsArray will fail)") + ) + lines.append( + "projection lookup : %s" % ("works" if self.projection_works else "FAILS") + ) + if self.problems: + lines.append("") + lines.append("problems:") + for code, message in self.problems: + lines.append(" [%s] %s" % (code, message)) + else: + lines.append("") + lines.append("GDAL is installed and working.") + return lines + + +def probe(python=None): + """Inspect ``python`` (default: this interpreter) for a working GDAL.""" + python = python or sys.executable + fields = {"python_executable": python} + + result = subprocess.run( + [python, "-c", _PROBE_SCRIPT], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + fields.update(json.loads(result.stdout.decode("utf8", "replace"))) + except ValueError: + # The probe did not even get to print: importing osgeo took the + # interpreter down with it. That is itself the finding. + fields["crashed"] = True + fields["bindings_error"] = ( + "the interpreter died while importing osgeo (exit %s). %s" + % (result.returncode, result.stderr.decode("utf8", "replace").strip()[:400]) + ) + + config = _gdal_config_info(sys_prefix=fields.get("sys_prefix")) + if config: + fields["gdal_config"] = config["path"] + fields["library_version"] = config["version"] + fields["prefix"] = config["prefix"] + + found = Probe(**fields) + found.problems = tuple(_diagnose(found, config)) + return found + + +def _diagnose(found, config): + """Turn a raw probe into a list of ``(code, message)`` problems.""" + problems = [] + + if found.bindings_version is None: + problems.append( + ( + "no-bindings", + "the osgeo Python bindings are not installed (%s)" + % (found.bindings_error or "import failed"), + ) + ) + + if config is None: + # Not a problem in itself when the bindings work -- a wheel or conda + # package can carry the library without shipping gdal-config -- but + # it is what blocks building bindings from an sdist. + if found.bindings_version is None: + problems.append( + ( + "no-library", + "no GDAL C library found: gdal-config is not on PATH", + ) + ) + else: + library = _parse_version(config["version"]) + if library and library[:2] < MIN_LIBRARY_VERSION: + problems.append( + ( + "old-library", + "GDAL %s is older than the %d.%d TopoAnalysis needs" + % (config["version"], MIN_LIBRARY_VERSION[0], MIN_LIBRARY_VERSION[1]), + ) + ) + + bindings = _parse_version(found.bindings_version) + if bindings and library and bindings[:2] != library[:2]: + problems.append( + ( + "version-mismatch", + "the bindings are %s but the C library is %s; they must " + "agree to at least major.minor" + % (found.bindings_version, config["version"]), + ) + ) + + mismatch = _architecture_mismatch(config["prefix"], found.machine) + if mismatch: + problems.append(("arch-mismatch", mismatch)) + + if found.bindings_version is not None and not found.has_gdal_array: + problems.append( + ( + "no-gdal-array", + "osgeo.gdal_array is missing, so Band.ReadAsArray() will fail " + "(%s). The bindings were built without numpy present." + % (found.gdal_array_error or "import failed"), + ) + ) + + if found.bindings_version is not None and not found.projection_works: + message = "looking up EPSG:4326 failed (%s)" % ( + found.projection_error or "unknown error" + ) + if found.proj_env: + message += ". These are set in the environment and override the " + message += "paths compiled into the library: " + ", ".join( + "%s=%s" % item for item in sorted(found.proj_env.items()) + ) + if found.proj_search_paths: + message += ". PROJ searched: " + ", ".join(found.proj_search_paths) + problems.append(("no-proj-data", message)) + + return problems + + +def _manifest_for(python_prefix): + """The GDAL manifest recorded inside ``python_prefix``, or None.""" + try: + with open(manifest_path(python_prefix)) as handle: + return json.load(handle) + except (OSError, ValueError): + return None + + +def _config_search_order(sys_prefix=None): + """Where to look for ``gdal-config``, most specific first. + + PATH is the *last* resort, not the first. ``--python`` can name an + interpreter in a different environment, and what matters is the library + that interpreter will load -- not whichever ``gdal-config`` happens to be + on the PATH of the shell this command was typed into. Getting that wrong + produces a confident and completely wrong version-mismatch diagnosis. + """ + candidates = [] + explicit = os.environ.get("GDAL_CONFIG") + if explicit: + candidates.append(explicit) + if sys_prefix: + manifest = _manifest_for(sys_prefix) + if manifest and manifest.get("prefix"): + candidates.append( + os.path.join(manifest["prefix"], "bin", "gdal-config") + ) + candidates.append(os.path.join(sys_prefix, "bin", "gdal-config")) + candidates.append("gdal-config") + return candidates + + +def _gdal_config_info(gdal_config=None, sys_prefix=None): + """Ask ``gdal-config`` about the installed C library, or return None.""" + if gdal_config: + candidates = [gdal_config] + else: + candidates = _config_search_order(sys_prefix) + + for program in candidates: + path = program if os.sep in program else shutil.which(program) + if not path or not os.path.exists(path): + continue + info = _interrogate_gdal_config(path) + if info is not None: + return info + return None + + +def _interrogate_gdal_config(path): + info = {"path": path} + for option in ("version", "prefix", "cflags", "libs"): + try: + result = subprocess.run( + [path, "--" + option], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + except OSError: + return None + if result.returncode != 0: + return None + info[option] = result.stdout.decode("utf8", "replace").strip() + return info if info.get("version") else None + + +def _parse_version(text): + """``'3.9.3'`` -> ``(3, 9, 3)``; None when it does not look like one.""" + if not text: + return None + match = re.match(r"(\d+)\.(\d+)(?:\.(\d+))?", str(text)) + if not match: + return None + return tuple(int(part) for part in match.groups(default="0")) + + +def _architecture_mismatch(prefix, machine=None): + """Describe an architecture clash between libgdal and the target Python. + + ``machine`` is the target interpreter's ``platform.machine()``, which is + not necessarily ours. Returns None when there is no clash, or when we + cannot tell -- an unknown answer must not block an install that would + have worked. + """ + if sys.platform not in ("darwin", "linux"): + return None + library = _find_library(prefix) + if library is None: + return None + + tool = shutil.which("file") + if tool is None: + return None + try: + result = subprocess.run( + [tool, "-L", library], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + except OSError: + return None + description = result.stdout.decode("utf8", "replace").lower() + + machine = (machine or platform.machine()).lower() + aliases = { + "x86_64": ("x86_64", "x86-64", "amd64"), + "amd64": ("x86_64", "x86-64", "amd64"), + "arm64": ("arm64", "aarch64"), + "aarch64": ("arm64", "aarch64"), + }.get(machine) + if aliases is None: + return None + if any(alias in description for alias in aliases): + return None + + # The library is a real binary of some architecture, just not ours. + for name, names in (("x86_64", ("x86_64", "x86-64")), ("arm64", ("arm64", "aarch64"))): + if any(alias in description for alias in names): + return ( + "the GDAL C library at %s is %s but this Python is %s; the " + "bindings cannot link against it. Install a GDAL built for " + "%s, or use a Python that matches the library." + % (library, name, machine, machine) + ) + return None + + +def _find_library(prefix): + if not prefix: + return None + suffixes = (".dylib",) if sys.platform == "darwin" else (".so",) + for libdir in ("lib", "lib64"): + directory = os.path.join(prefix, libdir) + if not os.path.isdir(directory): + continue + for entry in sorted(os.listdir(directory)): + if entry.startswith("libgdal") and any( + suffix in entry for suffix in suffixes + ): + return os.path.join(directory, entry) + return None + + +# --------------------------------------------------------------------------- +# Environments we might install into +# --------------------------------------------------------------------------- + + +def _conda_prefix(): + """The active conda environment's prefix, or None. + + ``conda-meta`` is the reliable marker: ``CONDA_PREFIX`` can be left over + from a shell that has since switched interpreters. + """ + if os.path.isdir(os.path.join(sys.prefix, "conda-meta")): + return sys.prefix + return None + + +def _conda_executable(): + """A conda-family program that can install into the current environment. + + ``CONDA_EXE`` comes first because ``conda`` itself is frequently only a + shell function, invisible to ``shutil.which``. mamba is preferred when + present purely because it is very much faster. + """ + for name in ("mamba", "micromamba"): + found = shutil.which(name) + if found: + return found + from_env = os.environ.get("CONDA_EXE") + if from_env and os.path.exists(from_env): + return from_env + return shutil.which("conda") + + +def default_source_prefix(): + """Per-user fallback prefix, used when there is no environment to use. + + :attr:`Probe.target_prefix` prefers the interpreter's own virtualenv or + conda environment; this is where it lands when there is neither. + """ + root = os.environ.get("XDG_DATA_HOME") or os.path.join( + os.path.expanduser("~"), ".local", "share" + ) + return os.path.join(root, "TopoAnalysis", "gdal") + + +#: System package managers, in the order we look for them. Each entry is +#: (program, install arguments, whether it needs administrator rights). +_SYSTEM_MANAGERS = ( + ("apt-get", ["install", "-y", "gdal-bin", "libgdal-dev"], True), + ("dnf", ["install", "-y", "gdal", "gdal-devel"], True), + ("yum", ["install", "-y", "gdal", "gdal-devel"], True), + ("zypper", ["install", "-y", "gdal", "libgdal-devel"], True), + ("pacman", ["-S", "--noconfirm", "gdal"], True), + ("apk", ["add", "--no-cache", "gdal", "gdal-dev"], True), + # Homebrew refuses to run as root, and does not need to. + ("brew", ["install", "gdal"], False), +) + + +def _system_manager(): + for program, arguments, needs_root in _SYSTEM_MANAGERS: + found = shutil.which(program) + if found: + return found, arguments, needs_root + return None + + +def _is_root(): + return hasattr(os, "geteuid") and os.geteuid() == 0 + + +# --------------------------------------------------------------------------- +# Plans +# --------------------------------------------------------------------------- + + +class Step(object): + """One thing an install does, describable without doing it.""" + + def __init__( + self, + summary, + command=None, + action=None, + env=None, + fallbacks=(), + needs_root=False, + minutes=1, + ): + self.summary = summary + self.command = command + self.action = action + self.env = env or {} + self.fallbacks = tuple(fallbacks) + self.needs_root = needs_root + self.minutes = minutes + + def __repr__(self): # pragma: no cover - debugging aid + return "" % self.summary + + +class Plan(object): + """An ordered list of :class:`Step` plus why it was chosen. + + ``blockers`` are reasons the plan cannot run at all. They are kept + separate from the steps so that ``plan`` can still show what *would* + happen -- which is the most useful thing to print when the answer is + "install these two tools first". + """ + + def __init__(self, strategy, steps=(), notes=(), blockers=()): + self.strategy = strategy + self.steps = tuple(steps) + self.notes = tuple(notes) + self.blockers = tuple(blockers) + + @property + def minutes(self): + return sum(step.minutes for step in self.steps) + + @property + def needs_root(self): + return any(step.needs_root for step in self.steps) + + def describe(self): + lines = ["strategy: %s" % self.strategy] + for note in self.notes: + lines.append(" note: %s" % note) + for blocker in self.blockers: + lines.append(" CANNOT RUN: %s" % blocker) + if not self.steps: + lines.append(" (nothing to do)") + return lines + lines.append("") + for number, step in enumerate(self.steps, 1): + lines.append(" %d. %s" % (number, step.summary)) + if step.command: + lines.append(" $ %s" % " ".join(step.command)) + lines.append("") + lines.append(" estimated time: about %d minute(s)" % max(1, self.minutes)) + return lines + + +def make_plan( + found=None, + strategy="auto", + python=None, + prefix=None, + allow_root=False, + jobs=None, +): + """Decide how to get a working GDAL, without doing anything yet. + + Parameters + ---------- + found : Probe, optional + A previous probe; taken fresh when omitted. + strategy : str + ``'auto'``, or one of ``'bindings'``, ``'conda'``, ``'wheel'``, + ``'system'``, ``'source'`` to force a route. + python : str, optional + Interpreter to install for; defaults to the current one. + prefix : str, optional + Where a source build should install; see + :func:`default_source_prefix`. + allow_root : bool + Permit steps that need administrator rights. Without it they are + still planned, so the user can see the command, but refused at + execution time. + jobs : int, optional + Parallel compile jobs for a source build. + """ + python = python or sys.executable + found = found if found is not None else probe(python) + notes = [] + + if strategy == "auto": + strategy, auto_notes = _choose_strategy(found, allow_root=allow_root) + notes.extend(auto_notes) + + if strategy == "none": + return Plan("none", (), ("GDAL is already installed and working.",)) + + if strategy == "proj-data": + return Plan( + "proj-data", + (), + ( + "GDAL and its bindings are fine; PROJ cannot read its " + "database. Nothing to install -- this is an environment " + "problem.", + ), + _proj_data_advice(found), + ) + + builder = { + "bindings": _plan_bindings, + "conda": _plan_conda, + "wheel": _plan_wheel, + "system": _plan_system, + "source": _plan_source, + }.get(strategy) + if builder is None: + raise ValueError("unknown strategy %r" % strategy) + + produced = builder( + found, python=python, prefix=prefix, allow_root=allow_root, jobs=jobs + ) + # Builders return (steps, notes), or (steps, notes, blockers) when they + # can tell in advance that this route cannot work on this machine. + steps, more_notes = produced[0], produced[1] + blockers = produced[2] if len(produced) > 2 else () + return Plan(strategy, steps, notes + list(more_notes), blockers) + + +def _proj_data_advice(found): + """What to actually do about a PROJ database that will not load.""" + advice = [] + if found is not None and found.proj_env: + advice.append( + "%s set in the environment, overriding the paths compiled into " + "the library. Activating a conda environment sets these, and they " + "then follow you into every other GDAL on the machine. Unset them " + "and try again: unset %s" + % ( + " and ".join(sorted(found.proj_env)), + " ".join(sorted(found.proj_env)), + ) + ) + else: + advice.append( + "PROJ's data files are missing from the installation itself. " + "Reinstall PROJ -- `conda install -c conda-forge proj` in a conda " + "environment, the system's proj-data package otherwise -- or " + "build GDAL with `topoanalysis-gdal install --strategy source`, " + "which installs a matching PROJ alongside it." + ) + return advice + + +def _choose_strategy(found, allow_root=False): + """Pick the cheapest route that stands a chance of working. + + ``allow_root`` matters here and not only at execution time: asking for it + is what makes the system package manager a usable route at all, and + without it a machine with apt would silently fall through to a half-hour + source build. + """ + notes = [] + + if found.ok: + return "none", notes + + # A pure numpy-bridge or PROJ-data failure is repaired by rebuilding the + # bindings against the library that is already here; no need to touch the + # C library at all. + # A PROJ data problem on its own is not fixed by installing anything: + # the library and bindings are fine, and something in the environment is + # pointing them at the wrong database. Reinstalling GDAL would leave + # that variable exactly where it was. + if set(found.codes) == {"no-proj-data"}: + return "proj-data", notes + + # ``no-proj-data`` *alongside* other problems is different: it usually + # just follows from bindings that are broken or mismatched, and goes away + # when they are rebuilt. + repairable = {"no-bindings", "no-gdal-array", "version-mismatch", "no-proj-data"} + wrong_architecture = found.has("arch-mismatch") + if wrong_architecture: + # Nothing installed on top of this library will fix it; say so plainly. + notes.append( + "the C library's architecture does not match this interpreter, so " + "the existing library cannot be reused" + ) + elif found.gdal_config and not found.has("old-library"): + if set(found.codes) <= repairable: + return "bindings", notes + + if found.is_conda and _conda_executable(): + notes.append("a conda environment is active, which is the most reliable route") + return "conda", notes + + if sys.platform == "win32": + notes.append( + "on Windows the only practical source of a GDAL C library is a " + "prebuilt wheel or conda" + ) + return "wheel", notes + + manager = _system_manager() + if manager is not None and found.has("old-library"): + # The too-old library almost certainly came from this very package + # manager -- a distro pinned to GDAL 2.x -- so asking it again just + # reinstalls the same version, and the bindings step would then + # faithfully build bindings for a library we have already rejected. + notes.append( + "%s is available but is where the too-old library came from; " + "a newer GDAL has to come from somewhere else" + % os.path.basename(manager[0]) + ) + elif manager is not None and wrong_architecture: + # The package manager is very often what produced the mismatch in the + # first place -- a Homebrew arm64 GDAL under a Rosetta x86_64 Python -- + # so asking it again would install the same wrong thing and loop. + notes.append( + "%s is available but would supply the same architecture again; " + "building from source is what actually resolves this" + % os.path.basename(manager[0]) + ) + elif manager is not None: + program, _, needs_root = manager + if not needs_root or _is_root() or allow_root: + return "system", notes + notes.append( + "%s could install the C library but needs administrator rights; " + "pass --allow-root to use it, or fall back to a source build" + % os.path.basename(program) + ) + + notes.append( + "no package manager can be used without help, so GDAL will be built " + "from source -- this needs only a compiler and CMake" + ) + return "source", notes + + +def _pip(python, *arguments): + return [python, "-m", "pip", "install"] + list(arguments) + + +def _bindings_requirements(library_version): + """Version specifiers to try for the bindings, best first. + + The bindings and the C library have to agree. An exact match is ideal; + failing that any patch release of the same major.minor will do, because + GDAL does not break its C API within a minor version. + """ + parsed = _parse_version(library_version) + if not parsed: + return ["gdal"] + major, minor, patch = parsed + return [ + "gdal==%d.%d.%d" % (major, minor, patch), + "gdal==%d.%d.*" % (major, minor), + ] + + +def _plan_bindings( + found, python, prefix=None, allow_root=False, jobs=None, config=None, force=None +): + """Build ``osgeo`` against a C library that is already installed. + + ``force`` re-installs even when the requested version is already + present. It has to default to on whenever we are *repairing* bindings, + because the commonest repair -- a missing ``gdal_array`` -- leaves an + ``osgeo`` of exactly the right version in place. Plain ``pip install + gdal==X.Y.Z`` would report "already satisfied" and change nothing, and + even with a reinstall pip would happily serve the broken wheel back out + of its own cache, since that wheel's name and version are correct. Only + ``--force-reinstall --no-cache-dir`` actually rebuilds. + """ + config = config or _gdal_config_info( + sys_prefix=found.sys_prefix if found is not None else None + ) + steps = [] + notes = [] + if config is None: + raise ValueError( + "the 'bindings' strategy needs a GDAL C library, but gdal-config " + "is not on PATH" + ) + + version = config["version"] + env = {} + # Make sure the gdal-config we chose is the one the sdist's setup.py + # finds, whatever else is on PATH. + env["PATH"] = os.path.dirname(config["path"]) + os.pathsep + os.environ.get("PATH", "") + libdir = _library_directory(config) + if libdir: + # Bake the library location into the extension so importing osgeo + # works without the user setting LD_LIBRARY_PATH first. + env["LDFLAGS"] = ( + "-L%s -Wl,-rpath,%s %s" % (libdir, libdir, os.environ.get("LDFLAGS", "")) + ).strip() + + steps.append( + Step( + "make sure the build tools and numpy are present", + command=_pip(python, "--upgrade", "setuptools", "wheel", "numpy"), + minutes=1, + ) + ) + + parsed = _parse_version(version) + if parsed and parsed[:2] < NUMPY_BUILD_REQUIRES_FROM: + notes.append( + "GDAL %s predates the sdist declaring numpy as a build dependency, " + "so the bindings are installed without build isolation -- otherwise " + "osgeo.gdal_array would be silently left out and ReadAsArray() " + "would fail at run time" % version + ) + + if force is None: + force = found is not None and found.bindings_version is not None + extra = ["--no-build-isolation"] + if force: + # --no-deps because step 1 has already put numpy in place, and + # --force-reinstall would otherwise tear down and rebuild the user's + # numpy as a side effect of repairing GDAL. + extra += ["--force-reinstall", "--no-cache-dir", "--no-deps"] + notes.append( + "the bindings are being rebuilt rather than merely requested: " + "this version pin can be satisfied by an install already present, " + "or by a wheel in pip's cache, that was built without numpy or " + "against a different C library" + ) + + requirements = _bindings_requirements(version) + steps.append( + Step( + "build the GDAL %s Python bindings against the installed C library" + % version, + command=_pip(python, *(extra + [requirements[0]])), + fallbacks=[ + _pip(python, *(extra + [requirement])) + for requirement in requirements[1:] + ], + env=env, + minutes=3, + ) + ) + return steps, notes + + +def _library_directory(config): + """Where libgdal actually lives, according to gdal-config. + + Taken from ``--libs`` rather than assumed to be ``/lib``: on + Fedora, RHEL and openSUSE a 64-bit library lands in ``lib64``, and an + RPATH pointing at an empty ``lib`` would leave the bindings unable to + find the library they were just built against. + """ + for token in (config.get("libs") or "").split(): + if token.startswith("-L"): + candidate = token[2:] + if candidate and os.path.isdir(candidate): + return candidate + # No usable -L: find the library itself rather than guess between lib + # and lib64, both of which commonly exist. + library = _find_library(config.get("prefix")) + return os.path.dirname(library) if library else None + + +def _plan_conda(found, python, prefix=None, allow_root=False, jobs=None): + executable = _conda_executable() + if executable is None: + raise ValueError("no conda, mamba or micromamba found") + target = (found.sys_prefix if found is not None else None) or _conda_prefix() or sys.prefix + command = [ + executable, + "install", + "-y", + "-p", + target, + "-c", + "conda-forge", + "gdal>=%d.%d" % MIN_LIBRARY_VERSION, + ] + notes = [ + "conda-forge ships the C library and the bindings as one package, " + "already matched, which removes both of the traps this module exists " + "to avoid" + ] + return [ + Step( + "install GDAL from conda-forge into %s" % target, + command=command, + minutes=3, + ) + ], notes + + +def _plan_wheel(found, python, prefix=None, allow_root=False, jobs=None): + """Windows: install a wheel that bundles the C library.""" + find_links = _windows_find_links() + notes = [ + "these wheels bundle the GDAL C library, so nothing else has to be " + "installed; they are built by Christoph Gohlke, who has maintained " + "the de-facto Windows builds of the scientific stack for years" + ] + return [ + Step( + "install prebuilt GDAL wheels for Windows", + command=_pip( + python, "--only-binary", ":all:", "--find-links", find_links, "gdal" + ), + minutes=2, + ) + ], notes + + +def _windows_find_links(): + """The newest cgohlke/geospatial-wheels release page, or a pinned one.""" + try: + import urllib.request + + with urllib.request.urlopen( + "https://api.github.com/repos/cgohlke/geospatial-wheels/releases/latest", + timeout=15, + ) as response: + tag = json.load(response)["tag_name"] + except Exception: + tag = CGOHLKE_FALLBACK_TAG + return ( + "https://github.com/cgohlke/geospatial-wheels/releases/expanded_assets/%s" % tag + ) + + +def _plan_system(found, python, prefix=None, allow_root=False, jobs=None): + manager = _system_manager() + if manager is None: + raise ValueError("no supported system package manager found") + program, arguments, needs_root = manager + + elevate = needs_root and not _is_root() + + def _command(arguments): + return (["sudo"] if elevate else []) + [program] + arguments + + steps = [] + notes = [] + if os.path.basename(program) == "apt-get": + # On a container image whose package lists were stripped -- which is + # every official Debian and Ubuntu image -- `apt-get install` fails + # outright with "Unable to locate package" until the lists are + # refreshed. + steps.append( + Step( + "refresh the apt package lists", + command=_command(["update"]), + needs_root=elevate, + minutes=1, + ) + ) + + steps.append( + Step( + "install the GDAL C library with %s" % os.path.basename(program), + command=_command(arguments), + needs_root=elevate, + minutes=3, + ) + ) + + # The bindings step cannot be planned in detail yet -- gdal-config does + # not exist until the step above has run -- so it is deferred and built + # when we get there. + steps.append( + Step( + "build the Python bindings against the newly installed C library", + action=_DeferredBindings(python), + minutes=3, + ) + ) + return steps, notes + + +def _plan_source(found, python, prefix=None, allow_root=False, jobs=None): + prefix = prefix or ( + found.target_prefix if found is not None else default_source_prefix() + ) + # So the plan quotes the versions the build will actually fetch. + _gdal_source.apply_environment_overrides() + missing = _gdal_source.missing_build_tools() + notes = [ + "GDAL will be built from pinned, checksummed tarballs and installed " + "into %s; nothing outside that prefix is modified" % prefix + ] + blockers = [] + if missing: + blockers.append( + "a source build needs %s, which %s not installed. %s" + % ( + " and ".join(missing), + "are" if len(missing) > 1 else "is", + _build_tool_hint(), + ) + ) + + steps = [] + for component in _gdal_source.components_needed(): + spec = _gdal_source.SOURCES[component] + steps.append( + Step( + "download and build %s %s" % (component, spec["version"]), + action=_SourceBuild(component, prefix, jobs), + minutes=spec["minutes"], + ) + ) + if "sqlite" not in _gdal_source.components_needed(): + notes.append("the system SQLite is usable, so it will not be rebuilt") + + steps.append( + Step( + "record the prefix so TopoAnalysis can find GDAL's data files", + action=_RecordPrefix( + prefix, found.sys_prefix if found is not None else None + ), + minutes=1, + ) + ) + steps.append( + Step( + "build the Python bindings against the GDAL we just built", + action=_DeferredBindings(python, prefix=prefix), + minutes=3, + ) + ) + return steps, notes, blockers + + +def _build_tool_hint(): + """How to get a compiler and CMake on this platform.""" + if sys.platform == "darwin": + return "Install them with: xcode-select --install && brew install cmake" + if sys.platform == "win32": + return ( + "Install the Visual Studio C++ build tools and CMake, or use " + "--strategy conda instead." + ) + manager = _system_manager() + if manager and os.path.basename(manager[0]) == "apt-get": + return "Install them with: sudo apt-get install build-essential cmake" + if manager and os.path.basename(manager[0]) in ("dnf", "yum"): + return "Install them with: sudo dnf install gcc-c++ make cmake" + return "Install a C/C++ compiler and CMake, or use --strategy conda." + + +class _SourceBuild(object): + """Deferred call into :mod:`TopoAnalysis._gdal_source`.""" + + def __init__(self, component, prefix, jobs): + self.component = component + self.prefix = prefix + self.jobs = jobs + + def __call__(self, log): + _gdal_source.build_one(self.component, self.prefix, jobs=self.jobs, log=log) + + +class _RecordPrefix(object): + """Note which environment's ``osgeo`` belongs to this GDAL prefix.""" + + def __init__(self, prefix, python_prefix=None): + self.prefix = prefix + self.python_prefix = python_prefix + + def __call__(self, log): + path = write_manifest(self.prefix, self.python_prefix) + log(" recorded %s" % path) + + +class _DeferredBindings(object): + """Plan and run the bindings step once the C library actually exists. + + ``system`` and ``source`` both install a C library as an earlier step, so + ``gdal-config`` cannot be interrogated when the plan is drawn up. + """ + + def __init__(self, python, prefix=None): + self.python = python + self.prefix = prefix + + def __call__(self, log): + search = None + if self.prefix: + search = os.path.join(self.prefix, "bin", "gdal-config") + if not os.path.exists(search): + raise RuntimeError( + "expected gdal-config at %s after the build, but it is not " + "there" % search + ) + config = _gdal_config_info(search) + if config is None: + raise RuntimeError( + "the C library was installed but gdal-config is still not on " + "PATH, so the bindings cannot be built" + ) + # The C library changed underneath whatever bindings may already be + # installed, so they must be rebuilt even if their version still + # nominally matches. + steps, notes = _plan_bindings(None, self.python, config=config, force=True) + for note in notes: + log(" note: %s" % note) + for step in steps: + log(" %s" % step.summary) + _run_step(step, log) + + +# --------------------------------------------------------------------------- +# Execution +# --------------------------------------------------------------------------- + + +def apply_plan(plan, allow_root=False, log=None, dry_run=False): + """Carry out ``plan``. Returns True when every step succeeded.""" + log = log or (lambda line: print(line, flush=True)) + + if dry_run: + for line in plan.describe(): + log(line) + return True + + if plan.blockers: + # Stopping here turns "cmake: command not found" three minutes into a + # build into a sentence that says what to install. + log("") + log("This route cannot run on this machine:") + for blocker in plan.blockers: + log(" %s" % blocker) + return False + + if plan.needs_root and not allow_root and not _is_root(): + log("") + log("This route needs administrator rights. Either run:") + for step in plan.steps: + if step.needs_root and step.command: + log(" %s" % " ".join(step.command)) + log("") + log("and then re-run this command, or pass --allow-root to let it use") + log("sudo directly, or choose --strategy source to build GDAL without") + log("any administrator rights at all.") + return False + + for number, step in enumerate(plan.steps, 1): + log("[%d/%d] %s" % (number, len(plan.steps), step.summary)) + _run_step(step, log) + return True + + +def _run_step(step, log): + if step.action is not None: + step.action(log) + return + + attempts = [step.command] + list(step.fallbacks) + environment = dict(os.environ) + environment.update(step.env) + + last = None + for index, command in enumerate(attempts): + log(" $ %s" % " ".join(command)) + result = subprocess.run(command, env=environment) + if result.returncode == 0: + return + last = result.returncode + if index + 1 < len(attempts): + log(" that failed (exit %d); trying %s" % (last, attempts[index + 1][-1])) + raise RuntimeError("%s failed (exit %s)" % (" ".join(attempts[0]), last)) + + +# --------------------------------------------------------------------------- +# Remembering a source-built prefix +# --------------------------------------------------------------------------- + + +MANIFEST_NAME = "gdal-prefix.json" + + +def manifest_path(python_prefix=None): + """Where the record of a TopoAnalysis-built GDAL lives. + + Preferably inside the environment the bindings were installed into, so + that two virtualenvs can each have their own build. The per-user cache + is the fallback for the case where that environment is not writable. + """ + if python_prefix: + return os.path.join(python_prefix, "share", "topoanalysis", MANIFEST_NAME) + root = os.environ.get("XDG_CACHE_HOME") or os.path.join( + os.path.expanduser("~"), ".cache" + ) + return os.path.join(root, "TopoAnalysis", MANIFEST_NAME) + + +def write_manifest(prefix, python_prefix=None): + """Record a GDAL prefix that TopoAnalysis built, and return the path. + + ``python_prefix`` is the environment whose ``osgeo`` was built against + ``prefix``. It is what makes :func:`activate_environment` safe: the data + files are only forced on an interpreter that is actually running the + library they belong to. + """ + python_prefix = python_prefix or sys.prefix + payload = { + "prefix": prefix, + "gdal_data": os.path.join(prefix, "share", "gdal"), + "proj_lib": os.path.join(prefix, "share", "proj"), + "gdal_version": _gdal_source.SOURCES["gdal"]["version"], + "python_prefix": python_prefix, + } + + # Inside the environment if we can, and only otherwise in the per-user + # cache. Writing both would leave a cache copy describing an environment + # it does not belong to, which is exactly the confusion the + # ``python_prefix`` field exists to prevent. + for candidate in (manifest_path(python_prefix), manifest_path()): + try: + os.makedirs(os.path.dirname(candidate), exist_ok=True) + with open(candidate, "w") as handle: + json.dump(payload, handle, indent=2) + except OSError: + continue + return candidate + raise RuntimeError( # pragma: no cover - both locations unwritable + "could not record the GDAL prefix in %s or %s" + % (manifest_path(python_prefix), manifest_path()) + ) + + +def read_manifest(): + """The manifest for this interpreter, or None.""" + for candidate in (manifest_path(sys.prefix), manifest_path()): + try: + with open(candidate) as handle: + return json.load(handle) + except (OSError, ValueError): + continue + return None + + +def activate_environment(): + """Point GDAL and PROJ at the data files belonging to the library in use. + + This exists because of a failure that is easy to hit and very hard to + read. Activating a conda environment exports ``GDAL_DATA`` and + ``PROJ_LIB``, and those variables outrank the paths compiled into any + *other* GDAL on the machine. A PROJ 9 library handed PROJ 8's + ``proj.db`` does not fall back gracefully; it refuses every coordinate + lookup with a message about ``DATABASE.LAYOUT.VERSION`` metadata that + names neither the environment variable nor the installation that set it. + + So these variables are *overridden*, not merely filled in when unset -- + a stale value is precisely the problem, and an unset one is comparatively + harmless. Two things keep that from being reckless: + + * it only happens when the manifest was written for this very + interpreter, which means its ``osgeo`` was built against the prefix + being pointed at, so these really are the matching data files; and + * ``TOPOANALYSIS_NO_GDAL_ENV=1`` turns it off entirely. + + Any failure here is swallowed: this must never be the reason an import + breaks. + """ + try: + if os.environ.get("TOPOANALYSIS_NO_GDAL_ENV"): + return + manifest = read_manifest() + if not manifest: + return + # A manifest from a different environment describes a library this + # interpreter is not running; its data files would be as wrong as + # the ones we are trying to displace. + recorded = manifest.get("python_prefix") + if recorded is not None and recorded != sys.prefix: + return + + proj_lib = manifest.get("proj_lib") + if proj_lib and os.path.isfile(os.path.join(proj_lib, "proj.db")): + # PROJ 9.1 renamed PROJ_LIB to PROJ_DATA and still honours both. + os.environ["PROJ_LIB"] = proj_lib + os.environ["PROJ_DATA"] = proj_lib + + gdal_data = manifest.get("gdal_data") + if gdal_data and os.path.isdir(gdal_data): + os.environ["GDAL_DATA"] = gdal_data + except Exception: # pragma: no cover - defensive by design + pass + + +# --------------------------------------------------------------------------- +# Convenience entry point +# --------------------------------------------------------------------------- + + +def ensure( + strategy="auto", + python=None, + prefix=None, + allow_root=False, + jobs=None, + dry_run=False, + log=None, +): + """Make GDAL work, and return the resulting :class:`Probe`. + + This is what :file:`install.py` calls. It is a no-op when GDAL already + works, so it is safe to call on every install. + """ + log = log or (lambda line: print(line, flush=True)) + python = python or sys.executable + + before = probe(python) + if before.ok: + log("GDAL %s is already installed and working." % before.bindings_version) + return before + + plan = make_plan( + before, + strategy=strategy, + python=python, + prefix=prefix, + allow_root=allow_root, + jobs=jobs, + ) + # apply_plan prints the plan itself on a dry run, so describing it here + # too would say everything twice. + if not dry_run: + for line in plan.describe(): + log(line) + log("") + + if not apply_plan(plan, allow_root=allow_root, log=log, dry_run=dry_run): + return before + if dry_run: + return before + + after = probe(python) + if after.ok: + log("") + log("GDAL %s is installed and working." % after.bindings_version) + return after + + +# --------------------------------------------------------------------------- +# Command line +# --------------------------------------------------------------------------- + + +def main(argv=None): + parser = argparse.ArgumentParser( + prog="topoanalysis-gdal", + description=( + "Find, diagnose and install GDAL -- both the C library and the " + "Python bindings." + ), + ) + parser.add_argument( + "command", + nargs="?", + default="doctor", + choices=("doctor", "plan", "install"), + help=( + "doctor: report what is installed and what is wrong (default). " + "plan: show what an install would do. install: do it." + ), + ) + parser.add_argument( + "--strategy", + default="auto", + choices=("auto", "bindings", "conda", "wheel", "system", "source"), + help="force a particular route instead of choosing automatically", + ) + parser.add_argument( + "--prefix", + help="where a source build installs (default: the active virtualenv)", + ) + parser.add_argument( + "--python", + default=sys.executable, + help="install for this interpreter instead of the current one", + ) + parser.add_argument( + "--allow-root", + action="store_true", + help="permit steps that need sudo, instead of only printing them", + ) + parser.add_argument( + "--jobs", type=int, help="parallel compile jobs for a source build" + ) + arguments = parser.parse_args(argv) + + found = probe(arguments.python) + + if arguments.command == "doctor": + for line in found.describe(): + print(line) + if not found.ok: + print("") + print("Run 'topoanalysis-gdal plan' to see how this would be fixed.") + return 0 if found.ok else 1 + + try: + plan = make_plan( + found, + strategy=arguments.strategy, + python=arguments.python, + prefix=arguments.prefix, + allow_root=arguments.allow_root, + jobs=arguments.jobs, + ) + except ValueError as exc: + print("cannot plan an install: %s" % exc, file=sys.stderr) + return 2 + + if arguments.command == "plan": + for line in plan.describe(): + print(line) + return 0 + + try: + ok = apply_plan(plan, allow_root=arguments.allow_root) + except Exception as exc: + print("") + print("install failed: %s" % exc, file=sys.stderr) + return 1 + if not ok: + return 1 + + after = probe(arguments.python) + print("") + for line in after.describe(): + print(line) + return 0 if after.ok else 1 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/gui_tools.py b/gui_tools.py index b684d48..83a5721 100644 --- a/gui_tools.py +++ b/gui_tools.py @@ -1,16 +1,20 @@ -import matplotlib -matplotlib.use('TKAgg') +"""Interactive outlet picking. + +Needs an interactive matplotlib backend; if figures do not appear, select +one before importing, e.g. ``matplotlib.use('TkAgg')``. +""" + +import pickle as p -import TopoAnalysis.dem as d import matplotlib.pylab as plt import numpy as np -import pickle as p -import matplotlib.cm as cm -import sys -if sys.version_info[0] >= 3: - raw_input = input - +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d +except ImportError: # pragma: no cover + import dem as d + + def plot_dem(dem, hs): plt.close('all') @@ -33,8 +37,9 @@ def select_outlets(dem, fd, prefix, hs = None, outlet_filename = 'outlets.p', co print('\nClick on point upstream of outlet.') xy = plt.ginput(1)[0] xy_path = fd.search_down_flow_direction_from_xy_location(xy) - plt.plot(xy) - plt.plot(xy_path) + if not xy_path: + print('No flow path from that point; try again.') + continue xy_path_plot = list(zip(*xy_path)) path = plt.plot(xy_path_plot[0],xy_path_plot[1], color+'-') print('\nClick on the outlet location.') @@ -52,7 +57,7 @@ def select_outlets(dem, fd, prefix, hs = None, outlet_filename = 'outlets.p', co plt.plot(outlet_loc[0], outlet_loc[1], color+'o') path.pop(0).remove() - outlet_prefix = raw_input('Type a name for this outlet (leaving this blank will prevent outlet from being saved and will complete the selection process: ') + outlet_prefix = input('Type a name for this outlet (leaving this blank will prevent outlet from being saved and will complete the selection process: ') if outlet_prefix != '': import os.path diff --git a/install.py b/install.py new file mode 100644 index 0000000..9062a72 --- /dev/null +++ b/install.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +"""Install TopoAnalysis and everything it needs, from a fresh clone. + + git clone + cd TopoAnalysis + python3 install.py + +That is the whole procedure. The script creates an environment, installs the +dependencies, compiles the Priority-Flood kernels, installs GDAL -- the C +library *and* the Python bindings -- and then checks that the result actually +works by reading and writing a GeoTIFF. + +It uses nothing but the standard library, so it runs on a machine where +nothing has been set up yet. + +Why it makes a virtualenv by default +------------------------------------ +Because installing into a system Python usually fails, and when it does not +fail it is a bad idea. Debian, Ubuntu, Fedora and Homebrew all now mark their +Python installations "externally managed" (PEP 668), so ``pip install`` +refuses to touch them. Creating ``.venv`` sidesteps that, keeps this project +from disturbing anything else, and gives the from-source GDAL build somewhere +self-contained to live. If you are already inside a virtualenv or a conda +environment, that one is used instead and nothing new is created. + +Useful options +-------------- +``--no-gdal`` skip GDAL entirely; everything except file I/O works +``--gdal-strategy`` force a route: conda, system, source, bindings, wheel +``--allow-root`` let the system package manager use sudo +``--system`` install into the current interpreter, no virtualenv +``--jobs N`` parallel compile jobs +``--dry-run`` print what would happen and stop +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import sysconfig +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +MIN_PYTHON = (3, 8) + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + +_STEP = [0] + + +def heading(text): + _STEP[0] += 1 + print("") + print("=" * 72) + print("%d. %s" % (_STEP[0], text)) + print("=" * 72) + sys.stdout.flush() + + +def say(text=""): + print(text) + sys.stdout.flush() + + +def die(text): + print("") + print("ERROR: %s" % text, file=sys.stderr) + sys.exit(1) + + +def run(command, env=None, check=True): + say(" $ %s" % " ".join(command)) + result = subprocess.run(command, env=env) + if check and result.returncode != 0: + die("command failed (exit %d):\n %s" % (result.returncode, " ".join(command))) + return result.returncode + + +# --------------------------------------------------------------------------- +# Environment +# --------------------------------------------------------------------------- + + +def in_virtualenv(): + return sys.prefix != getattr(sys, "base_prefix", sys.prefix) + + +def in_conda(): + return os.path.isdir(os.path.join(sys.prefix, "conda-meta")) + + +def venv_python(directory): + """The interpreter inside a virtualenv directory.""" + if sys.platform == "win32": + return os.path.join(directory, "Scripts", "python.exe") + return os.path.join(directory, "bin", "python") + + +def venv_is_usable(python): + """True when a virtualenv's interpreter exists and has a working pip. + + Checking pip matters because a *partial* virtualenv is a real and common + thing. ``venv`` creates the interpreter before it bootstraps pip, and on + Debian and Ubuntu -- where pip's bootstrap lives in a separate + ``python3-venv`` package -- it fails in between, leaving a directory that + has ``bin/python`` and no way to install anything into it. + """ + if not os.path.exists(python): + return False + result = subprocess.run( + [python, "-c", "import pip"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + return result.returncode == 0 + + +def create_venv(directory): + """Make a virtualenv at ``directory``, reusing a usable one already there.""" + python = venv_python(directory) + if venv_is_usable(python): + say(" reusing the existing environment at %s" % directory) + return python + + existed = os.path.exists(directory) + if existed and os.path.exists(python): + die( + "there is a virtual environment in %s but it has no working pip,\n" + "so nothing can be installed into it. Remove it and re-run:\n" + " rm -rf %s\n" + "On Debian and Ubuntu the usual cause is a missing python3-venv:\n" + " sudo apt install python3-venv" % (directory, directory) + ) + + say(" creating a virtual environment in %s" % directory) + result = subprocess.run([sys.executable, "-m", "venv", directory]) + if result.returncode != 0 or not venv_is_usable(python): + # We made this directory a moment ago and it is not usable, so it is + # ours to clean up. Leaving the half-built shell behind would make + # the next run take the "reuse" path into the same dead end. + if not existed: + shutil.rmtree(directory, ignore_errors=True) + die( + "could not create a working virtual environment in %s.\n" + "On Debian and Ubuntu this is almost always a missing package:\n" + " sudo apt install python3-venv\n" + "Alternatively re-run with --system to install into this\n" + "interpreter instead." % directory + ) + return python + + +def externally_managed(): + """The PEP 668 marker path if this interpreter forbids pip installs. + + Debian, Ubuntu, Fedora and Homebrew all ship one. pip refuses to install + anything into such an interpreter, with an error that does not mention + this script at all -- so it is worth saying plainly up front. + """ + for key in ("stdlib", "platstdlib"): + directory = sysconfig.get_path(key) + if not directory: + continue + marker = os.path.join(directory, "EXTERNALLY-MANAGED") + if os.path.exists(marker): + return marker + return None + + +def activation_hint(directory): + if sys.platform == "win32": + return "%s\\Scripts\\activate" % directory + return "source %s/bin/activate" % directory + + +# --------------------------------------------------------------------------- +# The steps +# --------------------------------------------------------------------------- + + +def check_python(): + if sys.version_info[:2] < MIN_PYTHON: + die( + "TopoAnalysis needs Python %d.%d or newer; this is %s" + % (MIN_PYTHON[0], MIN_PYTHON[1], sys.version.split()[0]) + ) + say(" Python %s at %s" % (sys.version.split()[0], sys.executable)) + + +#: pip needs to understand PEP 660 to do an editable install of this project. +MIN_PIP = (21, 3) + + +def install_build_tools(python): + """setuptools, wheel, pybind11 and numpy, before anything else. + + pybind11 has to be present for the C++ kernels to compile, and numpy has + to be present before the GDAL bindings are built -- otherwise the numpy + bridge is silently skipped. Both are build-time requirements of things + installed later, so they go in first. + + pip itself is upgraded only when it is too old to do the job. This + script may be pointed at an environment the user uses for other work, and + replacing their pip as a side effect of installing one package is not + ours to do. + """ + version = pip_version(python) + if version is None or version < MIN_PIP: + say( + " pip %s is older than %d.%d; upgrading it" + % (".".join(map(str, version)) if version else "(unknown)", *MIN_PIP) + ) + run([python, "-m", "pip", "install", "--upgrade", "pip"]) + else: + say(" pip %s is recent enough; leaving it alone" % ".".join(map(str, version))) + + run([python, "-m", "pip", "install", "--upgrade", "setuptools", "wheel"]) + run([python, "-m", "pip", "install", "pybind11>=2.6", "numpy>=1.20"]) + + +def pip_version(python): + """``(major, minor)`` of the target interpreter's pip, or None.""" + result = subprocess.run( + [python, "-c", "import pip, sys; sys.stdout.write(pip.__version__)"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + return None + parts = result.stdout.decode("utf8", "replace").strip().split(".") + try: + return int(parts[0]), int(parts[1]) + except (IndexError, ValueError): + return None + + +def install_package(python, extras, editable): + target = "." if not extras else ".[%s]" % extras + command = [python, "-m", "pip", "install"] + if editable: + command.append("-e") + command.append(target) + run(command) + + +def install_gdal(python, strategy, prefix, allow_root, jobs, dry_run): + """Hand off to the GDAL bootstrapper, which lives with the package.""" + sys.path.insert(0, HERE) + try: + import gdal_setup + except ImportError as exc: # pragma: no cover - only a broken checkout + die("could not load the GDAL bootstrapper from %s: %s" % (HERE, exc)) + + try: + return gdal_setup.ensure( + strategy=strategy, + python=python, + prefix=prefix, + allow_root=allow_root, + jobs=jobs, + dry_run=dry_run, + log=say, + ) + except Exception as exc: + # A failed GDAL install is a partial failure, not a crash: everything + # except file I/O still works, and a traceback here would obscure the + # fact that the rest of the installation succeeded. + say("") + say("GDAL could not be installed: %s" % exc) + return None + + +_VERIFY = r""" +import sys, tempfile, os +import numpy as np +import TopoAnalysis +from TopoAnalysis import Elevation, FilledElevation, FlowDirectionD8, Area + +print(" TopoAnalysis %s" % TopoAnalysis.__version__) +print(" compute backend : %s" % TopoAnalysis.backend()) + +# A small DEM with a pit, so the fill has something to do. +z = np.array([[5.0, 4.0, 3.0, 4.0, 5.0], + [4.0, 3.0, 2.0, 3.0, 4.0], + [3.0, 2.0, 0.5, 2.0, 3.0], + [2.0, 1.0, 1.5, 1.0, 2.0], + [1.0, 0.0, 1.0, 0.0, 1.0]]) +dem = Elevation(dx=30.0, grid=z) +filled = FilledElevation(elevation=dem) +d8 = FlowDirectionD8(flooded_dem=filled) +area = Area(flow_direction=d8) +assert np.isfinite(area._griddata).any(), "flow accumulation produced nothing" +print(" in-memory pipeline : ok (max area %.0f m^2)" % np.nanmax(area._griddata)) + +available = TopoAnalysis.gdal_is_available() +print(" GDAL available : %s" % available) +if not available: + print(" (file I/O is unavailable; everything else works)") + sys.exit(0) + +from osgeo import gdal, osr +print(" GDAL version : %s" % gdal.__version__) + +# The real test: write a georeferenced GeoTIFF and read it back through +# TopoAnalysis. This exercises the C library, the numpy bridge and PROJ. +tmp = tempfile.mkdtemp() +path = os.path.join(tmp, "probe.tif") +try: + driver = gdal.GetDriverByName("GTiff") + ds = driver.Create(path, 5, 5, 1, gdal.GDT_Float32) + ds.SetGeoTransform((500000.0, 30.0, 0.0, 4000000.0, 0.0, -30.0)) + srs = osr.SpatialReference() + srs.ImportFromEPSG(32611) + ds.SetProjection(srs.ExportToWkt()) + ds.GetRasterBand(1).WriteArray(z.astype("float32")) + ds = None + + back = Elevation(gdal_filename=path) + assert back._georef_info.nx == 5 and back._georef_info.ny == 5, "wrong shape" + assert np.allclose(back._griddata, z), "values did not survive the round trip" + print(" GeoTIFF round trip : ok (EPSG:32611, %dx%d)" + % (back._georef_info.nx, back._georef_info.ny)) +finally: + import shutil as _sh + _sh.rmtree(tmp, ignore_errors=True) +""" + + +def verify(python): + """Prove the install works rather than assuming it does. + + Run from outside the source tree: importing from the checkout directory + would succeed even if the install had not, because the package directory + *is* the repository. Running elsewhere makes the check mean something. + """ + result = subprocess.run([python, "-c", _VERIFY], cwd=tempfile.gettempdir()) + return result.returncode == 0 + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main(argv=None): + parser = argparse.ArgumentParser( + description="Install TopoAnalysis and all of its dependencies, GDAL included.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--venv", + default=None, + help="virtualenv directory to create or reuse (default: .venv). " + "Giving this forces a virtualenv even from inside a conda " + "environment, which is otherwise used as-is.", + ) + parser.add_argument( + "--system", + action="store_true", + help="install into the current interpreter instead of a virtualenv", + ) + parser.add_argument( + "--break-system-packages", + action="store_true", + help="with --system, install into an externally managed (PEP 668) " + "Python anyway", + ) + parser.add_argument( + "--extras", + default="test", + help="optional dependency groups to install (default: test; 'all' " + "adds statsmodels, dask and rasterio; '' for none). GDAL is not an " + "extra -- this script installs it separately, C library and all.", + ) + parser.add_argument( + "--no-editable", + dest="editable", + action="store_false", + help="install a copy instead of linking to this checkout", + ) + parser.add_argument("--no-gdal", action="store_true", help="do not install GDAL") + parser.add_argument( + "--gdal-strategy", + default="auto", + choices=("auto", "bindings", "conda", "wheel", "system", "source"), + help="force a particular route for GDAL (default: auto)", + ) + parser.add_argument( + "--gdal-prefix", help="where a from-source GDAL build should install" + ) + parser.add_argument( + "--allow-root", + action="store_true", + help="permit system package manager steps that need sudo", + ) + parser.add_argument("--jobs", type=int, help="parallel compile jobs") + parser.add_argument( + "--dry-run", action="store_true", help="show what would happen, then stop" + ) + arguments = parser.parse_args(argv) + + say("TopoAnalysis installer") + say("running in %s" % HERE) + + if arguments.break_system_packages: + # Set in the environment rather than passed as a flag so that it also + # reaches the pip calls made later by the GDAL bootstrapper. + os.environ["PIP_BREAK_SYSTEM_PACKAGES"] = "1" + + # -- 1 ---------------------------------------------------------------- + heading("Checking the interpreter") + check_python() + + # -- 2 ---------------------------------------------------------------- + heading("Choosing an environment") + if arguments.system: + python = sys.executable + marker = externally_managed() + if marker and not arguments.break_system_packages: + die( + "this Python is marked externally managed (PEP 668), so pip\n" + "will refuse to install into it:\n" + " %s\n" + "It is managed by the operating system's package manager, and\n" + "installing over it can break system tools. Drop --system to\n" + "use a virtual environment instead, or pass\n" + "--break-system-packages if you really mean it." % marker + ) + say(" --system given: installing into %s" % python) + activate = None + elif not arguments.venv and (in_virtualenv() or in_conda()): + # An environment the user already chose and activated. Respect it -- + # but only when they did not name one explicitly, because --venv is + # how you ask for a separate environment from inside conda. + python = sys.executable + say( + " already inside %s; installing into it" + % ("a conda environment" if in_conda() else "a virtualenv") + ) + say(" %s" % sys.prefix) + activate = None + else: + directory = arguments.venv or os.path.join(HERE, ".venv") + if arguments.dry_run: + # A real run would reuse a usable environment that is already + # there, so a dry run has to report on that one -- describing the + # outer interpreter instead would be describing a different + # machine state than the one an install would act on. + existing = venv_python(directory) + if venv_is_usable(existing): + say(" would reuse the existing environment at %s" % directory) + python = existing + else: + say(" would create a virtualenv in %s" % directory) + python = sys.executable + else: + python = create_venv(directory) + activate = directory + + if arguments.dry_run: + say("") + say("--dry-run: stopping before anything is installed.") + if activate and python == sys.executable: + say("Note: that environment does not exist yet, so the GDAL report") + say("below describes %s instead. A fresh" % sys.executable) + say("virtualenv starts with no GDAL bindings at all, whatever it") + say("says here.") + say("") + say("The GDAL plan for this machine would be:") + sys.path.insert(0, HERE) + import gdal_setup + + found = gdal_setup.probe(python) + for line in found.describe(): + say(" " + line) + if not found.ok: + say("") + plan = gdal_setup.make_plan( + found, + strategy=arguments.gdal_strategy, + python=python, + prefix=arguments.gdal_prefix, + allow_root=arguments.allow_root, + jobs=arguments.jobs, + ) + for line in plan.describe(): + say(" " + line) + return 0 + + # -- 3 ---------------------------------------------------------------- + heading("Installing build tools") + install_build_tools(python) + + # -- 4 ---------------------------------------------------------------- + heading("Installing TopoAnalysis and its dependencies") + os.chdir(HERE) + install_package(python, arguments.extras, arguments.editable) + + # -- 5 ---------------------------------------------------------------- + gdal_ok = True + if arguments.no_gdal: + heading("Skipping GDAL (--no-gdal)") + say(" everything except reading and writing raster files will work") + else: + heading("Installing GDAL (C library and Python bindings)") + found = install_gdal( + python, + arguments.gdal_strategy, + arguments.gdal_prefix, + arguments.allow_root, + arguments.jobs, + dry_run=False, + ) + gdal_ok = found is not None and found.ok + if not gdal_ok: + say("") + if found is not None: + say("GDAL is still not working:") + for line in found.describe(): + say(" " + line) + say("") + say("TopoAnalysis itself is installed and everything except file") + say("I/O will work. To retry just the GDAL part:") + say(" %s -m TopoAnalysis.gdal_setup install --strategy source" % python) + + # -- 6 ---------------------------------------------------------------- + heading("Checking the installation") + ok = verify(python) + + say("") + say("=" * 72) + if ok and gdal_ok: + say("TopoAnalysis is installed.") + elif ok: + say("TopoAnalysis is installed, but GDAL is not -- see above.") + else: + say("TopoAnalysis is installed, but the check did not pass -- see above.") + say("=" * 72) + if activate: + say("") + say("Activate the environment with:") + say(" %s" % activation_hint(activate)) + say("") + say("Then try:") + say(" topoanalysis-process --info") + say(" topoanalysis-gdal doctor") + + # A GDAL that was asked for and did not arrive is a partial failure, and + # a script that called this needs to be able to tell. The message above + # makes clear that the rest of the installation is fine; --no-gdal is how + # you say you did not want it in the first place. + return 0 if (ok and gdal_ok) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/kernels.py b/kernels.py new file mode 100644 index 0000000..0edb6bf --- /dev/null +++ b/kernels.py @@ -0,0 +1,264 @@ +"""Backend dispatch for the TopoAnalysis grid kernels. + +Import the kernels from here rather than from :mod:`TopoAnalysis.fastops` or +the compiled extension directly:: + + from TopoAnalysis import kernels + kernels.priority_flood(z, mode="flat") + +If the C++ extension built successfully it is used; otherwise the pure-NumPy +implementations in :mod:`TopoAnalysis.fastops` take over. Both produce the +same numbers -- the test suite asserts it -- so the choice is purely about +speed. + +Set the environment variable ``TOPOANALYSIS_PURE_PYTHON=1`` to force the +NumPy path, which is useful when debugging or comparing the two. +""" + +from __future__ import annotations + +import importlib.machinery +import importlib.util +import os +import warnings + +try: # pragma: no cover - exercised by whichever import style is in use + from . import fastops +except ImportError: # pragma: no cover + import fastops + +__all__ = [ + "backend", + "have_extension", + "priority_flood", + "flow_directions", + "receivers", + "topological_order", + "accumulate", + "flow_length", + "propagate_along_main_stem", + "upstream_mask", + "chi", + "drainage_basins", + "downstream_distance", + "imposemin", + "stream_order", + "default_seeds", + "D8_DI", + "D8_DJ", + "D8_DIST", + "D8_CODES", + "PRIORITY_FLOOD_REFERENCE", +] + +PRIORITY_FLOOD_REFERENCE = ( + "Barnes, R., Lehman, C., Mulla, D. (2014). Priority-flood: An optimal " + "depression-filling and watershed-labeling algorithm for digital elevation " + "models. Computers & Geosciences 62, 117-127. " + "doi:10.1016/j.cageo.2013.04.024" +) + +# Geometry constants are always taken from the NumPy module: both backends use +# the same neighbour ordering, and having one definition avoids them drifting. +D8_DI = fastops.D8_DI +D8_DJ = fastops.D8_DJ +D8_DIST = fastops.D8_DIST +D8_CODES = fastops.D8_CODES + +_FORCE_PYTHON = os.environ.get("TOPOANALYSIS_PURE_PYTHON", "").strip() not in ("", "0", "false", "False") + +def _load_extension(): + """Import the compiled kernels, or return None if they were not built.""" + try: + from . import _topoanalysis # type: ignore[attr-defined] + return _topoanalysis + except ImportError: + pass + + # Flat import (`import kernels` with this directory on sys.path): load the + # extension from *this file's* directory by path. A bare + # `import _topoanalysis` would search all of sys.path and could pick up an + # unrelated -- or stale, or ABI-incompatible -- module of the same name + # left lying around somewhere else. + here = os.path.dirname(os.path.abspath(__file__)) + for suffix in importlib.machinery.EXTENSION_SUFFIXES: + candidate = os.path.join(here, "_topoanalysis" + suffix) + if not os.path.exists(candidate): + continue + spec = importlib.util.spec_from_file_location("_topoanalysis", candidate) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except ImportError: # pragma: no cover - a broken build + return None + return module + return None + + +_ext = None if _FORCE_PYTHON else _load_extension() + + +def have_extension() -> bool: + """True when the compiled kernels are in use.""" + return _ext is not None + + +def backend() -> str: + """``'c++'`` or ``'python'``.""" + return "c++" if _ext is not None else "python" + + +def warn_if_slow(n_cells: int, threshold: int = 250_000) -> None: + """Warn once when a large grid is about to be processed in pure Python.""" + if _ext is None and n_cells >= threshold: + warnings.warn( + "TopoAnalysis is using its pure-Python kernels on a grid of " + "{0:,} cells; this will be slow. Build the C++ extension with " + "`pip install -e .` from the TopoAnalysis directory for a large " + "speed-up.".format(n_cells), + RuntimeWarning, + stacklevel=3, + ) + + +default_seeds = fastops.default_seeds + + +def priority_flood( + elevations, + closed=None, + seeds=None, + mode="flat", + epsilon=0.0, + cellsize=1.0, + max_pit_depth=0.0, + track_visited=False, + reference_algorithm=False, +): + """Fill depressions in place; see :func:`TopoAnalysis.fastops.priority_flood`.""" + if _ext is None: + return fastops.priority_flood( + elevations, + closed=closed, + seeds=seeds, + mode=mode, + epsilon=epsilon, + cellsize=cellsize, + max_pit_depth=max_pit_depth, + track_visited=track_visited, + reference_algorithm=reference_algorithm, + ) + return _ext.priority_flood( + elevations, + closed, + seeds, + mode, + epsilon, + cellsize, + max_pit_depth, + track_visited, + reference_algorithm, + ) + + +def flow_directions(elevations, cellsize_grid=None, cellsize=1.0): + """Steepest-descent D8 codes; see :func:`TopoAnalysis.fastops.flow_directions`.""" + if _ext is None: + return fastops.flow_directions(elevations, cellsize_grid, cellsize) + return _ext.flow_directions(elevations, cellsize_grid, cellsize) + + +def receivers(codes): + """Receiver index per cell; see :func:`TopoAnalysis.fastops.receivers`.""" + if _ext is None: + return fastops.receivers(codes) + return _ext.receivers(codes) + + +def topological_order(recv): + """``(order, n_cells_on_cycles)``; see :func:`TopoAnalysis.fastops.topological_order`.""" + if _ext is None: + return fastops.topological_order(recv) + return _ext.topological_order(recv) + + +def accumulate(recv, order, weights, gate=None): + """Downstream accumulation; see :func:`TopoAnalysis.fastops.accumulate`.""" + if _ext is None: + return fastops.accumulate(recv, order, weights, gate) + return _ext.accumulate(recv, order, weights, gate) + + +def flow_length(recv, order, step_length): + """``(length, from_codes)``; see :func:`TopoAnalysis.fastops.flow_length`.""" + if _ext is None: + return fastops.flow_length(recv, order, step_length) + return _ext.flow_length(recv, order, step_length) + + +def propagate_along_main_stem(recv, order, main_stem_from_codes, gate, values, mode): + """See :func:`TopoAnalysis.fastops.propagate_along_main_stem`.""" + if _ext is None: + return fastops.propagate_along_main_stem( + recv, order, main_stem_from_codes, gate, values, mode + ) + return _ext.propagate_along_main_stem( + recv, order, main_stem_from_codes, gate, values, mode + ) + + +def upstream_mask(recv, outlets): + """See :func:`TopoAnalysis.fastops.upstream_mask`.""" + if _ext is None: + return fastops.upstream_mask(recv, outlets) + return _ext.upstream_mask(recv, outlets) + + +def chi( + recv, + order, + area, + step_length, + outlets, + A0, + theta, + trapezoid=False, + max_length=0.0, + mask=None, +): + """``(chi, distance)``; see :func:`TopoAnalysis.fastops.chi`.""" + if _ext is None: + return fastops.chi( + recv, order, area, step_length, outlets, A0, theta, trapezoid, max_length, mask + ) + return _ext.chi( + recv, order, area, step_length, outlets, A0, theta, trapezoid, max_length, mask + ) + + +def drainage_basins(recv, order, outlets=None, valid=None): + """See :func:`TopoAnalysis.fastops.drainage_basins`.""" + if _ext is None: + return fastops.drainage_basins(recv, order, outlets, valid) + return _ext.drainage_basins(recv, order, outlets, valid) + + +def downstream_distance(recv, order, step_length): + """See :func:`TopoAnalysis.fastops.downstream_distance`.""" + if _ext is None: + return fastops.downstream_distance(recv, order, step_length) + return _ext.downstream_distance(recv, order, step_length) + + +def imposemin(recv, order, step_length, sl, elevations): + """Carve in place; see :func:`TopoAnalysis.fastops.imposemin`.""" + if _ext is None: + return fastops.imposemin(recv, order, step_length, sl, elevations) + return _ext.imposemin(recv, order, step_length, sl, elevations) + + +def stream_order(recv, order, is_stream, kind="strahler"): + """See :func:`TopoAnalysis.fastops.stream_order`.""" + if _ext is None: + return fastops.stream_order(recv, order, is_stream, kind) + return _ext.stream_order(recv, order, is_stream, kind) diff --git a/plotting.py b/plotting.py index bf6cbd2..83d64fa 100644 --- a/plotting.py +++ b/plotting.py @@ -1,13 +1,21 @@ -from . import dem as d +"""Longitudinal- and chi-profile plotting helpers.""" + import matplotlib.pylab as plt import numpy as np +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d + from . import demRecursionTools +except ImportError: # pragma: no cover + import dem as d + import demRecursionTools + def plot_downstream_profile(elevation, flow_direction, outlet, plot_code, downstream = True, start_at = 0.0, mean_pixel_dimension = None, figure = None): rc = flow_direction.search_down_flow_direction(outlet) - rows = np.array(list(zip(*rc))[0], dtype = np.int) - cols = np.array(list(zip(*rc))[1], dtype = np.int) + rows = np.array(list(zip(*rc))[0], dtype=int) + cols = np.array(list(zip(*rc))[1], dtype=int) if downstream: direction = 1.0 @@ -40,10 +48,7 @@ def plot_downstream_profile(elevation, flow_direction, outlet, plot_code, downst def plot_recursive_upstream_profiles(elevation, flow_direction, area, outlet, plot_code, downstream = False, start_at = 0.0, figure = None, minimum_area = 1.0E6): def plot_ld_link(current_length, ld_list, plot_code, downstream_sign, minimum_area): - if len(ld_list['index']) == 1: - (current_row, current_column) = ld_list['index'][0] - else: - (current_row, current_column) = ld_list['index'] + (current_row, current_column) = ld_list['index'] if ld_list.get('next') is None: @@ -149,9 +154,7 @@ def interactive_chi_profiles_and_map_view(prefix, code, plot_code, dem, fd, area outlets = pickle.load(open('outlets.p', 'rb')) outlet = outlets[prefix][code] - from demRecursionTools import map_chi_profiles - - chi_map = map_chi_profiles(dem, fd, area, outlet, minimum_area = minimum_area, theta = theta, Ao = Ao) + chi_map = demRecursionTools.map_chi_profiles(dem, fd, area, outlet, minimum_area = minimum_area, theta = theta, Ao = Ao) indexes = chi_map.keys() import operator (chi, _) = list(zip(*operator.itemgetter(*indexes)(chi_map))) diff --git a/process_dem.py b/process_dem.py index f8f90ef..ac5e0d7 100644 --- a/process_dem.py +++ b/process_dem.py @@ -1,4 +1,7 @@ -import dem as d +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as d +except ImportError: # pragma: no cover + import dem as d def process_dem(dem_name, EPSGcode, folder_name = '.'): @@ -39,8 +42,16 @@ def calc_ks_and_associated_grids(dem_name, Ao, theta, v, folder_name = '.', use_ else: mask = None - from denudationRateAnalysis import calculate_ks_for_sample - return calculate_ks_for_sample(v, d8, ksi, relief, area, Ao = Ao, mask = mask) + # denudationRateAnalysis is a separate, unpublished module; import it + # lazily so that the rest of process_dem works without it. + try: + from denudationRateAnalysis import calculate_ks_for_sample + except ImportError as exc: # pragma: no cover - external dependency + raise ImportError( + "calc_ks_and_associated_grids needs the separate " + "denudationRateAnalysis module, which is not part of TopoAnalysis." + ) from exc + return calculate_ks_for_sample(v, d8, ksi, relief, area, Ao=Ao, mask=mask) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f31cfdb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,119 @@ +[build-system] +requires = ["setuptools>=64", "wheel", "pybind11>=2.6"] +build-backend = "setuptools.build_meta" + +[project] +name = "TopoAnalysis" +version = "1.0.0" +description = "Digital elevation model analysis for tectonic geomorphology" +readme = "README.md" +requires-python = ">=3.8" +authors = [ + { name = "Sam Johnstone" }, + { name = "George Hilley" }, +] +keywords = [ + "geomorphology", + "digital elevation model", + "flow routing", + "priority flood", + "chi analysis", + "channel steepness", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: C++", + "Topic :: Scientific/Engineering :: GIS", +] + +dependencies = [ + "numpy>=1.20", + "scipy>=1.7", + "matplotlib>=3.3", +] + +[project.optional-dependencies] +# GDAL is deliberately not a hard dependency, and this extra will not work on +# its own. There is no GDAL wheel on PyPI at any version for any platform: +# every release is an sdist that compiles the bindings against a GDAL C +# library which must already exist on the machine. Declaring it here would +# therefore turn a missing C library into a failed install of TopoAnalysis +# itself, rather than the missing-file-I/O it actually is. +# +# Use one of these instead, both of which install the C library too: +# python install.py (from a clone -- does everything) +# topoanalysis-gdal install (once TopoAnalysis is installed) +# +# The extra remains for the case where the C library is already present and +# only the bindings are wanted. +gdal = ["GDAL>=3.0"] +stats = ["statsmodels>=0.12"] +parallel = ["dask[delayed]>=2021.0"] +quadrats = ["rasterio>=1.2"] +test = ["pytest>=7.0", "pytest-cov"] +# The docs are Markdown and are read as-is on the repository page; +# these are only needed to build them as a Sphinx site. +docs = ["sphinx>=5.0", "sphinx-rtd-theme", "myst-parser"] +# Deliberately without GDAL. Including it would make `pip install .[all]` +# fail on any machine without a GDAL C library already installed -- which is +# most of them -- and take the rest of the package down with it. GDAL is +# installed by `python install.py` or `topoanalysis-gdal install`, both of +# which supply the C library as well. +all = [ + "statsmodels>=0.12", + "dask[delayed]>=2021.0", + "rasterio>=1.2", +] + +[project.urls] +Homepage = "https://github.com/stanford-geomorph/TopoAnalysis" + +[project.scripts] +topoanalysis-process = "TopoAnalysis.cli:main" +topoanalysis-gdal = "TopoAnalysis.gdal_setup:main" + +# --------------------------------------------------------------------------- +# The repository root *is* the package directory, so the package is mapped +# explicitly rather than discovered. This keeps ``import TopoAnalysis.dem`` +# working for anyone who already has the parent directory on their path, while +# still producing a normal installable distribution. +# --------------------------------------------------------------------------- +[tool.setuptools] +include-package-data = true +packages = ["TopoAnalysis"] + +[tool.setuptools.package-dir] +TopoAnalysis = "." + +[tool.setuptools.package-data] +# The test suite ships with the package so an installed copy can be +# checked: `pytest --pyargs` needs a package, so point pytest at the +# directory instead -- `python -m pytest $(python -c "import +# TopoAnalysis, os; print(os.path.dirname(TopoAnalysis.__file__))")/tests`. +TopoAnalysis = ["_cpp/*.hpp", "_cpp/*.cpp", "tests/*.py", "conftest.py"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +# Import test modules by path rather than by dotted name. The checkout +# directory is itself a package, and under the default "prepend" mode pytest +# derives a module name from the directory's *name* -- which is not a valid +# identifier when the checkout is called TopoAnalysis-main (a GitHub zip) or +# anything else with a hyphen in it. +addopts = "--import-mode=importlib" +# install.py puts .venv inside this directory, because the directory is the +# package. Nothing good comes of pytest walking into it. +norecursedirs = [".venv", "venv", "build", "dist", "*.egg-info", ".git"] +filterwarnings = [ + "error::DeprecationWarning:TopoAnalysis.*", +] +markers = [ + "gdal: requires the GDAL Python bindings", + "slow: takes more than a second", + "parity: checks numerical agreement with TopoToolbox", +] + +[tool.coverage.run] +source = ["TopoAnalysis"] +omit = ["*/tests/*", "*/gui_tools.py"] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..42e45c2 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,16 @@ +-r requirements.txt + +# Building the C++ kernels. +pybind11>=2.6 +setuptools>=64 +wheel + +# Optional runtime extras that the test suite exercises. +statsmodels>=0.12 + +# Testing and docs. +pytest>=7.0 +pytest-cov +sphinx>=5.0 +sphinx-rtd-theme +myst-parser diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..67b5ac4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +# Runtime dependencies. +# +# GDAL is deliberately absent: it cannot be installed reliably from PyPI on +# every platform, and TopoAnalysis only needs it for file I/O. Install it +# from conda-forge (see environment.yml) or your system package manager. +numpy>=1.20 +scipy>=1.7 +matplotlib>=3.3 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b8b71d5 --- /dev/null +++ b/setup.py @@ -0,0 +1,71 @@ +"""Build script for TopoAnalysis. + +Almost all configuration lives in ``pyproject.toml``. This file exists only +to describe the optional C++ extension, which cannot be expressed +declaratively. + +The extension is marked ``optional``: if no C++ compiler is available, or the +build fails for any other reason, installation still succeeds and TopoAnalysis +falls back to the pure-NumPy kernels in ``TopoAnalysis.fastops``. Results are +identical either way; only the speed differs. +""" + +from __future__ import annotations + +import os +import sys + +from setuptools import Extension, setup + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _pybind11_include_dirs(): + """Locate pybind11's headers, tolerating its absence.""" + try: + import pybind11 + except ImportError: # pragma: no cover - only hit on a broken build env + return None + return [pybind11.get_include()] + + +def _extensions(): + include_dirs = _pybind11_include_dirs() + if include_dirs is None: + sys.stderr.write( + "TopoAnalysis: pybind11 not found; building without the compiled " + "kernels. Install pybind11 and reinstall for a large speed-up.\n" + ) + return [] + + if sys.platform == "win32": + extra_compile_args = ["/O2", "/std:c++17", "/EHsc"] + extra_link_args = [] + else: + extra_compile_args = ["-O3", "-std=c++17", "-funroll-loops"] + extra_link_args = [] + if sys.platform == "darwin": + # Structured bindings and std::optional need a modern libc++. + extra_compile_args.append("-mmacosx-version-min=10.14") + extra_link_args.append("-mmacosx-version-min=10.14") + + return [ + Extension( + "TopoAnalysis._topoanalysis", + sources=[os.path.join("_cpp", "module.cpp")], + # Without this a header-only edit would not trigger a rebuild. + depends=[ + os.path.join("_cpp", "priority_flood.hpp"), + os.path.join("_cpp", "flow_routing.hpp"), + ], + include_dirs=include_dirs + [os.path.join(HERE, "_cpp")], + language="c++", + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + # Never fail the install because the compiler is unhappy. + optional=True, + ) + ] + + +setup(ext_modules=_extensions()) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..bfed24c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,169 @@ +"""Shared fixtures: small synthetic landscapes with known answers. + +This file also makes ``import TopoAnalysis`` resolve when the suite is run +from a checkout rather than from an installed copy. It lives in ``tests/`` +rather than beside ``__init__.py`` deliberately: a ``conftest.py`` next to a +package's ``__init__.py`` makes pytest import that package by the checkout +directory's *name*, which fails whenever the directory is called anything +other than ``TopoAnalysis`` -- a GitHub zip unpacks to ``TopoAnalysis-main``. +""" + +from __future__ import annotations + +import importlib.util +import os +import sys + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_PACKAGE_DIR = os.path.dirname(_TESTS_DIR) +_PARENT = os.path.dirname(_PACKAGE_DIR) + + +def _register_package(): + """Make ``import TopoAnalysis`` resolve to this checkout.""" + try: + import TopoAnalysis # noqa: F401 + return + except ImportError: + pass + + if os.path.basename(_PACKAGE_DIR) == "TopoAnalysis": + if _PARENT not in sys.path: + sys.path.insert(0, _PARENT) + return + + # The checkout has some other name; load it under the one the package + # expects rather than asking the user to rename their directory. + spec = importlib.util.spec_from_file_location( + "TopoAnalysis", os.path.join(_PACKAGE_DIR, "__init__.py"), + submodule_search_locations=[_PACKAGE_DIR]) + module = importlib.util.module_from_spec(spec) + sys.modules["TopoAnalysis"] = module + spec.loader.exec_module(module) + + +_register_package() + +import numpy as np # noqa: E402 +import pytest # noqa: E402 + +from TopoAnalysis import dem, kernels # noqa: E402 + + +# --------------------------------------------------------------------------- +# Surfaces whose correct answers can be written down +# --------------------------------------------------------------------------- + + +@pytest.fixture +def east_plane(): + """A 5x5 plane dropping 1 m per 1 m eastward. + + Everything flows east, gradient is exactly 1, aspect is exactly 90. + """ + return dem.Elevation(dx=1.0, grid=np.tile(np.arange(5, 0, -1, dtype=float), (5, 1))) + + +@pytest.fixture +def south_plane(): + """A 5x5 plane dropping 1 m per 1 m southward (row-increasing).""" + return dem.Elevation(dx=1.0, grid=np.tile( + np.arange(5, 0, -1, dtype=float).reshape(-1, 1), (1, 5))) + + +@pytest.fixture +def single_pit(): + """A 5x5 bowl: a rim at 10, a ring at 4-5 and a 1 m pit in the middle. + + Filling raises every interior cell to the 10 m rim. + """ + return dem.Elevation(dx=1.0, grid=np.array([ + [10, 10, 10, 10, 10], + [10, 5, 4, 5, 10], + [10, 4, 1, 4, 10], + [10, 5, 4, 5, 10], + [10, 10, 10, 10, 10]], dtype=float)) + + +@pytest.fixture +def plateau(): + """A 5x5 grid with a flat plateau at 3 and one outlet cell at 2.""" + return dem.Elevation(dx=1.0, grid=np.array([ + [5, 5, 5, 5, 5], + [5, 3, 3, 3, 5], + [5, 3, 3, 3, 5], + [5, 3, 3, 2, 5], + [5, 5, 5, 5, 5]], dtype=float)) + + +@pytest.fixture +def v_valley(): + """A 7-column V-shaped valley draining south along the middle column. + + The channel is column 3; every row drains inward then down. + """ + ny, nx = 9, 7 + col = np.abs(np.arange(nx) - 3.0) + z = np.tile(col, (ny, 1)) * 2.0 + z += np.arange(ny, dtype=float).reshape(-1, 1)[::-1] * 0.5 + return dem.Elevation(dx=10.0, grid=z) + + +@pytest.fixture +def rough_dome(): + """A 40x40 noisy dome with real depressions. + + The noise amplitude is chosen so that the surface genuinely has closed + depressions -- a smoother dome makes every filling test vacuous, because + there is nothing to fill. ``test_dome_fixture_has_depressions`` pins + that down. + """ + rng = np.random.default_rng(20240921) + n = 40 + y, x = np.mgrid[0:n, 0:n] / n + z = (300.0 * (1.0 - ((x - 0.5) ** 2 + (y - 0.5) ** 2) / 0.5) + + 12.0 * np.sin(5 * np.pi * x) * np.sin(5 * np.pi * y) + + rng.normal(0.0, 6.0, (n, n))) + # A handful of deliberate pits, so the depressions are not an accident of + # the random seed. + for (i, j) in ((9, 12), (22, 8), (30, 27), (15, 31)): + z[i, j] -= 40.0 + return dem.Elevation(dx=30.0, grid=z) + + +@pytest.fixture +def dome_network(rough_dome): + """``(elevation, filled, flow_direction, area, flow_length)`` for the dome.""" + filled = dem.FilledElevation(elevation=rough_dome) + flow_direction = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=flow_direction) + flow_length = dem.FlowLength(flow_direction=flow_direction) + return rough_dome, filled, flow_direction, area, flow_length + + +@pytest.fixture +def geographic_grid(): + """A 10x10 lat/lon grid near 40 degrees north, 0.01 degree cells.""" + grid = dem.GeographicElevation(dx=0.01, grid=np.zeros((10, 10))) + grid._georef_info.geoTransform = (-120.0, 0.01, 0.0, 40.1, 0.0, -0.01) + grid._georef_info.xllcenter = -119.995 + grid._georef_info.yllcenter = 40.005 + return grid + + +# --------------------------------------------------------------------------- +# Environment markers +# --------------------------------------------------------------------------- + + +def pytest_runtest_setup(item): + """Skip GDAL-dependent tests when the bindings are missing.""" + if any(mark.name == 'gdal' for mark in item.iter_markers()): + if not dem.gdal_is_available(): + pytest.skip('the GDAL Python bindings are not installed') + + +@pytest.fixture +def backend_name(): + """Which kernel backend the test run is exercising.""" + return kernels.backend() diff --git a/tests/test_flow.py b/tests/test_flow.py new file mode 100644 index 0000000..40cc497 --- /dev/null +++ b/tests/test_flow.py @@ -0,0 +1,572 @@ +"""Tests for the flow-routing grid classes.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from TopoAnalysis import dem + + +# --------------------------------------------------------------------------- +# Filling +# --------------------------------------------------------------------------- + + +def test_filled_elevation_removes_the_pit(single_pit): + filled = dem.FilledElevation(elevation=single_pit) + assert np.all(filled._griddata >= single_pit._griddata - 1e-12) + # With the default aggradation slope the fill is just above the rim. + assert filled._griddata[2, 2] > 10.0 + assert filled._griddata[2, 2] == pytest.approx(10.0, abs=1e-6) + assert filled.fill_report['cells_filled'] == 9 + + +def test_flat_fill_option_matches_the_rim_exactly(single_pit): + filled = dem.FilledElevation(elevation=single_pit, aggradation_slope=0.0) + assert np.all(filled._griddata == 10.0) + + +def test_fill_restricted_to_a_mask(single_pit): + """Only masked cells take part; the rest keep their exact values.""" + mask = dem.Mask(dx=1.0, grid=np.zeros((5, 5), dtype=np.uint8)) + mask._griddata[1:4, 1:4] = 1 + masked = dem.FilledElevation(elevation=single_pit, mask=mask) + unmasked = dem.FilledElevation(elevation=single_pit) + + outside = mask._griddata == 0 + assert np.array_equal(masked._griddata[outside], single_pit._griddata[outside]) + # And the mask must actually change the answer. Confined to the 3x3 + # interior, the flood spills over that block's own rim of 4 rather than + # over the grid's 10 m rim. + assert not np.allclose(masked._griddata, unmasked._griddata) + assert masked._griddata[2, 2] == pytest.approx(4.0, abs=1e-6) + assert unmasked._griddata[2, 2] == pytest.approx(10.0, abs=1e-6) + + +def test_fill_from_explicit_outlets(single_pit): + outlet = single_pit._rowscols_to_xy(((2, 0),)) + filled = dem.FilledElevation(elevation=single_pit, outlets=outlet) + assert np.all(filled._griddata >= single_pit._griddata - 1e-12) + + +def test_clip_to_fill_marks_unreached_cells(): + """Cells the flood cannot reach become no-data.""" + # Two basins separated by a wall of NaN: flooding from the left outlet + # cannot reach the right half. + z = np.array([ + [5., 4., 3., np.nan, 3., 4., 5.], + [5., 4., 3., np.nan, 3., 4., 5.], + [5., 4., 3., np.nan, 3., 4., 5.]]) + grid = dem.Elevation(dx=1.0, grid=z) + outlet = grid._rowscols_to_xy(((1, 2),)) + + filled = dem.FilledElevation(elevation=grid, outlets=outlet, clip_to_fill=True) + assert np.isfinite(filled._griddata[:, :3]).all(), 'the left basin is reached' + assert np.isnan(filled._griddata[:, 4:]).all(), 'the right basin is not' + + +def test_clip_to_fill_excises_depressions_left_by_a_depth_limit(): + """A depression too deep to fill is marked no-data, as it always was.""" + ny, nx = 11, 21 + z = np.tile(np.linspace(100.0, 80.0, nx), (ny, 1)) + z[4:7, 9:12] = 50.0 # a 40 m pit + grid = dem.Elevation(dx=10.0, grid=z) + + clipped = dem.FilledElevation(elevation=grid, maximum_pit_depth=5.0, + clip_to_fill=True) + assert int(np.isnan(clipped._griddata).sum()) == 9 + assert np.isnan(clipped._griddata[4:7, 9:12]).all() + + # Without the limit the pit is filled and nothing is clipped. + filled = dem.FilledElevation(elevation=grid, clip_to_fill=True) + assert np.isfinite(filled._griddata).all() + assert filled._griddata[5, 10] > 80.0 + + +def test_findDEMedge_matches_the_flood_seeds(rough_dome): + rows, cols = rough_dome.findDEMedge() + n = rough_dome._georef_info.nx + assert set((rows * n + cols).tolist()) == set( + dem.kernels.default_seeds(rough_dome._griddata).tolist()) + + +def test_randomize_subbasins_with_mask_runs(single_pit): + """The method used to call a name that Python had mangled away.""" + filled = dem.FilledElevation(elevation=single_pit) + mask = dem.Mask(dx=1.0, grid=np.ones((5, 5), dtype=np.uint8)) + outlets = single_pit._rowscols_to_xy(((0, 0),)) + filled.randomize_subbasins_with_mask(mask, outlets) + assert filled._griddata.shape == (5, 5) + + +# --------------------------------------------------------------------------- +# Flow direction +# --------------------------------------------------------------------------- + + +def test_flow_direction_on_a_plane(east_plane): + filled = dem.FilledElevation(elevation=east_plane) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + # Everything flows east except the last column, which leaves the grid. + assert np.all(d8._griddata[:, :-1] == 1) + assert np.all(d8._griddata[:, -1] == 0) + + +def test_flow_direction_covers_the_last_row_and_column(rough_dome): + """The old windowed implementation left the last row and column unrouted. + + On a dome every cell drains outwards, so the only cells without a + direction are those on the perimeter whose steepest neighbour is off the + grid -- never a whole row or column. + """ + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + assert (d8._griddata[1:-1, 1:-1] == 0).sum() == 0 + # The last row and the last column are what the old code skipped + # entirely; each must now be routed except where flow leaves the grid. + for label, line in (('last row', d8._griddata[-1, :]), + ('last column', d8._griddata[:, -1]), + ('first row', d8._griddata[0, :]), + ('first column', d8._griddata[:, 0])): + assert (line != 0).any(), '{0} is entirely unrouted'.format(label) + + +def test_flow_direction_from_an_unfilled_elevation(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + assert np.all(d8._griddata[:, :-1] == 1) + + +def test_get_flow_to_cell_round_trips(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + receivers = d8.receivers + nx = d8._georef_info.nx + + for i in (5, 17, 30): + for j in (4, 22, 38): + row, col, good = d8.get_flow_to_cell(i, j) + if good: + assert receivers[i, j] == row * nx + col + else: + assert receivers[i, j] == -1 + + +def test_upstream_cells_are_the_inverse_of_downstream(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + for (i, j) in ((10, 10), (20, 25), (33, 7)): + for (ui, uj) in d8.get_upstream_cell_indexes(i, j): + assert d8.get_flow_to_cell(ui, uj)[:2] == (i, j) + + +def test_upstream_cell_set_is_closed(rough_dome): + """Everything upstream of a cell must itself drain to that cell.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + upstream = set(d8.get_indexes_of_upstream_cells(20, 20)) + assert (20, 20) in upstream + for (i, j) in upstream: + if (i, j) == (20, 20): + continue + row, col, good = d8.get_flow_to_cell(i, j) + assert good and (row, col) in upstream + + +def test_get_indexes_of_upstream_cells_can_be_iterated_twice(rough_dome): + """The result used to be a single-use zip object.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + indexes = d8.get_indexes_of_upstream_cells(20, 20) + assert len(list(indexes)) == len(list(indexes)) > 0 + + +def test_deep_basins_do_not_need_recursion(): + """A 4000-cell snake would have overflowed the recursive traversal.""" + n = 4000 + z = np.arange(n, 0, -1, dtype=float).reshape(1, n) + grid = dem.Elevation(dx=1.0, grid=z) + d8 = dem.FlowDirectionD8(elevation=grid) + assert len(d8.get_indexes_of_upstream_cells(0, n - 1)) == n + + +def test_pixel_scale_uses_the_true_diagonal(): + codes = np.array([[1, 2], [4, 32]], dtype=np.uint8) + d8 = dem.FlowDirectionD8(dx=1.0, grid=codes) + scale = d8.pixel_scale(np.float64) + assert scale[0, 0] == 1.0 and scale[1, 0] == 1.0 + assert scale[0, 1] == pytest.approx(np.sqrt(2.0)) + assert scale[1, 1] == pytest.approx(np.sqrt(2.0)) + + +def test_rivertools_conversion_rotates_the_codes(): + # RiverTools starts its numbering one position anticlockwise of ArcGIS. + codes = np.array([[1, 2, 4, 8], [16, 32, 64, 128]], dtype=np.uint8) + d8 = dem.FlowDirectionD8(dx=1.0, grid=codes.copy()) + d8.convert_rivertools_directions_to_arc() + assert d8._griddata.tolist() == [[128, 1, 2, 4], [8, 16, 32, 64]] + + +def test_search_down_flow_direction_follows_the_plane(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + start = east_plane._rowscols_to_xy(((2, 0),))[0] + cells, lengths = d8.search_down_flow_direction_with_length(start) + assert [c[1] for c in cells] == [0, 1, 2, 3, 4] + assert list(lengths) == pytest.approx([0.0, 1.0, 2.0, 3.0, 4.0]) + + +def test_divides_have_no_upstream_neighbours(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + divides = d8.divides() + + rows, cols = np.where(divides._griddata == 1) + for (i, j) in list(zip(rows.tolist(), cols.tolist()))[:50]: + assert d8.get_upstream_cell_indexes(i, j) == [] + + +def test_bounds_of_basin_for_outlet(rough_dome): + """Comparing a float with None used to raise on the first iteration.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + ((x, y),) = d8._rowscols_to_xy(((20, 20),)) + + (xr, yr) = d8.bounds_of_basin_for_outlet((y, x)) + assert xr[0] <= x <= xr[1] + assert yr[0] <= y <= yr[1] + + +def test_divides_for_outlets_returns_reusable_sequences(v_valley): + """Two adjoining basins must share a non-empty divide, listed twice over.""" + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + # The valley's outlet, and a point part-way up it: the second basin is + # nested inside the first, so the two adjoin along its whole rim. + a = d8._rowscols_to_xy(((8, 3),))[0] + b = d8._rowscols_to_xy(((0, 3),))[0] + + first, second = d8.divides_for_outlets(a, b) + assert len(first) > 0 and len(second) > 0, 'the basins must actually meet' + # Both must survive a second pass; they used to be single-use zips. + assert len(list(first)) == len(list(first)) == len(first) + assert len(list(second)) == len(list(second)) == len(second) + # Every reported cell belongs to the basin it was reported for. + in_a = d8.basin_mask((a,)) + in_b = d8.basin_mask((b,)) + assert all(in_a[i, j] for (i, j) in first) + assert all(in_b[i, j] for (i, j) in second) + + +# --------------------------------------------------------------------------- +# Accumulation and length +# --------------------------------------------------------------------------- + + +def test_area_conserves_the_total(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + + cell_area = rough_dome._georef_info.dx ** 2 + assert area._griddata.min() == pytest.approx(cell_area) + + outlets = d8.receivers < 0 + assert area._griddata[outlets].sum() == pytest.approx( + cell_area * rough_dome._griddata.size) + + +def test_area_on_a_plane_grows_downstream(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + area = dem.Area(flow_direction=d8) + # Each row is an independent flow line: 1, 2, 3, 4, 5 cells. + assert area._griddata[2].tolist() == [1.0, 2.0, 3.0, 4.0, 5.0] + + +def test_area_ignores_a_loaded_grids_missing_sort_order(rough_dome, tmp_path): + """Accumulation must not depend on an elevation sort being cached.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + expected = dem.Area(flow_direction=d8)._griddata + + # A flow-direction grid built straight from codes has no sort cache, so + # sort() would have ordered cells by their direction *code*. + fresh = dem.FlowDirectionD8(dx=rough_dome._georef_info.dx, + grid=d8._griddata.copy()) + assert np.allclose(dem.Area(flow_direction=fresh)._griddata, expected) + + +def test_area_gate_restricts_accumulation(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + + gate = dem.Mask(dx=rough_dome._georef_info.dx, + grid=np.zeros(rough_dome._griddata.shape, dtype=np.uint8)) + gate._griddata[10:30, 10:30] = 1 + + gated = dem.Area(flow_direction=d8, evaluate_at=gate) + plain = dem.Area(flow_direction=d8) + + # A mask that used to be ignored entirely must now reduce the totals ... + assert gated._griddata.max() < plain._griddata.max() + # ... and no cell can accumulate more than the gate could deliver. + cell = rough_dome._georef_info.dx ** 2 + assert gated._griddata.max() <= cell * int(gate._griddata.sum()) + # Cells outside the gate contribute nothing of their own, so any value + # they hold came from inside it. + assert np.all(gated._griddata <= plain._griddata + 1e-9) + + +def test_areas_between_finds_channel_heads(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + + cell = rough_dome._georef_info.dx ** 2 + heads = area.areas_between(d8, 5 * cell, 20 * cell) + assert len(heads) > 0 + for (x, y) in heads: + ((i, j),) = area._xy_to_rowscols(((x, y),)) + assert 5 * cell <= area._griddata[i, j] <= 20 * cell + + +def test_flow_length_on_a_plane(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + length = dem.FlowLength(flow_direction=d8) + assert length._griddata[2].tolist() == [0.0, 1.0, 2.0, 3.0, 4.0] + + +def test_flow_length_main_stem_uses_the_shared_code_convention(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + length = dem.FlowLength(flow_direction=d8) + # Cell (2,2)'s longest path comes from the west, so the code stored + # there must be 16 (W) in the same convention FlowDirectionD8 uses. + assert length.main_stem_directions[2, 2] == 16 + + +def test_flow_length_path_walks_upstream(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + length = dem.FlowLength(flow_direction=d8) + outlet = east_plane._rowscols_to_xy(((2, 4),))[0] + path = length.indexes_along_flow_path_from_outlet(outlet) + assert [c[1] for c in path] == [4, 3, 2, 1, 0] + + +def test_is_along_flow_length_agrees_with_the_stored_codes(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + length = dem.FlowLength(flow_direction=d8) + + hits = 0 + for i in range(5, 35, 7): + for j in range(5, 35, 7): + row, col, good = d8.get_flow_to_cell(i, j) + if good and length.is_along_flow_length((i, j), (row, col)): + hits += 1 + assert hits > 0 + + +@pytest.mark.gdal +def test_flow_length_round_trip(rough_dome, tmp_path): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + length = dem.FlowLength(flow_direction=d8) + + path = str(tmp_path / 'length.tif') + length.save(path) + back = dem.FlowLength.load(path) + + assert np.allclose(back._griddata, length._griddata) + assert np.array_equal(back.main_stem_directions, length.main_stem_directions) + + +# --------------------------------------------------------------------------- +# Chi, relief and steepness +# --------------------------------------------------------------------------- + + +def test_chi_is_zero_at_the_outlet(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,)) + chi = dem.Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e4, outlets=outlet) + + assert chi._griddata[outlet_rc] == 0.0 + assert chi._griddata.max() > 0.0 + + +def test_chi_trapezoid_option(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,)) + + left = dem.Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e4, outlets=outlet) + trap = dem.Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e4, outlets=outlet, + trapezoid=True) + assert not np.allclose(left._griddata, trap._griddata) + assert trap._griddata.max() <= left._griddata.max() + 1e-9 + + +def test_ksi_uses_the_ratio_of_areas(v_valley): + """chi must integrate (Ao/A)^theta, not (Ao/(A-Ao))^theta.""" + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + length = dem.FlowLength(flow_direction=d8) + + Ao, theta = 200.0, 0.45 + ksi = dem.Ksi(area=area, flow_direction=d8, flow_length=length, + theta=theta, Ao=Ao) + + # Pin the integrand exactly. At a cell where the channel network + # *starts* -- nothing above it on the main stem is on the network -- chi + # is one step of (Ao/A)^theta * step_length and nothing more. The old + # form, (Ao/(A - Ao))^theta, is a different number and diverges as A + # approaches Ao. + step = d8.step_lengths() + a = area._griddata + on_network = a > Ao + + sources = on_network.copy() + codes = length.main_stem_directions + ny, nx = a.shape + for i in range(ny): + for j in range(nx): + code = codes[i, j] + if not code: + continue + for d in range(8): + if code == dem.kernels.D8_CODES[d]: + ui = i + int(dem.kernels.D8_DI[d]) + uj = j + int(dem.kernels.D8_DJ[d]) + if 0 <= ui < ny and 0 <= uj < nx and on_network[ui, uj]: + sources[i, j] = False + break + assert sources.any(), 'the test needs at least one network source' + + expected = (Ao / a[sources]) ** theta * step[sources] + assert np.allclose(ksi._griddata[sources], expected) + + buggy = (Ao / (a[sources] - Ao)) ** theta * step[sources] + assert not np.allclose(ksi._griddata[sources], buggy) + assert np.isfinite(ksi._griddata[on_network]).all() + + +def test_relief_measures_the_drop_to_the_path_head(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + length = dem.FlowLength(flow_direction=d8) + + relief = dem.Relief(flow_direction=d8, elevation=v_valley, flow_length=length) + assert relief._griddata.min() >= -1e-9 + # The outlet of the longest path sees the largest relief. + assert relief._griddata.max() > 0.0 + + +def test_scaled_relief_accepts_a_flooded_dem(v_valley): + """The flooded_dem branch called a method that did not exist.""" + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + length = dem.FlowLength(flow_direction=d8) + + scaled = dem.ScaledRelief(flow_direction=d8, flooded_dem=filled, + elevation=v_valley, flow_length=length, + Ao=1e4, theta=0.45) + assert np.isfinite(scaled._griddata).all() + + +def test_chi_scaled_relief(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,)) + + relief = dem.ChiScaledRelief(elevation=v_valley, flow_direction=d8, + theta=0.45, Ao=1e4, outlets=outlet) + assert relief._griddata[outlet_rc] == pytest.approx(0.0) + assert relief._griddata.max() > 0.0 + + +def test_cross_divide_dchi_runs(rough_dome): + """zip() objects are not subscriptable; this used to raise immediately.""" + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlets = area.areas_greater_than(0.5 * area._griddata.max()) + chi = dem.Chi(area=area, flow_direction=d8, theta=0.45, Ao=1e4, + outlets=outlets[:1]) + + result = dem.CrossDivideDChi(chi=chi, flow_direction=d8) + assert result._griddata.shape == rough_dome._griddata.shape + normalized = dem.NormalizedCrossDivideDChi(chi=chi, flow_direction=d8) + assert np.isfinite(normalized._griddata).all() + + +# --------------------------------------------------------------------------- +# Masks and basins +# --------------------------------------------------------------------------- + + +def test_mask_matches_the_upstream_set(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + outlet = d8._rowscols_to_xy(((25, 25),)) + + mask = dem.Mask(flow_direction=d8, outlets=outlet) + expected = set(d8.get_indexes_of_upstream_cells(25, 25)) + assert int(mask._griddata.sum()) == len(expected) + for (i, j) in list(expected)[:100]: + assert mask._griddata[i, j] == 1 + + +def test_channel_slope_is_positive_downhill(east_plane): + d8 = dem.FlowDirectionD8(elevation=east_plane) + slope = dem.ChannelSlope(flow_direction=d8, elevation=east_plane) + assert np.all(slope._griddata[:, :-1] == pytest.approx(1.0)) + assert np.all(slope._griddata[:, -1] == 0.0) + + +def test_discrete_flow_accumulation_walks_downhill(east_plane): + # The walk needs a full 8-neighbourhood at every step, so it must start + # away from the grid edge. + outlet = east_plane._rowscols_to_xy(((2, 1),)) + dfa = dem.DiscreteFlowAccumulation(elevation=east_plane, outlets=outlet) + # The walk descends east, ties going to the first neighbour in the + # search order, so the path is diagonal; all that matters is that it + # advanced and that area accumulated along it. + visited = dfa._griddata > 0 + assert visited.sum() > 1 + assert dfa._griddata.max() > dfa._griddata[2, 1] + + +def test_restored_elevation_converges(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,)) + + restored = dem.RestoredElevation(flow_direction=d8, elevation=v_valley, + area=area, theta=0.45, ks=20.0, + outlets=outlet, iterations=2, verbose=False) + assert len(restored.convergence) == 2 + assert np.isfinite(restored._griddata).all() + + +def test_deflection_of_a_uniform_load(): + """A flat load must deflect uniformly, at the Airy value.""" + load = dem.Elevation(dx=1000.0, grid=np.full((32, 32), 1000.0)) + rho_c, rho_m, g = 2700.0, 3300.0, 9.81 + + w = dem.Deflection(elevation=load, D=1e23, rho_m=rho_m, rho_c=rho_c, g=g) + airy = -1000.0 * rho_c / (rho_m - rho_c) + assert np.allclose(w._griddata, airy, rtol=1e-6) diff --git a/tests/test_gdal_setup.py b/tests/test_gdal_setup.py new file mode 100644 index 0000000..cf745d1 --- /dev/null +++ b/tests/test_gdal_setup.py @@ -0,0 +1,688 @@ +"""Tests for the GDAL bootstrapper. + +Everything here is offline and installs nothing. The parts that would touch +the network or a package manager are exercised through :func:`make_plan`, +which decides what to do without doing any of it -- that separation is the +main reason the module is shaped the way it is. +""" + +from __future__ import annotations + +import io +import json +import os +import subprocess +import sys +import tarfile + +import pytest + +from TopoAnalysis import _gdal_source, gdal_setup + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _isolate_gdal_environment(): + """Undo every environment change these tests can cause. + + ``activate_environment`` assigns to ``os.environ`` directly, so + ``monkeypatch`` only restores the variables it was itself asked to touch. + ``PROJ_DATA`` in particular was being left behind pointing at a fixture's + deliberately-invalid ``proj.db``, which then broke unrelated tests -- + invisibly on GDAL 3.4, which ignores ``PROJ_DATA``, and visibly on 3.13, + which does not. + """ + names = ("PROJ_LIB", "PROJ_DATA", "GDAL_DATA", "TOPOANALYSIS_NO_GDAL_ENV") + saved = {name: os.environ.get(name) for name in names} + yield + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def make_probe(**fields): + """A Probe with the given raw fields and its problems worked out.""" + config = fields.pop("_config", None) + found = gdal_setup.Probe(**fields) + found.problems = tuple(gdal_setup._diagnose(found, config)) + return found + + +WORKING = { + "bindings_version": "3.9.3", + "has_gdal_array": True, + "projection_works": True, + "gdal_config": "/opt/env/bin/gdal-config", + "_config": {"version": "3.9.3", "prefix": "/nonexistent-prefix", "path": "/opt/env/bin/gdal-config"}, +} + + +# --------------------------------------------------------------------------- +# Version handling +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "text, expected", + [ + ("3.9.3", (3, 9, 3)), + ("3.4.1", (3, 4, 1)), + ("3.10", (3, 10, 0)), + ("3.13.3dev", (3, 13, 3)), + ("", None), + (None, None), + ("not a version", None), + ], +) +def test_parse_version(text, expected): + assert gdal_setup._parse_version(text) == expected + + +def test_bindings_requirements_prefer_an_exact_match(): + """The bindings must match the C library; exact first, then the series.""" + assert gdal_setup._bindings_requirements("3.4.1") == ["gdal==3.4.1", "gdal==3.4.*"] + + +def test_bindings_requirements_without_a_version(): + assert gdal_setup._bindings_requirements(None) == ["gdal"] + + +# --------------------------------------------------------------------------- +# Diagnosis +# --------------------------------------------------------------------------- + + +def test_a_working_install_has_no_problems(): + found = make_probe(**WORKING) + assert found.ok + assert found.problems == () + assert "GDAL is installed and working." in found.describe() + + +def test_missing_bindings_and_library_are_both_reported(): + found = make_probe(bindings_error="ModuleNotFoundError: osgeo", _config=None) + assert set(found.codes) == {"no-bindings", "no-library"} + + +def test_missing_numpy_bridge_is_caught(): + """The trap this module exists for: osgeo imports but ReadAsArray fails.""" + fields = dict(WORKING, has_gdal_array=False, gdal_array_error="ImportError: numpy") + found = make_probe(**fields) + assert found.has("no-gdal-array") + assert any("ReadAsArray" in message for _, message in found.problems) + + +def test_version_mismatch_between_bindings_and_library(): + fields = dict(WORKING, bindings_version="3.4.1") + found = make_probe(**fields) + assert found.has("version-mismatch") + + +def test_patch_level_differences_are_not_a_mismatch(): + """GDAL does not break its C API within a minor version.""" + fields = dict(WORKING, bindings_version="3.9.1") + found = make_probe(**fields) + assert not found.has("version-mismatch") + + +def test_broken_projection_lookup_is_reported(): + fields = dict(WORKING, projection_works=False, projection_error="proj.db missing") + found = make_probe(**fields) + assert found.has("no-proj-data") + + +def test_library_older_than_supported(): + fields = dict( + WORKING, + bindings_version="2.4.4", + _config={"version": "2.4.4", "prefix": "/nonexistent-prefix", "path": "/x/gdal-config"}, + ) + found = make_probe(**fields) + assert found.has("old-library") + + +def test_describe_survives_a_completely_empty_probe(): + found = make_probe() + assert any("not installed" in line for line in found.describe()) + + +# --------------------------------------------------------------------------- +# Choosing a strategy +# --------------------------------------------------------------------------- + + +def test_a_working_install_plans_nothing(): + plan = gdal_setup.make_plan(make_probe(**WORKING)) + assert plan.strategy == "none" + assert plan.steps == () + + +def test_an_existing_library_is_reused_rather_than_reinstalled(monkeypatch): + """Only the bindings are missing, so only the bindings get built.""" + monkeypatch.setattr( + gdal_setup, + "_gdal_config_info", + lambda *a, **k: { + "version": "3.9.3", + "prefix": "/opt/env", + "path": "/opt/env/bin/gdal-config", + }, + ) + found = make_probe( + bindings_error="ModuleNotFoundError: osgeo", + gdal_config="/opt/env/bin/gdal-config", + _config={"version": "3.9.3", "prefix": "/nonexistent-prefix", "path": "/opt/env/bin/gdal-config"}, + ) + plan = gdal_setup.make_plan(found) + assert plan.strategy == "bindings" + joined = " ".join(" ".join(step.command or []) for step in plan.steps) + assert "gdal==3.9.3" in joined + # The whole point: build against the runtime numpy, not an isolated one. + assert "--no-build-isolation" in joined + + +def test_an_architecture_mismatch_does_not_go_back_to_the_package_manager(monkeypatch): + """Homebrew supplied the wrong architecture; asking it again would loop.""" + # Windows short-circuits to the prebuilt-wheel route before any of this, + # so pin the platform rather than let the assertion depend on the runner. + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: None) + monkeypatch.setattr( + gdal_setup, "_system_manager", lambda: ("/opt/homebrew/bin/brew", ["install", "gdal"], False) + ) + found = make_probe(**WORKING) + found.problems = (("arch-mismatch", "library is arm64 but Python is x86_64"),) + strategy, notes = gdal_setup._choose_strategy(found) + assert strategy == "source" + assert any("same architecture" in note for note in notes) + + +def test_a_conda_environment_is_preferred_over_a_source_build(tmp_path, monkeypatch): + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: "/opt/conda/bin/mamba") + # conda-meta is what marks a prefix as a conda environment. + (tmp_path / "conda-meta").mkdir() + found = make_probe(bindings_error="no osgeo", sys_prefix=str(tmp_path)) + assert found.is_conda + strategy, _ = gdal_setup._choose_strategy(found) + assert strategy == "conda" + + +def test_root_is_not_assumed_for_system_packages(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: None) + monkeypatch.setattr( + gdal_setup, "_system_manager", lambda: ("/usr/bin/apt-get", ["install", "-y", "libgdal-dev"], True) + ) + monkeypatch.setattr(gdal_setup, "_is_root", lambda: False) + found = make_probe(bindings_error="no osgeo") + strategy, notes = gdal_setup._choose_strategy(found) + assert strategy == "source" + assert any("administrator rights" in note for note in notes) + + +def test_allow_root_makes_the_package_manager_usable(monkeypatch): + """Otherwise --allow-root would still fall through to a half-hour build.""" + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: None) + monkeypatch.setattr( + gdal_setup, + "_system_manager", + lambda: ("/usr/bin/apt-get", ["install", "-y", "libgdal-dev"], True), + ) + monkeypatch.setattr(gdal_setup, "_is_root", lambda: False) + found = make_probe(bindings_error="no osgeo") + strategy, _ = gdal_setup._choose_strategy(found, allow_root=True) + assert strategy == "system" + + +def test_windows_uses_a_prebuilt_wheel(monkeypatch): + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(gdal_setup, "_conda_executable", lambda: None) + found = make_probe(bindings_error="no osgeo") + strategy, _ = gdal_setup._choose_strategy(found) + assert strategy == "wheel" + + +def test_apt_refreshes_its_package_lists_first(monkeypatch): + """A stock Debian image has no package lists; install alone would fail.""" + monkeypatch.setattr( + gdal_setup, + "_system_manager", + lambda: ("/usr/bin/apt-get", ["install", "-y", "libgdal-dev"], True), + ) + monkeypatch.setattr(gdal_setup, "_is_root", lambda: True) + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="system") + assert plan.steps[0].command == ["/usr/bin/apt-get", "update"] + assert "install" in plan.steps[1].command + + +def test_repairing_broken_bindings_forces_a_rebuild(monkeypatch): + """pip would otherwise say 'already satisfied' and change nothing. + + The commonest repair is a missing gdal_array, which leaves osgeo at + exactly the right version -- and pip's wheel cache holds the bad build. + """ + monkeypatch.setattr( + gdal_setup, + "_gdal_config_info", + lambda *a, **k: { + "version": "3.9.3", + "prefix": "/opt/env", + "path": "/opt/env/bin/gdal-config", + }, + ) + fields = dict(WORKING, has_gdal_array=False, gdal_array_error="ImportError: numpy") + found = make_probe(**fields) + plan = gdal_setup.make_plan(found, strategy="bindings") + command = plan.steps[-1].command + assert "--force-reinstall" in command + assert "--no-cache-dir" in command + + +def test_a_first_install_of_the_bindings_does_not_force(monkeypatch): + """Nothing to displace, so leave pip's cache alone.""" + monkeypatch.setattr( + gdal_setup, + "_gdal_config_info", + lambda *a, **k: { + "version": "3.9.3", + "prefix": "/opt/env", + "path": "/opt/env/bin/gdal-config", + }, + ) + found = make_probe( + bindings_error="ModuleNotFoundError: osgeo", + gdal_config="/opt/env/bin/gdal-config", + _config={ + "version": "3.9.3", + "prefix": "/nonexistent-prefix", + "path": "/opt/env/bin/gdal-config", + }, + ) + plan = gdal_setup.make_plan(found, strategy="bindings") + assert "--force-reinstall" not in plan.steps[-1].command + + +# --------------------------------------------------------------------------- +# Plans +# --------------------------------------------------------------------------- + + +def test_a_source_plan_builds_the_library_then_the_bindings(): + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="source", prefix="/tmp/example-prefix") + summaries = [step.summary for step in plan.steps] + assert any("proj" in text for text in summaries) + assert any("gdal" in text for text in summaries) + # The bindings are always last: they need the library to exist first. + assert "bindings" in summaries[-1] + assert "/tmp/example-prefix" in " ".join(plan.notes) + + +def test_a_source_plan_can_be_described_without_running_anything(): + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="source", prefix="/tmp/example-prefix") + text = "\n".join(plan.describe()) + assert "strategy: source" in text + assert "estimated time" in text + + +def test_a_source_plan_refuses_to_start_without_a_compiler(monkeypatch, capsys): + """Better a sentence naming the missing tool than cmake failing later.""" + monkeypatch.setattr( + _gdal_source, "missing_build_tools", lambda: ["cmake", "a C/C++ compiler"] + ) + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="source", prefix="/tmp/example-prefix") + assert plan.blockers + assert "cmake" in plan.blockers[0] + # The steps are still listed, so `plan` can show what would happen. + assert plan.steps + assert gdal_setup.apply_plan(plan) is False + assert "cannot run on this machine" in capsys.readouterr().out + + +def test_a_source_plan_has_no_blockers_when_the_tools_are_present(monkeypatch): + monkeypatch.setattr(_gdal_source, "missing_build_tools", lambda: []) + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="source", prefix="/tmp/example-prefix") + assert plan.blockers == () + + +def test_forcing_the_bindings_strategy_without_a_library_is_an_error(monkeypatch): + monkeypatch.setattr(gdal_setup, "_gdal_config_info", lambda *a, **k: None) + found = make_probe(bindings_error="no osgeo") + with pytest.raises(ValueError, match="gdal-config"): + gdal_setup.make_plan(found, strategy="bindings") + + +def test_a_system_plan_marks_the_step_that_needs_root(monkeypatch): + monkeypatch.setattr( + gdal_setup, "_system_manager", lambda: ("/usr/bin/apt-get", ["install", "-y", "libgdal-dev"], True) + ) + monkeypatch.setattr(gdal_setup, "_is_root", lambda: False) + found = make_probe(bindings_error="no osgeo") + plan = gdal_setup.make_plan(found, strategy="system") + assert plan.needs_root + assert plan.steps[0].command[0] == "sudo" + + +def test_a_plan_needing_root_refuses_to_run_without_permission(capsys): + plan = gdal_setup.Plan( + "system", + [gdal_setup.Step("install things", command=["sudo", "apt-get", "install"], needs_root=True)], + ) + assert gdal_setup.apply_plan(plan, allow_root=False) is False + printed = capsys.readouterr().out + assert "sudo apt-get install" in printed + assert "--strategy source" in printed + + +def test_dry_run_executes_nothing(capsys): + def explode(log): + raise AssertionError("a dry run must not run anything") + + plan = gdal_setup.Plan("source", [gdal_setup.Step("build the world", action=explode)]) + assert gdal_setup.apply_plan(plan, dry_run=True) is True + assert "build the world" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- +# The prefix manifest +# --------------------------------------------------------------------------- + + +@pytest.fixture +def built_gdal(tmp_path, monkeypatch): + """A recorded source-built GDAL belonging to a fake environment.""" + prefix = tmp_path / "gdal" + (prefix / "share" / "proj").mkdir(parents=True) + (prefix / "share" / "proj" / "proj.db").write_bytes(b"not really a database") + (prefix / "share" / "gdal").mkdir(parents=True) + + environment = tmp_path / "env" + environment.mkdir() + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.delenv("TOPOANALYSIS_NO_GDAL_ENV", raising=False) + # activate_environment only acts for the interpreter the bindings were + # built for, so the test has to be that interpreter. + monkeypatch.setattr(sys, "prefix", str(environment)) + gdal_setup.write_manifest(str(prefix), str(environment)) + return prefix, environment + + +def test_manifest_round_trip(built_gdal): + prefix, environment = built_gdal + manifest = gdal_setup.read_manifest() + assert manifest["prefix"] == str(prefix) + assert manifest["python_prefix"] == str(environment) + assert manifest["proj_lib"].endswith(os.path.join("share", "proj")) + + +def test_the_manifest_is_written_inside_the_environment(built_gdal): + """So two virtualenvs can each have their own build.""" + _, environment = built_gdal + assert (environment / "share" / "topoanalysis" / "gdal-prefix.json").exists() + + +def test_activate_environment_sets_data_paths(built_gdal, monkeypatch): + prefix, _ = built_gdal + monkeypatch.delenv("PROJ_LIB", raising=False) + monkeypatch.delenv("GDAL_DATA", raising=False) + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == str(prefix / "share" / "proj") + assert os.environ["PROJ_DATA"] == str(prefix / "share" / "proj") + assert os.environ["GDAL_DATA"] == str(prefix / "share" / "gdal") + + +def test_activate_environment_displaces_a_stale_setting(built_gdal, monkeypatch): + """The case this exists for: conda exported another GDAL's data paths. + + A PROJ 9 library handed PROJ 8's proj.db refuses every lookup, so leaving + the inherited value in place is not the conservative choice -- it is the + broken one. + """ + prefix, _ = built_gdal + monkeypatch.setenv("PROJ_LIB", "/opt/conda/envs/other/share/proj") + monkeypatch.setenv("GDAL_DATA", "/opt/conda/envs/other/share/gdal") + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == str(prefix / "share" / "proj") + assert os.environ["GDAL_DATA"] == str(prefix / "share" / "gdal") + + +def test_activate_environment_can_be_switched_off(built_gdal, monkeypatch): + monkeypatch.setenv("TOPOANALYSIS_NO_GDAL_ENV", "1") + monkeypatch.setenv("PROJ_LIB", "/somewhere/the/user/chose") + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == "/somewhere/the/user/chose" + + +def test_a_manifest_from_another_environment_is_ignored(built_gdal, monkeypatch): + """Those data files belong to a library this interpreter is not running.""" + monkeypatch.setattr(sys, "prefix", "/some/entirely/different/env") + monkeypatch.setenv("PROJ_LIB", "/opt/conda/envs/other/share/proj") + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == "/opt/conda/envs/other/share/proj" + + +def test_activate_environment_ignores_a_prefix_without_a_database( + tmp_path, monkeypatch +): + """An empty share/proj is not a usable replacement, so leave things be.""" + prefix = tmp_path / "gdal" + (prefix / "share" / "proj").mkdir(parents=True) # no proj.db + environment = tmp_path / "env" + environment.mkdir() + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr(sys, "prefix", str(environment)) + gdal_setup.write_manifest(str(prefix), str(environment)) + monkeypatch.setenv("PROJ_LIB", "/opt/conda/envs/other/share/proj") + gdal_setup.activate_environment() + assert os.environ["PROJ_LIB"] == "/opt/conda/envs/other/share/proj" + + +def test_activate_environment_is_silent_without_a_manifest(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + monkeypatch.setattr(sys, "prefix", str(tmp_path / "nothing-here")) + gdal_setup.activate_environment() # must not raise + + +# --------------------------------------------------------------------------- +# Source pins +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("component", sorted(_gdal_source.SOURCES)) +def test_every_source_is_pinned_by_checksum_over_https(component): + spec = _gdal_source.SOURCES[component] + assert spec["url"].startswith("https://") + assert len(spec["sha256"]) == 64 + assert set(spec["sha256"]) <= set("0123456789abcdef") + + +@pytest.mark.parametrize("component", sorted(_gdal_source.SOURCES)) +def test_the_pinned_url_matches_the_pinned_version(component): + """A version bumped without its URL would download the old tarball.""" + spec = _gdal_source.SOURCES[component] + major, minor, patch = gdal_setup._parse_version(spec["version"]) + plain = "%d.%d.%d" % (major, minor, patch) + # SQLite encodes 3.53.4 in its filenames as 3530400. + encoded = "%d%02d%02d00" % (major, minor, patch) + assert plain in spec["url"] or encoded in spec["url"], spec["url"] + + +def test_library_directory_prefers_what_gdal_config_reports(tmp_path): + """On Fedora and RHEL a 64-bit libgdal lives in lib64, not lib.""" + lib64 = tmp_path / "lib64" + lib64.mkdir() + (tmp_path / "lib").mkdir() + suffix = ".dylib" if sys.platform == "darwin" else ".so" + (lib64 / ("libgdal" + suffix)).write_bytes(b"") + + assert gdal_setup._library_directory( + {"libs": "-L%s -lgdal" % lib64, "prefix": str(tmp_path)} + ) == str(lib64) + + +def test_library_directory_finds_the_library_when_there_is_no_minus_l(tmp_path): + """Guessing 'lib' would point the RPATH at an empty directory.""" + lib64 = tmp_path / "lib64" + lib64.mkdir() + (tmp_path / "lib").mkdir() # exists but holds nothing + suffix = ".dylib" if sys.platform == "darwin" else ".so" + (lib64 / ("libgdal" + suffix)).write_bytes(b"") + + assert gdal_setup._library_directory( + {"libs": "-lgdal", "prefix": str(tmp_path)} + ) == str(lib64) + + +def test_a_too_old_cmake_is_caught_before_the_build_starts(monkeypatch): + """PROJ 9 and GDAL 3.13 both declare a 3.16 floor.""" + monkeypatch.setattr(_gdal_source, "_cmake_version", lambda cmake: (3, 10)) + monkeypatch.setattr(_gdal_source, "_c_compiler", lambda: "/usr/bin/cc") + monkeypatch.setattr(_gdal_source, "_cxx_compiler", lambda: "/usr/bin/c++") + monkeypatch.setattr(_gdal_source.shutil, "which", lambda name: "/usr/bin/" + name) + missing = _gdal_source.missing_build_tools() + assert any("3.16" in item and "3.10" in item for item in missing) + + +def test_a_recent_enough_cmake_is_accepted(monkeypatch): + monkeypatch.setattr(_gdal_source, "_cmake_version", lambda cmake: (3, 16)) + monkeypatch.setattr(_gdal_source, "_c_compiler", lambda: "/usr/bin/cc") + monkeypatch.setattr(_gdal_source, "_cxx_compiler", lambda: "/usr/bin/c++") + monkeypatch.setattr(_gdal_source.shutil, "which", lambda name: "/usr/bin/" + name) + assert _gdal_source.missing_build_tools() == [] + + +def test_an_unreadable_cmake_version_does_not_block_the_build(monkeypatch): + """An unknown answer must not veto a build that would have worked.""" + monkeypatch.setattr(_gdal_source, "_cmake_version", lambda cmake: None) + monkeypatch.setattr(_gdal_source, "_c_compiler", lambda: "/usr/bin/cc") + monkeypatch.setattr(_gdal_source, "_cxx_compiler", lambda: "/usr/bin/c++") + monkeypatch.setattr(_gdal_source.shutil, "which", lambda name: "/usr/bin/" + name) + assert _gdal_source.missing_build_tools() == [] + + +def test_a_source_build_needs_a_cxx_compiler_not_just_a_c_one(monkeypatch): + """PROJ and GDAL are C++17; a bare gcc without g++ is a real machine.""" + monkeypatch.setattr(_gdal_source, "_cxx_compiler", lambda: None) + monkeypatch.setattr(_gdal_source, "_c_compiler", lambda: "/usr/bin/cc") + monkeypatch.setattr(_gdal_source.shutil, "which", lambda name: "/usr/bin/" + name) + assert "a C++ compiler" in _gdal_source.missing_build_tools() + + +def test_a_stale_proj_variable_is_diagnosed_not_reinstalled(): + """Reinstalling GDAL would leave the offending variable exactly where it is.""" + found = make_probe( + bindings_version="3.9.3", + has_gdal_array=True, + projection_works=False, + projection_error="RuntimeError: proj.db lacks DATABASE.LAYOUT.VERSION", + gdal_config="/opt/env/bin/gdal-config", + proj_env={"PROJ_LIB": "/opt/conda/envs/other/share/proj"}, + _config={ + "version": "3.9.3", + "prefix": "/nonexistent-prefix", + "path": "/opt/env/bin/gdal-config", + }, + ) + assert found.codes == ("no-proj-data",) + plan = gdal_setup.make_plan(found) + assert plan.strategy == "proj-data" + assert plan.steps == () + assert any("unset PROJ_LIB" in blocker for blocker in plan.blockers) + # The offending value is named, because that is the whole diagnosis. + assert any("/opt/conda/envs/other/share/proj" in m for _, m in found.problems) + + +def test_build_order_covers_every_source(): + assert set(_gdal_source.BUILD_ORDER) == set(_gdal_source.SOURCES) + + +def test_overriding_a_pin_requires_its_checksum(monkeypatch): + monkeypatch.setenv("TOPOANALYSIS_GDAL_SOURCE_GDAL_URL", "https://example.org/g.tar.gz") + monkeypatch.delenv("TOPOANALYSIS_GDAL_SOURCE_GDAL_SHA256", raising=False) + with pytest.raises(_gdal_source.BuildError, match="must be set together"): + _gdal_source.apply_environment_overrides() + + +def test_components_needed_is_a_subset_of_the_build_order(): + needed = _gdal_source.components_needed() + assert set(needed) <= set(_gdal_source.BUILD_ORDER) + # PROJ and GDAL are never optional. + assert "proj" in needed and "gdal" in needed + + +def test_extracting_a_hostile_archive_is_refused(tmp_path): + """A tarball that writes outside its directory must not be unpacked.""" + archive = tmp_path / "evil.tar.gz" + payload = io.BytesIO(b"pwned") + with tarfile.open(archive, "w:gz") as tar: + info = tarfile.TarInfo("../escaped.txt") + info.size = len(payload.getvalue()) + tar.addfile(info, payload) + + with pytest.raises((_gdal_source.BuildError, tarfile.TarError)): + _gdal_source._extract(str(archive), str(tmp_path / "into")) + assert not (tmp_path / "escaped.txt").exists() + + +def test_a_corrupt_download_is_rejected(tmp_path): + """A cached file with the wrong checksum is discarded, not built.""" + cache = tmp_path / "cache" + cache.mkdir() + (cache / "thing.tar.gz").write_bytes(b"not what was pinned") + with pytest.raises(_gdal_source.BuildError): + # The URL is unreachable, so the only way this could succeed is by + # trusting the bad cached copy. + _gdal_source._download( + "https://localhost:1/thing.tar.gz", "0" * 64, str(cache), lambda line: None + ) + + +# --------------------------------------------------------------------------- +# The command line +# --------------------------------------------------------------------------- + + +def test_doctor_reports_on_the_running_interpreter(capsys): + code = gdal_setup.main(["doctor"]) + printed = capsys.readouterr().out + assert "GDAL bindings" in printed + assert code in (0, 1) + + +def test_plan_prints_a_plan_without_installing(capsys): + code = gdal_setup.main(["plan", "--strategy", "source", "--prefix", "/tmp/x"]) + printed = capsys.readouterr().out + assert code == 0 + assert "strategy: source" in printed + + +def test_installer_dry_run_is_harmless(tmp_path): + """``install.py --dry-run`` must not create or modify anything.""" + import TopoAnalysis + + root = os.path.dirname(os.path.abspath(TopoAnalysis.__file__)) + script = os.path.join(root, "install.py") + if not os.path.exists(script): # an installed copy may not ship it + pytest.skip("install.py is not part of this installation") + + before = set(os.listdir(root)) + result = subprocess.run( + [sys.executable, script, "--dry-run"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=root, + ) + assert result.returncode == 0, result.stdout.decode("utf8", "replace") + assert "stopping before anything is installed" in result.stdout.decode("utf8", "replace") + assert set(os.listdir(root)) == before diff --git a/tests/test_grid.py b/tests/test_grid.py new file mode 100644 index 0000000..5949e1c --- /dev/null +++ b/tests/test_grid.py @@ -0,0 +1,402 @@ +"""Tests for :class:`TopoAnalysis.dem.BaseSpatialGrid` and its geometry.""" + +from __future__ import annotations + +import os + +import numpy as np +import pytest + +from TopoAnalysis import dem, error + + +# --------------------------------------------------------------------------- +# Construction and dispatch +# --------------------------------------------------------------------------- + + +def test_grid_from_array_sets_a_usable_geotransform(): + data = np.arange(12, dtype=float).reshape(3, 4) + grid = dem.BaseSpatialGrid(dx=2.0, grid=data) + + assert grid._georef_info.nx == 4 + assert grid._georef_info.ny == 3 + assert grid._georef_info.dx == 2.0 + # A grid built in memory must still be georeferenced well enough to save. + gt = grid._georef_info.geoTransform + assert gt[1] == 2.0 and gt[5] == -2.0 + assert gt[0] == pytest.approx(grid._georef_info.xllcenter - 1.0) + assert gt[3] == pytest.approx(grid._georef_info.yllcenter + 2.5 * 2.0) + + +def test_unsatisfiable_keywords_raise_a_helpful_error(): + with pytest.raises(error.InputError) as excinfo: + dem.BaseSpatialGrid(not_a_real_keyword=1) + # The message must list what the class does accept. + assert 'gdal_filename' in excinfo.value.msg + + +def test_random_grid_is_bounded_and_seedable(): + """Values lie in [0, 1); seeding NumPy makes the grid reproducible.""" + grid = dem.BaseSpatialGrid(nx=6, ny=4, dx=1.0) + assert grid._griddata.shape == (4, 6) + assert grid._griddata.min() >= 0.0 and grid._griddata.max() < 1.0 + # Two grids drawn in a row differ -- it is a random grid, after all ... + assert not np.array_equal(grid._griddata, + dem.BaseSpatialGrid(nx=6, ny=4, dx=1.0)._griddata) + # ... but reseeding NumPy's global generator reproduces one exactly. + np.random.seed(1234) + first = dem.BaseSpatialGrid(nx=6, ny=4, dx=1.0)._griddata.copy() + np.random.seed(1234) + second = dem.BaseSpatialGrid(nx=6, ny=4, dx=1.0)._griddata + assert np.array_equal(first, second) + + +# --------------------------------------------------------------------------- +# Coordinates +# --------------------------------------------------------------------------- + + +def test_xy_and_rowcol_round_trip(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((7, 5))) + for row in range(7): + for col in range(5): + ((x, y),) = grid._rowscols_to_xy(((row, col),)) + assert grid._xy_to_rowscols(((x, y),)) == ((row, col),) + + +def test_row_zero_is_the_north_edge(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + ((_, y_top),) = grid._rowscols_to_xy(((0, 0),)) + ((_, y_bottom),) = grid._rowscols_to_xy(((2, 0),)) + assert y_top > y_bottom + + +def test_points_outside_the_grid_are_rejected(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + info = grid._georef_info + # One whole cell beyond the eastern edge -- this used to come back as a + # valid subscript of nx, which then raised IndexError downstream. + outside = (info.xllcenter + info.nx * info.dx, info.yllcenter) + assert grid._xy_to_rowscols((outside,)) == ((None, None),) + + +def test_getitem_returns_none_outside_the_grid(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.ones((3, 3))) + assert grid[1, 1] == 1.0 + assert grid[-1, 0] is None + assert grid[3, 0] is None + assert grid[0, 3] is None + + +def test_setitem_ignores_out_of_range_writes(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + grid[3, 0] = 5.0 # one past the last row + grid[0, 3] = 5.0 # one past the last column + grid[-1, 0] = 5.0 + assert not grid._griddata.any() + + +def test_coordinate_vectors_have_one_entry_per_cell(): + grid = dem.BaseSpatialGrid(dx=2.0, grid=np.zeros((4, 6))) + x, y = grid.coordinate_vectors() + # np.arange(start, start + (n-1)*dx, dx) yields n-1 values, which is what + # get_XY_matricies used to return. + assert x.size == 6 and y.size == 4 + X, Y = grid.get_XY_matricies() + assert X.shape == (4, 6) and Y.shape == (4, 6) + assert grid.get_XY_matrices()[0].shape == (4, 6) + + +def test_extent_covers_the_pixel_edges(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((3, 4))) + xmin, xmax, ymin, ymax = grid.extent() + assert xmax - xmin == pytest.approx(4 * 10.0) + assert ymax - ymin == pytest.approx(3 * 10.0) + + +# --------------------------------------------------------------------------- +# Clipping, tiling, resampling +# --------------------------------------------------------------------------- + + +def test_clip_to_extent_keeps_the_requested_cells(): + data = np.arange(100, dtype=float).reshape(10, 10) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + ((x0, y0),) = grid._rowscols_to_xy(((7, 2),)) # south-west corner + ((x1, y1),) = grid._rowscols_to_xy(((3, 6),)) # north-east corner + clipped = grid.clip_to_extent((x0, x1, y0, y1)) + + assert clipped._griddata.shape == (5, 5) + assert np.array_equal(clipped._griddata, data[3:8, 2:7]) + # The clip must be georeferenced where it was cut from. + assert clipped._rowscols_to_xy(((0, 0),))[0] == pytest.approx((x0, y1)) + + +def test_clip_rejects_an_extent_that_misses_the_grid(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((5, 5))) + with pytest.raises(error.InputError): + grid.clip_to_extent((1000.0, 1010.0, 1000.0, 1010.0)) + + +def test_extent_of_data_ignores_nodata_margins(): + data = np.full((8, 10), np.nan) + data[2:5, 3:7] = 1.0 + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + xmin, xmax, ymin, ymax = grid.extent_of_data() + ((x_left, y_bottom),) = grid._rowscols_to_xy(((4, 3),)) + ((x_right, y_top),) = grid._rowscols_to_xy(((2, 6),)) + assert (xmin, xmax, ymin, ymax) == pytest.approx( + (x_left, x_right, y_bottom, y_top)) + + +def test_tile_then_mosaic_reproduces_the_grid(): + rng = np.random.default_rng(4) + data = rng.random((37, 41)) + grid = dem.BaseSpatialGrid(dx=5.0, grid=data) + + padding = 4 + tiles = grid.tile(tile_xdim=16, tile_ydim=16, + tile_xpadding=padding, tile_ypadding=padding) + unpadded = [t.remove_padding(xpadding=padding, ypadding=padding) for t in tiles] + rebuilt = dem.BaseSpatialGrid.mosaic(unpadded) + + assert rebuilt._griddata.shape == data.shape + assert np.allclose(rebuilt._griddata, data) + assert rebuilt._georef_info.xllcenter == pytest.approx(grid._georef_info.xllcenter) + assert rebuilt._georef_info.yllcenter == pytest.approx(grid._georef_info.yllcenter) + + +def test_resample_preserves_a_linear_ramp(): + """A plane must resample to the same plane at any spacing.""" + nx = ny = 21 + x = np.arange(nx) * 4.0 + data = np.tile(x, (ny, 1)) + grid = dem.BaseSpatialGrid(dx=4.0, grid=data) + + coarse = grid.resample(8.0, interpolation='linear') + assert coarse._georef_info.dx == 8.0 + assert coarse._griddata.shape[1] == pytest.approx(nx // 2, abs=1) + + # The value at each new cell centre must equal the plane's value there. + xs, _ = coarse.coordinate_vectors() + expected = np.tile(xs - grid._georef_info.xllcenter, (coarse._georef_info.ny, 1)) + assert np.allclose(coarse._griddata, expected, atol=1e-8) + + +def test_resample_rejects_an_unknown_interpolation(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((6, 6))) + with pytest.raises(error.InputError): + grid.resample(2.0, interpolation='bilinear-ish') + + +# --------------------------------------------------------------------------- +# Sorting and value lookup +# --------------------------------------------------------------------------- + + +def test_sort_orders_cells_by_value(): + data = np.array([[3.0, 1.0], [4.0, 2.0]]) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + ascending = grid.sort(reverse=False) + assert data.reshape(-1)[ascending].tolist() == [1.0, 2.0, 3.0, 4.0] + descending = grid.sort(reverse=True) + assert data.reshape(-1)[descending].tolist() == [4.0, 3.0, 2.0, 1.0] + + +def test_sort_with_a_mask_keeps_only_masked_cells(): + data = np.array([[3.0, 1.0], [4.0, 2.0]]) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + mask = dem.BaseSpatialGrid(dx=1.0, grid=np.array([[1.0, 0.0], [1.0, 0.0]])) + + indices = grid.sort(reverse=False, force=True, mask=mask) + # Only the left column (values 3 and 4) survives, in ascending order. + assert data.reshape(-1)[indices].tolist() == [3.0, 4.0] + + +def test_find_nearest_cell_searches_the_full_radius(): + data = np.zeros((9, 9)) + data[4, 8] = 10.0 # four cells east of the centre + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + # range(i-r, i+r) stopped one short of the radius, so the far cell was + # invisible at exactly r = 4. + assert grid.find_nearest_cell_with_greatest_value((4, 4), pixel_radius=4) == (4, 8) + + +def test_find_nearest_cell_does_not_wrap_around_the_grid(): + data = np.zeros((5, 5)) + data[0, 0] = 1.0 + data[4, 4] = 99.0 # would be reached by a negative index + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + assert grid.find_nearest_cell_with_greatest_value((0, 0), pixel_radius=2) == (0, 0) + + +def test_find_nearest_cell_returns_none_when_there_is_no_data(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.full((5, 5), np.nan)) + assert grid.find_nearest_cell_with_greatest_value((2, 2), 2) == (None, None) + + +def test_set_value_at_rowscols_handles_an_empty_list(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + grid.set_value_at_rowscols(1.0, []) + assert not grid._griddata.any() + grid.set_value_at_rowscols(1.0, [(0, 0), (2, 2)]) + assert grid._griddata[0, 0] == 1.0 and grid._griddata[2, 2] == 1.0 + + +def test_value_grid_set_value_at_indexes(): + grid = dem.ValueGrid(dx=1.0, grid=np.zeros((3, 3))) + # zip() is a lazy iterator in Python 3; using one as a subscript raised. + grid.set_value_at_indexes([(0, 1), (2, 2)], 7.0) + assert grid._griddata[0, 1] == 7.0 and grid._griddata[2, 2] == 7.0 + + +# --------------------------------------------------------------------------- +# Smoothing and derivatives +# --------------------------------------------------------------------------- + + +def test_average_over_distance_preserves_a_constant_field(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.full((21, 21), 5.0)) + averaged = grid.average_over_distance(3.0) + assert averaged.shape == (21, 21) + assert np.allclose(averaged, 5.0) + + +@pytest.mark.parametrize('shape', [(21, 21), (20, 20), (20, 25)]) +def test_average_over_distance_is_centred(shape): + """A point source must smear symmetrically about its own cell.""" + data = np.zeros(shape) + centre = (shape[0] // 2, shape[1] // 2) + data[centre] = 1.0 + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + averaged = grid.average_over_distance(2.0) + + # The answer is the normalised disc, centred on the source cell. An + # off-by-half-a-cell kernel would place it one cell away. + ny, nx = shape + di = np.abs(np.arange(ny) - centre[0]) + dj = np.abs(np.arange(nx) - centre[1]) + di = np.minimum(di, ny - di) + dj = np.minimum(dj, nx - dj) + inside = np.hypot(di[:, None], dj[None, :]) <= 2.0 + expected = inside / inside.sum() + + assert np.allclose(averaged, expected, atol=1e-12) + assert averaged.sum() == pytest.approx(1.0) + + +def test_average_over_distance_rejects_a_radius_that_selects_nothing(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((5, 5))) + with pytest.raises(error.InputError): + grid.average_over_distance(-1.0) + + +def test_gradient_over_length_scale_on_a_plane(): + """A plane of slope 0.5 must give 0.5 whatever the measuring length.""" + nx = ny = 21 + data = np.tile(np.arange(nx) * 0.5, (ny, 1)) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + for length in (1.0, 3.0, 5.0): + sx, sy = grid.calculate_gradient_over_length_scale(length) + n = int(np.ceil(length)) + assert np.allclose(sx[n:-n, n:-n], 0.5) + assert np.allclose(sy[n:-n, n:-n], 0.0, atol=1e-12) + assert np.isnan(sx[0, 0]) + + +def test_laplacian_over_length_scale_on_a_quadratic(): + """d2/dx2 of x^2 is 2, at any measuring length.""" + nx = ny = 25 + x = np.arange(nx, dtype=float) + data = np.tile(x ** 2, (ny, 1)) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + for length in (1.0, 2.0, 4.0): + curvature = grid.calculate_laplacian_over_length_scale(length) + n = int(np.ceil(length)) + assert np.allclose(curvature[n:-n, n:-n], 2.0) + + +def test_length_scale_larger_than_the_grid_is_rejected(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((5, 5))) + with pytest.raises(error.InputError): + grid.calculate_gradient_over_length_scale(10.0) + + +def test_principal_curvatures_of_a_sphere_cap(): + """Both principal curvatures of a paraboloid of revolution are equal.""" + n = 41 + y, x = np.mgrid[-20:21, -20:21].astype(float) + data = -0.001 * (x ** 2 + y ** 2) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + k1, k2 = grid.principal_curvatures() + centre = (slice(15, 26), slice(15, 26)) + assert np.allclose(k1._griddata[centre], k2._griddata[centre], atol=2e-4) + assert np.all(np.isfinite(k1._griddata[centre])) + + +def test_assign_bcs_replicates_every_corner(): + grid = dem.Elevation(dx=1.0, grid=np.arange(9, dtype=float).reshape(3, 3)) + padded = grid.assignBCs(grid._griddata, 3, 3) + assert padded[0, 0] == grid._griddata[0, 0] + assert padded[0, -1] == grid._griddata[0, -1] + assert padded[-1, 0] == grid._griddata[-1, 0] + # The south-east corner used to copy the south-west one. + assert padded[-1, -1] == grid._griddata[-1, -1] + + +# --------------------------------------------------------------------------- +# File round-trips +# --------------------------------------------------------------------------- + + +@pytest.mark.gdal +def test_geotiff_round_trip(tmp_path): + rng = np.random.default_rng(9) + data = rng.random((11, 13)) * 100.0 + data[3, 4] = np.nan + grid = dem.Elevation(dx=25.0, grid=data) + + path = str(tmp_path / 'grid.tif') + grid.save(path) + back = dem.Elevation.load(path) + + assert np.array_equal(back._griddata, data, equal_nan=True) + assert back._georef_info.nx == 13 and back._georef_info.ny == 11 + assert back._georef_info.dx == pytest.approx(25.0) + assert back._georef_info.xllcenter == pytest.approx(grid._georef_info.xllcenter) + assert back._georef_info.yllcenter == pytest.approx(grid._georef_info.yllcenter) + + +@pytest.mark.gdal +def test_ascii_grid_round_trip_maps_nodata(tmp_path): + data = np.array([[1.0, 2.0], [np.nan, 4.0]]) + grid = dem.Elevation(dx=1.0, grid=data) + + path = str(tmp_path / 'grid.asc') + grid.write_to_ai(path) + text = open(path).read() + assert 'NODATA_value -9999' in text + assert 'nan' not in text.lower() + + back = dem.Elevation(ai_ascii_filename=path, EPSGprojectionCode=32610) + # The no-data flag must come back as NaN, not as an elevation of -9999. + assert np.isnan(back._griddata[1, 0]) + assert back._griddata[0, 0] == pytest.approx(1.0) + + +@pytest.mark.gdal +def test_save_flushes_to_disk(tmp_path): + grid = dem.Elevation(dx=1.0, grid=np.ones((4, 4))) + path = str(tmp_path / 'flushed.tif') + grid.save(path) + assert os.path.getsize(path) > 0 diff --git a/tests/test_kernels.py b/tests/test_kernels.py new file mode 100644 index 0000000..3ea262f --- /dev/null +++ b/tests/test_kernels.py @@ -0,0 +1,365 @@ +"""Tests for the compute kernels. + +Two things are checked here: + +1. that the compiled C++ kernels and the pure-NumPy ones agree exactly, so + the two backends are interchangeable; and +2. that the Priority-Flood implementation has the properties Barnes et al. + (2014) prove it has -- the filled surface is minimal, monotone, and + independent of which of the paper's algorithms produced it. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from TopoAnalysis import fastops, kernels + + +FILL_CASES = [ + ('flat', 0.0, 0.0), + ('epsilon', 0.0, 0.0), + ('epsilon', 1e-3, 0.0), + ('flat', 0.0, 3.0), + ('epsilon', 1e-4, 5.0), +] + + +def random_surface(seed, ny=17, nx=23, nodata_fraction=0.0): + rng = np.random.default_rng(seed) + z = rng.random((ny, nx)) * 20.0 + if nodata_fraction: + z[rng.random(z.shape) < nodata_fraction] = np.nan + return z + + +def build_network(z, cellsize=1.0): + """``(filled, codes, receivers, order)`` for a surface.""" + filled = np.ascontiguousarray(z, dtype=float) + kernels.priority_flood(filled, mode='epsilon', epsilon=1e-9, cellsize=cellsize) + codes = kernels.flow_directions(filled, cellsize=cellsize) + receivers = kernels.receivers(codes) + order, cycles = kernels.topological_order(receivers) + assert cycles == 0 + return filled, codes, np.asarray(receivers), np.asarray(order) + + +# --------------------------------------------------------------------------- +# Backend agreement +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('seed', [0, 1, 2, 3]) +@pytest.mark.parametrize('mode,epsilon,max_depth', FILL_CASES) +def test_fill_backends_agree(seed, mode, epsilon, max_depth): + z = random_surface(seed, nodata_fraction=0.08 if seed % 2 else 0.0) + + a, b = z.copy(), z.copy() + ra = kernels.priority_flood(a, mode=mode, epsilon=epsilon, + max_pit_depth=max_depth, track_visited=True) + rb = fastops.priority_flood(b, mode=mode, epsilon=epsilon, + max_pit_depth=max_depth, track_visited=True) + + assert np.array_equal(a, b, equal_nan=True) + assert np.array_equal(np.asarray(ra['visited']), np.asarray(rb['visited'])) + assert ra['cells_filled'] == rb['cells_filled'] + assert ra['max_fill_depth'] == pytest.approx(rb['max_fill_depth']) + + +@pytest.mark.parametrize('seed', [4, 5]) +def test_routing_backends_agree(seed): + z = random_surface(seed) + filled, codes, receivers, order = build_network(z) + + assert np.array_equal(codes, fastops.flow_directions(filled, cellsize=1.0)) + assert np.array_equal(receivers, fastops.receivers(codes)) + + weights = np.ones_like(filled) + gate = (filled > np.median(filled)).astype(np.uint8) + py_order, _ = fastops.topological_order(receivers) + + assert np.allclose(kernels.accumulate(receivers, order, weights), + fastops.accumulate(receivers, py_order, weights)) + assert np.allclose(kernels.accumulate(receivers, order, weights, gate), + fastops.accumulate(receivers, py_order, weights, gate)) + + length, codes_c = kernels.flow_length(receivers, order, weights) + length_py, codes_py = fastops.flow_length(receivers, py_order, weights) + assert np.allclose(length, length_py) + assert np.array_equal(codes_c, codes_py) + + outlets = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + assert np.array_equal(kernels.upstream_mask(receivers, outlets), + fastops.upstream_mask(receivers, outlets)) + assert np.array_equal(kernels.drainage_basins(receivers, order), + fastops.drainage_basins(receivers, py_order)) + + accumulated = np.asarray(kernels.accumulate(receivers, order, weights)) + for trapezoid in (False, True): + chi_c, dist_c = kernels.chi(receivers, order, accumulated, weights, + outlets, 1.0, 0.45, trapezoid, 0.0, None) + chi_py, dist_py = fastops.chi(receivers, py_order, accumulated, weights, + outlets, 1.0, 0.45, trapezoid, 0.0, None) + assert np.allclose(chi_c, chi_py, equal_nan=True) + assert np.allclose(dist_c, dist_py, equal_nan=True) + + assert np.allclose(kernels.downstream_distance(receivers, order, weights), + fastops.downstream_distance(receivers, py_order, weights)) + + carved_c, carved_py = filled.copy(), filled.copy() + kernels.imposemin(receivers, order, weights, 1e-3, carved_c) + fastops.imposemin(receivers, py_order, weights, 1e-3, carved_py) + assert np.allclose(carved_c, carved_py) + + streams = (accumulated > 5).astype(np.uint8) + for kind in ('strahler', 'shreve'): + assert np.array_equal(kernels.stream_order(receivers, order, streams, kind), + fastops.stream_order(receivers, py_order, streams, kind)) + + +# --------------------------------------------------------------------------- +# Priority-Flood properties (Barnes et al. 2014) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize('seed', [10, 11, 12]) +@pytest.mark.parametrize('mode,epsilon', [('flat', 0.0), ('epsilon', 1e-6)]) +def test_improved_algorithm_matches_the_reference(seed, mode, epsilon): + """Algorithm 2 must return exactly what Algorithm 1 returns.""" + z = random_surface(seed) + improved, reference = z.copy(), z.copy() + kernels.priority_flood(improved, mode=mode, epsilon=epsilon) + kernels.priority_flood(reference, mode=mode, epsilon=epsilon, + reference_algorithm=True) + assert np.array_equal(improved, reference) + + +@pytest.mark.parametrize('seed', [20, 21]) +def test_filling_never_lowers_a_cell(seed): + z = random_surface(seed) + filled = z.copy() + kernels.priority_flood(filled, mode='flat') + assert np.all(filled >= z - 1e-12) + + +def test_flat_fill_produces_the_spill_elevation(single_pit): + """Every interior cell of the bowl ends up at the 10 m rim.""" + filled = np.ascontiguousarray(single_pit._griddata, dtype=float) + report = kernels.priority_flood(filled, mode='flat') + assert np.all(filled == 10.0) + assert report['cells_filled'] == 9 + assert report['max_fill_depth'] == pytest.approx(9.0) + + +def test_epsilon_fill_leaves_no_flats(single_pit): + """Priority-Flood+epsilon must leave a strictly descending path.""" + filled = np.ascontiguousarray(single_pit._griddata, dtype=float) + kernels.priority_flood(filled, mode='epsilon', epsilon=1e-3, cellsize=1.0) + interior = filled[1:-1, 1:-1] + assert np.all(interior > 10.0) + # The deepest cell must end up highest: it is furthest from the spill. + assert filled[2, 2] == interior.max() + + +def test_max_pit_depth_leaves_deep_depressions_alone(single_pit): + filled = np.ascontiguousarray(single_pit._griddata, dtype=float) + kernels.priority_flood(filled, mode='flat', max_pit_depth=2.0) + # The 9 m deep centre stays put ... + assert filled[2, 2] == pytest.approx(1.0) + # ... but the flood still reached the far side of it. + assert np.all(np.isfinite(filled)) + + +def test_nodata_acts_as_an_outlet(): + """A depression touching no-data drains into it and is not filled.""" + z = np.array([ + [10, 10, 10, 10, 10], + [10, 5, 5, 5, 10], + [np.nan, 5, 1, 5, 10], + [10, 5, 5, 5, 10], + [10, 10, 10, 10, 10]], dtype=float) + filled = z.copy() + kernels.priority_flood(filled, mode='flat') + # (2,1) sits next to the NaN, so it keeps its own elevation ... + assert filled[2, 1] == pytest.approx(5.0) + # ... and the pit only rises to the level it must cross to get there. + assert filled[2, 2] == pytest.approx(5.0) + assert np.isnan(filled[2, 0]) + + +def test_default_seeds_are_the_perimeter_and_nodata_fringe(): + z = np.arange(25, dtype=float).reshape(5, 5) + z[2, 2] = np.nan + seeds = set(kernels.default_seeds(z).tolist()) + + perimeter = {i * 5 + j for i in range(5) for j in range(5) + if i in (0, 4) or j in (0, 4)} + nan_fringe = {(2 + di) * 5 + (2 + dj) + for di in (-1, 0, 1) for dj in (-1, 0, 1)} - {2 * 5 + 2} + + assert seeds == perimeter | nan_fringe + + +# --------------------------------------------------------------------------- +# Flow routing +# --------------------------------------------------------------------------- + + +def test_d8_codes_follow_the_arcgis_convention(): + """Each of the eight codes must point at the neighbour it names.""" + # 32 64 128 + # 16 X 1 + # 8 4 2 + expected = {(0, 1): 1, (1, 1): 2, (1, 0): 4, (1, -1): 8, + (0, -1): 16, (-1, -1): 32, (-1, 0): 64, (-1, 1): 128} + for (di, dj), code in expected.items(): + z = np.ones((3, 3)) + z[1 + di, 1 + dj] = 0.0 + codes = kernels.flow_directions(z, cellsize=1.0) + assert codes[1, 1] == code, 'offset {0} should give code {1}'.format((di, dj), code) + + +def test_diagonals_compete_on_gradient_not_on_drop(): + """A diagonal neighbour only wins if its *gradient* is steeper.""" + z = np.ones((3, 3)) * 10.0 + z[1, 2] = 9.0 # east: drop 1 over 1 -> gradient 1.00 + z[2, 2] = 8.6 # south-east: drop 1.4 over sqrt(2) -> gradient 0.99 + assert kernels.flow_directions(z, cellsize=1.0)[1, 1] == 1 + + z[2, 2] = 8.5 # gradient 1.06, now steeper than east + assert kernels.flow_directions(z, cellsize=1.0)[1, 1] == 2 + + +def test_accumulation_conserves_weight(rough_dome): + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=30.0) + weights = np.ones_like(z) + accumulated = np.asarray(kernels.accumulate(receivers, order, weights)) + + outlets = receivers.reshape(-1) < 0 + # Everything that enters the grid must leave through some outlet. + assert accumulated.reshape(-1)[outlets].sum() == pytest.approx(weights.sum()) + assert accumulated.min() >= 1.0 + + +def test_accumulation_ignores_the_order_it_is_given(rough_dome): + """Any valid topological order must give the same accumulation.""" + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=30.0) + other_order, _ = fastops.topological_order(receivers) + weights = np.ones_like(z) + assert np.allclose(kernels.accumulate(receivers, order, weights), + kernels.accumulate(receivers, other_order, weights)) + + +def test_flow_length_tie_break_is_deterministic(): + """Equal-length paths must resolve the same way whatever the order.""" + z = np.array([ + [2.0, 3.0, 2.0], + [3.0, 1.0, 3.0], + [2.0, 3.0, 2.0]]) + _, _, receivers, order = build_network(z) + other_order, _ = fastops.topological_order(receivers) + step = np.ones_like(z) + _, codes_a = kernels.flow_length(receivers, order, step) + _, codes_b = kernels.flow_length(receivers, other_order, step) + assert np.array_equal(codes_a, codes_b) + + +def test_upstream_mask_covers_the_whole_basin(rough_dome): + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=30.0) + outlets = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + mask = np.asarray(kernels.upstream_mask(receivers, outlets)) + # Every cell drains to some outlet, so the union covers the grid. + assert mask.all() + + +def test_chi_is_zero_at_the_outlet_and_grows_upstream(v_valley): + z = np.ascontiguousarray(v_valley._griddata, dtype=float) + filled, _, receivers, order = build_network(z, cellsize=10.0) + step = np.full(z.shape, 10.0) + area = np.asarray(kernels.accumulate(receivers, order, np.full(z.shape, 100.0))) + + outlets = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + chi, distance = kernels.chi(receivers, order, area, step, outlets, + 1e4, 0.45, True, 0.0, None) + chi = np.asarray(chi) + + assert np.nanmin(chi) == 0.0 + # chi must increase monotonically upstream: every cell exceeds its receiver. + flat_chi = chi.reshape(-1) + for k, r in enumerate(receivers.reshape(-1)): + if r >= 0 and np.isfinite(flat_chi[k]) and np.isfinite(flat_chi[r]): + assert flat_chi[k] >= flat_chi[r] - 1e-12 + + +def test_trapezoid_and_left_endpoint_chi_differ_predictably(v_valley): + """The trapezoid rule must sit between the two endpoint rules.""" + z = np.ascontiguousarray(v_valley._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=10.0) + step = np.full(z.shape, 10.0) + area = np.asarray(kernels.accumulate(receivers, order, np.full(z.shape, 100.0))) + outlets = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + + left, _ = kernels.chi(receivers, order, area, step, outlets, 1e4, 0.45, False, 0.0, None) + trap, _ = kernels.chi(receivers, order, area, step, outlets, 1e4, 0.45, True, 0.0, None) + valid = np.isfinite(left) & np.isfinite(trap) + # Area grows downstream, so the integrand shrinks downstream and the + # left-endpoint (upstream) value is the larger of the two. + assert np.all(np.asarray(trap)[valid] <= np.asarray(left)[valid] + 1e-9) + + +def test_imposemin_only_lowers_and_enforces_the_gradient(rough_dome): + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + filled, _, receivers, order = build_network(z, cellsize=30.0) + step = np.full(z.shape, 30.0) + + carved = filled.copy() + kernels.imposemin(receivers, order, step, 1e-3, carved) + + assert np.all(carved <= filled + 1e-12) + flat = carved.reshape(-1) + for k, r in enumerate(receivers.reshape(-1)): + if r >= 0: + assert flat[k] - flat[r] >= 1e-3 * 30.0 - 1e-9 + + +def test_strahler_order_of_a_simple_confluence(): + """Two order-1 streams meeting make an order-2 stream.""" + z = np.array([[3.0, 9.0, 3.0], + [9.0, 2.0, 9.0], + [9.0, 1.0, 9.0]]) + _, _, receivers, order = build_network(z) + streams = np.array([[1, 0, 1], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) + + strahler = np.asarray(kernels.stream_order(receivers, order, streams, 'strahler')) + assert strahler[0, 0] == 1 and strahler[0, 2] == 1 + assert strahler[1, 1] == 2 and strahler[2, 1] == 2 + + shreve = np.asarray(kernels.stream_order(receivers, order, streams, 'shreve')) + assert shreve[1, 1] == 2 and shreve[2, 1] == 2 + + +def test_drainage_basins_partition_the_grid(rough_dome): + z = np.ascontiguousarray(rough_dome._griddata, dtype=float) + _, _, receivers, order = build_network(z, cellsize=30.0) + labels = np.asarray(kernels.drainage_basins(receivers, order)) + + assert labels.min() >= 1, 'every cell drains somewhere on this surface' + # Each cell must carry the same label as the cell it drains into. + flat = labels.reshape(-1) + for k, r in enumerate(receivers.reshape(-1)): + if r >= 0: + assert flat[k] == flat[r] + + +def test_topological_order_reports_cycles(): + """A hand-made loop must be reported, not silently mis-ordered.""" + # Two cells pointing at each other: 1 -> east, 16 -> west. + codes = np.array([[1, 16]], dtype=np.uint8) + receivers = kernels.receivers(codes) + order, cycles = kernels.topological_order(receivers) + assert cycles == 2 + assert sorted(np.asarray(order).tolist()) == [0, 1] diff --git a/tests/test_modules.py b/tests/test_modules.py new file mode 100644 index 0000000..595667b --- /dev/null +++ b/tests/test_modules.py @@ -0,0 +1,319 @@ +"""Tests for the supporting modules: windows, datasets, profiles, the CLI.""" + +from __future__ import annotations + +import os + +import numpy as np +import pytest + +import matplotlib +matplotlib.use('Agg') # no display in CI + +from TopoAnalysis import (MovingWindow, analysis, datasets, dem, + demRecursionTools, error, plotting, utils) +from TopoAnalysis import cli + + +# --------------------------------------------------------------------------- +# MovingWindow +# --------------------------------------------------------------------------- + + +class WindowMean(MovingWindow.CircularMovingWindow): + function = staticmethod(np.mean) + + +class WindowRange(MovingWindow.RectangularMovingWindow): + function = staticmethod(lambda values: values.max() - values.min()) + + +def test_moving_window_needs_a_radius(): + with pytest.raises(error.InputError): + WindowMean() + + +def test_abstract_windows_cannot_be_instantiated(): + with pytest.raises(error.Error): + MovingWindow.MovingWindow(window_radius=1.0) + with pytest.raises(error.Error): + MovingWindow.RectangularMovingWindow(window_radius=1.0) + with pytest.raises(error.Error): + MovingWindow.CircularMovingWindow(window_radius=1.0) + + +def test_circular_window_is_a_disc_not_a_band(): + """The kernel used to square the row offset twice, giving a column.""" + window = WindowMean(window_radius=2.0) + rows, cols = window._build_search_kernel(1.0) + offsets = set(zip(rows.tolist(), cols.tolist())) + assert (0, 2) in offsets and (2, 0) in offsets + assert (2, 2) not in offsets, 'the corner is outside a radius-2 disc' + + +def test_moving_window_mean_of_a_constant_field(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.full((9, 9), 4.0)) + # The method used to call a mangled name that did not exist, and to pass + # the grid where the cell size was expected. + result = grid.apply_moving_window(WindowMean(window_radius=2.0)) + assert np.allclose(result._griddata, 4.0) + + +def test_moving_window_ignores_nodata(): + data = np.ones((7, 7)) + data[3, 3] = np.nan + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + result = grid.apply_moving_window(WindowMean(window_radius=1.0)) + # The NaN is dropped from the window rather than poisoning it. + assert result._griddata[3, 3] == pytest.approx(1.0) + + +def test_rectangular_window_range(): + data = np.tile(np.arange(9, dtype=float), (9, 1)) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + result = grid.apply_moving_window(WindowRange(window_radius=1.0)) + assert result._griddata[4, 4] == pytest.approx(2.0) + + +def test_window_dimension_is_still_accepted(): + """The original keyword name must keep working.""" + window = WindowMean(window_dimension=3.0) + assert window.window_radius == 3.0 + + +# --------------------------------------------------------------------------- +# datasets +# --------------------------------------------------------------------------- + + +def test_triangle_grid_has_the_requested_shape_and_relief(): + grid = datasets.triangle_grid(30, 40, width=10, amp=5.0) + assert grid._griddata.shape == (30, 40) + assert grid._griddata.min() == pytest.approx(0.0) + assert grid._griddata.max() == pytest.approx(10.0, rel=0.1) + + +def test_sinusoid_grid_spans_the_requested_amplitude(): + grid = datasets.sinusoid_grid(20, 40, width=10, amp=3.0) + row = grid._griddata[0] + assert np.allclose(grid._griddata, row, atol=1e-12), 'every row is the same' + assert row.min() == pytest.approx(0.0) + assert row.max() == pytest.approx(6.0, rel=0.05) + # Four ridges across 40 columns at a spacing of 10. + peaks = np.flatnonzero((row[1:-1] > row[:-2]) & (row[1:-1] > row[2:])) + 1 + assert len(peaks) == 4 + + +def test_slope_adds_a_ramp_rather_than_scaling_the_relief(): + """`triangle *= tilt` zeroed row 0 and amplified the rest.""" + flat = datasets.triangle_grid(20, 20, width=5, amp=1.0) + sloped = datasets.triangle_grid(20, 20, width=5, amp=1.0, slope_y=2.0) + assert not np.allclose(sloped._griddata[0], 0.0) + # The added ramp is the only difference between the two. + difference = sloped._griddata - flat._griddata + assert np.allclose(difference, difference[:, [0]]) + assert difference[1, 0] - difference[0, 0] == pytest.approx( + 2.0 * 20.0 / 19.0, rel=1e-6) + + +def test_synthetic_grids_are_georeferenced_enough_to_route(): + grid = datasets.triangle_grid(24, 24, width=6, amp=4.0, sig=0.05) + filled = dem.FilledElevation(elevation=grid) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + assert dem.Area(flow_direction=d8)._griddata.max() > 1.0 + # A grid built in memory used to have no geotransform, so save() failed. + assert grid._georef_info.geoTransform != 0 + + +# --------------------------------------------------------------------------- +# Recursive profile tools +# --------------------------------------------------------------------------- + + +@pytest.fixture +def profile_network(v_valley): + filled = dem.FilledElevation(elevation=v_valley) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlet = area._rowscols_to_xy((outlet_rc,))[0] + return v_valley, d8, area, outlet + + +def test_map_values_to_recursive_list_structure(profile_network): + elevation, d8, area, outlet = profile_network + tree = d8.map_values_to_recursive_list(outlet, elevation=elevation, area=area) + + assert 'index' in tree and 'distance_scale' in tree + assert tree['distance_scale'] == 1.0, 'the root has no parent step' + assert tree['area'] == pytest.approx(area._griddata.max()) + + +def test_distance_scale_describes_the_step_from_the_parent(profile_network): + """It used to be overwritten on the parent by each child in turn.""" + elevation, d8, area, outlet = profile_network + tree = d8.map_values_to_recursive_list(outlet, elevation=elevation, area=area) + + stack = [tree] + checked = 0 + while stack: + node = stack.pop() + parent_index = node['index'] + for child in node.get('next', []): + di = child['index'][0] - parent_index[0] + dj = child['index'][1] - parent_index[1] + expected = np.sqrt(2.0) if (di and dj) else 1.0 + assert child['distance_scale'] == pytest.approx(expected) + checked += 1 + stack.append(child) + assert checked > 0 + + +def test_recursive_map_covers_the_whole_basin(profile_network): + elevation, d8, area, outlet = profile_network + tree = d8.map_values_to_recursive_list(outlet, elevation=elevation, area=area) + + seen = set() + stack = [tree] + while stack: + node = stack.pop() + seen.add(tuple(node['index'])) + stack.extend(node.get('next', [])) + + ((row, col),) = d8._xy_to_rowscols((outlet,)) + assert seen == set(d8.get_indexes_of_upstream_cells(row, col)) + + +def test_chi_elevation_and_ks_fit(profile_network): + elevation, d8, area, outlet = profile_network + de = area._mean_pixel_dimension() + tree = d8.map_values_to_recursive_list(outlet, elevation=elevation, area=area) + + e, c = demRecursionTools.chi_elevation(tree, de, np.array([0.45]), xo=10.0) + assert e.size == c.size > 0 + assert c.min() == pytest.approx(0.0) + + ks, r2 = demRecursionTools.best_ks_with_r2_list(tree, de, np.array([0.45]), xo=10.0) + assert np.isfinite(ks).all() + + +def test_hypsometric_integral_is_between_zero_and_one(profile_network): + elevation, d8, area, outlet = profile_network + dA = dem.BaseSpatialGrid(dx=area._georef_info.dx, + grid=area._area_per_pixel()) + hi = demRecursionTools.hi(elevation, d8, dA, outlet) + assert 0.0 <= hi <= 1.0 + + +def test_calc_ks_for_outlet_without_an_xo(profile_network): + """kwargs.pop('xo') used to raise KeyError when xo was left at default.""" + elevation, d8, area, outlet = profile_network + ks, r2 = utils.calc_ks_for_outlet(outlet, 0.45, xo=10.0, flow_direction=d8, + elevation=elevation, area=area) + assert np.isfinite(ks).all() + + +def test_map_chi_profiles(profile_network): + elevation, d8, area, outlet = profile_network + chi_map = demRecursionTools.map_chi_profiles( + elevation, d8, area, outlet, minimum_area=100.0, theta=0.45) + assert len(chi_map) > 0 + for (chi, relief) in chi_map.values(): + assert np.isfinite(chi) + + +# --------------------------------------------------------------------------- +# Plotting (smoke tests -- Agg backend, no display) +# --------------------------------------------------------------------------- + + +def test_plot_downstream_profile(profile_network): + elevation, d8, area, outlet = profile_network + rows, cols, length, profile = plotting.plot_downstream_profile( + elevation, d8, outlet, 'k-') + assert rows.size == cols.size == length.size == profile.size > 0 + + +def test_plot_two_grids(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + slope = dem.MaxSlope(elevation=rough_dome) + + # Passing xlabel used to leave the local name unbound and raise. + axes = dem.plot(area, slope, xlabel='area', ylabel='slope', + interactive=False, decimation_factor=7) + assert axes.get_xlabel() == 'area' + + +def test_grid_plot_accepts_an_axes(rough_dome): + import matplotlib.pyplot as plt + + fig, ax = plt.subplots() + returned = rough_dome.plot(ax=ax, interactive=False, colorbar=False) + assert returned is fig + plt.close(fig) + + +# --------------------------------------------------------------------------- +# Quadrats +# --------------------------------------------------------------------------- + + +def test_quadrats_partition_the_data(): + data = np.arange(100, dtype=float).reshape(10, 10) + quadrats = analysis.Quadrats(data=data, dx=5, dy=5) + assert len(quadrats.quadrats) == 4 + assert all(q.shape == (5, 5) for q in quadrats.quadrats) + assert quadrats.map_quadrats(np.sum) == [ + float(data[r:r + 5, c:c + 5].sum()) for r in (0, 5) for c in (0, 5)] + + +def test_quadrats_without_rasterio_message(): + quadrats = analysis.Quadrats(data=np.zeros((4, 4))) + try: + import rasterio # noqa: F401 + except ImportError: + with pytest.raises(ImportError, match='rasterio'): + quadrats.load_data('nowhere.tif') + + +# --------------------------------------------------------------------------- +# Command line +# --------------------------------------------------------------------------- + + +def test_cli_info(capsys): + assert cli.main(['--info']) == 0 + out = capsys.readouterr().out + assert 'compute backend' in out + assert 'Barnes' in out + + +def test_cli_rejects_an_unknown_product(): + with pytest.raises(SystemExit): + cli.main(['dem.tif', '--products', 'not-a-product']) + + +def test_cli_requires_a_dem(): + with pytest.raises(SystemExit): + cli.main([]) + + +@pytest.mark.gdal +def test_cli_end_to_end(tmp_path, rough_dome): + source = str(tmp_path / 'dem.tif') + rough_dome.save(source) + + outdir = str(tmp_path / 'out') + assert cli.main([source, '-o', outdir, '--products', + 'filled,flowdir,area,length,slope,hillshade', + '--min-area', '9000']) == 0 + + produced = sorted(os.listdir(outdir)) + for name in ('dem_filled.tif', 'dem_flowdir.tif', 'dem_area.tif', + 'dem_length.tif', 'dem_slope.tif', 'dem_hillshade.tif'): + assert name in produced + + back = dem.Area.load(os.path.join(outdir, 'dem_area.tif')) + assert back._griddata.max() > 0.0 diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..543ad43 --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,642 @@ +"""One test per repaired defect, so none of them can quietly return. + +Each test names the specific failure it guards against. Defects already +covered by the behavioural suites (filling, routing, terrain attributes) are +not repeated here. +""" + +from __future__ import annotations + +import importlib +import os +import subprocess +import sys + +import numpy as np +import pytest + +from TopoAnalysis import dem, error, fastops, kernels + +#: The package directory, and the directory that must be on sys.path for +#: ``import TopoAnalysis`` to resolve. +PACKAGE_DIR = os.path.dirname(os.path.abspath(dem.__file__)) +IMPORT_ROOT = os.path.dirname(PACKAGE_DIR) + + +def run_in_subprocess(script, **extra_env): + """Run ``script`` in a fresh interpreter that can import TopoAnalysis.""" + env = dict(os.environ, PYTHONPATH=IMPORT_ROOT, **extra_env) + return subprocess.run([sys.executable, '-c', script], capture_output=True, + text=True, cwd=IMPORT_ROOT, env=env) + + +# --------------------------------------------------------------------------- +# Module-level hygiene +# --------------------------------------------------------------------------- + + +def test_no_removed_numpy_or_scipy_aliases_remain(): + """np.NAN, np.float and scipy.ndimage.morphology were all removed upstream.""" + package = PACKAGE_DIR + banned = ('np.NAN', 'np.NaN', 'np.float)', 'np.int)', 'np.bool)', + 'scipy.ndimage.morphology', 'scipy.ndimage.filters', + 'interp2d', 'plt.hold') + + offenders = [] + for name in sorted(os.listdir(package)): + if not name.endswith('.py'): + continue + text = open(os.path.join(package, name)).read() + for token in banned: + # The tests themselves may name these tokens when describing them. + if token in text and not name.startswith('test_'): + offenders.append('{0}: {1}'.format(name, token)) + assert offenders == [] + + +def test_import_does_not_require_gdal(): + """GDAL must be imported lazily, so the package loads without it.""" + script = ( + 'import sys, types;' + 'blocker = types.ModuleType("osgeo");' + 'blocker.__getattr__ = lambda name: (_ for _ in ()).throw(ImportError());' + 'sys.modules["osgeo"] = blocker;' + 'import TopoAnalysis;' + 'print(TopoAnalysis.gdal_is_available())' + ) + result = run_in_subprocess(script) + assert result.returncode == 0, result.stderr + + +def test_recursion_limit_is_left_alone(): + """The module used to raise it to a million, which segfaults before it trips.""" + assert sys.getrecursionlimit() < 100000 + + +def test_constructor_dispatch_does_not_use_eval(): + """Constructors are selected with getattr, not by compiling source text.""" + import ast + + tree = ast.parse(open(dem.__file__).read()) + called = {node.func.id for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} + assert 'eval' not in called + assert 'exec' not in called + + +def test_both_import_styles_work(): + """`import dem` (flat) and `from TopoAnalysis import dem` must both work.""" + script = ('import sys; sys.path.insert(0, {0!r}); ' + 'import dem, kernels; print(kernels.backend())').format(PACKAGE_DIR) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() in ('c++', 'python') + + +# --------------------------------------------------------------------------- +# Geometry +# --------------------------------------------------------------------------- + + +def test_extent_of_data_handles_a_non_square_grid(): + """The column scan used to index rows, and ran off the end when nx != ny.""" + data = np.full((4, 12), np.nan) + data[1:3, 5:9] = 1.0 + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + xmin, xmax, ymin, ymax = grid.extent_of_data() + assert xmax > xmin and ymax > ymin + + +def test_tile_geotransform_is_self_consistent(): + """The tile transform put the north edge one cell too far north.""" + grid = dem.BaseSpatialGrid(dx=3.0, grid=np.arange(400, dtype=float).reshape(20, 20)) + for tile in grid.tile(tile_xdim=8, tile_ydim=8, tile_xpadding=2, tile_ypadding=2): + info = tile._georef_info + # yllcenter must invert out of the transform the same way the reader + # derives it. + assert info.geoTransform[3] - info.dx * (info.ny - 0.5) == pytest.approx( + info.yllcenter) + + +def test_flow_length_clip_keeps_the_direction_codes(): + """clip_to_extent used to copy the *lengths* into the direction grid.""" + n = 12 + z = np.tile(np.arange(n, 0, -1, dtype=float), (n, 1)) + grid = dem.Elevation(dx=1.0, grid=z) + d8 = dem.FlowDirectionD8(elevation=grid) + length = dem.FlowLength(flow_direction=d8) + + ((x0, y0),) = length._rowscols_to_xy(((9, 2),)) + ((x1, y1),) = length._rowscols_to_xy(((2, 9),)) + clipped = length.clip_to_extent((x0, x1, y0, y1)) + + codes = clipped.main_stem_directions + assert codes.shape == clipped._griddata.shape + assert codes.dtype == np.uint8 + # Direction codes are powers of two (or 0), never flow lengths. + assert set(np.unique(codes).tolist()) <= {0, 1, 2, 4, 8, 16, 32, 64, 128} + + +# --------------------------------------------------------------------------- +# Flow routing +# --------------------------------------------------------------------------- + + +def test_flow_code_for_position_prefers_the_steeper_gradient(): + """The per-cell re-router divided cardinals, not diagonals, by sqrt(2).""" + z = np.ones((3, 3)) * 10.0 + z[1, 2] = 9.0 # east, gradient 1.00 + z[2, 2] = 8.6 # south-east, gradient 0.99 + grid = dem.Elevation(dx=1.0, grid=z) + + d8 = dem.FlowDirectionD8(dx=1.0, grid=np.zeros((3, 3), dtype=np.uint8)) + mask = dem.Mask(dx=1.0, grid=np.zeros((3, 3), dtype=np.uint8)) + mask._griddata[1, 1] = 1 + d8.update_flow_codes_in_mask(grid, mask) + assert d8._griddata[1, 1] == 1, 'the steeper cardinal must win' + + z[2, 2] = 8.5 # south-east, gradient 1.06 + grid = dem.Elevation(dx=1.0, grid=z) + d8.update_flow_codes_in_mask(grid, mask) + assert d8._griddata[1, 1] == 2, 'the steeper diagonal must win' + + +def test_update_flow_codes_invalidates_the_cached_network(): + z = np.tile(np.arange(5, 0, -1, dtype=float), (5, 1)) + grid = dem.Elevation(dx=1.0, grid=z) + d8 = dem.FlowDirectionD8(elevation=grid) + first = d8.receivers.copy() + + mask = dem.Mask(dx=1.0, grid=np.ones((5, 5), dtype=np.uint8)) + flipped = dem.Elevation(dx=1.0, grid=np.fliplr(z).copy()) + d8.update_flow_codes_in_mask(flipped, mask) + + assert not np.array_equal(d8.receivers, first) + + +def test_mainstem_valley_area_differs_from_valley_area(): + """The override was name-mangled out of reach, so the two were identical.""" + n = 24 + y, x = np.mgrid[0:n, 0:n].astype(float) + z = 100.0 - y * 2.0 + np.abs(x - n / 2) * 1.5 + elevation = dem.Elevation(dx=10.0, grid=z) + + filled = dem.FilledElevation(elevation=elevation) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + laplace = dem.Laplacian(elevation=elevation) + + common = dict(flow_direction=d8, area=area, laplace=laplace, + valley_laplace_value=-1e9, min_area_value=1e3) + whole = dem.ValleyArea(**common) + mainstem = dem.MainstemValleyArea(**common) + + assert not np.allclose(whole._griddata, mainstem._griddata) + # Only the trunk contributes, so the mainstem totals cannot exceed the + # full accumulation anywhere. + assert np.all(mainstem._griddata <= whole._griddata + 1e-9) + + +def test_area_gate_actually_gates(): + """`mask[i, j] is not None` was always true, so the mask did nothing.""" + n = 10 + z = np.tile(np.arange(n, 0, -1, dtype=float), (n, 1)) + d8 = dem.FlowDirectionD8(elevation=dem.Elevation(dx=1.0, grid=z)) + + gate = dem.Mask(dx=1.0, grid=np.zeros((n, n), dtype=np.uint8)) + gate._griddata[:, :5] = 1 + + gated = dem.Area(flow_direction=d8, evaluate_at=gate) + plain = dem.Area(flow_direction=d8) + + # Inside the gate, accumulation is unaffected ... + assert gated._griddata[0, 4] == pytest.approx(plain._griddata[0, 4]) + # ... the first cell outside it still receives what the gate delivers ... + assert gated._griddata[0, 5] == pytest.approx(plain._griddata[0, 4]) + # ... but passes nothing further on, and adds nothing of its own. + assert gated._griddata[0, 6] == 0.0 + assert gated._griddata[0, 9] == 0.0 + assert plain._griddata[0, 9] == pytest.approx(10.0) + + +# --------------------------------------------------------------------------- +# Regression grids that fit a model +# --------------------------------------------------------------------------- + + +@pytest.fixture +def smoothing_inputs(): + n = 30 + y, x = np.mgrid[0:n, 0:n].astype(float) + z = 200.0 - y * 3.0 + np.abs(x - n / 2) * 2.0 + elevation = dem.Elevation(dx=20.0, grid=z) + filled = dem.FilledElevation(elevation=elevation) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + return elevation, d8, area + + +def test_ks_from_chi_with_smoothing_counts_each_crossing(smoothing_inputs): + """`self._n[rows, cols] += 1` only counted a repeated cell once.""" + pytest.importorskip('statsmodels') + elevation, d8, area = smoothing_inputs + + ks = dem.KsFromChiWithSmoothing( + elevation=elevation, area=area, flow_direction=d8, theta=0.45, + vertical_interval=15.0, area_threshold=2000.0, verbose=False) + + assert ks._n.sum() > 0 + assert ks._n.max() > 1, 'busy cells must be counted once per profile' + assert np.isfinite(ks._griddata[~np.isnan(ks._griddata)]).all() + + +def test_channel_slope_with_smoothing_accepts_a_horizontal_interval(smoothing_inputs): + """The horizontal_interval entry pointed at a method that did not exist.""" + elevation, d8, area = smoothing_inputs + slope = dem.ChannelSlopeWithSmoothing( + elevation=elevation, area=area, flow_direction=d8, + horizontal_interval=100.0, min_area=2000.0, verbose=False) + assert np.isfinite(slope._griddata[~np.isnan(slope._griddata)]).all() + + +def test_channel_up_and_down_slope_variants(smoothing_inputs): + elevation, d8, area = smoothing_inputs + common = dict(elevation=elevation, area=area, flow_direction=d8, + vertical_interval=15.0, min_area=2000.0, verbose=False) + down = dem.ChannelDownSlopeWithSmoothing(**common) + up = dem.ChannelUpSlopeWithSmoothing(**common) + assert down._griddata.shape == up._griddata.shape + assert not np.array_equal(np.nan_to_num(down._griddata), + np.nan_to_num(up._griddata)) + + +def test_theta_from_chi_with_smoothing(smoothing_inputs): + pytest.importorskip('statsmodels') + elevation, d8, area = smoothing_inputs + theta = dem.ThetaFromChiWithSmoothing( + elevation=elevation, area=area, flow_direction=d8, + vertical_interval=15.0, min_area=2000.0, verbose=False) + fitted = theta._griddata[~np.isnan(theta._griddata)] + assert fitted.size > 0 + assert np.all(np.abs(fitted) <= 10.0), 'the optimiser is bounded at +/-10' + + +def test_geographic_theta_inherits_the_theta_class(): + """It used to inherit KsFromChiWithSmoothing and compute ks instead.""" + assert issubclass(dem.GeographicThetaFromChiWithSmoothing, + dem.ThetaFromChiWithSmoothing) + assert issubclass(dem.GeographicKsFromChiWithSmoothing, + dem.KsFromChiWithSmoothing) + + +# --------------------------------------------------------------------------- +# Profile tools +# --------------------------------------------------------------------------- + + +def test_hypsometric_integral_detects_nodata(): + """`np.nan in array` is always False, so NaNs slipped through.""" + from TopoAnalysis import demRecursionTools + + tree = {'elevation': np.nan, 'dA': 1.0, + 'next': [{'elevation': 5.0, 'dA': 1.0}]} + assert demRecursionTools.hi_list(tree) == 0.0 + + +def test_tributary_step_length_multiplies_the_scale(): + """`trb_ld['de'] + trb_ld['distance_scale']` should have been a product.""" + from TopoAnalysis import demRecursionTools + + source = open(demRecursionTools.__file__).read() + assert "trb_ld['de'] * trb_ld['distance_scale']" in source + + +# --------------------------------------------------------------------------- +# Backend equivalence at the library level +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not kernels.have_extension(), + reason='the compiled extension is not built') +def test_pure_python_backend_gives_the_same_pipeline(tmp_path): + """The whole pipeline must agree between the two backends.""" + script = ''' +import os +os.environ["TOPOANALYSIS_PURE_PYTHON"] = "1" +import numpy as np +import TopoAnalysis.kernels as kernels +assert kernels.backend() == "python" +import TopoAnalysis.dem as dem + +rng = np.random.default_rng(99) +n = 25 +y, x = np.mgrid[0:n, 0:n] / n +z = 100*(1 - ((x-0.5)**2 + (y-0.5)**2)) + rng.normal(0, 0.5, (n, n)) +e = dem.Elevation(dx=30.0, grid=z) +f = dem.FilledElevation(elevation=e) +d8 = dem.FlowDirectionD8(flooded_dem=f) +a = dem.Area(flow_direction=d8) +l = dem.FlowLength(flow_direction=d8) +np.save(os.environ["OUT"], np.stack([f._griddata, d8._griddata.astype(float), + a._griddata, l._griddata])) +''' + out = str(tmp_path / 'python_backend.npy') + result = run_in_subprocess(script, OUT=out) + assert result.returncode == 0, result.stderr + + rng = np.random.default_rng(99) + n = 25 + y, x = np.mgrid[0:n, 0:n] / n + z = 100 * (1 - ((x - 0.5) ** 2 + (y - 0.5) ** 2)) + rng.normal(0, 0.5, (n, n)) + e = dem.Elevation(dx=30.0, grid=z) + f = dem.FilledElevation(elevation=e) + d8 = dem.FlowDirectionD8(flooded_dem=f) + a = dem.Area(flow_direction=d8) + l = dem.FlowLength(flow_direction=d8) + + expected = np.stack([f._griddata, d8._griddata.astype(float), + a._griddata, l._griddata]) + assert np.allclose(np.load(out), expected) + + +def test_kernels_expose_the_barnes_reference(): + assert 'Barnes' in kernels.PRIORITY_FLOOD_REFERENCE + assert '2014' in kernels.PRIORITY_FLOOD_REFERENCE + assert kernels.backend() in ('c++', 'python') + + +def test_fastops_and_kernels_expose_the_same_api(): + shared = [name for name in fastops.__all__ if not name.startswith('D8_')] + for name in shared: + assert hasattr(kernels, name), name + + +# --------------------------------------------------------------------------- +# Defects found by the post-rewrite review +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not kernels.have_extension(), + reason='the compiled extension is not built') +def test_kernels_are_thread_safe(): + """Every binding must resolve its buffers before releasing the GIL. + + ``py::array::request()`` calls ``PyObject_GetBuffer``, which touches + reference counts. Calling it inside a ``gil_scoped_release`` block + corrupted the interpreter -- leaked references, and reproducible aborts -- + as soon as two threads ran a kernel at once. + """ + import threading + + rng = np.random.default_rng(1) + z = np.ascontiguousarray(rng.random((40, 40)) * 10) + kernels.priority_flood(z, mode='epsilon', epsilon=1e-9) + codes = kernels.flow_directions(z, cellsize=1.0) + recv = np.ascontiguousarray(kernels.receivers(codes)) + order = np.ascontiguousarray(kernels.topological_order(recv)[0]) + weights = np.ones_like(z) + streams = np.ones(z.shape, dtype=np.uint8) + outlets = np.flatnonzero(recv.reshape(-1) < 0).astype(np.int64) + + watched = (recv, order, weights, streams, outlets) + before = [sys.getrefcount(a) for a in watched] + + def hammer(): + for _ in range(400): + kernels.accumulate(recv, order, weights) + kernels.downstream_distance(recv, order, weights) + kernels.stream_order(recv, order, streams, 'strahler') + kernels.flow_length(recv, order, weights) + kernels.drainage_basins(recv, order, outlets, None) + kernels.upstream_mask(recv, outlets) + + threads = [threading.Thread(target=hammer) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert [sys.getrefcount(a) for a in watched] == before + + +@pytest.mark.skipif(not kernels.have_extension(), + reason='the compiled extension is not built') +def test_kernels_reject_a_mismatched_order_array(): + """A wrong-length ``order`` must raise, not read off the end of it.""" + recv = np.full((4, 4), -1, dtype=np.int64) + weights = np.ones((4, 4)) + with pytest.raises((ValueError, RuntimeError)): + kernels.accumulate(recv, np.arange(3, dtype=np.int64), weights) + + +def test_strahler_order_does_not_depend_on_the_topological_order(): + """Both backends, and any valid ordering, must agree. + + The incremental rule gives {2, 2, 3} an order of 4 or 3 according to + which donor arrives first; the junction is now resolved as a whole. + """ + recv = np.array([[5, 6, 1, -1], [1, -1, 5, 6], [5, 10, 7, 6]], dtype=np.int64) + streams = np.ones(recv.shape, dtype=np.uint8) + + cpp_order, _ = kernels.topological_order(recv) + py_order, _ = fastops.topological_order(recv) + results = [ + np.asarray(kernels.stream_order(recv, cpp_order, streams, 'strahler')), + np.asarray(kernels.stream_order(recv, py_order, streams, 'strahler')), + np.asarray(fastops.stream_order(recv, cpp_order, streams, 'strahler')), + np.asarray(fastops.stream_order(recv, py_order, streams, 'strahler')), + ] + for other in results[1:]: + assert np.array_equal(results[0], other) + + +def test_strahler_of_a_mixed_junction(): + """Orders {2, 2, 3} meeting give 3, not 4: only the largest can pair.""" + # Four headwaters feeding one cell: two combine to 2, another two to 2, + # those combine to 3, and a bare order-1 also arrives. + recv = np.array([ + [4, 4, 4, 4], + [-1, 4, 4, 4], + ], dtype=np.int64) + streams = np.ones(recv.shape, dtype=np.uint8) + order, _ = kernels.topological_order(recv) + result = np.asarray(kernels.stream_order(recv, order, streams, 'strahler')) + # Cell 4 receives four order-1 donors, so it is order 2. + assert result.reshape(-1)[4] == 2 + + +def test_drainage_basins_honours_valid_with_outlets(): + """The C++ path used to drop the mask whenever outlets were supplied.""" + z = np.add.outer(np.arange(9.0), np.arange(9.0)) + codes = kernels.flow_directions(z, cellsize=1.0) + recv = kernels.receivers(codes) + order, _ = kernels.topological_order(recv) + + valid = np.zeros((9, 9), dtype=np.uint8) + valid[:4, :4] = 1 + outlets = np.array([0], dtype=np.int64) + + cpp = np.asarray(kernels.drainage_basins(recv, order, outlets, valid)) + py = np.asarray(fastops.drainage_basins(recv, order, outlets, valid)) + assert np.array_equal(cpp, py) + assert cpp[valid == 0].sum() == 0 + assert (cpp > 0).sum() <= int(valid.sum()) + + +def test_priority_flood_rejects_a_non_contiguous_grid(): + """reshape(-1) copies, so the fill would be silently thrown away.""" + base = np.array([[5., 5., 5., 5., 5.], + [5., 0., 0., 0., 5.], + [5., 0., 0., 0., 5.], + [5., 0., 0., 0., 5.], + [5., 5., 5., 5., 5.]]) + for awkward in (np.asfortranarray(base), np.repeat(base, 2, axis=0)[::2]): + with pytest.raises(TypeError): + fastops.priority_flood(awkward) + if kernels.have_extension(): + with pytest.raises(TypeError): + kernels.priority_flood(awkward) + + +def test_imposemin_rejects_a_non_contiguous_grid(): + recv = np.full((4, 4), -1, dtype=np.int64) + order = np.arange(16, dtype=np.int64) + step = np.ones((4, 4)) + awkward = np.asfortranarray(np.zeros((4, 4))) + with pytest.raises(TypeError): + fastops.imposemin(recv, order, step, 1e-3, awkward) + + +def test_default_seeds_of_an_empty_grid(): + assert fastops.default_seeds(np.zeros((0, 5))).size == 0 + assert fastops.default_seeds(np.zeros((5, 0))).size == 0 + + +def test_create_from_geotransform_places_the_grid_correctly(): + """yllcenter was read from ny before ny was assigned.""" + gt = (1000.0, 10.0, 0.0, 2000.0, 0.0, -10.0) + grid = dem.Elevation(nx=15, ny=12, projection='', geo_transform=gt, + grid=np.zeros((12, 15))) + + # The south edge of a 12-row, 10 m grid whose north edge is 2000 is 1880, + # so the southern row of cell centres sits at 1885. + assert grid._georef_info.yllcenter == pytest.approx(1885.0) + assert grid._rowscols_to_xy(((0, 0),))[0] == pytest.approx((1005.0, 1995.0)) + assert grid._rowscols_to_xy(((11, 0),))[0] == pytest.approx((1005.0, 1885.0)) + + +@pytest.mark.gdal +def test_create_from_geotransform_round_trips_through_gdal(tmp_path): + """A grid built from a geotransform must agree with the file it writes.""" + gt = (1000.0, 10.0, 0.0, 2000.0, 0.0, -10.0) + rng = np.random.default_rng(6) + grid = dem.Elevation(nx=15, ny=12, projection='', geo_transform=gt, + grid=rng.random((12, 15))) + + path = str(tmp_path / 'grid.tif') + grid.save(path) + back = dem.Elevation.load(path) + + assert back._georef_info.yllcenter == pytest.approx(grid._georef_info.yllcenter) + assert back._rowscols_to_xy(((0, 0),))[0] == pytest.approx( + grid._rowscols_to_xy(((0, 0),))[0]) + assert back._rowscols_to_xy(((11, 14),))[0] == pytest.approx( + grid._rowscols_to_xy(((11, 14),))[0]) + + +def test_editing_a_flow_code_invalidates_the_cached_network(): + """Routing is cached now, so a write has to drop the cache.""" + rng = np.random.default_rng(2) + grid = dem.Elevation(dx=10.0, grid=rng.random((20, 20)) * 10) + fd = dem.FlowDirectionD8(flooded_dem=dem.FilledElevation(elevation=grid)) + + before = dem.Area(flow_direction=fd)._griddata.copy() + + current = int(fd._griddata[10, 10]) + fd[10, 10] = 1 if current != 1 else 16 + assert not np.array_equal(dem.Area(flow_direction=fd)._griddata, before) + + after_setitem = dem.Area(flow_direction=fd)._griddata.copy() + current = int(fd._griddata[5, 5]) + fd.set_value_at_rowscols(1 if current != 1 else 16, [(5, 5)]) + assert not np.array_equal(dem.Area(flow_direction=fd)._griddata, after_setitem) + + +def test_geographic_laplacian_uses_metres_not_degrees(): + """dx is in DEGREES on a lat/lon grid; dividing by it was 1e10 wrong.""" + n = 11 + y, x = np.mgrid[0:n, 0:n].astype(float) + grid = dem.GeographicElevation(dx=0.01, grid=0.25 * (x ** 2 + y ** 2)) + grid._georef_info.geoTransform = (-120.0, 0.01, 0.0, 40.11, 0.0, -0.01) + grid._georef_info.xllcenter = -119.995 + grid._georef_info.yllcenter = 40.005 + + curvature = dem.GeographicLaplacian(elevation=grid)._griddata[5, 5] + cell = grid._mean_pixel_dimension()[5, 5] + # The Laplacian of 0.25*(x^2 + y^2) is 1 per cell squared. + assert curvature == pytest.approx(1.0 / cell ** 2, rel=1e-6) + assert curvature < 1e-4, 'a degree-scaled result would be about 1e4' + + +def test_tile_with_zero_padding_does_not_duplicate_the_edges(): + """`griddata[:, -0:]` is the whole array, not an empty slice.""" + data = np.arange(100, dtype=float).reshape(10, 10) + grid = dem.BaseSpatialGrid(dx=1.0, grid=data) + + tiles = grid.tile(tile_xdim=5, tile_ydim=5, tile_xpadding=0, tile_ypadding=0) + assert [t._griddata.shape for t in tiles] == [(5, 5)] * 4 + rebuilt = dem.BaseSpatialGrid.mosaic([t.remove_padding(0, 0) for t in tiles]) + assert np.array_equal(rebuilt._griddata, data) + + +def test_star_import_names_only_things_that_exist(): + import TopoAnalysis + + missing = [name for name in TopoAnalysis.__all__ + if not hasattr(TopoAnalysis, name)] + assert missing == [] + + namespace = {} + exec('from TopoAnalysis import *', namespace) # noqa: S102 - that is the test + assert 'Elevation' in namespace and 'FlowDirectionD8' in namespace + + +def test_the_test_suite_runs_from_a_differently_named_checkout(tmp_path): + """A GitHub zip unpacks to TopoAnalysis-main, not TopoAnalysis.""" + import shutil + + target = tmp_path / 'TopoAnalysis-main' + shutil.copytree(PACKAGE_DIR, str(target), + ignore=shutil.ignore_patterns('__pycache__', 'build', '.git', + '.pytest_cache', 'docs')) + + env = dict(os.environ) + env.pop('PYTHONPATH', None) + result = subprocess.run( + [sys.executable, '-m', 'pytest', '-q', 'tests/test_grid.py', + '-p', 'no:cacheprovider'], + capture_output=True, text=True, cwd=str(target), env=env) + assert result.returncode == 0, result.stdout[-4000:] + result.stderr[-2000:] + + +def test_extension_is_loaded_only_from_the_package_directory(tmp_path): + """A stray `_topoanalysis` elsewhere on sys.path must not be picked up. + + The flat-import fallback used a bare ``import _topoanalysis``, which + searches all of sys.path; a stale or unrelated module of that name would + be loaded instead of the package's own. + """ + decoy = tmp_path / '_topoanalysis.py' + decoy.write_text('MARKER = "decoy"\n') + + script = ( + 'import sys, os;' + 'sys.path.insert(0, {0!r});' + 'sys.path.insert(0, {1!r});' + 'import kernels;' + 'print(getattr(kernels._ext, "MARKER", "not-the-decoy"))' + ).format(str(tmp_path), PACKAGE_DIR) + result = subprocess.run([sys.executable, '-c', script], capture_output=True, + text=True, cwd=str(tmp_path)) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == 'not-the-decoy' diff --git a/tests/test_terrain.py b/tests/test_terrain.py new file mode 100644 index 0000000..ddc7d55 --- /dev/null +++ b/tests/test_terrain.py @@ -0,0 +1,175 @@ +"""Tests for the terrain-attribute grids, against analytic answers.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from TopoAnalysis import dem + + +# --------------------------------------------------------------------------- +# Slope and hillshade +# --------------------------------------------------------------------------- + + +def test_max_slope_on_a_plane(east_plane): + slope = dem.MaxSlope(elevation=east_plane) + # Centred differences reproduce a plane exactly in the interior. + assert np.allclose(slope._griddata[1:-1, 1:-1], 1.0) + + +def test_max_slope_scales_with_cell_size(): + data = np.tile(np.arange(7, dtype=float), (7, 1)) + fine = dem.MaxSlope(elevation=dem.Elevation(dx=1.0, grid=data)) + coarse = dem.MaxSlope(elevation=dem.Elevation(dx=2.0, grid=data)) + assert np.allclose(fine._griddata[1:-1, 1:-1], + 2.0 * coarse._griddata[1:-1, 1:-1]) + + +def test_hillshade_lights_the_slope_that_faces_the_sun(east_plane): + """An east-facing slope is lit from the east and shadowed from the west.""" + lit = dem.Hillshade(elevation=east_plane, azimuth=90, inclination=45) + shadowed = dem.Hillshade(elevation=east_plane, azimuth=270, inclination=45) + assert lit._griddata[2, 2] == 255 + assert shadowed._griddata[2, 2] == 0 + + +def test_hillshade_of_a_south_facing_slope(south_plane): + lit = dem.Hillshade(elevation=south_plane, azimuth=180, inclination=45) + shadowed = dem.Hillshade(elevation=south_plane, azimuth=0, inclination=45) + assert lit._griddata[2, 2] > shadowed._griddata[2, 2] + + +def test_hillshade_is_a_byte_grid_without_wraparound(rough_dome): + hs = dem.Hillshade(elevation=rough_dome, azimuth=315, inclination=45) + assert hs._griddata.dtype == np.uint8 + # Negative cosines used to wrap round the byte into bright speckle. + assert hs._griddata.min() >= 0 and hs._griddata.max() <= 255 + + +def test_hillshade_of_a_flat_surface_is_the_sine_of_the_altitude(): + flat = dem.Elevation(dx=1.0, grid=np.zeros((7, 7))) + hs = dem.Hillshade(elevation=flat, azimuth=315, inclination=30) + assert hs._griddata[3, 3] == pytest.approx(255 * np.cos(np.radians(60)), abs=1) + + +# --------------------------------------------------------------------------- +# Curvature +# --------------------------------------------------------------------------- + + +def test_laplacian_of_a_quadratic_bowl(): + """The Laplacian of a(x^2 + y^2) is 4a.""" + n = 21 + y, x = np.mgrid[0:n, 0:n].astype(float) + a = 0.25 + grid = dem.Elevation(dx=1.0, grid=a * (x ** 2 + y ** 2)) + + curvature = dem.Laplacian(elevation=grid) + assert np.allclose(curvature._griddata[2:-2, 2:-2], 4 * a) + + +def test_laplacian_scales_with_cell_size(): + n = 15 + y, x = np.mgrid[0:n, 0:n].astype(float) + data = x ** 2 + fine = dem.Laplacian(elevation=dem.Elevation(dx=1.0, grid=data)) + coarse = dem.Laplacian(elevation=dem.Elevation(dx=2.0, grid=data)) + assert np.allclose(fine._griddata[2:-2, 2:-2], + 4.0 * coarse._griddata[2:-2, 2:-2]) + + +def test_local_relief_window_is_centred(): + """A single peak must raise the relief symmetrically about itself.""" + data = np.zeros((21, 21)) + data[10, 10] = 100.0 + grid = dem.Elevation(dx=1.0, grid=data) + + relief = dem.LocalRelief(elevation=grid, pixel_radius=3) + # np.ogrid[-r:r] gave a window one cell short on the far side, so the + # footprint was off-centre. + assert relief._griddata[10, 7] == pytest.approx(relief._griddata[10, 13]) + assert relief._griddata[7, 10] == pytest.approx(relief._griddata[13, 10]) + assert relief._griddata[10, 10] == pytest.approx(100.0) + + +def test_local_relief_equals_the_range_over_the_disc(rough_dome): + """The value is exactly the range over the disc, not over its bounding box.""" + radius = 2 + relief = dem.LocalRelief(elevation=rough_dome, pixel_radius=radius) + + yy, xx = np.ogrid[-radius:radius + 1, -radius:radius + 1] + disc = (xx * xx + yy * yy) <= radius * radius + for (i, j) in ((10, 10), (20, 15), (31, 27)): + window = rough_dome._griddata[i - radius:i + radius + 1, + j - radius:j + radius + 1][disc] + assert relief._griddata[i, j] == pytest.approx(window.max() - window.min()) + + # The bounding box is strictly larger, so it is a different answer. + box = rough_dome._griddata[8:13, 8:13] + assert relief._griddata[10, 10] <= box.max() - box.min() + 1e-9 + assert relief._griddata.min() >= 0.0 + + +def test_log_area_is_a_float_grid(rough_dome): + filled = dem.FilledElevation(elevation=rough_dome) + d8 = dem.FlowDirectionD8(flooded_dem=filled) + area = dem.Area(flow_direction=d8) + + log_area = dem.LogArea(area=area) + # Declaring this uint8 truncated log10(area) to a handful of integers. + assert log_area._griddata.dtype == np.float64 + finite = np.isfinite(log_area._griddata) + assert np.allclose(10 ** log_area._griddata[finite], area._griddata[finite]) + + +def test_gradient_grid_components(east_plane): + gradient = dem.Gradient(elevation=east_plane) + assert np.allclose(gradient._gx[1:-1, 1:-1], -1.0) + assert np.allclose(gradient._gy[1:-1, 1:-1], 0.0) + + +# --------------------------------------------------------------------------- +# Geographic grids +# --------------------------------------------------------------------------- + + +def test_geographic_cell_area_shrinks_polewards(geographic_grid): + areas = geographic_grid._area_per_pixel() + # Row 0 is the north edge, so its cells are the smallest. + assert areas[0, 0] < areas[-1, 0] + assert np.allclose(areas[:, 0], areas[:, -1]) + + +def test_geographic_cell_area_matches_the_spherical_formula(geographic_grid): + areas = geographic_grid._area_per_pixel() + re = geographic_grid.earth_radius + dlon = dlat = 0.01 + lat = geographic_grid._georef_info.yllcenter + 9 * dlat # northernmost row + expected = (re ** 2) * np.radians(dlon) * ( + np.sin(np.radians(lat + dlat / 2)) - np.sin(np.radians(lat - dlat / 2))) + assert areas[0, 0] == pytest.approx(expected, rel=1e-9) + + +def test_geographic_mean_pixel_dimension_is_the_area_root(geographic_grid): + assert np.allclose(geographic_grid._mean_pixel_dimension() ** 2, + geographic_grid._area_per_pixel()) + + +def test_geographic_area_uses_spherical_cells(): + """Drainage area on a lat/lon grid must use the true cell areas.""" + n = 12 + z = np.tile(np.arange(n, 0, -1, dtype=float), (n, 1)) + grid = dem.GeographicElevation(dx=0.01, grid=z) + grid._georef_info.geoTransform = (-120.0, 0.01, 0.0, 40.12, 0.0, -0.01) + grid._georef_info.xllcenter = -119.995 + grid._georef_info.yllcenter = 40.005 + + d8 = dem.FlowDirectionD8(flooded_dem=dem.FilledElevation(elevation=grid)) + area = dem.GeographicArea(flow_direction=d8) + + per_pixel = grid._area_per_pixel() + assert area._griddata[5, 0] == pytest.approx(per_pixel[5, 0]) + # Cells accumulate the true, latitude-dependent areas. + assert area._griddata[5, 4] == pytest.approx(per_pixel[5, 0:5].sum()) diff --git a/tests/test_topotoolbox.py b/tests/test_topotoolbox.py new file mode 100644 index 0000000..560d0dd --- /dev/null +++ b/tests/test_topotoolbox.py @@ -0,0 +1,751 @@ +"""Tests for the TopoToolbox-equivalent functions. + +Without MATLAB on hand, parity is established three ways: + +* against closed-form answers on surfaces whose value is known exactly + (planes, quadratics, spherical cells); +* against the documented definitions transcribed from the MATLAB source + (stencils, constants, conventions); and +* against TopoAnalysis' own results where the two are meant to agree, with + the documented conversion applied. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from TopoAnalysis import dem, error, topotoolbox as tt + + +# --------------------------------------------------------------------------- +# Filling and flats +# --------------------------------------------------------------------------- + + +def test_fillsinks_is_a_flat_fill(single_pit): + filled = tt.fillsinks(single_pit) + # TopoToolbox's fillsinks adds no epsilon: the depression becomes an + # exactly flat plateau at the spill elevation. + assert np.all(filled._griddata == 10.0) + assert single_pit._griddata[2, 2] == 1.0, 'the input must not be modified' + + +def test_fillsinks_respects_maxdepth(single_pit): + filled = tt.fillsinks(single_pit, maxdepth=2.0) + assert filled._griddata[2, 2] == pytest.approx(1.0) + + +def test_fillsinks_treats_nodata_as_an_outlet(): + z = np.array([ + [10, 10, 10, 10, 10], + [10, 5, 5, 5, 10], + [np.nan, 5, 1, 5, 10], + [10, 5, 5, 5, 10], + [10, 10, 10, 10, 10]], dtype=float) + filled = tt.fillsinks(dem.Elevation(dx=1.0, grid=z)) + assert filled._griddata[2, 1] == pytest.approx(5.0) + assert np.isnan(filled._griddata[2, 0]) + + +def test_identifyflats_finds_the_plateau_and_its_sills(plateau): + flats, sills = tt.identifyflats(plateau) + # Every plateau cell has an equal neighbour and no lower one ... + assert flats._griddata[1, 1] + assert flats._griddata[1, 2] + # ... except those beside the 2 m outlet, which do have a lower one. + assert not flats._griddata[3, 2] + assert sills._griddata[3, 2] + assert not sills._griddata[1, 1] + + +def test_identifyflats_single_output(plateau): + only_flats = tt.identifyflats(plateau, output='flats') + both = tt.identifyflats(plateau) + assert np.array_equal(only_flats._griddata, both[0]._griddata) + + +def test_imposemin_only_lowers(v_valley): + fd = tt.flowobj(v_valley) + carved = tt.imposemin(fd, v_valley, sl=1e-3) + assert np.all(carved._griddata <= v_valley._griddata + 1e-12) + + +def test_imposemin_enforces_the_gradient(v_valley): + fd = tt.flowobj(v_valley) + sl = 1e-3 + carved = tt.imposemin(fd, v_valley, sl=sl) + + z = carved._griddata.reshape(-1) + step = fd.step_lengths().reshape(-1) + for k, r in enumerate(fd.receivers.reshape(-1)): + if r >= 0: + assert z[k] - z[r] >= sl * step[k] - 1e-9 + + +# --------------------------------------------------------------------------- +# Flow +# --------------------------------------------------------------------------- + + +def test_flowacc_counts_cells_not_area(east_plane): + fd = tt.flowobj(east_plane) + counts = tt.flowacc(fd) + # TopoToolbox's flowacc has unit weight per cell and counts the cell + # itself, so the minimum is 1 -- not dx**2. + assert counts._griddata.min() == 1.0 + assert counts._griddata[2].tolist() == [1.0, 2.0, 3.0, 4.0, 5.0] + + +def test_flowacc_relates_to_Area_by_the_cell_area(rough_dome): + fd = tt.flowobj(rough_dome) + counts = tt.flowacc(fd) + area = dem.Area(flow_direction=fd) + cell = rough_dome._georef_info.dx ** 2 + assert np.allclose(counts._griddata * cell, area._griddata) + + +def test_flowacc_accepts_weights(east_plane): + fd = tt.flowobj(east_plane) + weights = np.full(east_plane._griddata.shape, 2.0) + assert tt.flowacc(fd, weights)._griddata[2, 4] == pytest.approx(10.0) + + +def test_flowdistance_upstream_is_zero_at_the_outlet(east_plane): + fd = tt.flowobj(east_plane) + distance = tt.flowdistance(fd, 'upstream') + # Distance to the outlet: 4 cells away at the western edge, 0 at the east. + assert distance._griddata[2].tolist() == [4.0, 3.0, 2.0, 1.0, 0.0] + + +def test_flowdistance_downstream_is_the_flow_length(east_plane): + fd = tt.flowobj(east_plane) + tt_length = tt.flowdistance(fd, 'downstream') + ta_length = dem.FlowLength(flow_direction=fd) + assert np.allclose(tt_length._griddata, ta_length._griddata) + + +def test_flowdistance_rejects_an_unknown_direction(east_plane): + fd = tt.flowobj(east_plane) + with pytest.raises(error.InputError): + tt.flowdistance(fd, 'sideways') + + +def test_drainagebasins_partition_matches_the_flow_network(rough_dome): + fd = tt.flowobj(rough_dome) + labels = tt.drainagebasins(fd)._griddata + + flat = labels.reshape(-1) + for k, r in enumerate(fd.receivers.reshape(-1)): + if r >= 0: + assert flat[k] == flat[r] + assert labels.min() >= 1 + + +def test_drainagebasins_from_outlets(rough_dome): + fd = tt.flowobj(rough_dome) + outlets = fd._rowscols_to_xy(((20, 20), (10, 30))) + labels = tt.drainagebasins(fd, outlets=outlets)._griddata + + assert set(np.unique(labels).tolist()) <= {0, 1, 2} + assert labels[20, 20] == 1 + assert labels[10, 30] == 2 + + +def test_streamorder_of_a_confluence(): + z = np.array([[3.0, 9.0, 3.0], + [9.0, 2.0, 9.0], + [9.0, 1.0, 9.0]]) + grid = dem.Elevation(dx=1.0, grid=z) + fd = dem.FlowDirectionD8(elevation=grid) + streams = np.array([[1, 0, 1], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) + + strahler = tt.streamorder(fd, streams)._griddata + assert strahler[0, 0] == 1 and strahler[1, 1] == 2 + shreve = tt.streamorder(fd, streams, 'shreve')._griddata + assert shreve[1, 1] == 2 + + +def test_streamorder_rejects_an_unknown_type(east_plane): + fd = tt.flowobj(east_plane) + with pytest.raises(error.InputError): + tt.streamorder(fd, np.ones((5, 5)), 'horton') + + +def test_flowobj_rejects_carve(east_plane): + """'carve' is not implemented and must say so rather than guess.""" + with pytest.raises(error.InputError): + tt.flowobj(east_plane, preprocess='carve') + + +# --------------------------------------------------------------------------- +# Terrain attributes against closed-form answers +# --------------------------------------------------------------------------- + + +def test_gradient8_on_a_cardinal_plane(east_plane): + assert tt.gradient8(east_plane)._griddata[2, 2] == pytest.approx(1.0) + + +def test_gradient8_on_a_diagonal_plane(): + """A 2 m drop over a sqrt(2) m step is a gradient of sqrt(2).""" + z = np.add.outer(np.arange(5, 0, -1, dtype=float), + np.arange(5, 0, -1, dtype=float)) + grid = dem.Elevation(dx=1.0, grid=z) + assert tt.gradient8(grid)._griddata[2, 2] == pytest.approx(np.sqrt(2.0)) + + +def test_gradient8_is_never_negative(rough_dome): + """The structuring elements include the centre, so pits give 0, not < 0.""" + assert tt.gradient8(rough_dome)._griddata.min() >= 0.0 + + +def test_gradient8_is_zero_on_a_flat(): + flat = dem.Elevation(dx=1.0, grid=np.full((5, 5), 7.0)) + assert np.all(tt.gradient8(flat)._griddata == 0.0) + + +@pytest.mark.parametrize('unit,expected', [ + ('tangent', 1.0), + ('degree', 45.0), + ('radian', np.pi / 4), + ('sine', np.sqrt(0.5)), + ('percent', 100.0), +]) +def test_gradient8_units(east_plane, unit, expected): + assert tt.gradient8(east_plane, unit)._griddata[2, 2] == pytest.approx(expected) + + +def test_gradient8_rejects_an_unknown_unit(east_plane): + with pytest.raises(error.InputError): + tt.gradient8(east_plane, 'furlongs') + + +def test_arcslope_on_a_plane(east_plane): + """The Horn 1-2-1 stencil reproduces a plane exactly.""" + assert tt.arcslope(east_plane)._griddata[2, 2] == pytest.approx(1.0) + + +def test_arcslope_has_no_sqrt2_factor(): + """The 8*cellsize denominator carries all the geometry.""" + z = np.add.outer(np.arange(5, 0, -1, dtype=float), + np.arange(5, 0, -1, dtype=float)) + grid = dem.Elevation(dx=1.0, grid=z) + # dz/dx = dz/dy = 1, so the Horn magnitude is sqrt(2). + assert tt.arcslope(grid)._griddata[2, 2] == pytest.approx(np.sqrt(2.0)) + + +@pytest.mark.parametrize('descent,expected', [ + ('north', 0.0), + ('east', 90.0), + ('south', 180.0), + ('west', 270.0), +]) +def test_aspect_points_down_slope(descent, expected): + """Aspect is the compass bearing the surface descends towards.""" + n = 7 + y, x = np.mgrid[0:n, 0:n].astype(float) + surface = { + 'north': y, # elevation grows with row, i.e. downhill to north + 'south': -y, + 'east': -x, + 'west': x, + }[descent] + grid = dem.Elevation(dx=1.0, grid=surface) + assert tt.aspect(grid)._griddata[3, 3] == pytest.approx(expected) + + +def test_aspect_of_a_flat_cell_is_ninety(): + """atan2(0, 0) is 0, so TopoToolbox reports flats as east-facing.""" + flat = dem.Elevation(dx=1.0, grid=np.zeros((5, 5))) + assert tt.aspect(flat)._griddata[2, 2] == pytest.approx(90.0) + + +def test_aspect_classification(): + n = 7 + y, x = np.mgrid[0:n, 0:n].astype(float) + grid = dem.Elevation(dx=1.0, grid=-x) # descends east, aspect 90 + classes = tt.aspect(grid, classify=True)._griddata + assert classes.dtype == np.uint8 + assert classes[3, 3] == 5 # [90, 135) -> class 5 (E) + + +def test_aspect_is_independent_of_cell_size(): + n = 7 + y, x = np.mgrid[0:n, 0:n].astype(float) + fine = dem.Elevation(dx=1.0, grid=-x) + coarse = dem.Elevation(dx=30.0, grid=-x) + assert np.allclose(tt.aspect(fine)._griddata, tt.aspect(coarse)._griddata) + + +def test_curvature_of_a_plane_is_zero(east_plane): + for ctype in ('profc', 'planc', 'tangc', 'meanc', 'total'): + curv = tt.curvature(east_plane, ctype)._griddata + assert np.allclose(curv[1:-1, 1:-1], 0.0, atol=1e-12), ctype + + +def test_total_curvature_of_a_quadratic(): + """'total' is fxx^2 + 2 fxy^2 + fyy^2, with no minus and no denominator.""" + n = 15 + y, x = np.mgrid[0:n, 0:n].astype(float) + a = 0.02 + grid = dem.Elevation(dx=1.0, grid=a * x ** 2) + curv = tt.curvature(grid, 'total')._griddata + # fxx = 2a, fyy = fxy = 0. + assert np.allclose(curv[2:-2, 2:-2], (2 * a) ** 2) + + +def test_curvature_rejects_an_unknown_type(east_plane): + with pytest.raises(error.InputError): + tt.curvature(east_plane, 'gaussian') + + +def test_curvature_zeroes_the_zero_gradient_cells(): + """0/0 becomes 0, not NaN -- matching TopoToolbox.""" + flat = dem.Elevation(dx=1.0, grid=np.zeros((7, 7))) + assert np.all(tt.curvature(flat, 'profc')._griddata == 0.0) + + +def test_hillshade_is_a_cosine_not_a_byte(east_plane): + shade = tt.hillshade(east_plane, azimuth=90, altitude=45)._griddata + # TopoToolbox scales to 0-255 only when plotting. + assert shade.dtype == np.float64 + assert -1.0 - 1e-9 <= shade.min() and shade.max() <= 1.0 + 1e-9 + + +def test_hillshade_is_brightest_facing_the_light(east_plane): + lit = tt.hillshade(east_plane, azimuth=90, altitude=45)._griddata[2, 2] + shadowed = tt.hillshade(east_plane, azimuth=270, altitude=45)._griddata[2, 2] + assert lit == pytest.approx(1.0) + assert shadowed == pytest.approx(0.0, abs=1e-9) + + +def test_hillshade_defaults_match_topotoolbox(rough_dome): + """azimuth 315, altitude 60 -- the code's defaults, not the docstring's.""" + default = tt.hillshade(rough_dome)._griddata + explicit = tt.hillshade(rough_dome, azimuth=315, altitude=60)._griddata + assert np.array_equal(default, explicit) + + +def test_hillshade_of_a_flat_surface_is_the_sine_of_the_altitude(): + flat = dem.Elevation(dx=1.0, grid=np.zeros((7, 7))) + for altitude in (30.0, 45.0, 60.0): + shade = tt.hillshade(flat, altitude=altitude)._griddata[3, 3] + assert shade == pytest.approx(np.sin(np.radians(altitude))) + + +def test_hillshade_agrees_in_sign_with_the_esri_version(rough_dome): + """The two formulations must at least light the same slopes.""" + tt_shade = tt.hillshade(rough_dome, azimuth=315, altitude=45)._griddata + esri = dem.Hillshade(elevation=rough_dome, azimuth=315, + inclination=45)._griddata.astype(float) + interior = (slice(2, -2), slice(2, -2)) + correlation = np.corrcoef(tt_shade[interior].ravel(), esri[interior].ravel())[0, 1] + assert correlation > 0.97 + + +def test_localtopography_range_matches_local_relief(rough_dome): + tt_relief = tt.localtopography(rough_dome, radius=60.0, type='range')._griddata + ta_relief = dem.LocalRelief(elevation=rough_dome, pixel_radius=2)._griddata + assert np.allclose(tt_relief, ta_relief) + + +# --------------------------------------------------------------------------- +# Cell area +# --------------------------------------------------------------------------- + + +def test_cellarea_uses_the_topotoolbox_sphere(geographic_grid): + areas = tt.cellarea(geographic_grid)._griddata + radius_km = np.sqrt(tt.EARTH_SURFACE_AREA_KM2 / (4 * np.pi)) + dlon = dlat = 0.01 + lat = geographic_grid._georef_info.yllcenter + 9 * dlat + expected = (radius_km * 1000.0) ** 2 * np.radians(dlon) * ( + np.sin(np.radians(lat + dlat / 2)) - np.sin(np.radians(lat - dlat / 2))) + assert areas[0, 0] == pytest.approx(expected, rel=1e-9) + + +def test_cellarea_km_units(geographic_grid): + metres = tt.cellarea(geographic_grid, 'm')._griddata + kilometres = tt.cellarea(geographic_grid, 'km')._griddata + assert np.allclose(metres, kilometres * 1e6) + + +def test_cellarea_matches_topoanalysis_to_the_radius_difference(geographic_grid): + """The two differ only through their Earth radii, by ~1.5e-5.""" + tt_area = tt.cellarea(geographic_grid)._griddata + ta_area = geographic_grid._area_per_pixel() + ratio = (np.sqrt(tt.EARTH_SURFACE_AREA_KM2 / (4 * np.pi)) * 1000.0 + / geographic_grid.earth_radius) ** 2 + # TopoAnalysis evaluates the longitude difference per cell, so its + # areas vary in the last few bits along a row; TopoToolbox's do not. + assert np.allclose(tt_area, ta_area * ratio, rtol=1e-9) + + +def test_cellarea_rejects_an_unknown_unit(geographic_grid): + with pytest.raises(error.InputError): + tt.cellarea(geographic_grid, 'miles') + + +# --------------------------------------------------------------------------- +# Chi and steepness +# --------------------------------------------------------------------------- + + +def test_chitransform_is_zero_at_the_outlet(v_valley): + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + chi = tt.chitransform(fd, counts, mn=0.45, a0=1e6)._griddata + + outlets = fd.receivers < 0 + assert np.allclose(chi[outlets], 0.0) + assert np.nanmax(chi) > 0.0 + + +def test_chitransform_correctcellsize(v_valley): + """With correctcellsize the area is taken to be in cells, not m^2.""" + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + area = dem.Area(flow_direction=fd) # already m^2 + + corrected = tt.chitransform(fd, counts, correctcellsize=True)._griddata + direct = tt.chitransform(fd, area, correctcellsize=False)._griddata + assert np.allclose(corrected, direct, equal_nan=True) + + +def test_chitransform_scales_with_a0(v_valley): + """chi is proportional to a0**mn.""" + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + mn = 0.45 + one = tt.chitransform(fd, counts, mn=mn, a0=1.0)._griddata + million = tt.chitransform(fd, counts, mn=mn, a0=1e6)._griddata + valid = np.isfinite(one) & (one > 0) + assert np.allclose(million[valid] / one[valid], 1e6 ** mn) + + +def test_chitransform_uses_the_trapezoid_rule(v_valley): + """It must agree with Chi(trapezoid=True), not with the default rule.""" + fd = tt.flowobj(v_valley) + area = dem.Area(flow_direction=fd) + outlet_rc = np.unravel_index(int(np.argmax(area._griddata)), area._griddata.shape) + outlets = area._rowscols_to_xy((outlet_rc,)) + + tt_chi = tt.chitransform(fd, area, mn=0.45, a0=1e6, outlets=outlets, + correctcellsize=False)._griddata + ta_chi = dem.Chi(area=area, flow_direction=fd, theta=0.45, Ao=1e6, + outlets=outlets, trapezoid=True)._griddata + + finite = np.isfinite(tt_chi) + assert np.allclose(tt_chi[finite], ta_chi[finite]) + + +def test_ksn_is_slope_times_area_to_theta(v_valley): + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + theta = 0.45 + k = tt.ksn(fd, v_valley, counts, theta=theta)._griddata + + carved = tt.imposemin(fd, v_valley, sl=1e-5) + z = carved._griddata.reshape(-1) + step = fd.step_lengths().reshape(-1) + receivers = fd.receivers.reshape(-1) + area = (counts._griddata * fd._georef_info.dx ** 2).reshape(-1) + + donors = np.flatnonzero(receivers >= 0) + gradient = (z[donors] - z[receivers[donors]]) / step[donors] + assert np.allclose(k.reshape(-1)[donors], gradient * area[donors] ** theta) + + +def test_mchi_is_the_elevation_gradient_in_chi_space(v_valley): + """M_chi is dz/dchi, a plain forward difference with no carving.""" + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + chi = tt.chitransform(fd, counts, mn=0.45, a0=1.0) + mc = tt.mchi(fd, v_valley, chi)._griddata.reshape(-1) + + z = np.asarray(v_valley._griddata, dtype=float).reshape(-1) + c = chi._griddata.reshape(-1) + receivers = fd.receivers.reshape(-1) + donors = np.flatnonzero(receivers >= 0) + expected = (z[donors] - z[receivers[donors]]) / (c[donors] - c[receivers[donors]]) + + finite = np.isfinite(expected) + assert np.allclose(mc[donors][finite], expected[finite]) + + +def test_mchi_scales_inversely_with_a0(v_valley): + """M_chi depends on the a0 baked into chi: mchi(a0) * a0**mn is invariant.""" + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + mn = 0.45 + + one = tt.mchi(fd, v_valley, tt.chitransform(fd, counts, mn=mn, a0=1.0))._griddata + million = tt.mchi(fd, v_valley, tt.chitransform(fd, counts, mn=mn, a0=1e6))._griddata + + finite = np.isfinite(one) & np.isfinite(million) & (np.abs(one) > 1e-12) + assert np.allclose(million[finite] * (1e6 ** mn), one[finite], rtol=1e-9) + + +def test_mchi_and_ksn_are_close_but_not_identical(v_valley): + """TopoToolbox's docstring claims they are the same; the code differs. + + ``ksn`` carves the DEM first and takes the integrand at the upstream + cell, while ``mchi`` uses the raw DEM and chi's trapezoidal averaging. + """ + fd = tt.flowobj(v_valley) + counts = tt.flowacc(fd) + mn = 0.45 + + carved = tt.imposemin(fd, v_valley, sl=1e-5) + mc = tt.mchi(fd, carved, tt.chitransform(fd, counts, mn=mn, a0=1.0))._griddata + k = tt.ksn(fd, v_valley, counts, theta=mn)._griddata + + receivers = fd.receivers.reshape(-1) + donors = np.flatnonzero(receivers >= 0) + a = mc.reshape(-1)[donors] + b = k.reshape(-1)[donors] + finite = np.isfinite(a) & np.isfinite(b) & (b > 0) + + ratio = a[finite] / b[finite] + assert np.all(ratio > 0.5) and np.all(ratio < 2.0) + assert not np.allclose(a[finite], b[finite]) + + +# --------------------------------------------------------------------------- +# Referencing +# --------------------------------------------------------------------------- + + +def test_getcoordinates_runs_north_to_south(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((4, 6))) + x, y = tt.getcoordinates(grid) + assert x.size == 6 and y.size == 4 + assert y[0] > y[-1], 'row 0 is the north edge' + assert x[1] - x[0] == pytest.approx(10.0) + + +def test_getextent_is_cell_centres_not_edges(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((4, 6))) + centres = tt.getextent(grid) + edges = grid.extent() + assert centres[0] == pytest.approx(edges[0] + 5.0) + assert centres[1] == pytest.approx(edges[1] - 5.0) + + +def test_coord2sub_and_sub2coord_round_trip(): + grid = dem.BaseSpatialGrid(dx=10.0, grid=np.zeros((7, 5))) + for row in range(7): + for col in range(5): + x, y = tt.sub2coord(grid, row, col) + r, c = tt.coord2sub(grid, x, y) + assert (int(r[0]), int(c[0])) == (row, col) + + +def test_coord2sub_returns_nan_outside_the_grid(): + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((3, 3))) + r, c = tt.coord2sub(grid, [1000.0], [1000.0]) + assert np.isnan(r[0]) and np.isnan(c[0]) + + +def test_coord2sub_rounds_halves_away_from_zero(): + """MATLAB's round, not Python's round-half-to-even.""" + grid = dem.BaseSpatialGrid(dx=1.0, grid=np.zeros((5, 5))) + info = grid._georef_info + # Exactly half a cell east of column 0. + r, c = tt.coord2sub(grid, [info.xllcenter + 0.5], [info.yllcenter]) + assert int(c[0]) == 1 + + +# --------------------------------------------------------------------------- +# Checks against literal ports of the MATLAB source +# +# These transcribe the .m files directly rather than restating the intent, so +# they catch a drift in the Python that a behavioural test would not. +# --------------------------------------------------------------------------- + + +def matlab_surfnorm(z): + """A literal port of MATLAB's ``surfnorm(Z)`` (graph3d/surfnorm.m). + + The boundary is expanded by the quadratic ghost cell ``3*a1 - 3*a2 + a3`` + -- *not* the linear ``2*a1 - a2`` -- which makes the edge derivative the + second-order one-sided difference. + """ + m, n = z.shape + x, y = np.meshgrid(np.arange(1.0, n + 1), np.arange(1.0, m + 1)) + + def expand(a): + a = np.vstack([3 * a[0:1] - 3 * a[1:2] + a[2:3], a, + 3 * a[m - 1:m] - 3 * a[m - 2:m - 1] + a[m - 3:m - 2]]) + return np.hstack([3 * a[:, 0:1] - 3 * a[:, 1:2] + a[:, 2:3], a, + 3 * a[:, n - 1:n] - 3 * a[:, n - 2:n - 1] + a[:, n - 3:n - 2]]) + + xx, yy, zz = expand(x), expand(y), expand(z) + + def filter_row(a): # filter2([1 0 -1]/2, a) -- correlation + out = np.zeros_like(a) + out[:, 1:-1] = 0.5 * (a[:, :-2] - a[:, 2:]) + return out + + def filter_col(a): # filter2([-1;0;1]/2, a) + out = np.zeros_like(a) + out[1:-1, :] = 0.5 * (a[2:, :] - a[:-2, :]) + return out + + rows, cols = slice(1, m + 1), slice(1, n + 1) + ax, ay, az = (filter_row(v)[rows, cols] for v in (xx, yy, zz)) + bx, by, bz = (filter_col(v)[rows, cols] for v in (xx, yy, zz)) + + nx = -(ay * bz - az * by) + ny = -(az * bx - ax * bz) + nz = -(ax * by - ay * bx) + magnitude = np.sqrt(nx * nx + ny * ny + nz * nz) + magnitude[magnitude == 0] = np.finfo(float).eps + return nx / magnitude, ny / magnitude, nz / magnitude + + +@pytest.mark.parity +@pytest.mark.parametrize('shape', [(7, 8), (12, 12), (5, 20)]) +def test_surfnorm_matches_matlab_including_the_border(shape): + from TopoAnalysis.topotoolbox import _surfnorm + + z = np.random.default_rng(7).random(shape) * 100 + for ours, theirs in zip(_surfnorm(z), matlab_surfnorm(z)): + assert np.allclose(ours, theirs, atol=1e-12) + + +def matlab_identifyflats(z): + """A literal port of ``@GRIDobj/identifyflats.m``.""" + from scipy.ndimage import binary_dilation, grey_dilation, grey_erosion + + log_nans = np.isnan(z) + dem_ = np.where(log_nans, -np.inf, z) + + flats = grey_erosion(dem_, size=(3, 3), mode='constant', cval=np.inf) == dem_ + if log_nans.any(): + flats &= ~log_nans + flats[:, [0, -1]] = False + flats[[0, -1], :] = False + if log_nans.any(): + flats[binary_dilation(log_nans, structure=np.ones((3, 3), bool))] = False + + marker = np.full(dem_.shape, -np.inf) + marker[flats] = dem_[flats] + sills = (grey_dilation(marker, size=(3, 3), mode='constant', + cval=-np.inf) == dem_) & ~flats + if log_nans.any(): + sills[log_nans] = False + return flats, sills + + +@pytest.mark.parity +@pytest.mark.parametrize('seed', [0, 1, 2, 3]) +def test_identifyflats_matches_matlab(seed): + """Including the cleared rim, the NaN halo, and isolated pits.""" + from scipy.ndimage import gaussian_filter + + rng = np.random.default_rng(seed) + n = int(rng.integers(20, 45)) + # Rounded, so plateaus and ties actually occur. + z = np.round(gaussian_filter(rng.random((n, n)) * 50, 2.0), 1) + if seed % 2: + z[rng.random(z.shape) < 0.05] = np.nan + + filled = tt.fillsinks(dem.Elevation(dx=1.0, grid=z)) + expected_flats, expected_sills = matlab_identifyflats(filled._griddata) + flats, sills = tt.identifyflats(filled) + + assert np.array_equal(flats._griddata, expected_flats) + assert np.array_equal(sills._griddata, expected_sills) + + +@pytest.mark.parity +def test_identifyflats_clears_the_rim(): + """A plateau reaching the grid edge is flat only in the interior.""" + z = np.full((6, 6), 3.0) + z[5, 5] = 2.0 # the plateau's only outlet, at a corner + flats = tt.identifyflats(dem.Elevation(dx=1.0, grid=z), output='flats')._griddata + + # The rim is cleared unconditionally ... + assert not flats[0, :].any() and not flats[-1, :].any() + assert not flats[:, 0].any() and not flats[:, -1].any() + # ... and interior cells away from the outlet are flat. + assert flats[1, 1] and flats[2, 3] + # (4, 4) touches the 2, so it has a lower neighbour and is not flat. + assert not flats[4, 4] + + +@pytest.mark.parity +def test_identifyflats_clears_the_nodata_halo(): + """MATLAB drops the 3x3 dilation of the no-data mask from the flats.""" + z = np.full((7, 7), 3.0) + z[3, 3] = np.nan + flats = tt.identifyflats(dem.Elevation(dx=1.0, grid=z), output='flats')._griddata + + for di in (-1, 0, 1): + for dj in (-1, 0, 1): + assert not flats[3 + di, 3 + dj], 'the NaN halo must be cleared' + assert flats[1, 1], 'the rest of the plateau is still flat' + + +@pytest.mark.parity +def test_identifyflats_counts_an_isolated_pit_as_a_flat(): + """MATLAB's test is 'no strictly lower neighbour', with no equal-neighbour rule.""" + z = np.full((5, 5), 99.0) + z[1:4, 1:4] = 5.0 + z[2, 2] = 1.0 # lower than all eight neighbours + flats = tt.identifyflats(dem.Elevation(dx=1.0, grid=z), output='flats')._griddata + assert flats[2, 2] + + +@pytest.mark.parity +def test_closed_basins_exclude_the_grid_edge(): + z = np.full((7, 7), 50.0) + z[3, 3] = 10.0 # an interior closed basin + z[0, 3] = 1.0 # touches the edge, so not closed + closed = tt.identifyflats(dem.Elevation(dx=1.0, grid=z), + output='closedbasins')._griddata + assert closed[3, 3] + assert not closed[0, 3] + + +@pytest.mark.parity +def test_fillsinks_maxdepth_is_all_or_nothing_per_depression(): + """MATLAB restores the whole depression, not just its deepest cells.""" + z = np.full((7, 7), 10.0) + z[1:6, 1:6] = 9.0 + z[2:5, 2:5] = 6.0 + z[3, 3] = 4.0 # one depression, spill 10, depth 6 + grid = dem.Elevation(dx=1.0, grid=z) + + # Below the depression's depth: nothing is touched at all. + for maxdepth in (1.0, 3.0, 5.0): + assert np.array_equal(tt.fillsinks(grid, maxdepth=maxdepth)._griddata, z), \ + 'maxdepth={0} must leave the whole depression alone'.format(maxdepth) + + # At or above it: the depression fills completely. + for maxdepth in (6.0, 7.0): + filled = tt.fillsinks(grid, maxdepth=maxdepth)._griddata + assert np.all(filled[1:6, 1:6] == 10.0) + + +@pytest.mark.parity +def test_localtopography_mean_uses_the_disc_not_its_bounding_box(): + data = np.zeros((15, 15)) + data[4, 4] = 100.0 # 4.24 cells from the centre + grid = dem.Elevation(dx=1.0, grid=data) + # A radius-3 disc excludes it; the 7x7 bounding box does not. + assert tt.localtopography(grid, radius=3.0, type='mean')._griddata[7, 7] == \ + pytest.approx(0.0) + assert tt.localtopography(grid, radius=6.0, type='mean')._griddata[7, 7] > 0.0 + + +@pytest.mark.parity +def test_localtopography_std_is_the_sample_standard_deviation(): + """MATLAB's stdfilt normalises by N-1.""" + values = np.arange(25.0).reshape(5, 5) + grid = dem.Elevation(dx=1.0, grid=values) + window = [7.0, 11.0, 12.0, 13.0, 17.0] # the radius-1 disc at (2, 2) + assert tt.localtopography(grid, radius=1.0, type='std')._griddata[2, 2] == \ + pytest.approx(np.std(window, ddof=1)) diff --git a/topotoolbox.py b/topotoolbox.py new file mode 100644 index 0000000..1b79469 --- /dev/null +++ b/topotoolbox.py @@ -0,0 +1,1039 @@ +"""TopoToolbox-equivalent terrain analysis. + +Every function here reproduces the numerical result of the identically named +function in TopoToolbox 2 (Schwanghart & Scherler 2014), the MATLAB package, +so that a result computed in TopoAnalysis can be checked against one computed +in MATLAB. + +The rest of TopoAnalysis keeps its own idioms -- drainage area in square +metres, ESRI-style hillshade, chi integrated with a left-endpoint rule. This +module exists for the cases where you need the *same number*, not just the +same quantity. :doc:`docs/topotoolbox_parity` tabulates the correspondence +and lists the places where exact agreement is not achievable. + +Example +------- +:: + + from TopoAnalysis import dem, topotoolbox as tt + + z = dem.Elevation(gdal_filename='srtm.tif') + zf = tt.fillsinks(z) # == GRIDobj/fillsinks + fd = tt.flowobj(z, preprocess='fill') # == FLOWobj(DEM,'preprocess','fill') + a = tt.flowacc(fd) # == flowacc(FD), in CELLS + c = tt.chitransform(fd, a, mn=0.45) # == chitransform(S,A) + +Conventions +----------- +* Grids are TopoAnalysis grid objects; results are returned as grid objects + of the same shape and georeferencing. +* Drainage area is in **number of cells**, as TopoToolbox reports it -- not + in square metres, which is what :class:`TopoAnalysis.dem.Area` returns. +* Row 0 is the north edge, matching both packages. + +References +---------- +Schwanghart, W. and Scherler, D. (2014). "TopoToolbox 2 -- MATLAB-based +software for topographic analysis and modeling in Earth surface sciences." +*Earth Surface Dynamics* 2, 1-7. +""" + +from __future__ import annotations + +import numpy as np + +try: # pragma: no cover - exercised by whichever import style is in use + from . import dem as _dem + from . import error as Error + from . import kernels +except ImportError: # pragma: no cover + import dem as _dem + import error as Error + import kernels + +__all__ = [ + "fillsinks", + "identifyflats", + "imposemin", + "flowobj", + "flowacc", + "flowdistance", + "drainagebasins", + "gradient8", + "arcslope", + "hillshade", + "aspect", + "curvature", + "cellarea", + "localtopography", + "streamorder", + "chitransform", + "ksn", + "mchi", + "getcoordinates", + "getextent", + "coord2sub", + "sub2coord", + "STREAM_UNITS", +] + +#: Unit conversions shared by ``gradient8`` and ``arcslope``. +_UNIT_FUNCS = { + 'tangent': lambda g: g, + 'degree': lambda g: np.degrees(np.arctan(g)), + 'radian': lambda g: np.arctan(g), + 'sine': lambda g: np.sin(np.arctan(g)), + 'percent': lambda g: g * 100.0, +} + +STREAM_UNITS = tuple(sorted(_UNIT_FUNCS)) + + +def _like(source, values, cls=None): + """A new grid with ``source``'s georeferencing and ``values`` as data.""" + target = (cls or _dem.BaseSpatialGrid)() + target._copy_info_from_grid(source, set_zeros=True) + target._griddata = values + return target + + +def _convert_unit(values, unit): + try: + return _UNIT_FUNCS[unit](values) + except KeyError: + raise Error.InputError( + 'unit', "unit must be one of {0}; got {1!r}".format(STREAM_UNITS, unit)) + + +# --------------------------------------------------------------------------- +# Depression handling +# --------------------------------------------------------------------------- + + +def fillsinks(elevation, maxdepth=None): + """Fill topographic depressions to a flat surface. + + Equivalent to ``GRIDobj/fillsinks(DEM)`` and, with ``maxdepth``, to + ``fillsinks(DEM, maxdepth)``. + + Unlike :class:`TopoAnalysis.dem.FilledElevation`, whose default adds a + tiny gradient across filled areas so that D8 routing has a direction to + follow, this fills to an exactly flat plateau -- which is what + TopoToolbox does, and what ``identifyflats`` then relies on. + + Parameters + ---------- + elevation : BaseSpatialGrid + The DEM. ``NaN`` is no-data and, as in TopoToolbox, acts as an + outlet: water reaching a NaN cell leaves the grid. + maxdepth : float, optional + Leave depressions deeper than this unfilled. + + Returns + ------- + Elevation + A new grid; the input is not modified. + """ + from scipy.ndimage import generate_binary_structure, label + + original = np.array(elevation._griddata, dtype=np.float64, copy=True) + + filled = _dem.Elevation() + filled._copy_info_from_grid(elevation) + # An explicit copy: np.ascontiguousarray returns the *same* array when the + # input is already float64 and contiguous, and the flood works in place. + filled._griddata = original.copy() + + if maxdepth is None: + kernels.priority_flood(filled._griddata, mode='flat') + return filled + + # MATLAB's depth-limited branch is all-or-nothing per *connected + # depression*, not per cell: whenever a depression's deepest point + # exceeds maxdepth, that cell is added to the marker set, which restores + # the whole depression to its original elevations. Unblocking it can + # split the depression into shallower ones, which then do get filled, so + # the process repeats until nothing exceeds the limit. + maxdepth = float(maxdepth) + structure = generate_binary_structure(2, 2) + seeds = list(kernels.default_seeds(original).tolist()) + + for _ in range(original.size + 1): + surface = original.copy() + kernels.priority_flood(surface, seeds=np.array(seeds, dtype=np.int64), + mode='flat') + depth = surface - original + labels, count = label(depth > 0, structure=structure) + if count == 0: + break + + added = [] + for component in range(1, count + 1): + inside = labels == component + if depth[inside].max() <= maxdepth: + continue + # Seed the deepest cell of the component. + flat_index = int(np.flatnonzero((labels == component).reshape(-1))[ + int(np.argmax(depth[inside]))]) + added.append(flat_index) + if not added: + break + seeds.extend(added) + + filled._griddata = surface + return filled + + +def identifyflats(elevation, output='both'): + """Locate flat areas, their sills and the closed basins. + + Equivalent to ``GRIDobj/identifyflats``. + + * A **flat** is a cell with no strictly lower neighbour -- the MATLAB test + is ``imerode(dem, ones(3)) == dem``. An isolated pit therefore counts + as a flat. The grid's outer ring is cleared, as is the one-cell halo + around any no-data. + * A **sill** is a non-flat cell at the same elevation as an adjacent + flat: the point a flat drains through. + * A **closed basin** is a regional minimum that does not touch the grid + edge. + + Parameters + ---------- + elevation : BaseSpatialGrid + Normally the output of :func:`fillsinks`. + output : {'both', 'flats', 'sills', 'closedbasins', 'all'} + ``'both'`` returns ``(flats, sills)``; ``'all'`` adds the closed + basins. + + Returns + ------- + BaseSpatialGrid or tuple + Boolean grids. + """ + from scipy.ndimage import binary_dilation, generate_binary_structure, label + + z = np.asarray(elevation._griddata, dtype=np.float64) + nodata = np.isnan(z) + work = np.where(nodata, -np.inf, z) + structure = generate_binary_structure(2, 2) # ones(3) + + # imerode with a 3x3 of ones is the local minimum over the 3x3 INCLUDING + # the centre, padded with +inf; `== dem` therefore means "no strictly + # lower neighbour". There is deliberately no requirement of an *equal* + # neighbour, so a cell lower than all eight of its neighbours is a flat. + local_min = work.copy() + for d in range(8): + di, dj = int(kernels.D8_DI[d]), int(kernels.D8_DJ[d]) + local_min = np.minimum(local_min, _shift(work, di, dj, np.inf)) + flats = (local_min == work) + if nodata.any(): + flats &= ~nodata + + # Regional minima, before the rim is cleared: connected components of + # `flats` are equal-valued plateaus with no lower neighbour outside them. + regional_minima = flats.copy() + + flats[0, :] = False + flats[-1, :] = False + flats[:, 0] = False + flats[:, -1] = False + if nodata.any(): + flats[binary_dilation(nodata, structure=structure)] = False + + # Sills: the 3x3 maximum taken over the flat cells only, compared with + # the cell's own elevation, excluding the flats themselves. + marker = np.full(z.shape, -np.inf) + marker[flats] = work[flats] + dilated = marker.copy() + for d in range(8): + di, dj = int(kernels.D8_DI[d]), int(kernels.D8_DJ[d]) + dilated = np.maximum(dilated, _shift(marker, di, dj, -np.inf)) + sills = (dilated == work) & ~flats + if nodata.any(): + sills[nodata] = False + + if output in ('closedbasins', 'all'): + # imclearborder: drop the components that touch the grid edge. + labels, count = label(regional_minima, structure=structure) + on_border = set(labels[0, :].tolist()) | set(labels[-1, :].tolist()) \ + | set(labels[:, 0].tolist()) | set(labels[:, -1].tolist()) + on_border.discard(0) + closed = regional_minima & ~np.isin(labels, list(on_border)) + if nodata.any(): + closed |= nodata + labels, _ = label(closed, structure=structure) + on_border = set(labels[0, :].tolist()) | set(labels[-1, :].tolist()) \ + | set(labels[:, 0].tolist()) | set(labels[:, -1].tolist()) + on_border.discard(0) + closed &= ~np.isin(labels, list(on_border)) + closed[nodata] = False + + flats_grid = _like(elevation, flats) + sills_grid = _like(elevation, sills) + if output == 'flats': + return flats_grid + if output == 'sills': + return sills_grid + if output == 'closedbasins': + return _like(elevation, closed) + if output == 'all': + return flats_grid, sills_grid, _like(elevation, closed) + return flats_grid, sills_grid + + +def _shift(grid, di, dj, fill): + """``grid`` translated so element (i, j) holds ``grid[i+di, j+dj]``.""" + out = np.full_like(grid, fill) + ny, nx = grid.shape + src_i = slice(max(di, 0), ny + min(di, 0)) + src_j = slice(max(dj, 0), nx + min(dj, 0)) + dst_i = slice(max(-di, 0), ny + min(-di, 0)) + dst_j = slice(max(-dj, 0), nx + min(-dj, 0)) + out[dst_i, dst_j] = grid[src_i, src_j] + return out + + +def imposemin(flow_direction, elevation, sl=0.0): + """Carve elevations so every flow path descends at gradient >= ``sl``. + + Equivalent to ``FLOWobj/imposemin(FD, DEM, sl)``. Cells are only ever + lowered, so channels are cut through obstructions instead of being + drowned by them. + + Returns + ------- + Elevation + A new grid. + """ + carved = _dem.Elevation() + carved._copy_info_from_grid(elevation) + carved._griddata = np.array(elevation._griddata, dtype=np.float64, copy=True) + kernels.imposemin( + flow_direction.receivers, + flow_direction.topological_order, + np.ascontiguousarray(flow_direction.step_lengths(), dtype=np.float64), + float(sl), + carved._griddata) + return carved + + +# --------------------------------------------------------------------------- +# Flow +# --------------------------------------------------------------------------- + + +def flowobj(elevation, preprocess='fill'): + """Build a D8 flow-direction grid the way ``FLOWobj`` does. + + Parameters + ---------- + elevation : BaseSpatialGrid + preprocess : {'fill', 'none'} + ``'fill'`` removes depressions first (TopoToolbox's + ``'preprocess','fill'``). ``'carve'`` is not implemented; see the + note below. + + Returns + ------- + FlowDirectionD8 + + Notes + ----- + TopoToolbox routes flow across the flat areas left by filling using an + auxiliary "geodesic" topography, so that flat cells drain towards the + nearest sill. TopoAnalysis instead adds a small gradient while filling + (:attr:`~TopoAnalysis.dem.PriorityQueueMixIn.aggradation_slope`). Both + produce a valid, connected drainage network with identical basin + boundaries wherever the terrain is not flat, but **inside flats the two + networks differ**. TopoToolbox's ``'carve'`` preprocessing lowers the + DEM instead of raising it and is a different surface again. + """ + if preprocess not in ('fill', 'none'): + raise Error.InputError( + 'preprocess', "preprocess must be 'fill' or 'none'; 'carve' is not " + "implemented (see the module docstring)") + + if preprocess == 'fill': + # A small gradient across filled flats, so every cell has a defined + # downslope neighbour. A pure flat fill would leave plateaus with + # no direction at all. + surface = _dem.FilledElevation(elevation=elevation) + else: + surface = elevation + + return _dem.FlowDirectionD8(flooded_dem=surface) + + +def flowacc(flow_direction, weights=None): + """Flow accumulation in **number of cells**. + + Equivalent to ``FLOWobj/flowacc(FD)``: every cell counts itself, so the + minimum value is 1. + + Parameters + ---------- + flow_direction : FlowDirectionD8 + weights : ndarray or BaseSpatialGrid, optional + Per-cell weight; defaults to 1. + + Returns + ------- + BaseSpatialGrid + + See Also + -------- + TopoAnalysis.dem.Area : the same accumulation weighted by cell area. + """ + shape = flow_direction._griddata.shape + if weights is None: + w = np.ones(shape, dtype=np.float64) + else: + w = np.ascontiguousarray( + getattr(weights, '_griddata', weights), dtype=np.float64) + + accumulated = kernels.accumulate( + flow_direction.receivers, flow_direction.topological_order, w) + return _like(flow_direction, np.asarray(accumulated)) + + +def flowdistance(flow_direction, direction='upstream'): + """Along-flow distance in map units. + + Equivalent to ``FLOWobj/flowdistance(FD, direction)`` with no seed + points. + + Parameters + ---------- + direction : {'upstream', 'downstream', 'maxdownstream'} + ``'upstream'`` gives the distance from each cell down to its outlet. + ``'downstream'`` (a synonym for ``'maxdownstream'``) gives the length + of the longest flow path reaching each cell -- the quantity + :class:`TopoAnalysis.dem.FlowLength` computes. + """ + step = np.ascontiguousarray(flow_direction.step_lengths(), dtype=np.float64) + receivers = flow_direction.receivers + order = flow_direction.topological_order + + if direction == 'upstream': + distance = kernels.downstream_distance(receivers, order, step) + elif direction in ('downstream', 'maxdownstream'): + distance, _ = kernels.flow_length(receivers, order, step) + else: + raise Error.InputError( + 'direction', + "direction must be 'upstream', 'downstream' or 'maxdownstream'") + + return _like(flow_direction, np.asarray(distance)) + + +def drainagebasins(flow_direction, outlets=None, valid=None): + """Label drainage basins. + + Equivalent to ``FLOWobj/drainagebasins``. Without ``outlets`` every + terminal cell that something drains into seeds its own basin; with them, + each cell is labelled by the 1-based position of the outlet it drains to. + + Parameters + ---------- + outlets : sequence of (x, y), optional + valid : ndarray or BaseSpatialGrid, optional + Restrict labelling to these cells (e.g. ``~np.isnan(dem._griddata)``). + + Returns + ------- + BaseSpatialGrid + Integer labels; 0 means "reaches none of the outlets". + + Notes + ----- + TopoToolbox numbers basins in the order its edge list happens to meet + them, so the *partition* matches but the integer labels generally do + not. Compare partitions, not label values. + """ + outlet_indices = None + if outlets is not None: + flat = [] + for (row, col) in flow_direction._xy_to_rowscols(outlets): + if row is not None: + flat.append(row * flow_direction._georef_info.nx + col) + outlet_indices = np.array(flat, dtype=np.int64) + + valid_mask = None + if valid is not None: + valid_mask = np.ascontiguousarray( + (np.asarray(getattr(valid, '_griddata', valid)) != 0).astype(np.uint8)) + + labels = kernels.drainage_basins( + flow_direction.receivers, flow_direction.topological_order, + outlet_indices, valid_mask) + return _like(flow_direction, np.asarray(labels)) + + +def streamorder(flow_direction, stream_mask, order_type='strahler'): + """Strahler or Shreve stream order. + + Equivalent to ``FLOWobj/streamorder`` / ``STREAMobj/streamorder``. + + Parameters + ---------- + stream_mask : ndarray or BaseSpatialGrid + Non-zero on the channel network, e.g. ``flowacc(fd)._griddata > 1000``. + order_type : {'strahler', 'shreve'} + + Returns + ------- + BaseSpatialGrid + Integer orders; 0 off the network. + """ + if order_type not in ('strahler', 'shreve'): + raise Error.InputError('order_type', "order_type must be 'strahler' or 'shreve'") + + mask = np.ascontiguousarray( + (np.asarray(getattr(stream_mask, '_griddata', stream_mask)) != 0).astype(np.uint8)) + orders = kernels.stream_order( + flow_direction.receivers, flow_direction.topological_order, mask, order_type) + return _like(flow_direction, np.asarray(orders)) + + +# --------------------------------------------------------------------------- +# Terrain attributes +# --------------------------------------------------------------------------- + + +def gradient8(elevation, unit='tangent'): + """Steepest downward gradient over the 8 neighbours. + + Equivalent to ``GRIDobj/gradient8``. The cardinal and diagonal drops are + divided by ``cellsize`` and ``sqrt(2)*cellsize`` respectively, and the + result is clamped at 0, so pits and flats give exactly 0 rather than a + negative gradient. + + Parameters + ---------- + unit : {'tangent', 'degree', 'radian', 'sine', 'percent'} + """ + z = np.asarray(elevation._griddata, dtype=np.float64) + cs = elevation._georef_info.dx + nodata = np.isnan(z) + work = np.where(nodata, np.inf, z) + + gradient = np.zeros(z.shape, dtype=np.float64) + for d in range(8): + di, dj = int(kernels.D8_DI[d]), int(kernels.D8_DJ[d]) + # +inf outside the grid, so an off-grid neighbour never wins the + # minimum (TopoToolbox erodes with an infinite pad for the same + # reason). + neighbour = _shift(work, di, dj, np.inf) + distance = cs * kernels.D8_DIST[d] + gradient = np.maximum(gradient, (work - neighbour) / distance) + + gradient[nodata] = np.nan + return _like(elevation, _convert_unit(gradient, unit)) + + +def arcslope(elevation, unit='tangent'): + """ArcGIS/Horn 3x3 slope magnitude. + + Equivalent to ``GRIDobj/arcslope``. Uses the 1-2-1 weighted 3x3 stencil + with an ``8 * cellsize`` denominator -- there is no ``sqrt(2)`` anywhere; + the diagonals enter through the weights. + + ``NaN`` holes are filled from their nearest valid neighbour before + differencing, then restored, which is how TopoToolbox handles them here + (and is the opposite of what :func:`gradient8` does). + """ + from scipy.ndimage import distance_transform_edt + from scipy.signal import convolve2d + + z = np.asarray(elevation._griddata, dtype=np.float64) + cs = elevation._georef_info.dx + nodata = np.isnan(z) + + padded = np.pad(z, 1, mode='edge') + padded_nodata = np.pad(nodata, 1, mode='edge') + if padded_nodata.any(): + _, indices = distance_transform_edt(padded_nodata, return_indices=True) + padded = padded[tuple(indices)] + + kernel_y = np.array([[-1.0, -2.0, -1.0], [0.0, 0.0, 0.0], [1.0, 2.0, 1.0]]) + kernel_x = kernel_y.T + gx = convolve2d(padded, kernel_x, mode='valid') / (8.0 * cs) + gy = convolve2d(padded, kernel_y, mode='valid') / (8.0 * cs) + + slope = np.hypot(gx, gy) + slope[nodata] = np.nan + return _like(elevation, _convert_unit(slope, unit)) + + +def _surfnorm(z): + """MATLAB ``surfnorm(Z)`` for the single-argument case. + + Returns ``(Nx, Ny, Nz)``, the unit surface normal with derivatives taken + per cell index: ``Nx`` positive where elevation decreases eastward, + ``Ny`` positive where it decreases southward (row-increasing), ``Nz`` + always positive. + + The edge treatment is the second-order one-sided difference, which is + what MATLAB's quadratic ghost-cell extrapolation reduces to. + """ + z = np.asarray(z, dtype=np.float64) + if min(z.shape) < 3: + raise Error.InputError('surfnorm', 'the grid must be at least 3x3') + + dz_drow, dz_dcol = np.gradient(z, edge_order=2) + nx = -dz_dcol + ny = -dz_drow + nz = np.ones_like(z) + + magnitude = np.sqrt(nx * nx + ny * ny + nz * nz) + magnitude[magnitude == 0] = np.finfo(np.float64).eps + return nx / magnitude, ny / magnitude, nz / magnitude + + +def hillshade(elevation, azimuth=315.0, altitude=60.0, exaggerate=1.0): + """Shaded relief as the cosine of the illumination incidence angle. + + Equivalent to ``GRIDobj/hillshade`` with its default ``'surfnorm'`` + method, including its defaults of ``azimuth=315`` and ``altitude=60``. + + Returns + ------- + BaseSpatialGrid + Values in ``[-1, 1]``. TopoToolbox only scales to 0-255 when it is + plotting, so neither does this. + + See Also + -------- + TopoAnalysis.dem.Hillshade : the ESRI formulation, returning 0-255. + """ + cs = elevation._georef_info.dx + z = np.asarray(elevation._griddata, dtype=np.float64) + + # The compass bearing is converted by subtracting 90, which is correct + # here because surfnorm's +y axis points along increasing row (south). + azimuth_rad = np.radians(azimuth - 90.0) + altitude_rad = np.radians(altitude) + cos_elevation = np.cos(altitude_rad) + sx = cos_elevation * np.cos(azimuth_rad) + sy = cos_elevation * np.sin(azimuth_rad) + sz = np.sin(altitude_rad) + + nx, ny, nz = _surfnorm(z / cs * exaggerate) + return _like(elevation, nx * sx + ny * sy + nz * sz) + + +def aspect(elevation, classify=False): + """Down-slope direction in degrees clockwise from north. + + Equivalent to ``GRIDobj/aspect``. 0 = north, 90 = east, 180 = south, + 270 = west. Perfectly flat cells come out as 90, exactly as they do in + TopoToolbox. + + Parameters + ---------- + classify : bool + Return the 8-class Gomez-Plaza wetness ranking (1 = N, 3 = NE, + 5 = E, 7 = SE, 8 = S, 6 = SW, 4 = W, 2 = NW) instead of degrees. + """ + nx, ny, _ = _surfnorm(np.asarray(elevation._griddata, dtype=np.float64)) + degrees = np.mod(90.0 + np.degrees(np.arctan2(ny, nx)), 360.0) + # atan2(0, 0) is 0 in MATLAB, so a flat cell comes out as 90 (east). + # Reproducing that through NumPy depends on the sign of zero, which + # np.gradient does not preserve, so set it explicitly. + degrees[(nx == 0.0) & (ny == 0.0)] = 90.0 + + if not classify: + return _like(elevation, degrees) + + edges = np.arange(0, 361, 45) + classes = np.array([1, 3, 5, 7, 8, 6, 4, 2], dtype=np.uint8) + bins = np.digitize(degrees, edges, right=False) - 1 + out = np.zeros(degrees.shape, dtype=np.uint8) + inside = (bins >= 0) & (bins < classes.size) & ~np.isnan(degrees) + out[inside] = classes[bins[inside]] + return _like(elevation, out) + + +def curvature(elevation, ctype='profc'): + """Surface curvature from the Evans 3x3 quadratic fit. + + Equivalent to ``GRIDobj/curvature``. + + Parameters + ---------- + ctype : {'profc', 'planc', 'tangc', 'meanc', 'total'} + Profile, plan, tangential, mean, or total curvature. ``'profc'`` + (the default) is positive where a slope steepens downhill. + + Returns + ------- + BaseSpatialGrid + Units are 1/length, except ``'total'`` which is 1/length^2. + + Notes + ----- + Cells where the gradient vanishes give 0/0; TopoToolbox sets those to 0, + and so does this. Cells adjacent to no-data therefore come out as 0 + rather than ``NaN`` -- again matching TopoToolbox. + """ + from scipy.signal import convolve2d + + valid_types = ('profc', 'planc', 'tangc', 'meanc', 'total') + if ctype not in valid_types: + raise Error.InputError('ctype', 'ctype must be one of {0}'.format(valid_types)) + + z = np.asarray(elevation._griddata, dtype=np.float64) + cs = elevation._georef_info.dx + padded = np.pad(z, 1, mode='symmetric') + + def conv(kernel): + return convolve2d(padded, kernel, mode='valid') + + fx = conv(np.array([[-1.0, 0, 1.0]] * 3) / (6 * cs)) + fy = conv(np.array([[1.0, 1.0, 1.0], [0, 0, 0], [-1.0, -1.0, -1.0]]) / (6 * cs)) + kxx = np.array([[1.0, -2.0, 1.0]] * 3) / (3 * cs ** 2) + fxx = conv(kxx) + fyy = conv(kxx.T) + fxy = conv(np.array([[-1.0, 0, 1.0], [0, 0, 0], [1.0, 0, -1.0]]) / (4 * cs ** 2)) + + p2 = fx ** 2 + fy ** 2 + with np.errstate(divide='ignore', invalid='ignore'): + if ctype == 'profc': + curv = -(fx ** 2 * fxx + 2 * fx * fy * fxy + fy ** 2 * fyy) / ( + p2 * (1 + p2) ** 1.5) + elif ctype == 'tangc': + curv = -(fy ** 2 * fxx - 2 * fx * fy * fxy + fx ** 2 * fyy) / ( + p2 * (1 + p2) ** 0.5) + elif ctype == 'planc': + curv = -(fy ** 2 * fxx - 2 * fx * fy * fxy + fx ** 2 * fyy) / (p2 ** 1.5) + elif ctype == 'meanc': + curv = -((1 + fy ** 2) * fxx - 2 * fxy * fx * fy + (1 + fx ** 2) * fyy) / ( + 2 * (p2 + 1) ** 1.5) + else: + curv = fxx ** 2 + 2 * fxy ** 2 + fyy ** 2 + + # Order matters: zero the 0/0 cells first, then restore NaN only where + # the DEM itself was no-data. + curv[~np.isfinite(curv)] = 0.0 + curv[np.isnan(z)] = np.nan + return _like(elevation, curv) + + +#: Earth's total surface area in km^2, the constant TopoToolbox's ``cellarea`` +#: uses. It implies a spherical radius of 6 371 047 m. +EARTH_SURFACE_AREA_KM2 = 510072000.0 + + +def cellarea(grid, unit='m'): + """Area of each cell of a geographic (lat/lon) grid. + + Equivalent to ``GRIDobj/cellarea``. Cells are treated as spherical + quadrangles on a sphere whose total area is + :data:`EARTH_SURFACE_AREA_KM2`. + + Parameters + ---------- + unit : {'m', 'km'} + Square metres (the default) or square kilometres. + + Returns + ------- + BaseSpatialGrid + Constant along each row, since area depends only on latitude. + """ + unit = unit.lower() + if unit not in ('m', 'km'): + raise Error.InputError('unit', "unit must be 'm' or 'km'") + scale = 1e6 if unit == 'm' else 1.0 + + info = grid._georef_info + dlon = abs(info.geoTransform[1]) + dlat = abs(info.geoTransform[5]) + + # Row 0 is the north edge, so latitudes descend with row index. + lat_center = info.yllcenter + (info.ny - 1 - np.arange(info.ny)) * dlat + lat_north = np.radians(lat_center + dlat / 2.0) + lat_south = np.radians(lat_center - dlat / 2.0) + + fraction = (np.radians(dlon) * (np.sin(lat_north) - np.sin(lat_south))) / (4 * np.pi) + per_row = EARTH_SURFACE_AREA_KM2 * fraction * scale + return _like(grid, np.repeat(per_row[:, None], info.nx, axis=1)) + + +def localtopography(elevation, radius=5000.0, type='range'): + """Statistic of elevation within a circular window. + + Equivalent to ``GRIDobj/localtopography``. The window radius is given in + map units and rounded up to a whole number of cells. + + Parameters + ---------- + type : {'range', 'max', 'min', 'mean', 'median', 'std'} + + Notes + ----- + TopoToolbox's default disk (``strel('disk', r)`` with ``N=8``) is a + periodic-line approximation whose exact shape is an internal MATLAB + detail. This uses an exact disc -- TopoToolbox's ``N=0`` -- so values + can differ slightly near the window scale. + """ + from scipy.ndimage import (convolve, distance_transform_edt, generic_filter, + maximum_filter, median_filter, minimum_filter) + + z = np.asarray(elevation._griddata, dtype=np.float64) + cs = elevation._georef_info.dx + radius_px = int(np.ceil(radius / cs)) + yy, xx = np.ogrid[-radius_px:radius_px + 1, -radius_px:radius_px + 1] + footprint = (xx * xx + yy * yy) <= radius_px * radius_px + + nodata = np.isnan(z) + if nodata.any() and type in ('mean', 'median', 'std'): + _, indices = distance_transform_edt(nodata, return_indices=True) + work = z[tuple(indices)] + else: + work = z + + if type == 'max': + result = maximum_filter(np.where(nodata, -np.inf, work), footprint=footprint) + elif type == 'min': + result = minimum_filter(np.where(nodata, np.inf, work), footprint=footprint) + elif type == 'range': + result = (maximum_filter(np.where(nodata, -np.inf, work), footprint=footprint) + - minimum_filter(np.where(nodata, np.inf, work), footprint=footprint)) + elif type in ('mean', 'average'): + # The disc, not the bounding square: uniform_filter(size=2r+1) would + # include the four corners, which lie outside the window. + kernel = footprint.astype(np.float64) + kernel /= kernel.sum() + result = convolve(work, kernel, mode='reflect') + elif type == 'median': + result = median_filter(work, footprint=footprint, mode='reflect') + elif type == 'std': + # ddof=1: MATLAB's stdfilt is the sample standard deviation. + result = generic_filter(work, lambda v: np.std(v, ddof=1), + footprint=footprint, mode='reflect') + else: + raise Error.InputError( + 'type', "type must be one of 'range', 'max', 'min', 'mean', 'median', 'std'") + + result = np.asarray(result, dtype=np.float64) + result[nodata] = np.nan + return _like(elevation, result) + + +# --------------------------------------------------------------------------- +# Chi and channel steepness +# --------------------------------------------------------------------------- + + +def chitransform(flow_direction, area, mn=0.45, a0=1e6, outlets=None, + correctcellsize=True, stream_mask=None): + r"""The chi integral transform. + + Equivalent to ``STREAMobj/chitransform``: + + .. math:: + \chi = \int_{\mathrm{outlet}}^{x} + \left(\frac{A_0}{A(x')}\right)^{m/n}\,\mathrm{d}x' + + integrated upstream with the trapezoidal rule, so chi is 0 at each outlet + and increases upstream. + + Parameters + ---------- + area : BaseSpatialGrid + Flow accumulation. With ``correctcellsize=True`` (the default, as in + TopoToolbox) it is taken to be in **number of cells** and multiplied + by ``cellsize**2``; pass ``False`` if it is already in m^2, e.g. if + it came from :class:`TopoAnalysis.dem.Area`. + mn : float + The m/n concavity ratio. TopoToolbox's default is 0.45. + a0 : float + Reference area in m^2. TopoToolbox's default is 1e6 (1 km^2). + outlets : sequence of (x, y), optional + Integration starts here. Defaults to every terminal cell of the + network, which is what a STREAMobj built over the whole grid gives. + stream_mask : ndarray or BaseSpatialGrid, optional + Restrict the integration to the channel network. + + Returns + ------- + BaseSpatialGrid + Chi in map units; ``NaN`` away from the integrated network. + """ + a = np.asarray(area._griddata, dtype=np.float64) + if correctcellsize: + # TopoToolbox multiplies by S.cellsize^2 unconditionally; feeding it + # an area that is already in m^2 squares the cell size twice. + a = a * (flow_direction._georef_info.dx ** 2) + + receivers = flow_direction.receivers + order = flow_direction.topological_order + step = np.ascontiguousarray(flow_direction.step_lengths(), dtype=np.float64) + + if outlets is None: + outlet_indices = np.flatnonzero(receivers.reshape(-1) < 0).astype(np.int64) + else: + flat = [] + for (row, col) in flow_direction._xy_to_rowscols(outlets): + if row is not None: + flat.append(row * flow_direction._georef_info.nx + col) + outlet_indices = np.array(flat, dtype=np.int64) + + mask = None + if stream_mask is not None: + mask = np.ascontiguousarray( + (np.asarray(getattr(stream_mask, '_griddata', stream_mask)) != 0).astype(np.uint8)) + + chi_values, _ = kernels.chi( + receivers, order, np.ascontiguousarray(a), step, outlet_indices, + float(a0), float(mn), True, 0.0, mask) + return _like(flow_direction, np.asarray(chi_values)) + + +def ksn(flow_direction, elevation, area, theta=0.45, correctcellsize=True, + min_gradient=1e-5): + r"""Normalized channel steepness index, :math:`k_{sn} = S A^{\theta}`. + + Equivalent to ``STREAMobj/ksn``, including its unconditional minima + imposition at a gradient of 1e-5 before the slope is measured. + + Parameters + ---------- + area : BaseSpatialGrid + Flow accumulation in cells (see ``correctcellsize``). + theta : float + Reference concavity; TopoToolbox's default is 0.45. + min_gradient : float + Gradient imposed by the carving step. TopoToolbox hard-codes 1e-5. + + Returns + ------- + BaseSpatialGrid + ``ksn`` with a reference area of 1 m^2 -- TopoToolbox's ``ksn`` has + no ``a0``. Multiply by ``a0**theta`` to compare against + ``chiplot``'s ``ks``. + """ + a = np.asarray(area._griddata, dtype=np.float64) + if correctcellsize: + a = a * (flow_direction._georef_info.dx ** 2) + + carved = imposemin(flow_direction, elevation, sl=min_gradient) + + receivers = flow_direction.receivers.reshape(-1) + z = np.asarray(carved._griddata, dtype=np.float64).reshape(-1) + step = flow_direction.step_lengths().reshape(-1) + + gradient = np.zeros(z.shape, dtype=np.float64) + donors = np.flatnonzero(receivers >= 0) + gradient[donors] = (z[donors] - z[receivers[donors]]) / step[donors] + + shape = flow_direction._griddata.shape + with np.errstate(divide='ignore', invalid='ignore'): + values = gradient.reshape(shape) / (a ** (-theta)) + return _like(flow_direction, values) + + +def mchi(flow_direction, elevation, chi): + r"""Gradient of elevation in chi space, :math:`M_{\chi}`. + + Equivalent to ``STREAMobj/mchi``: a forward difference assigned to the + upstream cell, with no carving and no smoothing. + + ``M_chi`` scales with the ``a0`` baked into ``chi``: computing chi with + ``a0=1`` puts it in the same units as :func:`ksn`, and + ``mchi(a0) * a0**mn`` is invariant. + + It is *close to* but not equal to :func:`ksn`, despite what TopoToolbox's + own docstring says: ``ksn`` carves the DEM first and evaluates the + integrand at the upstream cell, while ``mchi`` uses the raw elevations + and inherits chi's trapezoidal averaging. + """ + receivers = flow_direction.receivers.reshape(-1) + z = np.asarray(elevation._griddata, dtype=np.float64).reshape(-1) + c = np.asarray(getattr(chi, '_griddata', chi), dtype=np.float64).reshape(-1) + + values = np.zeros(z.shape, dtype=np.float64) + donors = np.flatnonzero(receivers >= 0) + with np.errstate(divide='ignore', invalid='ignore'): + values[donors] = ((z[donors] - z[receivers[donors]]) + / (c[donors] - c[receivers[donors]])) + return _like(flow_direction, values.reshape(flow_direction._griddata.shape)) + + +# --------------------------------------------------------------------------- +# Referencing +# --------------------------------------------------------------------------- + + +def getcoordinates(grid): + """``(x, y)`` cell-centre coordinate vectors. + + Equivalent to ``GRIDobj/getcoordinates``. ``x`` runs west to east; + ``y`` runs **north to south**, matching the row order of the grid. + """ + info = grid._georef_info + x = info.xllcenter + np.arange(info.nx) * info.dx + y = info.yllcenter + (info.ny - 1 - np.arange(info.ny)) * info.dx + return x, y + + +def getextent(grid): + """``(xmin, xmax, ymin, ymax)`` of the outermost **cell centres**. + + Equivalent to ``GRIDobj/getextent``. Note this is half a cell inside + :meth:`TopoAnalysis.dem.BaseSpatialGrid.extent`, which returns the pixel + edges that matplotlib wants. + """ + x, y = getcoordinates(grid) + return (float(x.min()), float(x.max()), float(y.min()), float(y.max())) + + +def _round_half_away(values): + """MATLAB's ``round``: halves go away from zero, not to even.""" + values = np.asarray(values, dtype=np.float64) + return np.where(values >= 0, np.floor(values + 0.5), np.ceil(values - 0.5)) + + +def coord2sub(grid, x, y): + """Map coordinates to 0-based ``(row, col)`` subscripts. + + Equivalent to ``GRIDobj/coord2sub`` apart from the index base. Points + outside the grid give ``NaN``. + + Notes + ----- + Halves are rounded away from zero, as MATLAB does. Python's built-in + ``round`` -- which + :meth:`TopoAnalysis.dem.BaseSpatialGrid._xy_to_rowscols` uses -- rounds + halves to even, so the two disagree for points landing exactly on a cell + boundary. + """ + info = grid._georef_info + x = np.atleast_1d(np.asarray(x, dtype=np.float64)) + y = np.atleast_1d(np.asarray(y, dtype=np.float64)) + + north = info.yllcenter + (info.ny - 1) * info.dx + col = _round_half_away((x - info.xllcenter) / info.dx) + row = _round_half_away((north - y) / info.dx) + + outside = (col < 0) | (col > info.nx - 1) | (row < 0) | (row > info.ny - 1) \ + | np.isnan(x) | np.isnan(y) + row = np.where(outside, np.nan, row) + col = np.where(outside, np.nan, col) + return row, col + + +def sub2coord(grid, row, col): + """0-based ``(row, col)`` subscripts to cell-centre map coordinates. + + Equivalent to ``GRIDobj/sub2coord``. Subscripts outside the grid are + extrapolated rather than rejected, as in TopoToolbox. + """ + info = grid._georef_info + row = np.asarray(row, dtype=np.float64) + col = np.asarray(col, dtype=np.float64) + x = info.xllcenter + col * info.dx + y = info.yllcenter + (info.ny - 1 - row) * info.dx + return x, y diff --git a/utils.py b/utils.py index 96523e8..ff8209b 100644 --- a/utils.py +++ b/utils.py @@ -1,27 +1,60 @@ +"""Convenience wrappers for fitting channel steepness at a single outlet.""" + +import numpy as np + +try: # pragma: no cover - exercised by whichever import style is in use + from . import demRecursionTools +except ImportError: # pragma: no cover + import demRecursionTools + + def calc_ks_for_outlet(outlet, theta, **kwargs): - fd = kwargs['flow_direction'] - xo = kwargs.get('xo') if kwargs.get('xo') is not None else 500.0 - kwargs.pop('xo') + """Fit channel steepness to the basin draining to ``outlet``. + + Parameters + ---------- + outlet : (float, float) + Map coordinates of the outlet. + theta : float + Reference concavity. + flow_direction : FlowDirectionD8 + Required, passed by keyword. + xo : float, optional + Reference length; the reference area is ``xo**2``. Default 500. + plot : float, optional + If equal to ``theta``, also draw the chi-elevation plot. + **kwargs + Any other keyword is a grid to sample along the profiles, e.g. + ``elevation=dem, area=area``. + + Returns + ------- + (ndarray, float) + ``(ks, r_squared)``. + """ + kwargs = dict(kwargs) + fd = kwargs.pop('flow_direction') + # kwargs.pop('xo') without a default raised KeyError whenever xo was + # left at its default. + xo = kwargs.pop('xo', None) + if xo is None: + xo = 500.0 plot = kwargs.pop('plot', None) - + de = fd._mean_pixel_dimension() ld_list = fd.map_values_to_recursive_list(outlet, **kwargs) - - from demRecursionTools import best_ks_with_r2_list - from numpy import array - ret = best_ks_with_r2_list(ld_list, de, array([theta]), xo = xo) + + ret = demRecursionTools.best_ks_with_r2_list(ld_list, de, np.array([theta]), xo=xo) if plot is not None and plot == theta: import matplotlib.pylab as plt - from demRecursionTools import chi_elevation - import numpy as np - e, c = chi_elevation(ld_list, de, array([theta]), xo=xo) + + e, c = demRecursionTools.chi_elevation(ld_list, de, np.array([theta]), xo=xo) plt.figure() plt.plot(c, e, 'k.') - plt.hold(True) + # Matplotlib 3.0 removed the explicit hold call; overlaying is now + # the default, so no equivalent is needed here. (ks, _) = ret - plt.plot([0, np.max(c)],[0, ks*np.max(c)], 'k-') - + plt.plot([0, np.max(c)], [0, ks * np.max(c)], 'k-') + return ret - - \ No newline at end of file