Repository: makepath/xarray-spatial

Issue:
Modernise build system
We need to modernise our build system to use PEP517 and PEP518, i.e. a migration from configuration in `setup.py` to `setup.cfg` and `pyproject.toml` instead.  See https://setuptools.pypa.io/en/latest/build_meta.html


Discussion:

Patch:
diff --git a/.appveyor.yml b/.appveyor.yml
index 7a0fb97..a237c5e 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -10,9 +10,7 @@ environment:
 build: off
 
 install:
-  - "%PYTHON%\\python.exe -m pip install wheel"
-  - "%PYTHON%\\python.exe -m pip install -r requirements.txt"
-  - "%PYTHON%\\python.exe -m pip install -r requirements-dev.txt"
+  - "%PYTHON%\\python.exe -m pip install .[tests]"
 
 test_script:
   - "%PYTHON%\\python.exe -m pytest"
diff --git a/.github/workflows/docs-publish.yml b/.github/workflows/docs-publish.yml
index 48562e6..ff37149 100644
--- a/.github/workflows/docs-publish.yml
+++ b/.github/workflows/docs-publish.yml
@@ -21,7 +21,7 @@ jobs:
         run: |
           pip install .[docs,tests]
           pip list
-      - name: Build
+      - name: Build docs
         run: |
           make -C docs html
       - uses: lauchacarro/Azure-Storage-Action@v1.0
diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml
index baa0d4a..d8d98e1 100644
--- a/.github/workflows/pypi-publish.yml
+++ b/.github/workflows/pypi-publish.yml
@@ -5,13 +5,13 @@ on:
        - '*'
 jobs:
   publish:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20-04
     steps:
       - uses: actions/checkout@master
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Get release version
         run: |
           echo "CHANGELOG_VERSION=$(cat CHANGELOG.md | grep -oP '(?<=###\s)(.*)(?=\s\-)' | head -n 1 | sed 's/Version\s/v/')" >> $GITHUB_ENV
@@ -24,14 +24,12 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pyct \
-                      wheel \
-                      setuptools
+          pip install build
       - name: Get all git tags
         run: git fetch --tags -f
       - name: Build package
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build --sdist --wheel
       - name: Get package size
         run: echo "PKG_SIZE=$(find dist -maxdepth 1 -regex '.*gz' | xargs stat --format='%s')" >> $GITHUB_ENV
       - name: Check package size
@@ -44,4 +42,4 @@ jobs:
         uses: pypa/gh-action-pypi-publish@master
         with:
           skip_existing: true
-          password: ${{ secrets.PYPI_API_TOKEN }}
\ No newline at end of file
+          password: ${{ secrets.PYPI_API_TOKEN }}
diff --git a/pyproject.toml b/pyproject.toml
index 648b754..cc7ca4f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,10 +1,11 @@
 [build-system]
 requires = [
-    'param >=1.6.1',
-    'pyct >= 0.4.8',
+    'build',
+    'param',
+    'pyct',
     'setuptools >= 54.1.2',
-    'wheel >= 0.36.2',
 ]
+
 [tool.check-manifest]
 ignore = [
     '.appveyor.yml',
diff --git a/requirements-dev.txt b/requirements-dev.txt
index b124a60..e2d3f12 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -2,5 +2,3 @@ check-manifest
 pytest
 setuptools_scm
 tox
-twine
-wheel
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 0cd6101..895cae3 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-bokeh>2
+bokeh >2
 datashader
 noise >=1.2.2
 numba
diff --git a/setup.cfg b/setup.cfg
index bc5f57c..38c05a5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,61 @@
 [metadata]
+author = makepath
+classifiers =
+    Development Status :: 4 - Beta
+    Intended Audience :: Developers
+    Intended Audience :: Science/Research
+    License :: OSI Approved :: MIT License
+    Operating System :: OS Independent
+    Programming Language :: Python :: 3
+description = xarray-based spatial analysis tools
+license = MIT
+license_file = LICENSE.txt
 long_description = file: README.md
 long_description_content_type = text/markdown
+name = xarray-spatial
+platforms = any
+url = https://github.com/makepath/xarray-spatial
+
+[options]
+include_package_data = True
+install_requires =
+    datashader
+    numba
+    xarray
+packages = find:
+python_requires = >=3.6
+zip_safe = False
+
+[options.entry_points]
+console_scripts =
+    xrspatial = xrspatial.__main__:main
+
+[options.extras_require]
+doc =
+    geopandas
+    Jinja2 >=2.11
+    ipykernel
+    matplotlib
+    nbsphinx
+    numpydoc
+    pandoc
+    pydata_sphinx_theme
+    sphinx
+    sphinx-multiversion
+    sphinx-panels
+    sphinx_rtd_theme
+examples =
+optional =
+    # Optional for polygonize return types.
+    awkward>=1.4
+    geopandas
+    shapely
+    spatialpandas
+    # Optional for gpu_rtx functions. Also requires cupy.
+    rtxpy
+tests =
+    noise >= 1.2.2
+    pytest
 
 [flake8]
 exclude =
diff --git a/setup.py b/setup.py
index 6792cbe..16fed30 100755
--- a/setup.py
+++ b/setup.py
@@ -1,72 +1,17 @@
 import os
+import param
+import pyct.build
 import sys
 import shutil
 from setuptools import setup
 
 
-# build dependencies
-import pyct.build
-import param
-
-# dependencies
-
-# datashader first, then pyct unless pyct version compatible with ds
-# is specified
-# spatialpandas may not be required in final pharmacy_desert version
-# pyct may not be required after pyctdev is released
-install_requires = [
-    'dask',
-    'datashader',
-    'numba',
-    'pandas',
-    'pillow',
-    'requests',
-    'xarray',
-    'pyct <=0.4.6',
-    'param >=1.6.1',
-    'distributed >=2021.03.0',
-]
-
-examples = [
-]
-
-# Additional tests dependencies and examples_extra may be needed in the future
-extras_require = {
-    'tests': [
-        'pytest',
-        'noise >=1.2.2',
-    ],
-    'examples': examples,
-    'optional': [
-        # Optional for polygonize return types.
-        'awkward>=1.4',
-        'geopandas',
-        'shapely',
-        'spatialpandas',
-        # Optional for gpu_rtx functions. Also requires cupy.
-        "rtxpy",
-    ],
-    'docs': [
-        'geopandas',
-        'Jinja2 >=2.11',
-        'ipykernel',
-        'matplotlib',
-        'nbsphinx',
-        'numpydoc',
-        'pandoc',
-        'pydata_sphinx_theme',
-        'sphinx',
-        'sphinx-multiversion',
-        'sphinx-panels',
-        'sphinx_rtd_theme',
-    ],
-}
-
-extras_require['all'] = sorted(set(sum(extras_require.values(), [])))
-
-version = param.version.get_setup_version(__file__, 'xarray-spatial',
-                                          pkgname='xrspatial',
-                                          archive_commit="$Format:%h$")
+version = param.version.get_setup_version(
+    __file__,
+    'xarray-spatial',
+    pkgname='xrspatial',
+    archive_commit="$Format:%h$",
+)
 
 if 'sdist' in sys.argv and 'bdist_wheel' in sys.argv:
     try:
@@ -78,37 +23,12 @@ if 'sdist' in sys.argv and 'bdist_wheel' in sys.argv:
         sys.exit('invalid version')
 
 
-# metadata for setuptools
-
-setup_args = dict(
-    name='xarray-spatial',
-    version=version,
-    description='xarray-based spatial analysis tools',
-    install_requires=install_requires,
-    extras_require=extras_require,
-    tests_require=extras_require['tests'],
-    zip_safe=False,
-    classifiers=['Programming Language :: Python :: 3',
-                 'License :: OSI Approved :: MIT License',
-                 'Operating System :: OS Independent'],
-    packages=['xrspatial',
-              'xrspatial.tests'
-              ],
-    include_package_data=True,
-    entry_points={
-        'console_scripts': [
-            'xrspatial = xrspatial.__main__:main'
-        ]
-    },
-)
-
-
 if __name__ == '__main__':
     example_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                 'xrspatial', 'examples')
     if 'develop' not in sys.argv:
         pyct.build.examples(example_path, __file__, force=True)
-    setup(**setup_args)
+    setup(version=version)
 
     if os.path.isdir(example_path):
         shutil.rmtree(example_path)
diff --git a/xrspatial/classify.py b/xrspatial/classify.py
index 7a09f44..a2492f6 100755
--- a/xrspatial/classify.py
+++ b/xrspatial/classify.py
@@ -16,8 +16,6 @@ from datashader.colors import rgb
 import numba as nb
 import dask.array as da
 
-from numpy.random import RandomState
-
 from xrspatial.utils import cuda_args
 from xrspatial.utils import ngjit
 from xrspatial.utils import ArrayTypeFunctionMapping
@@ -78,11 +76,13 @@ def _cpu_bin(data, bins, new_values):
                 # first bin
                 if b == 0:
                     if val <= bins[b]:
-                        val_bin = b
+                        if np.isfinite(val):
+                            val_bin = b
                         break
                 else:
                     if val > bins[b - 1] and val <= bins[b]:
-                        val_bin = b
+                        if np.isfinite(val):
+                            val_bin = b
                         break
 
             if val_bin > -1:
@@ -144,6 +144,10 @@ def _run_gpu_bin(data, bins, new_values, out):
 
 
 def _run_cupy_bin(data, bins, new_values):
+    # replace inf by nan to avoid classify these values as we want to treat them as outliers
+    data = cupy.where(data == cupy.inf, cupy.nan, data)
+    data = cupy.where(data == -cupy.inf, cupy.nan, data)
+
     bins_cupy = cupy.asarray(bins)
     new_values_cupy = cupy.asarray(new_values)
     out = cupy.empty(data.shape, dtype='f4')
@@ -542,12 +546,17 @@ def _run_jenks(data, n_classes, module):
 def _run_natural_break(agg, num_sample, k, module):
     data = agg.data
     num_data = data.size
+    max_data = module.max(data[module.isfinite(data)])
+    if module == cupy:
+        max_data = max_data.get()
 
     if num_sample is not None and num_sample < num_data:
         # randomly select sample from the whole dataset
         # create a pseudo random number generator
-        generator = RandomState(1234567890)
-        idx = module.linspace(
+        # Note: cupy and nupy generate different random numbers
+        # use numpy.random to ensure the same result
+        generator = np.random.RandomState(1234567890)
+        idx = np.linspace(
             0, data.size, data.size, endpoint=False, dtype=module.uint32
         )
         generator.shuffle(idx)
@@ -586,6 +595,7 @@ def _run_natural_break(agg, num_sample, k, module):
     else:
         centroids = _run_jenks(sample_data, k, module)
         bins = module.array(centroids[1:])
+        bins[-1] = max_data
 
     out = _bin(agg, bins, module.arange(uvk))
     return out
@@ -701,6 +711,8 @@ def _run_equal_interval(agg, k, module):
         inf = np.inf
 
     data = module.where(data == inf, nan, data)
+    data = module.where(data == -inf, nan, data)
+
     max_data = module.nanmax(data)
     min_data = module.nanmin(data)
     if module == cupy:


Test patch:
diff --git a/xrspatial/tests/test_classify.py b/xrspatial/tests/test_classify.py
index 55fc2bb..073ca66 100755
--- a/xrspatial/tests/test_classify.py
+++ b/xrspatial/tests/test_classify.py
@@ -15,7 +15,7 @@ from xrspatial.tests.general_checks import create_test_raster
 
 def input_data(backend='numpy'):
     elevation = np.array([
-        [1.,  2.,  3.,  4., np.nan],
+        [-np.inf,  2.,  3.,  4., np.nan],
         [5.,  6.,  7.,  8.,  9.],
         [10., 11., 12., 13., 14.],
         [15., 16., 17., 18., np.inf],
@@ -29,10 +29,10 @@ def result_reclassify():
     bins = [10, 15, np.inf]
     new_values = [1, 2, 3]
     expected_result = np.asarray([
-        [1., 1., 1., 1., np.nan],
+        [np.nan, 1., 1., 1., np.nan],
         [1., 1., 1., 1., 1.],
         [1., 2., 2., 2., 2.],
-        [2., 3., 3., 3., 3.]
+        [2., 3., 3., 3., np.nan]
     ], dtype=np.float32)
     return bins, new_values, expected_result
 
@@ -71,8 +71,8 @@ def test_reclassify_dask_cupy(result_reclassify):
 def result_quantile():
     k = 5
     expected_result = np.asarray([
-        [0., 0., 0., 0., np.nan],
-        [1., 1., 1., 2., 2.],
+        [np.nan, 0., 0., 0., np.nan],
+        [0., 1., 1., 1., 2.],
         [2., 2., 3., 3., 3.],
         [4., 4., 4., 4., np.nan]
     ], dtype=np.float32)
@@ -117,14 +117,27 @@ def test_quantile_cupy(result_quantile):
 def result_natural_breaks():
     k = 5
     expected_result = np.asarray([
-        [0., 0., 0., 1., np.nan],
-        [1., 1., 2., 2., 2.],
+        [np.nan, 0., 0., 0., np.nan],
+        [1., 1., 1., 2., 2.],
         [2., 3., 3., 3., 3.],
         [4., 4., 4., 4., np.nan]
     ], dtype=np.float32)
     return k, expected_result
 
 
+@pytest.fixture
+def result_natural_breaks_num_sample():
+    k = 5
+    num_sample = 8
+    expected_result = np.asarray([
+        [np.nan, 0., 0., 0., np.nan],
+        [0., 1., 1., 1., 2.],
+        [2., 3., 3., 3., 3.],
+        [4., 4., 4., 4., np.nan]
+    ], dtype=np.float32)
+    return k, num_sample, expected_result
+
+
 def test_natural_breaks_numpy(result_natural_breaks):
     numpy_agg = input_data()
     k, expected_result = result_natural_breaks
@@ -132,6 +145,13 @@ def test_natural_breaks_numpy(result_natural_breaks):
     general_output_checks(numpy_agg, numpy_natural_breaks, expected_result, verify_dtype=True)
 
 
+def test_natural_breaks_numpy_num_sample(result_natural_breaks_num_sample):
+    numpy_agg = input_data()
+    k, num_sample, expected_result = result_natural_breaks_num_sample
+    numpy_natural_breaks = natural_breaks(numpy_agg, k=k, num_sample=num_sample)
+    general_output_checks(numpy_agg, numpy_natural_breaks, expected_result, verify_dtype=True)
+
+
 def test_natural_breaks_cpu_deterministic():
     results = []
     elevation = np.arange(100).reshape(10, 10)
@@ -164,12 +184,20 @@ def test_natural_breaks_cupy(result_natural_breaks):
     general_output_checks(cupy_agg, cupy_natural_breaks, expected_result, verify_dtype=True)
 
 
+@pytest.mark.skipif(doesnt_have_cuda(), reason="CUDA Device not Available")
+def test_natural_breaks_cupy_num_sample(result_natural_breaks_num_sample):
+    cupy_agg = input_data('cupy')
+    k, num_sample, expected_result = result_natural_breaks_num_sample
+    cupy_natural_breaks = natural_breaks(cupy_agg, k=k, num_sample=num_sample)
+    general_output_checks(cupy_agg, cupy_natural_breaks, expected_result, verify_dtype=True)
+
+
 @pytest.fixture
 def result_equal_interval():
     k = 3
     expected_result = np.asarray([
-        [0., 0., 0., 0., np.nan],
-        [0., 0., 1., 1., 1.],
+        [np.nan, 0., 0., 0., np.nan],
+        [0., 0., 0., 1., 1.],
         [1., 1., 1., 2., 2.],
         [2., 2., 2., 2., np.nan]
     ], dtype=np.float32)