Repository: natcap/invest
Pull request #1209 (2023-02-28 20:10:43)

Resolve conflicts in PR #1208
Resolves conflicts from #1202 shown in #1208 


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index 30a60d4fa..b6111f4be 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -46,10 +46,20 @@ Unreleased Changes
 * Workbench
     * Added tooltips to the model tabs so that they can be identified even when
       several tabs are open (`#1071 <https://github.com/natcap/invest/issues/1088>`_)
+* HRA
+    * Fixed an issue where a cryptic exception was being thrown if the criteria
+      table's sections were not spelled exactly as expected.  There is now a
+      much more readable error if a section is obviously missing.  Leading and
+      trailing whitespace is also now removed from all string fields in the
+      criteria table, which should also help reduce the chance of errors.
+      https://github.com/natcap/invest/issues/1191
 * Scenic Quality
     * Any points over nodata (and therefore excluded from the viewshed
       analysis) will now correctly have their FID reported in the logging.
       https://github.com/natcap/invest/issues/1188
+    * Clarifying where the visual quality calculations' disk-based sorting
+      cache should be located, which addresses an interesting crash experienced
+      by some users on Windows. https://github.com/natcap/invest/issues/1189
 * SDR
     * The ``ws_id`` field is no longer a required field in the watershed vector.
       https://github.com/natcap/invest/issues/1201
@@ -57,6 +67,10 @@ Unreleased Changes
     * If a soil group raster contains any pixels that are not in the set of
       allowed soil groups (anything other than 1, 2, 3 or 4), a human readable
       exception will now be raised. https://github.com/natcap/invest/issues/1193
+* Visitation: Recreation and Tourism
+    * Fixed a ``FutureWarning`` when reading in CSVs. This fix does not
+      otherwise affect model behavior. https://github.com/natcap/invest/issues/1202
+
 
 3.12.1 (2022-12-16)
 -------------------
diff --git a/Makefile b/Makefile
index 2007e97a2..23add81cb 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ GIT_TEST_DATA_REPO_REV      := 29d8da596ff197d3cc6e355e7cd4313945b89b71
 
 GIT_UG_REPO                 := https://github.com/natcap/invest.users-guide
 GIT_UG_REPO_PATH            := doc/users-guide
-GIT_UG_REPO_REV             := a270a6662068b6ed576814e7060d424a83a21bb9
+GIT_UG_REPO_REV             := 019a80680601b8720c54a6a5f78751d50251d1f5
 
 ENV = "./env"
 ifeq ($(OS),Windows_NT)
diff --git a/scripts/userguide-filevalidator.py b/scripts/userguide-filevalidator.py
index a42023d6c..5f8f7d178 100644
--- a/scripts/userguide-filevalidator.py
+++ b/scripts/userguide-filevalidator.py
@@ -1,7 +1,7 @@
 #!python
 
-import os
 import argparse
+import os
 
 
 def main(userguide_dir):
@@ -19,7 +19,7 @@ def main(userguide_dir):
     from natcap.invest.model_metadata import MODEL_METADATA
 
     missing_files = []
-    userguide_dir_source = os.path.join(userguide_dir, 'source')
+    userguide_dir_source = os.path.join(userguide_dir, 'source', 'en')
 
     for data in MODEL_METADATA.values():
         # html referenced won't exist unless we actually built the UG,
diff --git a/src/natcap/invest/hra.py b/src/natcap/invest/hra.py
index 8f1b25a41..266e46fb1 100644
--- a/src/natcap/invest/hra.py
+++ b/src/natcap/invest/hra.py
@@ -1703,12 +1703,32 @@ def _parse_criteria_table(criteria_table_path, target_composite_csv_path):
                              engine='python')
     table = df.to_numpy()
 
+    # clean up any leading or trailing whitespace.
+    for row_num in range(table.shape[0]):
+        for col_num in range(table.shape[1]):
+            value = table[row_num][col_num]
+            if isinstance(value, str):
+                table[row_num][col_num] = value.strip()
+
+    # Some fields are required and will lead to cryptic errors if not found or
+    # slightly misspelled.
+    habitat_name_header = 'HABITAT NAME'
+    overlap_section_header = 'HABITAT STRESSOR OVERLAP PROPERTIES'
+    habitat_resilience_header = 'HABITAT RESILIENCE ATTRIBUTES'
+    required_section_headers = {
+        habitat_name_header, overlap_section_header,
+        habitat_resilience_header}
+    missing_sections = required_section_headers - set(table[:, 0])
+    if missing_sections:
+        raise AssertionError(
+            "The criteria table is missing these section headers: "
+            f"{', '.join(missing_sections)}")
+
     # Habitats are loaded from the top row (table[0])
     known_habitats = set(table[0]).difference(
-        {'HABITAT NAME', numpy.nan, 'CRITERIA TYPE'})
+        {habitat_name_header, numpy.nan, 'CRITERIA TYPE'})
 
     # Stressors are loaded from the first column (table[:, 0])
-    overlap_section_header = 'HABITAT STRESSOR OVERLAP PROPERTIES'
     known_stressors = set()
     for row_index, value in enumerate(table[:, 0]):
         try:
@@ -1731,7 +1751,7 @@ def _parse_criteria_table(criteria_table_path, target_composite_csv_path):
     criteria_col = None
     habitats = set()
     for col_index, value in enumerate(table[0]):
-        if value == 'HABITAT NAME':
+        if value == habitat_name_header:
             continue
         if value == 'CRITERIA TYPE':
             criteria_col = col_index
@@ -1753,8 +1773,8 @@ def _parse_criteria_table(criteria_table_path, target_composite_csv_path):
             continue
 
         if (row[0] in known_stressors or
-                row[0] == 'HABITAT RESILIENCE ATTRIBUTES'):
-            if row[0] == 'HABITAT RESILIENCE ATTRIBUTES':
+                row[0] == habitat_resilience_header):
+            if row[0] == habitat_resilience_header:
                 row[0] = 'RESILIENCE'  # Shorten for convenience
             current_stressor = row[0]
             current_stressor_header_row = row_index
diff --git a/src/natcap/invest/recreation/recmodel_client.py b/src/natcap/invest/recreation/recmodel_client.py
index 1a9b7dfbb..1016254e5 100644
--- a/src/natcap/invest/recreation/recmodel_client.py
+++ b/src/natcap/invest/recreation/recmodel_client.py
@@ -1,42 +1,41 @@
 """InVEST Recreation Client."""
 import json
-import os
-import zipfile
-import time
 import logging
 import math
+import os
 import pickle
-import urllib.request
-import tempfile
 import shutil
+import tempfile
+import time
+import urllib.request
+import zipfile
 
-import rtree
+import numpy
+import numpy.linalg
+import pygeoprocessing
 import Pyro4
-from osgeo import ogr
-from osgeo import gdal
-from osgeo import osr
+import rtree
 import shapely
 import shapely.geometry
-import shapely.wkt
 import shapely.prepared
-import pygeoprocessing
-import numpy
-import numpy.linalg
 import shapely.speedups
+import shapely.wkt
 import taskgraph
+from osgeo import gdal
+from osgeo import ogr
+from osgeo import osr
 
 if shapely.speedups.available:
     shapely.speedups.enable()
 
 # prefer to do intrapackage imports to avoid case where global package is
 # installed and we import the global version of it rather than the local
-from .. import utils
+from .. import gettext
 from .. import spec_utils
-from ..unit_registry import u
+from .. import utils
 from .. import validation
 from ..model_metadata import MODEL_METADATA
-from .. import gettext
-
+from ..unit_registry import u
 
 LOGGER = logging.getLogger(__name__)
 
@@ -1499,8 +1498,8 @@ def _validate_same_projection(base_vector_path, table_path):
     # This will load the table as a list of paths which we can iterate through
     # without bothering the rest of the table structure
     data_paths = utils.read_csv_to_dataframe(
-        table_path, to_lower=True, squeeze=True, expand_path_cols=['path']
-    )['path'].tolist()
+        table_path, to_lower=True, expand_path_cols=['path']
+    ).squeeze('columns')['path'].tolist()
 
     base_vector = gdal.OpenEx(base_vector_path, gdal.OF_VECTOR)
     base_layer = base_vector.GetLayer()
diff --git a/src/natcap/invest/scenic_quality/scenic_quality.py b/src/natcap/invest/scenic_quality/scenic_quality.py
index a13f6da49..aade09e68 100644
--- a/src/natcap/invest/scenic_quality/scenic_quality.py
+++ b/src/natcap/invest/scenic_quality/scenic_quality.py
@@ -981,8 +981,8 @@ def _calculate_visual_quality(source_raster_path, working_dir, target_path):
     raster_info = pygeoprocessing.get_raster_info(source_raster_path)
     raster_nodata = raster_info['nodata'][0]
 
-    temp_dir = tempfile.mkdtemp(dir=working_dir,
-                                prefix='visual_quality')
+    temp_dir = os.path.normpath(
+        tempfile.mkdtemp(dir=working_dir, prefix='visual_quality'))
 
     # phase 1: calculate percentiles from the visible_structures raster
     LOGGER.info('Determining percentiles for %s',
@@ -1006,8 +1006,11 @@ def _calculate_visual_quality(source_raster_path, working_dir, target_path):
         gdal.GDT_Float64, _VALUATION_NODATA,
         raster_driver_creation_tuple=FLOAT_GTIFF_CREATION_OPTIONS)
 
+    # The directory passed to raster_band_percentile() is expected to not exist
+    # or be completely empty.
+    percentiles_working_dir = os.path.join(temp_dir, 'percentiles_working_dir')
     percentile_values = pygeoprocessing.raster_band_percentile(
-        (masked_raster_path, 1), temp_dir, [0., 25., 50., 75.])
+        (masked_raster_path, 1), percentiles_working_dir, [0., 25., 50., 75.])
 
     shutil.rmtree(temp_dir, ignore_errors=True)
 


Test patch:
diff --git a/tests/test_hra.py b/tests/test_hra.py
index 30e5514a6..7ce7b3e9d 100644
--- a/tests/test_hra.py
+++ b/tests/test_hra.py
@@ -320,6 +320,40 @@ class HRAUnitTests(unittest.TestCase):
         self.assertIn("Criterion could not be opened as a spatial file",
                       str(cm.exception))
 
+    def test_criteria_table_missing_section_headers(self):
+        """HRA: verify exception when a required section is not found."""
+        from natcap.invest import hra
+
+        criteria_table_path = os.path.join(self.workspace_dir, 'criteria.csv')
+        with open(criteria_table_path, 'w') as criteria_table:
+            criteria_table.write(
+                textwrap.dedent(  # NOTE: also checking whitespace around
+                    """\
+                      HABITAT-NAME,eelgrass,,,hardbottom ,,,CRITERIA TYPE
+                    HABITAT-FOOOOO-ATTRIBUTES,RATING  ,DQ,WEIGHT,RATING,DQ,WEIGHT,E/C
+                    recruitment rate,2,2,2,2,2,2,C
+                    connectivity rate,foo/eelgrass_connectivity.shp,2,2,2,2,2,C
+                    ,,,,,,,
+                    HABITAT STRESSOR OVERLAP PROPERTIES,,,,,,,
+                    oil,RATING,DQ,WEIGHT,RATING,DQ,WEIGHT,E/C
+                    frequency of disturbance ,2,2,3,2,2,3,C
+                    management effectiveness,2,2,1,2,2,1,E
+                    ,,,,,,,
+                    fishing,RATING,DQ,WEIGHT,RATING,DQ,WEIGHT,E/C
+                    frequency of disturbance,2,2,3,2,2,3,C
+                    management effectiveness,2,2,1,2,2,1,E
+                    """
+                ))
+        target_composite_csv_path = os.path.join(self.workspace_dir,
+                                                 'composite.csv')
+        with self.assertRaises(AssertionError) as cm:
+            habitats, stressors = hra._parse_criteria_table(
+                criteria_table_path, target_composite_csv_path)
+        self.assertIn('The criteria table is missing these section headers',
+                      str(cm.exception))
+        self.assertIn('HABITAT NAME', str(cm.exception))
+        self.assertIn('HABITAT RESILIENCE ATTRIBUTES', str(cm.exception))
+
     def test_maximum_reclassified_score(self):
         """HRA: check maximum reclassed score given a stack of scores."""
         from natcap.invest import hra