Repository: natcap/invest (python)

Issue:
Criteria table from datastack archive produced by HRA doesn't work
To reproduce:

1. Open up the 3.12+ version of HRA (the one with the datastack override for spatial criteria)
2. Load this datastack (Stanford auth required): https://drive.google.com/file/d/1tqheKjtekvNxavlB9FekCqtCycLIQZDX/view?usp=sharing
3. Observe a validation error in the criteria table saying the table can't be opened as a CSV

The workaround for now is to:

1. Extract the datastack somewhere
2. Open the criteria table CSV in Excel
3. Save the CSV to a new location as a CSV.
4. Update the UI to use the new path
5. The input validates as expected and the model runs.

Pull request:
Converting path separators in `expand_path` if on a POSIX system
This patch fixes an issue most common when loading datastacks created on a Windows computer and then loaded on Linux/OSX, and it's only because python doesn't handle mixed path separators at all.

Fixes #1065

## Checklist
- [x] Updated HISTORY.rst and link to any relevant issue (if these changes are user-facing)
~~- [ ] Updated the user's guide (if needed)~~
~~- [ ] Tested the Workbench UI (if relevant)~~


Patch:
diff --git a/HISTORY.rst b/HISTORY.rst
index c28763a7a..3f8a02787 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -79,6 +79,8 @@ Unreleased Changes
     * Fixed an issue preventing the HRA criteria table from loading when the
       table was UTF-8 encoded with a Byte-Order Marker.
       https://github.com/natcap/invest/issues/1460
+    * Fixed an issue with the cross-OS loading of InVEST datastack files.
+      https://github.com/natcap/invest/issues/1065
     * Fixed an issue where habitats and stressors in the criteria table were
       not being lowercased causing a comparison issue against values from the
       info table that were being lowercased.
diff --git a/src/natcap/invest/utils.py b/src/natcap/invest/utils.py
index 9ee61a060..f3a076d50 100644
--- a/src/natcap/invest/utils.py
+++ b/src/natcap/invest/utils.py
@@ -2,8 +2,8 @@
 import codecs
 import contextlib
 import logging
-import math
 import os
+import platform
 import re
 import shutil
 import tempfile
@@ -405,6 +405,8 @@ def expand_path(path, base_path):
     """
     if not path:
         return None
+    if platform.system() in {'Darwin', 'Linux'} and '\\' in path:
+        path = path.replace('\\', '/')
     if os.path.isabs(path):
         return os.path.abspath(path)  # normalize path separators
     return os.path.abspath(os.path.join(os.path.dirname(base_path), path))


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 3006b44fa..4ebfdecab 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -4,6 +4,7 @@ import glob
 import logging
 import logging.handlers
 import os
+import platform
 import queue
 import re
 import shutil
@@ -11,6 +12,7 @@ import tempfile
 import textwrap
 import threading
 import unittest
+import unittest.mock
 import warnings
 
 import numpy
@@ -1176,3 +1178,47 @@ class ReclassifyRasterOpTests(unittest.TestCase):
             " the LULC raster but not the table are: [3].")
         self.assertTrue(
             expected_message in str(context.exception), str(context.exception))
+
+
+class ExpandPathTests(unittest.TestCase):
+    def setUp(self):
+        self.workspace_dir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.workspace_dir)
+
+    @unittest.skipIf(platform.system() == 'Windows',
+                     "Function behavior differs across systems.")
+    def test_os_path_normalization_linux(self):
+        """Utils: test path separator conversion Win to Linux."""
+        from natcap.invest import utils
+
+        # Assumption: a path was created on Windows and is now being loaded on
+        # a Mac or Linux computer.
+        rel_path = "foo\\bar.shp"
+        relative_to = os.path.join(self.workspace_dir, 'test.csv')
+        expected_path = os.path.join(self.workspace_dir, "foo/bar.shp")
+        path = utils.expand_path(rel_path, relative_to)
+        self.assertEquals(path, expected_path)
+
+    @unittest.skipIf(platform.system() != 'Windows',
+                     "Function behavior differs across systems.")
+    def test_os_path_normalization_windows(self):
+        """Utils: test path separator conversion Mac/Linux to Windows."""
+        from natcap.invest import utils
+
+        # Assumption: a path was created on mac/linux and is now being loaded
+        # on a Windows computer.
+        rel_path = "foo/bar.shp"
+        relative_to = os.path.join(self.workspace_dir, 'test.csv')
+        expected_path = os.path.join(self.workspace_dir, "foo\\bar.shp")
+        path = utils.expand_path(rel_path, relative_to)
+        self.assertEquals(path, expected_path)
+
+    def test_falsey(self):
+        """Utils: test return None when falsey."""
+        from natcap.invest import utils
+
+        for value in ('', None, False, 0):
+            self.assertEqual(
+                None, utils.expand_path(value, self.workspace_dir))
diff --git a/tests/test_validation.py b/tests/test_validation.py
index 2a5e4fa81..806da695e 100644
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -2,6 +2,7 @@
 import codecs
 import functools
 import os
+import platform
 import shutil
 import string
 import tempfile
@@ -1195,9 +1196,18 @@ class TestGetValidatedDataframe(unittest.TestCase):
         self.assertEqual(
             f'{self.workspace_dir}{os.sep}foo{os.sep}bar.txt',
             df['path'][1])
-        self.assertEqual(
-            f'{self.workspace_dir}{os.sep}foo\\bar.txt',
-            df['path'][2])
+
+        # utils.expand_path() will convert Windows path separators to linux if
+        # we're on mac/linux
+        if platform.system() == 'Windows':
+            self.assertEqual(
+                f'{self.workspace_dir}{os.sep}foo\\bar.txt',
+                df['path'][2])
+        else:
+            self.assertEqual(
+                f'{self.workspace_dir}{os.sep}foo/bar.txt',
+                df['path'][2])
+
         self.assertEqual(
             f'{self.workspace_dir}{os.sep}foo.txt',
             df['path'][3])