Repository: stan-dev/cmdstanpy
Pull request #671 (2023-06-12 17:08:37)

Handle tilde `~` in model paths.
Apple uses tildes in some paths for files that are synced through iCloud (see [here](https://apple.stackexchange.com/q/280338/102603), for example). It's of course a somewhat questionable choice. This PR allows models in these iCloud folders to be compiled and used.

#### Submission Checklist

- [x] Run unit tests
- [x] Declare copyright holder and open-source license: see below

#### Copyright and Licensing

Please list the copyright holder for the work you are submitting (this will be you or your assignee, such as a university or company): Harvard University



By submitting this pull request, the copyright holder is agreeing to license the submitted work under the following licenses:
- Code: BSD 3-clause (https://opensource.org/licenses/BSD-3-Clause)



codecov-commenter: ## [Codecov](https://app.codecov.io/gh/stan-dev/cmdstanpy/pull/671?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stan-dev) Report
> Merging [#671](https://app.codecov.io/gh/stan-dev/cmdstanpy/pull/671?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stan-dev) (83132a2) into [develop](https://app.codecov.io/gh/stan-dev/cmdstanpy/commit/7ad7dcc32048b4df7391a79e9045f54f45249d0e?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stan-dev) (7ad7dcc) will **decrease** coverage by `0.13%`.
> The diff coverage is `n/a`.

```diff
@@             Coverage Diff             @@
##           develop     #671      +/-   ##
===========================================
- Coverage    80.44%   80.32%   -0.13%     
===========================================
  Files           69       23      -46     
  Lines        10653     3553    -7100     
===========================================
- Hits          8570     2854    -5716     
+ Misses        2083      699    -1384     
```


[see 48 files with indirect coverage changes](https://app.codecov.io/gh/stan-dev/cmdstanpy/pull/671/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stan-dev)

:mega: We’re building smart automated test selection to slash your CI/CD build times. [Learn more](https://about.codecov.io/iterative-testing/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=stan-dev)

ahartikainen: Maybe pytest skipif for Windows is enough for the tests.

Patch:
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0ba662b..cf9c49c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -39,7 +39,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+        python-version: ["3.7.1 - 3.7.16", "3.8", "3.9", "3.10", "3.11"]
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     steps:
diff --git a/cmdstanpy/model.py b/cmdstanpy/model.py
index 54faa22..9504d6d 100644
--- a/cmdstanpy/model.py
+++ b/cmdstanpy/model.py
@@ -501,7 +501,8 @@ class CmdStanModel:
                 return
 
         compilation_failed = False
-        # if target path has space, use copy in a tmpdir (GNU-Make constraint)
+        # if target path has spaces or special characters, use a copy in a
+        # temporary directory (GNU-Make constraint)
         with SanitizedOrTmpFilePath(self._stan_file) as (stan_file, is_copied):
             exe_file = os.path.splitext(stan_file)[0] + EXTENSION
 
diff --git a/cmdstanpy/utils/filesystem.py b/cmdstanpy/utils/filesystem.py
index aae22b5..045a82d 100644
--- a/cmdstanpy/utils/filesystem.py
+++ b/cmdstanpy/utils/filesystem.py
@@ -4,6 +4,7 @@ Utilities for interacting with the filesystem on multiple platforms
 import contextlib
 import os
 import platform
+import re
 import shutil
 import tempfile
 from typing import Any, Iterator, List, Mapping, Tuple, Union
@@ -169,11 +170,22 @@ class MaybeDictToFilePath:
 
 
 class SanitizedOrTmpFilePath:
-    """Context manager for tmpfiles, handles spaces in filepath."""
+    """
+    Context manager for tmpfiles, handles special characters in filepath.
+    """
+    UNIXISH_PATTERN = re.compile(r"[\s~]")
+    WINDOWS_PATTERN = re.compile(r"\s")
+
+    @classmethod
+    def _has_special_chars(cls, file_path: str) -> bool:
+        if platform.system() == "Windows":
+            return bool(cls.WINDOWS_PATTERN.search(file_path))
+        return bool(cls.UNIXISH_PATTERN.search(file_path))
 
     def __init__(self, file_path: str):
         self._tmpdir = None
-        if ' ' in os.path.abspath(file_path) and platform.system() == 'Windows':
+
+        if self._has_special_chars(os.path.abspath(file_path)):
             base_path, file_name = os.path.split(os.path.abspath(file_path))
             os.makedirs(base_path, exist_ok=True)
             try:
@@ -183,12 +195,13 @@ class SanitizedOrTmpFilePath:
             except RuntimeError:
                 pass
 
-        if ' ' in os.path.abspath(file_path):
+        if self._has_special_chars(os.path.abspath(file_path)):
             tmpdir = tempfile.mkdtemp()
-            if ' ' in tmpdir:
+            if self._has_special_chars(tmpdir):
                 raise RuntimeError(
-                    'Unable to generate temporary path without spaces! \n'
-                    + 'Please move your stan file to location without spaces.'
+                    'Unable to generate temporary path without spaces or '
+                    'special characters! \n Please move your stan file to a '
+                    'location without spaces or special characters.'
                 )
 
             _, path = tempfile.mkstemp(suffix='.stan', dir=tmpdir)


Test patch:
diff --git a/test/data/path~with~tilde/bernoulli_path_with_tilde.stan b/test/data/path~with~tilde/bernoulli_path_with_tilde.stan
new file mode 100644
index 0000000..d925e62
--- /dev/null
+++ b/test/data/path~with~tilde/bernoulli_path_with_tilde.stan
@@ -0,0 +1,12 @@
+data {
+  int<lower=0> N;
+  int<lower=0,upper=1> y[N];
+}
+parameters {
+  real<lower=0,upper=1> theta;
+}
+model {
+  theta ~ beta(1,1);
+  for (n in 1:N)
+    y[n] ~ bernoulli(theta);
+}
diff --git a/test/test_model.py b/test/test_model.py
index 7127a8e..39355bb 100644
--- a/test/test_model.py
+++ b/test/test_model.py
@@ -426,16 +426,18 @@ def test_model_compile() -> None:
     assert exe_time == os.path.getmtime(model2.exe_file)
 
 
-def test_model_compile_space() -> None:
+@pytest.mark.parametrize("path", ["space in path", "tilde~in~path"])
+def test_model_compile_special_char(path: str) -> None:
     with tempfile.TemporaryDirectory(
         prefix="cmdstanpy_testfolder_"
     ) as tmp_path:
-        path_with_space = os.path.join(tmp_path, "space in path")
-        os.makedirs(path_with_space, exist_ok=True)
+        path_with_special_char = os.path.join(tmp_path, path)
+        os.makedirs(path_with_special_char, exist_ok=True)
         bern_stan_new = os.path.join(
-            path_with_space, os.path.split(BERN_STAN)[1]
+            path_with_special_char, os.path.split(BERN_STAN)[1]
         )
-        bern_exe_new = os.path.join(path_with_space, os.path.split(BERN_EXE)[1])
+        bern_exe_new = os.path.join(path_with_special_char,
+                                    os.path.split(BERN_EXE)[1])
         shutil.copyfile(BERN_STAN, bern_stan_new)
         model = CmdStanModel(stan_file=bern_stan_new)
 
@@ -451,7 +453,8 @@ def test_model_compile_space() -> None:
         assert exe_time == os.path.getmtime(model2.exe_file)
 
 
-def test_model_includes_space() -> None:
+@pytest.mark.parametrize("path", ["space in path", "tilde~in~path"])
+def test_model_includes_special_char(path: str) -> None:
     """Test model with include file in path with spaces."""
     stan = os.path.join(DATAFILES_PATH, 'bernoulli_include.stan')
     stan_divide = os.path.join(DATAFILES_PATH, 'divide_real_by_two.stan')
@@ -459,22 +462,23 @@ def test_model_includes_space() -> None:
     with tempfile.TemporaryDirectory(
         prefix="cmdstanpy_testfolder_"
     ) as tmp_path:
-        path_with_space = os.path.join(tmp_path, "space in path")
-        os.makedirs(path_with_space, exist_ok=True)
-        bern_stan_new = os.path.join(path_with_space, os.path.split(stan)[1])
+        path_with_special_char = os.path.join(tmp_path, path)
+        os.makedirs(path_with_special_char, exist_ok=True)
+        bern_stan_new = os.path.join(path_with_special_char,
+                                     os.path.split(stan)[1])
         stan_divide_new = os.path.join(
-            path_with_space, os.path.split(stan_divide)[1]
+            path_with_special_char, os.path.split(stan_divide)[1]
         )
         shutil.copyfile(stan, bern_stan_new)
         shutil.copyfile(stan_divide, stan_divide_new)
 
         model = CmdStanModel(
             stan_file=bern_stan_new,
-            stanc_options={'include-paths': path_with_space},
+            stanc_options={'include-paths': path_with_special_char},
         )
-        assert "space in path" in str(model.exe_file)
+        assert path in str(model.exe_file)
 
-        assert "space in path" in model.src_info()['included_files'][0]
+        assert path in model.src_info()['included_files'][0]
         assert (
             "divide_real_by_two.stan" in model.src_info()['included_files'][0]
         )
diff --git a/test/test_sample.py b/test/test_sample.py
index 5562a4d..b8c0633 100644
--- a/test/test_sample.py
+++ b/test/test_sample.py
@@ -49,7 +49,8 @@ BERNOULLI_COLS = SAMPLER_STATE + ['theta']
     [
         'bernoulli.stan',
         'bernoulli with space in name.stan',
-        'path with space/' + 'bernoulli_path_with_space.stan',
+        'path with space/bernoulli_path_with_space.stan',
+        'path~with~tilde/bernoulli_path_with_tilde.stan',
     ],
 )
 def test_bernoulli_good(stanfile: str):
diff --git a/test/test_utils.py b/test/test_utils.py
index 319c38a..fe42f86 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -75,7 +75,9 @@ def test_default_path() -> None:
         assert 'CMDSTAN' in os.environ
 
 
-def test_non_spaces_location() -> None:
+@pytest.mark.parametrize("bad_dir", ["bad dir", "bad~dir"])
+@pytest.mark.parametrize("bad_name", ["bad name", "bad~name"])
+def test_non_special_chars_location(bad_dir: str, bad_name: str) -> None:
     with tempfile.TemporaryDirectory(
         prefix="cmdstan_tests", dir=_TMPDIR
     ) as tmpdir:
@@ -86,10 +88,10 @@ def test_non_spaces_location() -> None:
             assert not is_changed
 
         # prepare files for test
-        bad_path = os.path.join(tmpdir, 'bad dir')
+        bad_path = os.path.join(tmpdir, bad_dir)
         os.makedirs(bad_path, exist_ok=True)
         stan = os.path.join(DATAFILES_PATH, 'bernoulli.stan')
-        stan_bad = os.path.join(bad_path, 'bad name.stan')
+        stan_bad = os.path.join(bad_path, bad_name)
         shutil.copy(stan, stan_bad)
 
         stan_copied = None
@@ -98,7 +100,20 @@ def test_non_spaces_location() -> None:
                 stan_copied = pth
                 assert os.path.exists(stan_copied)
                 assert ' ' not in stan_copied
-                assert is_changed
+
+                # Determine if the file should have been copied, i.e., we either
+                # are on a unix-ish system or on windows, the path contains a
+                # space, and there is no short path.
+                if platform.system() == 'Windows':
+                    should_change = ' ' in bad_name or (
+                        ' ' in bad_path
+                        and not os.path.exists(windows_short_path(bad_path))
+                    )
+                else:
+                    should_change = True
+                    assert '~' not in stan_copied
+
+                assert is_changed == should_change
                 raise RuntimeError
         except RuntimeError:
             pass