Repository: stan-dev/cmdstanpy
Pull request #301 (2020-10-01 03:26:45)

Feature/227 cmdstan install
#### Submission Checklist

- [x] Run unit tests
- [x] Declare copyright holder and open-source license: see below

#### Summary

Cleanup function `install_cmdstan` (`cmdstanpy/utils.py`) and corresponding script `install_cmdstan.py` per issues #227 and #262 

- Constants, logic, and documentation for directories for local install of CmdStan
- Added arguments "overwrite" and "verbose" to function `install_cmdstan`
- Function `install_cmdstan` now streams output from subprocess (was blocking)
- Added logic so that release candidate names sort properly w/r/t releases
- Added a few unit tests for functions in script `install_cmdstan.py`, (more needed)

#### Copyright and Licensing

Please list the copyright holder for the work you are submitting (this will be you or your assignee, such as a university or company):  Columbia University



By submitting this pull request, the copyright holder is agreeing to license the submitted work under the following licenses:
- Code: BSD 3-clause (https://opensource.org/licenses/BSD-3-Clause)




Patch:
diff --git a/cmdstanpy/__init__.py b/cmdstanpy/__init__.py
index 2d0c5bc..c6655b3 100644
--- a/cmdstanpy/__init__.py
+++ b/cmdstanpy/__init__.py
@@ -21,6 +21,8 @@ _TMPDIR = tempfile.mkdtemp()
 _CMDSTAN_WARMUP = 1000
 _CMDSTAN_SAMPLING = 1000
 _CMDSTAN_THIN = 1
+_DOT_CMDSTANPY = '.cmdstanpy'
+_DOT_CMDSTAN = '.cmdstan'
 
 
 def _cleanup_tmpdir():
diff --git a/cmdstanpy/install_cmdstan.py b/cmdstanpy/install_cmdstan.py
index 20f5b3e..03c0895 100644
--- a/cmdstanpy/install_cmdstan.py
+++ b/cmdstanpy/install_cmdstan.py
@@ -1,10 +1,18 @@
 #!/usr/bin/env python
 """
 Download and install a CmdStan release from GitHub.
+Downloads the release tar.gz file to temporary storage.
+Retries GitHub requests in order to allow for transient network outages.
+Builds CmdStan executables and tests the compiler by building
+example model ``bernoulli.stan``.
+
 Optional command line arguments:
-   -v, --version : version, defaults to latest
-   -d, --dir : install directory, defaults to '~/.cmdstanpy
-   -c --compiler : add C++ compiler to path (Windows only)
+   -v, --version <release> : version, defaults to latest release version
+   -d, --dir <path> : install directory, defaults to '$HOME/.cmdstan(py)
+   --overwrite: flag, when specified re-installs existing version.
+   --verbose: flag, when specified prints output from CmdStan build process.
+
+   -c, --compiler : add C++ compiler to path (Windows only)
 """
 import argparse
 import contextlib
@@ -14,14 +22,26 @@ import subprocess
 import sys
 import tarfile
 import urllib.request
+import urllib.error
 from pathlib import Path
 from time import sleep
 
+from cmdstanpy import _DOT_CMDSTAN, _DOT_CMDSTANPY
+from cmdstanpy.utils import validate_dir
+
 EXTENSION = '.exe' if platform.system() == 'Windows' else ''
 
 
+class CmdStanRetrieveError(RuntimeError):
+    pass
+
+
+class CmdStanInstallError(RuntimeError):
+    pass
+
+
 @contextlib.contextmanager
-def pushd(new_dir):
+def pushd(new_dir: str):
     """Acts like pushd/popd."""
     previous_dir = os.getcwd()
     os.chdir(new_dir)
@@ -35,18 +55,53 @@ def usage():
         """Arguments:
         -v (--version) :CmdStan version
         -d (--dir) : install directory
+        --overwrite : replace installed version
+        --verbose : show CmdStan build messages
         -h (--help) : this message
         """
     )
 
 
-def install_version(cmdstan_version):
-    """Build specified CmdStan version."""
+def install_version(
+    cmdstan_version: str, overwrite: bool = False, verbose: bool = False
+):
+    """
+    Build specified CmdStan version by spawning subprocesses to
+    run the Make utility on the downloaded CmdStan release src files.
+    Assumes that current working directory is parent of release dir.
+
+    :param cmdstan_version: CmdStan release, corresponds to release dirname.
+    :param overwrite: when ``True``, run ``make clean-all`` before building.
+    :param verbose: when ``True``, print build msgs to stdout.
+    """
     with pushd(cmdstan_version):
-        print('Building {} binaries'.format(cmdstan_version))
         make = os.getenv(
             'MAKE', 'make' if platform.system() != 'Windows' else 'mingw32-make'
         )
+        print('Building version {}'.format(cmdstan_version))
+        if overwrite:
+            print(
+                'Overwrite requested, remove existing build of version '
+                '{}'.format(cmdstan_version)
+            )
+            cmd = [make, 'clean-all']
+            proc = subprocess.Popen(
+                cmd,
+                cwd=None,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                env=os.environ,
+            )
+            while proc.poll() is None:
+                if verbose:
+                    print(proc.stdout.readline().decode('utf-8').strip())
+            _, stderr = proc.communicate()
+            if proc.returncode:
+                msgs = ['Command "make clean-all" failed']
+                if stderr:
+                    msgs.append(stderr.decode('utf-8').strip())
+                raise CmdStanInstallError('\n'.join(msgs))
+            print('Rebuilding version {}'.format(cmdstan_version))
         cmd = [make, 'build']
         proc = subprocess.Popen(
             cmd,
@@ -55,12 +110,15 @@ def install_version(cmdstan_version):
             stderr=subprocess.PIPE,
             env=os.environ,
         )
+        while proc.poll() is None:
+            if verbose:
+                print(proc.stdout.readline().decode('utf-8').strip())
         _, stderr = proc.communicate()
         if proc.returncode:
-            print('Command "make build" failed')
+            msgs = ['Command "make build" failed']
             if stderr:
-                print(stderr.decode('utf-8').strip())
-            sys.exit(3)
+                msgs.append(stderr.decode('utf-8').strip())
+            raise CmdStanInstallError('\n'.join(msgs))
         print('Test model compilation')
         cmd = [
             make,
@@ -77,40 +135,60 @@ def install_version(cmdstan_version):
         )
         _, stderr = proc.communicate()
         if proc.returncode:
-            print('Failed to compile example model bernoulli.stan')
+            msgs = ['Failed to compile example model bernoulli.stan']
             if stderr:
-                print(stderr.decode('utf-8').strip())
-            sys.exit(3)
+                msgs.append(stderr.decode('utf-8').strip())
+            raise CmdStanInstallError('\n'.join(msgs))
     print('Installed {}'.format(cmdstan_version))
 
 
-def is_installed(cmdstan_version):
-    """Check if CmdStan is already installed."""
-    if not os.path.exists(os.path.join(cmdstan_version, 'bin')):
-        return False
-    return os.path.exists(
-        os.path.join(
-            cmdstan_version, 'examples', 'bernoulli', 'bernoulli' + EXTENSION
-        )
+def is_version_available(version: str):
+    is_available = True
+    url = (
+        'https://github.com/stan-dev/cmdstan/releases/download/'
+        'v{0}/cmdstan-{0}.tar.gz'.format(version)
     )
+    for i in range(6):
+        try:
+            urllib.request.urlopen(url)
+        except urllib.error.HTTPError as err:
+            print('Release {} is unavailable from URL {}'.format(version, url))
+            print('HTTPError: {}'.format(err.code))
+            is_available = False
+            break
+        except urllib.error.URLError as e:
+            if i < 5:
+                print(
+                    'checking version {} availability, retry ({}/5)'.format(
+                        version, i + 1
+                    )
+                )
+                sleep(1)
+                continue
+            print('Release {} is unavailable from URL {}'.format(version, url))
+            print('URLError: {}'.format(e.reason))
+            is_available = False
+    return is_available
 
 
 def latest_version():
-    """Report most recent CmdStan verion installed."""
+    """Report latest CmdStan release version."""
     for i in range(6):
         try:
             file_tmp, _ = urllib.request.urlretrieve(
                 'https://api.github.com/repos/stan-dev/cmdstan/releases/latest'
             )
             break
-        except urllib.error.URLError as err:
+        except urllib.error.URLError as e:
             print('Cannot connect to github.')
-            print(err)
+            print(e)
             if i < 5:
                 print('retry ({}/5)'.format(i + 1))
                 sleep(1)
                 continue
-            sys.exit(3)
+            raise CmdStanRetrieveError(
+                'Cannot connect to CmdStan github repo.'
+            ) from e
     with open(file_tmp, 'r') as fd:
         response = fd.read()
         start_idx = response.find('\"tag_name\":\"v') + len('"tag_name":"v')
@@ -118,29 +196,39 @@ def latest_version():
     return response[start_idx:end_idx]
 
 
-def retrieve_latest_version(version):
+def retrieve_version(version: str):
     """Download specified CmdStan version."""
     print('Downloading CmdStan version {}'.format(version))
     url = (
         'https://github.com/stan-dev/cmdstan/releases/download/'
         'v{0}/cmdstan-{0}.tar.gz'.format(version)
     )
-    for i in range(6):
+    for i in range(6):  # always retry to allow for transient URLErrors
         try:
             file_tmp, _ = urllib.request.urlretrieve(url, filename=None)
             break
-        except urllib.error.URLError as err:
+        except urllib.error.HTTPError as e:
+            raise CmdStanRetrieveError(
+                'HTTPError: {}\n'
+                'Version {} not available from github.com.'.format(
+                    e.code, version
+                )
+            ) from e
+        except urllib.error.URLError as e:
             print(
                 'Failed to download CmdStan version {} from github.com'.format(
                     version
                 )
             )
-            print(err)
+            print(e)
             if i < 5:
                 print('retry ({}/5)'.format(i + 1))
                 sleep(1)
                 continue
-            sys.exit(3)
+            print('Version {} not available from github.com.'.format(version))
+            raise CmdStanRetrieveError(
+                'Version {} not available from github.com.'.format(version)
+            ) from e
     print('Download successful, file: {}'.format(file_tmp))
     try:
         tar = tarfile.open(file_tmp)
@@ -149,44 +237,22 @@ def retrieve_latest_version(version):
             # fixes long-path limitation on Windows
             target = r'\\?\{}'.format(target)
         tar.extractall(target)
-    except Exception as err:  # pylint: disable=broad-except
-        print('Failed to unpack download')
-        print(err)
-        sys.exit(3)
+    except Exception as e:  # pylint: disable=broad-except
+        raise CmdStanInstallError(
+            'Failed to unpack file {}'.format(file_tmp)
+        ) from e
     finally:
         tar.close()
     print('Unpacked download as cmdstan-{}'.format(version))
 
 
-def validate_dir(install_dir):
-    """Check that specified install directory exists, can write."""
-    if not os.path.exists(install_dir):
-        try:
-            os.makedirs(install_dir)
-        except (IOError, OSError, PermissionError) as e:
-            raise ValueError(
-                'Cannot create directory: {}'.format(install_dir)
-            ) from e
-    else:
-        if not os.path.isdir(install_dir):
-            raise ValueError(
-                'File exists, should be a directory: {}'.format(install_dir)
-            )
-        try:
-            with open('tmp_test_w', 'w'):
-                pass
-            os.remove('tmp_test_w')  # cleanup
-        except (IOError, OSError, PermissionError) as e:
-            raise ValueError(
-                'Cannot write files to directory {}'.format(install_dir)
-            ) from e
-
-
 def main():
     """Main."""
     parser = argparse.ArgumentParser()
     parser.add_argument('--version', '-v')
     parser.add_argument('--dir', '-d')
+    parser.add_argument('--overwrite', action='store_true')
+    parser.add_argument('--verbose', action='store_true')
     if platform.system() == 'Windows':
         # use compiler installed with install_cxx_toolchain
         # Install a new compiler if compiler not found
@@ -196,14 +262,27 @@ def main():
         )
     args = parser.parse_args(sys.argv[1:])
 
-    version = vars(args)['version']
-    if version is None:
-        version = latest_version()
-    print('CmdStan version: {}'.format(version))
+    version = latest_version()
+    if vars(args)['version']:
+        version = vars(args)['version']
+
+    if is_version_available(version):
+        print('Installing CmdStan version: {}'.format(version))
+    else:
+        raise ValueError(
+            'Invalid version requested: {}, cannot install.'.format(version)
+        )
+
+    cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))
+    if not os.path.exists(cmdstan_dir):
+        cmdstanpy_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTANPY))
+        if os.path.exists(cmdstanpy_dir):
+            cmdstan_dir = cmdstanpy_dir
+
+    install_dir = cmdstan_dir
+    if vars(args)['dir']:
+        install_dir = vars(args)['dir']
 
-    install_dir = vars(args)['dir']
-    if install_dir is None:
-        install_dir = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
     validate_dir(install_dir)
     print('Install directory: {}'.format(install_dir))
 
@@ -214,7 +293,7 @@ def main():
         )
         from .utils import cxx_toolchain_path
 
-        cxx_loc = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
+        cxx_loc = cmdstan_dir
         compiler_found = False
         for cxx_version in ['40', '35']:
             if _is_installed_cxx(cxx_loc, cxx_version):
@@ -233,12 +312,29 @@ def main():
 
     cmdstan_version = 'cmdstan-{}'.format(version)
     with pushd(install_dir):
-        if not os.path.exists(cmdstan_version):
-            retrieve_latest_version(version)
-        if is_installed(cmdstan_version):
-            print('CmdStan version {} already installed'.format(version))
+        if vars(args)['overwrite'] or not (
+            os.path.exists(cmdstan_version)
+            and os.path.exists(
+                os.path.join(
+                    cmdstan_version,
+                    'examples',
+                    'bernoulli',
+                    'bernoulli' + EXTENSION,
+                )
+            )
+        ):
+            try:
+                retrieve_version(version)
+                install_version(
+                    cmdstan_version=cmdstan_version,
+                    overwrite=vars(args)['overwrite'],
+                    verbose=vars(args)['verbose'],
+                )
+            except RuntimeError as e:
+                print(e)
+                sys.exit(3)
         else:
-            install_version(cmdstan_version)
+            print('CmdStan version {} already installed'.format(version))
 
 
 if __name__ == '__main__':
diff --git a/cmdstanpy/install_cxx_toolchain.py b/cmdstanpy/install_cxx_toolchain.py
index d7cfae4..a1561aa 100644
--- a/cmdstanpy/install_cxx_toolchain.py
+++ b/cmdstanpy/install_cxx_toolchain.py
@@ -7,7 +7,7 @@ Currently implemented platforms (platform.system)
     Linux: Not implemented
 Optional command line arguments:
    -v, --version : version, defaults to latest
-   -d, --dir : install directory, defaults to '~/.cmdstanpy
+   -d, --dir : install directory, defaults to '~/.cmdstan(py)
    -s (--silent) : install with /VERYSILENT instead of /SILENT for RTools
    -m --no-make : don't install mingw32-make (Windows RTools 4.0 only)
 """
@@ -22,6 +22,10 @@ import urllib.request
 from collections import OrderedDict
 from time import sleep
 
+from cmdstanpy import _DOT_CMDSTAN, _DOT_CMDSTANPY
+from cmdstanpy.utils import validate_dir
+
+
 EXTENSION = '.exe' if platform.system() == 'Windows' else ''
 IS_64BITS = sys.maxsize > 2 ** 32
 
@@ -206,30 +210,6 @@ def retrieve_toolchain(filename, url):
     print('Download successful, file: {}'.format(filename))
 
 
-def validate_dir(install_dir):
-    """Check that specified install directory exists, can write."""
-    if not os.path.exists(install_dir):
-        try:
-            os.makedirs(install_dir)
-        except (IOError, OSError, PermissionError) as e:
-            raise ValueError(
-                'Cannot create directory: {}'.format(install_dir)
-            ) from e
-    else:
-        if not os.path.isdir(install_dir):
-            raise ValueError(
-                'File exists, should be a directory: {}'.format(install_dir)
-            )
-        try:
-            with open('tmp_test_w', 'w'):
-                pass
-            os.remove('tmp_test_w')  # cleanup
-        except OSError as e:
-            raise ValueError(
-                'Cannot write files to directory {}'.format(install_dir)
-            ) from e
-
-
 def normalize_version(version):
     """Return maj.min part of version string."""
     if platform.system() == 'Windows':
@@ -297,7 +277,14 @@ def main():
 
     install_dir = vars(args)['dir']
     if install_dir is None:
-        install_dir = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
+        cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))
+        if not os.path.exists(cmdstan_dir):
+            cmdstanpy_dir = os.path.expanduser(
+                os.path.join('~', _DOT_CMDSTANPY)
+            )
+            if os.path.exists(cmdstanpy_dir):
+                cmdstan_dir = cmdstanpy_dir
+        install_dir = cmdstan_dir
     validate_dir(install_dir)
     print('Install directory: {}'.format(install_dir))
 
diff --git a/cmdstanpy/utils.py b/cmdstanpy/utils.py
index 97852b7..909ebae 100644
--- a/cmdstanpy/utils.py
+++ b/cmdstanpy/utils.py
@@ -24,7 +24,14 @@ import numpy as np
 import pandas as pd
 
 
-from cmdstanpy import _TMPDIR, _CMDSTAN_WARMUP, _CMDSTAN_SAMPLING, _CMDSTAN_THIN
+from cmdstanpy import (
+    _TMPDIR,
+    _CMDSTAN_WARMUP,
+    _CMDSTAN_SAMPLING,
+    _CMDSTAN_THIN,
+    _DOT_CMDSTAN,
+    _DOT_CMDSTANPY,
+)
 
 EXTENSION = '.exe' if platform.system() == 'Windows' else ''
 
@@ -37,19 +44,51 @@ def get_logger():
     return logger
 
 
-def get_latest_cmdstan(dot_dir: str) -> str:
+def validate_dir(install_dir: str):
+    """Check that specified install directory exists, can write."""
+    if not os.path.exists(install_dir):
+        try:
+            os.makedirs(install_dir)
+        except (IOError, OSError, PermissionError) as e:
+            raise ValueError(
+                'Cannot create directory: {}'.format(install_dir)
+            ) from e
+    else:
+        if not os.path.isdir(install_dir):
+            raise ValueError(
+                'File exists, should be a directory: {}'.format(install_dir)
+            )
+        try:
+            with open('tmp_test_w', 'w'):
+                pass
+            os.remove('tmp_test_w')  # cleanup
+        except OSError as e:
+            raise ValueError(
+                'Cannot write files to directory {}'.format(install_dir)
+            ) from e
+
+
+def get_latest_cmdstan(cmdstan_dir: str) -> str:
     """
     Given a valid directory path, find all installed CmdStan versions
     and return highest (i.e., latest) version number.
     Assumes directory populated via script `install_cmdstan`.
     """
     versions = [
-        name.split('-')[1]
-        for name in os.listdir(dot_dir)
-        if os.path.isdir(os.path.join(dot_dir, name))
+        ''.join(name.split('-')[1:])  # name may contain '-rc'
+        for name in os.listdir(cmdstan_dir)
+        if os.path.isdir(os.path.join(cmdstan_dir, name))
         and name.startswith('cmdstan-')
         and name[8].isdigit()
     ]
+    # munge rc for sort, e.g. 2.23-rc1 -> 2.23.-99
+    for i in range(len(versions)):  # # pylint: disable=C0200
+        tmp = versions[i].split('rc')
+        if len(tmp) == 1:
+            continue
+        rc_sortable = str(int(tmp[1]) - 100)
+        versions[i] = '.'.join([tmp[0], rc_sortable])
+
     versions.sort(key=lambda s: list(map(int, s.split('.'))))
     if len(versions) == 0:
         return None
@@ -57,54 +96,6 @@ def get_latest_cmdstan(dot_dir: str) -> str:
     return latest
 
 
-class MaybeDictToFilePath:
-    """Context manager for json files."""
-
-    def __init__(self, *objs: Union[str, dict], logger: logging.Logger = None):
-        self._unlink = [False] * len(objs)
-        self._paths = [''] * len(objs)
-        self._logger = logger or get_logger()
-        i = 0
-        for obj in objs:
-            if isinstance(obj, dict):
-                data_file = create_named_text_file(
-                    dir=_TMPDIR, prefix='', suffix='.json'
-                )
-                self._logger.debug('input tempfile: %s', data_file)
-                if any(
-                    not item
-                    for item in obj
-                    if isinstance(item, (Sequence, np.ndarray))
-                ):
-                    rdump(data_file, obj)
-                else:
-                    jsondump(data_file, obj)
-                self._paths[i] = data_file
-                self._unlink[i] = True
-            elif isinstance(obj, str):
-                if not os.path.exists(obj):
-                    raise ValueError("File doesn't exist {}".format(obj))
-                self._paths[i] = obj
-            elif obj is None:
-                self._paths[i] = None
-            elif i == 1 and isinstance(obj, (Integral, Real)):
-                self._paths[i] = obj
-            else:
-                raise ValueError('data must be string or dict')
-            i += 1
-
-    def __enter__(self):
-        return self._paths
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        for can_unlink, path in zip(self._unlink, self._paths):
-            if can_unlink and path:
-                try:
-                    os.remove(path)
-                except PermissionError:
-                    pass
-
-
 def validate_cmdstan_path(path: str) -> None:
     """
     Validate that CmdStan directory exists and binaries have been built.
@@ -119,46 +110,6 @@ def validate_cmdstan_path(path: str) -> None:
         )
 
 
-class TemporaryCopiedFile:
-    """Context manager for tmpfiles, handles spaces in filepath."""
-
-    def __init__(self, file_path: str):
-        self._path = None
-        self._tmpdir = None
-        if ' ' in os.path.abspath(file_path) and platform.system() == 'Windows':
-            base_path, file_name = os.path.split(os.path.abspath(file_path))
-            os.makedirs(base_path, exist_ok=True)
-            try:
-                short_base_path = windows_short_path(base_path)
-                if os.path.exists(short_base_path):
-                    file_path = os.path.join(short_base_path, file_name)
-            except RuntimeError:
-                pass
-
-        if ' ' in os.path.abspath(file_path):
-            tmpdir = tempfile.mkdtemp()
-            if ' ' in tmpdir:
-                raise RuntimeError(
-                    'Unable to generate temporary path without spaces! \n'
-                    + 'Please move your stan file to location without spaces.'
-                )
-
-            _, path = tempfile.mkstemp(suffix='.stan', dir=tmpdir)
-
-            shutil.copy(file_path, path)
-            self._path = path
-            self._tmpdir = tmpdir
-        else:
-            self._path = file_path
-
-    def __enter__(self):
-        return self._path, self._tmpdir is not None
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._tmpdir:
-            shutil.rmtree(self._tmpdir, ignore_errors=True)
-
-
 def set_cmdstan_path(path: str) -> None:
     """
     Validate, then set CmdStan directory path.
@@ -182,12 +133,14 @@ def cmdstan_path() -> str:
     if 'CMDSTAN' in os.environ:
         cmdstan = os.environ['CMDSTAN']
     else:
-        cmdstan_dir = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
+        cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))
         if not os.path.exists(cmdstan_dir):
-            raise ValueError(
-                'no CmdStan installation found, '
-                'run command line script "install_cmdstan"'
-            )
+            cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTANPY))
+            if not os.path.exists(cmdstan_dir):
+                raise ValueError(
+                    'no CmdStan installation found, '
+                    'run command line script "install_cmdstan"'
+                )
         latest_cmdstan = get_latest_cmdstan(cmdstan_dir)
         if latest_cmdstan is None:
             raise ValueError(
@@ -262,16 +215,19 @@ def cxx_toolchain_path(version: str = None) -> Tuple[str]:
                 )
                 toolchain_root = ''
     else:
+        cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))
+        if not os.path.exists(cmdstan_dir):
+            cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTANPY))
         rtools_dir = os.path.expanduser(
-            os.path.join('~', '.cmdstanpy', 'RTools40')
+            os.path.join('~', cmdstan_dir, 'RTools40')
         )
         if not os.path.exists(rtools_dir):
             rtools_dir = os.path.expanduser(
-                os.path.join('~', '.cmdstanpy', 'RTools35')
+                os.path.join('~', cmdstan_dir, 'RTools35')
             )
             if not os.path.exists(rtools_dir):
                 rtools_dir = os.path.expanduser(
-                    os.path.join('~', '.cmdstanpy', 'RTools')
+                    os.path.join('~', cmdstan_dir, 'RTools')
                 )
                 if not os.path.exists(rtools_dir):
                     raise ValueError(
@@ -279,9 +235,9 @@ def cxx_toolchain_path(version: str = None) -> Tuple[str]:
                         'run command line script "install_cxx_toolchain"'
                     )
             else:
-                rtools_dir = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
+                rtools_dir = os.path.expanduser(os.path.join('~', cmdstan_dir))
         else:
-            rtools_dir = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
+            rtools_dir = os.path.expanduser(os.path.join('~', cmdstan_dir))
         compiler_path = ''
         tool_path = ''
         if version not in ('35', '3.5', '3') and os.path.exists(
@@ -444,8 +400,8 @@ def parse_rdump_value(rhs: str) -> Union[int, float, np.array]:
             val = float(rhs)
         else:
             val = int(rhs)
-    except TypeError as exc:
-        raise ValueError('bad value in Rdump file: {}'.format(rhs)) from exc
+    except TypeError as e:
+        raise ValueError('bad value in Rdump file: {}'.format(rhs)) from e
     return val
 
 
@@ -698,10 +654,10 @@ def scan_metric(fd: TextIO, config_dict: Dict, lineno: int) -> int:
         )
     try:
         float(stepsize.strip())
-    except ValueError as exc:
+    except ValueError as e:
         raise ValueError(
             'line {}: invalid stepsize: {}'.format(lineno, stepsize)
-        ) from exc
+        ) from e
     line = fd.readline().strip()
     lineno += 1
     if not (
@@ -892,28 +848,146 @@ def create_named_text_file(dir: str, prefix: str, suffix: str) -> str:
     return path
 
 
-def install_cmdstan(version: str = None, dir: str = None) -> bool:
+def install_cmdstan(
+    version: str = None,
+    dir: str = None,
+    overwrite: bool = False,
+    verbose: bool = False,
+) -> bool:
     """
-    Run 'install_cmdstan' -script
+    Download and install a CmdStan release from GitHub by running
+    script ``install_cmdstan`` as a subprocess.  Downloads the release
+    tar.gz file to temporary storage.  Retries GitHub requests in order
+    to allow for transient network outages. Builds CmdStan executables
+    and tests the compiler by building example model ``bernoulli.stan``.
+
+    :param version: CmdStan version string, e.g. "2.24.1".
+        Defaults to latest CmdStan release.
+
+    :param dir: Path to install directory.  Defaults to hidden directory
+        ``$HOME/.cmdstan`` or ``$HOME/.cmdstanpy``, if the latter exists.
+        If no directory is specified and neither of the above directories
+        exist, directory ``$HOME/.cmdstan`` will be created and populated.
+
+    :param overwrite:  Boolean value; when ``True``, will overwrite and
+        rebuild an existing CmdStan installation.  Default is ``False``.
+
+    :param verbose:  Boolean value; when ``True``, output from CmdStan build
+        processes will be streamed to the console.  Default is ``False``.
+
+    :return: Boolean value; ``True`` for success.
     """
     logger = get_logger()
     python = sys.executable
     here = os.path.dirname(os.path.abspath(__file__))
     path = os.path.join(here, 'install_cmdstan.py')
-    cmd = [python, path]
+    cmd = [python, '-u', path]
     if version is not None:
         cmd.extend(['--version', version])
     if dir is not None:
         cmd.extend(['--dir', dir])
+    if overwrite:
+        cmd.extend(['--overwrite', 'TRUE'])
+    if verbose:
+        cmd.extend(['--verbose', 'TRUE'])
     proc = subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ
     )
     while proc.poll() is None:
-        output = proc.stdout.readline().decode('utf-8').strip()
-        if output:
-            logger.info(output)
-    proc.communicate()
+        print(proc.stdout.readline().decode('utf-8').strip())
+
+    _, stderr = proc.communicate()
     if proc.returncode:
-        logger.warning('CmdStan installation failed')
+        logger.warning('CmdStan installation failed.')
+        if stderr:
+            logger.warning(stderr.decode('utf-8').strip())
         return False
     return True
+
+
+class MaybeDictToFilePath:
+    """Context manager for json files."""
+
+    def __init__(self, *objs: Union[str, dict], logger: logging.Logger = None):
+        self._unlink = [False] * len(objs)
+        self._paths = [''] * len(objs)
+        self._logger = logger or get_logger()
+        i = 0
+        for obj in objs:
+            if isinstance(obj, dict):
+                data_file = create_named_text_file(
+                    dir=_TMPDIR, prefix='', suffix='.json'
+                )
+                self._logger.debug('input tempfile: %s', data_file)
+                if any(
+                    not item
+                    for item in obj
+                    if isinstance(item, (Sequence, np.ndarray))
+                ):
+                    rdump(data_file, obj)
+                else:
+                    jsondump(data_file, obj)
+                self._paths[i] = data_file
+                self._unlink[i] = True
+            elif isinstance(obj, str):
+                if not os.path.exists(obj):
+                    raise ValueError("File doesn't exist {}".format(obj))
+                self._paths[i] = obj
+            elif obj is None:
+                self._paths[i] = None
+            elif i == 1 and isinstance(obj, (Integral, Real)):
+                self._paths[i] = obj
+            else:
+                raise ValueError('data must be string or dict')
+            i += 1
+
+    def __enter__(self):
+        return self._paths
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        for can_unlink, path in zip(self._unlink, self._paths):
+            if can_unlink and path:
+                try:
+                    os.remove(path)
+                except PermissionError:
+                    pass
+
+
+class TemporaryCopiedFile:
+    """Context manager for tmpfiles, handles spaces in filepath."""
+
+    def __init__(self, file_path: str):
+        self._path = None
+        self._tmpdir = None
+        if ' ' in os.path.abspath(file_path) and platform.system() == 'Windows':
+            base_path, file_name = os.path.split(os.path.abspath(file_path))
+            os.makedirs(base_path, exist_ok=True)
+            try:
+                short_base_path = windows_short_path(base_path)
+                if os.path.exists(short_base_path):
+                    file_path = os.path.join(short_base_path, file_name)
+            except RuntimeError:
+                pass
+
+        if ' ' in os.path.abspath(file_path):
+            tmpdir = tempfile.mkdtemp()
+            if ' ' in tmpdir:
+                raise RuntimeError(
+                    'Unable to generate temporary path without spaces! \n'
+                    + 'Please move your stan file to location without spaces.'
+                )
+
+            _, path = tempfile.mkstemp(suffix='.stan', dir=tmpdir)
+
+            shutil.copy(file_path, path)
+            self._path = path
+            self._tmpdir = tmpdir
+        else:
+            self._path = file_path
+
+    def __enter__(self):
+        return self._path, self._tmpdir is not None
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._tmpdir:
+            shutil.rmtree(self._tmpdir, ignore_errors=True)
diff --git a/docsrc/api.rst b/docsrc/api.rst
index f8eb1a0..c0d40b4 100644
--- a/docsrc/api.rst
+++ b/docsrc/api.rst
@@ -57,3 +57,35 @@ RunSet
 
 .. autoclass:: cmdstanpy.stanfit.RunSet
    :members:
+
+*********
+Functions
+*********
+
+.. _function_cmdstan_path:
+
+cmdstan_path
+============
+
+.. autofunction:: cmdstanpy.cmdstan_path
+
+.. _function_install_cmdstan:
+
+install_cmstan
+==============
+
+.. autofunction:: cmdstanpy.install_cmdstan
+
+.. _function_set_cmdstan_path:
+
+set_cmdstan_path
+================
+
+.. autofunction:: cmdstanpy.set_cmdstan_path
+
+.. _function_set_make_env:
+
+set_make_env
+============
+
+.. autofunction:: cmdstanpy.set_make_env
diff --git a/docsrc/hello_world.rst b/docsrc/hello_world.rst
index 601a739..d60f7d0 100644
--- a/docsrc/hello_world.rst
+++ b/docsrc/hello_world.rst
@@ -41,7 +41,7 @@ The data file specifies the number of observations and their values.
 Specify a Stan model
 ^^^^^^^^^^^^^^^^^^^^
 
-The: :ref:`class_cmdstanmodel` class manages the Stan program and its corresponding compiled executable.
+The :ref:`class_cmdstanmodel` class manages the Stan program and its corresponding compiled executable.
 It provides properties and functions to inspect the model code and filepaths.
 By default, the Stan program is compiled on instantiation.
 
diff --git a/docsrc/installation.rst b/docsrc/installation.rst
index 904a93b..8eab960 100644
--- a/docsrc/installation.rst
+++ b/docsrc/installation.rst
@@ -21,6 +21,7 @@ command line using ``pip``:
 The optional packages are
 
   * ``tqdm`` which allows for progress bar display during sampling
+  * ``ujson`` which provides faster IO
 
 To install CmdStanPy with all the optional packages:
 
@@ -57,11 +58,14 @@ consisting of a modern C++ compiler and the GNU-Make utility.
 Function ``install_cmdstan``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-CmdStanPy provides the function ``install_cmdstan`` which
+CmdStanPy provides the function :ref:`function_install_cmdstan` which
 downloads CmdStan from GitHub and builds the CmdStan utilities.
 It can be can be called from within Python or from the command line.
-By default it installs the latest version of CmdStan into a directory named
-``.cmdstanpy`` in your ``$HOME`` directory:
+
+The default install location is a hidden directory in the user ``$HOME`` directory
+named ``.cmdstan``.  (In earlier versions, the hidden directory was named ``.cmdstanpy``,
+and if directory ``$HOME/.cmdstanpy`` exists, it will continue to be used as the
+default install dir.)  This directory will be created by the install script.
 
 + From Python
 
@@ -75,14 +79,14 @@ By default it installs the latest version of CmdStan into a directory named
 .. code-block:: bash
 
     install_cmdstan
-    ls -F ~/.cmdstanpy
+    ls -F ~/.cmdstan
 
 + On Windows
 
 .. code-block:: bash
 
     python -m cmdstanpy.install_cmdstan
-    dir "%HOME%/.cmdstanpy"
+    dir "%HOME%/.cmdstan"
 
 The named arguments: `-d <directory>` and  `-v <version>`
 can be used to override these defaults:
@@ -92,17 +96,20 @@ can be used to override these defaults:
     install_cmdstan -d my_local_cmdstan -v 2.20.0
     ls -F my_local_cmdstan
 
-DIY Installion 
-^^^^^^^^^^^^^^
+DIY Installation 
+^^^^^^^^^^^^^^^^
 
 If you with to install CmdStan yourself, follow the instructions
 in the `CmdStan User's Guide <https://mc-stan.org/docs/cmdstan-guide/cmdstan-installation.html>`__.
 
-Post Installion: Setting Environment Variables
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Post Installation: Setting Environment Variables
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The default for the CmdStan installation location
-is a directory named ``.cmdstanpy`` in your ``$HOME`` directory.
+is a directory named ``.cmdstan`` in your ``$HOME`` directory.
+(In earlier versions, the hidden directory was named ``.cmdstanpy``,
+and if directory ``$HOME/.cmdstanpy`` exists, it will continue to be used as the
+default install dir.)
 
 If you have installed CmdStan in a different directory,
 then you can set the environment variable ``CMDSTAN`` to this
@@ -130,5 +137,3 @@ To use custom ``make``-tool use ``set_make_env`` function.
 
     from cmdstanpy import set_make_env
     set_make_env("mingw32-make.exe") # On Windows with mingw32-make
-
-
diff --git a/requirements-optional.txt b/requirements-optional.txt
index 78620c4..2c1e2aa 100644
--- a/requirements-optional.txt
+++ b/requirements-optional.txt
@@ -1,1 +1,2 @@
 tqdm
+ujson


Test patch:
diff --git a/test/test_install_cmdstan.py b/test/test_install_cmdstan.py
new file mode 100644
index 0000000..69316d2
--- /dev/null
+++ b/test/test_install_cmdstan.py
@@ -0,0 +1,36 @@
+"""install_cmdstan test"""
+
+import unittest
+
+from cmdstanpy.install_cmdstan import (
+    is_version_available,
+    latest_version,
+    retrieve_version,
+    CmdStanRetrieveError,
+)
+
+
+class InstallCmdStanTest(unittest.TestCase):
+    def test_is_version_available(self):
+        # check http error for bad version
+        self.assertTrue(is_version_available('2.24.1'))
+        self.assertFalse(is_version_available('2.222.222-rc222'))
+
+    def test_latest_version(self):
+        # examples of known previous version:  2.24-rc1, 2.23.0
+        version = latest_version()
+        nums = version.split('.')
+        self.assertTrue(len(nums) >= 2)
+        self.assertTrue(nums[0][0].isdigit())
+        self.assertTrue(nums[1][0].isdigit())
+
+    def test_retrieve_version(self):
+        # check http error for bad version
+        with self.assertRaisesRegex(
+            CmdStanRetrieveError, 'not available from github.com'
+        ):
+            retrieve_version('no_such_version')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index b63eedd..db7848c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -4,18 +4,21 @@ import json
 import os
 import unittest
 import platform
+import random
 import shutil
 import string
-import random
+import tempfile
+import stat
 
 import numpy as np
 
-from cmdstanpy import _TMPDIR
+from cmdstanpy import _TMPDIR, _DOT_CMDSTAN, _DOT_CMDSTANPY
 from cmdstanpy.utils import (
     cmdstan_path,
     set_cmdstan_path,
     validate_cmdstan_path,
     get_latest_cmdstan,
+    validate_dir,
     check_sampler_csv,
     MaybeDictToFilePath,
     read_metric,
@@ -48,11 +51,15 @@ class CmdStanPathTest(unittest.TestCase):
                 self.assertEqual(cmdstan_path(), path)
                 self.assertTrue('CMDSTAN' in os.environ)
             else:
-                install_dir = os.path.expanduser(
-                    os.path.join('~', '.cmdstanpy')
+                cmdstan_dir = os.path.expanduser(
+                    os.path.join('~', _DOT_CMDSTAN)
                 )
-                install_version = os.path.expanduser(
-                    os.path.join(install_dir, get_latest_cmdstan(install_dir))
+                if not os.path.exists(cmdstan_dir):
+                    cmdstan_dir = os.path.expanduser(
+                        os.path.join('~', _DOT_CMDSTANPY)
+                    )
+                install_version = os.path.join(
+                    cmdstan_dir, get_latest_cmdstan(cmdstan_dir)
                 )
                 self.assertTrue(
                     os.path.samefile(cmdstan_path(), install_version)
@@ -99,18 +106,24 @@ class CmdStanPathTest(unittest.TestCase):
         if 'CMDSTAN' in os.environ:
             self.assertEqual(cmdstan_path(), os.environ['CMDSTAN'])
         else:
-            install_dir = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
-            install_version = os.path.expanduser(
-                os.path.join(install_dir, get_latest_cmdstan(install_dir))
+            cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))
+            if not os.path.exists(cmdstan_dir):
+                cmdstan_dir = os.path.expanduser(
+                    os.path.join('~', _DOT_CMDSTANPY)
+                )
+            install_version = os.path.join(
+                cmdstan_dir, get_latest_cmdstan(cmdstan_dir)
             )
             set_cmdstan_path(install_version)
             self.assertEqual(install_version, cmdstan_path())
             self.assertEqual(install_version, os.environ['CMDSTAN'])
 
     def test_validate_path(self):
-        install_dir = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
-        install_version = os.path.expanduser(
-            os.path.join(install_dir, get_latest_cmdstan(install_dir))
+        cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))
+        if not os.path.exists(cmdstan_dir):
+            cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTANPY))
+        install_version = os.path.join(
+            cmdstan_dir, get_latest_cmdstan(cmdstan_dir)
         )
         set_cmdstan_path(install_version)
         validate_cmdstan_path(install_version)
@@ -130,6 +143,31 @@ class CmdStanPathTest(unittest.TestCase):
             validate_cmdstan_path(path_test)
         shutil.rmtree(folder_name)
 
+    def test_validate_dir(self):
+        path = os.path.join(_TMPDIR, 'cmdstan-M.m.p')
+        self.assertFalse(os.path.exists(path))
+        validate_dir(path)
+        self.assertTrue(os.path.exists(path))
+
+        _, file = tempfile.mkstemp(dir=_TMPDIR)
+        with self.assertRaisesRegex(Exception, 'File exists'):
+            validate_dir(file)
+
+        if platform.system() != 'Windows':
+            with self.assertRaisesRegex(Exception, 'Cannot create directory'):
+                dir = tempfile.mkdtemp(dir=_TMPDIR)
+                os.chmod(dir, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+                validate_dir(os.path.join(dir, 'cmdstan-M.m.p'))
+
+    def test_munge_cmdstan_versions(self):
+        tdir = os.path.join(HERE, 'tmpdir_xxx')
+        os.mkdir(tdir)
+        os.mkdir(os.path.join(tdir, 'cmdstan-2.22-rc1'))
+        os.mkdir(os.path.join(tdir, 'cmdstan-2.22-rc2'))
+        os.mkdir(os.path.join(tdir, 'cmdstan-2.22.0'))
+        self.assertEqual(get_latest_cmdstan(tdir), 'cmdstan-2.22.0')
+        shutil.rmtree(tdir, ignore_errors=True)
+
     def test_dict_to_file(self):
         file_good = os.path.join(DATAFILES_PATH, 'bernoulli_output_1.csv')
         dict_good = {'a': 0.5}