Repository: stan-dev/cmdstanpy
Pull request #96 (2019-08-02 13:25:39)

C++ toolchain installation for Windows
#### Submission Checklist

- [x] Run unit tests
- [x] Declare copyright holder and open-source license: see below

#### Summary

Enable user to install C++ with script / from python session.

Needs still some streamlining and editing, so any system can be used. Currently many functions
work only with Windows.

Also needs functionality to cmdstanpy to easily set-up make/local to use the installed toolchain.

#### Copyright and Licensing

Please list the copyright holder for the work you are submitting (this will be you or your assignee, such as a university or company):

Ari Hartikainen

By submitting this pull request, the copyright holder is agreeing to license the submitted work under the following licenses:
- Code: BSD 3-clause (https://opensource.org/licenses/BSD-3-Clause)



ahartikainen: install part is ready. Still need some convenience function to add RTools bin folders to path.
ahartikainen: This is ready, needs some docs.

Also, this code needs some cleaning if we implement the same thing for the macOS and linux.
ahartikainen: Yes, I think we could install toolchain for appveyor with this and then add tests for win vs others.

After we get this in, it would be interesting to do the same for clang on linux/macos.

Patch:
diff --git a/.appveyor.yml b/.appveyor.yml
index 1810b99..c539c14 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -7,7 +7,7 @@ platform:
 
 cache:
   - C:\Users\appveyor\.cmdstanpy
-  #- C:\RTools
+  - C:\RTools
 
 init:
   - git config --global core.autocrlf input
@@ -46,9 +46,9 @@ install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
 
   # Install RTools
-  - ps: Start-FileDownload 'https://cran.r-project.org/bin/windows/Rtools/Rtools35.exe'
-  - "Rtools35.exe /VERYSILENT /SP- /NORESTART"  #TODO: dont download and install if c:\rtools folder exists.
-  - "SET PATH=C:\\Rtools\\bin;C:\\Rtools\\mingw_64\\bin;%PATH%"
+  - "python .\\cmdstanpy\\install_cxx_toolchain.py --silent --dir C:\\"
+  - "dir C:\\Rtools\\RTools35"
+  - "SET PATH=C:\\Rtools\\RTools35\\bin;C:\\Rtools\\RTools35\\mingw_64\\bin;%PATH%"
   - "SET MAKE=mingw32-make.exe"
   - "gcc --version"
   - "mingw32-make --version"
@@ -69,7 +69,7 @@ install:
 build: false
 
 test_script:
-  - "SET PATH=C:\\Rtools\\bin;C:\\Rtools\\mingw_64\\bin;%PATH%"
+  - "SET PATH=C:\\Rtools\\RTools35\\bin;C:\\Rtools\\RTools35\\mingw_64\\bin;%PATH%"
   - "SET MAKE=mingw32-make.exe"
   - "mkdir new_dir"
   - "cd new_dir"
diff --git a/cmdstanpy/install_cxx_toolchain.py b/cmdstanpy/install_cxx_toolchain.py
new file mode 100644
index 0000000..b333e01
--- /dev/null
+++ b/cmdstanpy/install_cxx_toolchain.py
@@ -0,0 +1,262 @@
+#!/usr/bin/env python
+"""
+Download and install a C++ toolchain.
+Currently implemented platforms (platform.system)
+    Windows: RTools 3.5 (default), 4.0
+    Darwin (macOS): Not implemented
+    Linux: Not implemented
+Optional command line arguments:
+   -v, --version : version, defaults to latest
+   -d, --dir : install directory, defaults to '~/.cmdstanpy
+   -s (--silent) : install with /VERYSILENT instead of /SILENT for RTools
+"""
+import argparse
+import contextlib
+import os
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import urllib.request
+from pathlib import Path
+from time import sleep
+
+EXTENSION = '.exe' if platform.system() == 'Windows' else ''
+IS_64BITS = sys.maxsize > 2 ** 32
+
+
+@contextlib.contextmanager
+def pushd(new_dir):
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    yield
+    os.chdir(previous_dir)
+
+
+def usage():
+    print(
+        """Arguments:
+        -v (--version) :CmdStan version
+        -d (--dir) : install directory
+        -s (--silent) : install with /VERYSILENT instead of /SILENT for RTools
+        -h (--help) : this message
+        """
+    )
+
+
+def get_config(dir, silent):
+    config = []
+    if platform.system() == 'Windows':
+        _, dir = os.path.splitdrive(os.path.abspath(dir))
+        if dir.startswith('\\'):
+            dir = dir[1:]
+        config = [
+            '/SP-',
+            '/VERYSILENT' if silent else '/SILENT',
+            '/SUPPRESSMSGBOXES',
+            '/CURRENTUSER',
+            'LANG="English"',
+            '/DIR="{}"'.format(dir),
+            '/NOICONS',
+            '/NORESTART',
+        ]
+    return config
+
+
+def install_version(installation_dir, installation_file, version, silent):
+    with pushd('.'):
+        print(
+            'Installing the C++ toolchain: {}'.format(
+                os.path.splitext(installation_file)[0]
+            )
+        )
+        cmd = [installation_file]
+        cmd.extend(get_config(installation_dir, silent))
+        print(' '.join(cmd))
+        proc = subprocess.Popen(
+            cmd, cwd=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        while proc.poll() is None:
+            output = proc.stdout.readline().decode('utf-8').strip()
+            if output:
+                print(output, flush=True)
+        stdout, stderr = proc.communicate()
+        if proc.returncode:
+            print('Installation failed: returncode={}'.format(proc.returncode))
+            if stderr:
+                print(stderr.decode('utf-8').strip())
+            if is_installed(installation_dir, version):
+                print('Installation files found at the installation location.')
+            sys.exit(3)
+    # check installation
+    if is_installed(installation_dir, version):
+        os.remove(installation_file)
+    print('Installed {}'.format(os.path.splitext(installation_file)[0]))
+
+
+def is_installed(toolchain_loc, version):
+    if platform.system() == 'Windows':
+        if version == '3.5':
+            if not os.path.exists(os.path.join(toolchain_loc, 'bin')):
+                return False
+            return os.path.exists(
+                os.path.join(
+                    toolchain_loc,
+                    'mingw_64' if IS_64BITS else 'mingw_32',
+                    'bin',
+                    'g++' + EXTENSION,
+                )
+            )
+        elif version == '4.0':
+            return os.path.exists(
+                os.path.join(
+                    toolchain_loc,
+                    'mingw64' if IS_64BITS else 'mingw32',
+                    'bin',
+                    'g++' + EXTENSION,
+                )
+            )
+        else:
+            return False
+    return False
+
+
+def latest_version():
+    if platform.system() == 'Windows':
+        return '3.5'
+
+
+def retrieve_toolchain(filename, url):
+    print('Downloading C++ toolchain: {}'.format(filename))
+    for i in range(6):
+        try:
+            _ = urllib.request.urlretrieve(url, filename=filename)
+            break
+        except urllib.error.URLError as err:
+            print('Failed to download C++ toolchain')
+            print(err)
+            if i < 5:
+                print('retry ({}/5)'.format(i + 1))
+                sleep(1)
+                continue
+            sys.exit(3)
+    print('Download successful, file: {}'.format(filename))
+
+
+def validate_dir(install_dir):
+    if not os.path.exists(install_dir):
+        try:
+            os.makedirs(install_dir)
+        except OSError as e:
+            raise ValueError(
+                'Cannot create directory: {}'.format(install_dir)
+            ) from e
+    else:
+        if not os.path.isdir(install_dir):
+            raise ValueError(
+                'File {} is not a directory: {}'.format(install_dir)
+            )
+        try:
+            with open('tmp_test_w', 'w') as fd:
+                pass
+            os.remove('tmp_test_w')  # cleanup
+        except OSError as e:
+            raise ValueError(
+                'Cannot write files to directory {}'.format(install_dir)
+            ) from e
+
+
+def normalize_version(version):
+    if platform.system() == 'Windows':
+        if version in ['4', '40']:
+            version = '4.0'
+        elif version == '35':
+            version = '3.5'
+    return version
+
+
+def get_toolchain_name():
+    if platform.system() == 'Windows':
+        return 'RTools'
+
+
+def get_url(version):
+    if platform.system() == 'Windows':
+        if version == '4.0':
+
+            if IS_64BITS:
+                url = 'https://cran.r-project.org/bin/windows/testing/rtools40-x86_64.exe'  # noqa: disable=E501
+            else:
+                url = 'https://cran.r-project.org/bin/windows/testing/rtools40-i686.exe'  # noqa: disable=E501
+        elif version == '3.5':
+            url = 'https://cran.r-project.org/bin/windows/Rtools/Rtools35.exe'
+    return url
+
+
+def get_toolchain_version(name, version):
+    root_folder = None
+    toolchain_folder = None
+    if platform.system() == 'Windows':
+        root_folder = 'RTools'
+        toolchain_folder = '{}{}'.format(name, version.replace('.', ''))
+
+    return root_folder, toolchain_folder
+
+
+def main():
+    if platform.system() not in {'Windows'}:
+        msg = (
+            'Download for the C++ toolchain'
+            ' on the current platform has not been implemented: %s'
+        )
+        raise NotImplementedError(msg % platform.system())
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--version', '-v')
+    parser.add_argument('--dir', '-d')
+    parser.add_argument('--silent', '-s', action='store_true')
+    args = parser.parse_args(sys.argv[1:])
+
+    toolchain = get_toolchain_name()
+    version = vars(args)['version']
+    if version is None:
+        version = latest_version()
+    version = normalize_version(version)
+    print("C++ toolchain '{}' version: {}".format(toolchain, version))
+
+    url = get_url(version)
+
+    install_dir = vars(args)['dir']
+    if install_dir is None:
+        install_dir = os.path.expanduser(os.path.join('~', '.cmdstanpy'))
+    validate_dir(install_dir)
+    print('Install directory: {}'.format(install_dir))
+
+    if platform.system() == 'Windows':
+        silent = 'silent' in vars(args)
+        # force silent == False for 4.0 version
+        if 'silent' not in vars(args) and version in ('4.0', '4', '40'):
+            silent = False
+    else:
+        silent = False
+
+    root_folder, toolchain_version = get_toolchain_version(toolchain, version)
+    toolchain_loc = os.path.join(root_folder, toolchain_version)
+    with pushd(install_dir):
+        if is_installed(toolchain_loc, version):
+            print(
+                'C++ toolchain {} already installed'.format(toolchain_version)
+            )
+        else:
+            if os.path.exists(toolchain_loc):
+                shutil.rmtree(toolchain_loc, ignore_errors=False)
+            retrieve_toolchain(toolchain_version + EXTENSION, url)
+            install_version(
+                toolchain_loc, toolchain_version + EXTENSION, version, silent
+            )
+
+
+if __name__ == '__main__':
+    main()
diff --git a/cmdstanpy/model.py b/cmdstanpy/model.py
index c47c2ed..fbd3ecd 100644
--- a/cmdstanpy/model.py
+++ b/cmdstanpy/model.py
@@ -819,7 +819,10 @@ class Model(object):
         self._logger.info('start chain %u', idx + 1)
         self._logger.debug('sampling: %s', cmd)
         proc = subprocess.Popen(
-            cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            cmd.split(),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=os.environ,
         )
         if pbar:
             stdout_pbar = self._read_progress(proc, pbar, idx)
diff --git a/cmdstanpy/utils.py b/cmdstanpy/utils.py
index ca5f2c3..a4c1fc6 100644
--- a/cmdstanpy/utils.py
+++ b/cmdstanpy/utils.py
@@ -2,6 +2,7 @@
 Utility functions
 """
 import os
+from numbers import Integral, Real
 
 try:
     import ujson as json
@@ -17,7 +18,7 @@ import tempfile
 import logging
 import sys
 
-from typing import Dict, TextIO, List, Union
+from typing import Dict, TextIO, List, Union, Tuple
 from cmdstanpy import TMPDIR
 
 EXTENSION = '.exe' if platform.system() == 'Windows' else ''
@@ -34,7 +35,7 @@ def get_latest_cmdstan(dot_dir: str) -> str:
     """
     Given a valid directory path, find all installed CmdStan versions
     and return highest (i.e., latest) version number.
-    Assumes directory populated via script `bin/install_cmdstan`.
+    Assumes directory populated via script `install_cmdstan`.
     """
     versions = [
         name.split('-')[1]
@@ -179,6 +180,137 @@ def cmdstan_path() -> str:
     return cmdstan_path
 
 
+def cxx_toolchain_path(version: str = None) -> Tuple[str]:
+    """
+    Validate, then activate C++ toolchain directory path.
+    """
+    if platform.system() != "Windows":
+        raise RuntimeError(
+            "Functionality is currently only supported on Windows"
+        )
+    if version is not None and not isinstance(version, str):
+        raise TypeError("Format version number as a string")
+    logger = get_logger()
+    toolchain_root = ''
+    if 'CMDSTAN_TOOLCHAIN' in os.environ:
+        toolchain_root = os.environ['CMDSTAN_TOOLCHAIN']
+        if os.path.exists(os.path.join(toolchain_root, "mingw_64")):
+            compiler_path = os.path.join(
+                toolchain_root,
+                'mingw_64' if (sys.maxsize > 2 ** 32) else 'mingw_32',
+                'bin',
+            )
+            if os.path.exists(compiler_path):
+                tool_path = os.path.join(toolchain_root, 'bin')
+                if not os.path.exists(tool_path):
+                    tool_path = ''
+                    compiler_path = ''
+                    logger.warning(
+                        "Found invalid installion for RTools35 on %",
+                        toolchain_root,
+                    )
+                    toolchain_root = ''
+            else:
+                compiler_path = ''
+                logger.warning(
+                    "Found invalid installion for RTools35 on %", toolchain_root
+                )
+                toolchain_root = ''
+        elif os.path.exist(os.path.join(toolchain_root, 'mingw64')):
+            compiler_path = os.path.join(
+                toolchain_root,
+                'mingw64' if (sys.maxsize > 2 ** 32) else 'mingw32',
+                'bin',
+            )
+            if os.path.exists(compiler_path):
+                tool_path = os.path.join(toolchain_root, 'usr', 'bin')
+                if not os.path.exists(tool_path):
+                    tool_path = ''
+                    compiler_path = ''
+                    logger.warning(
+                        "Found invalid installion for RTools40 on %",
+                        toolchain_root,
+                    )
+                    toolchain_root = ''
+            else:
+                compiler_path = ''
+                logger.warning(
+                    "Found invalid installion for RTools40 on %", toolchain_root
+                )
+                toolchain_root = ''
+    else:
+        rtools_dir = os.path.expanduser(
+            os.path.join('~', '.cmdstanpy', 'RTools')
+        )
+        if not os.path.exists(rtools_dir):
+            raise ValueError(
+                'no RTools installation found, '
+                'run command line script "install_cxx_toolchain"'
+            )
+        compiler_path = ''
+        tool_path = ''
+        if version not in ('4', '40', '4.0') and os.path.exists(
+            os.path.join(rtools_dir, "RTools35")
+        ):
+            toolchain_root = os.path.join(rtools_dir, "RTools35")
+            compiler_path = os.path.join(
+                toolchain_root,
+                'mingw_64' if (sys.maxsize > 2 ** 32) else 'mingw_32',
+                'bin',
+            )
+            if os.path.exists(compiler_path):
+                tool_path = os.path.join(toolchain_root, 'bin')
+                if not os.path.exists(tool_path):
+                    tool_path = ''
+                    compiler_path = ''
+                    logger.warning(
+                        "Found invalid installion for RTools35 on %",
+                        toolchain_root,
+                    )
+                    toolchain_root = ''
+            else:
+                compiler_path = ''
+                logger.warning(
+                    "Found invalid installion for RTools35 on %", toolchain_root
+                )
+                toolchain_root = ''
+        if (
+            not toolchain_root or version in ('4', '40', '4.0')
+        ) and os.path.exists(os.path.join(rtools_dir, 'RTools40')):
+            toolchain_root = os.path.join(rtools_dir, 'RTools40')
+            compiler_path = os.path.join(
+                toolchain_root,
+                'mingw64' if (sys.maxsize > 2 ** 32) else 'mingw32',
+                'bin',
+            )
+            if os.path.exists(compiler_path):
+                tool_path = os.path.join(toolchain_root, 'usr', 'bin')
+                if not os.path.exists(tool_path):
+                    tool_path = ''
+                    compiler_path = ''
+                    logger.warning(
+                        "Found invalid installion for RTools40 on %",
+                        toolchain_root,
+                    )
+                    toolchain_root = ''
+            else:
+                compiler_path = ''
+                logger.warning(
+                    "Found invalid installion for RTools40 on %", toolchain_root
+                )
+                toolchain_root = ''
+    if not toolchain_root:
+        raise ValueError(
+            'no C++ toolchain installation found, '
+            'run command line script "install_cxx_toolchain"'
+        )
+    logger.info("Adds C++ toolchain to $PATH: %s", toolchain_root)
+    os.environ["PATH"] = "{};{};{}".format(
+        compiler_path, tool_path, os.environ["PATH"]
+    )
+    return compiler_path, tool_path
+
+
 def _rdump_array(key: str, val: np.ndarray) -> str:
     """Flatten numpy ndarray, format as Rdump variable declaration."""
     c = 'c(' + ', '.join(map(str, val.T.flat)) + ')'
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
index ce8db4b..69b5744 100644
--- a/docs/getting_started.rst
+++ b/docs/getting_started.rst
@@ -26,8 +26,13 @@ To install CmdStanPy with all the optional packages
 
 CmdStanPy requires a local install of CmdStan.
 If you don't have CmdStan installed, you can run the CmdStanPy script ``install_cmdstan``
-which downloads CmdStan from GitHub and builds the CmdStan utilities.
-By default this script installs the latest version of CmdStan into a directory named
+which downloads CmdStan from GitHub and builds the CmdStan utilities. Latest CmdStan (2.19+)
+needs a C++14 compatible C++ -toolchain. On Windows, it is recommended to use RTools 3.5+.
+It is possible to install latest RTools with ``install_cxx_toolchain`` script.
+
+    python -m cmdstanpy.install_cxx_toolchain
+
+By default ``install_cmdstan`` script installs the latest version of CmdStan into a directory named
 ``.cmdstanpy`` in your ``$HOME`` directory:
 
 .. code-block:: bash
@@ -77,13 +82,14 @@ To use custom ``make``-tool use ``set_make_env`` function.
     from cmdstanpy import set_make_env
     set_make_env("mingw32-make.exe") # On Windows with mingw32-make
 
-User can install optional packages with pip with the CmdStanPy installation
-
-.. code-block:: bash
+On Windows, to use ``install_cxx_toolchain`` version of RTools user can
+call ``cmdstan.utils.cxx_toolchain_path`` function, which adds the correct
+subfolders on the $PATH
 
-    pip install --upgrade cmdstanpy[all]
+.. code-block:: python
 
-or by installing packages manually.
+    from cmdstanpy.utils import cxx_toolchain_path
+    cxx_toolchain_path()
 
 For faster IO cmdstanpy will use ``ujson`` package if it's installed
 
diff --git a/setup.py b/setup.py
index e677f61..b1b55d7 100644
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,10 @@ setuptools.setup(
     url='https://github.com/stan-dev/cmdstanpy',
     packages=['cmdstanpy'],
     entry_points={
-        'console_scripts': ['install_cmdstan=cmdstanpy.install_cmdstan:main']
+        'console_scripts': [
+            'install_cmdstan=cmdstanpy.install_cmdstan:main',
+            'install_cxx_toolchain=cmdstanpy.install_cxx_toolchain:main',
+        ]
     },
     install_requires=INSTALL_REQUIRES,
     extras_require=EXTRAS_REQUIRE,


Test patch:
diff --git a/test/test_cxx_installation.py b/test/test_cxx_installation.py
new file mode 100644
index 0000000..e9fed8f
--- /dev/null
+++ b/test/test_cxx_installation.py
@@ -0,0 +1,70 @@
+import argparse
+import os
+import unittest
+from unittest import mock
+import platform
+import shutil
+
+from cmdstanpy import TMPDIR
+from cmdstanpy import install_cxx_toolchain
+
+
+class install_cxx_script(unittest.TestCase):
+    def test_config(self):
+        """Test config output."""
+        if platform.system() != "Windows":
+            return
+        else:
+            config = install_cxx_toolchain.get_config("C:\\RTools", True)
+
+            config_reference = [
+                '/SP-',
+                '/VERYSILENT',
+                '/SUPPRESSMSGBOXES',
+                '/CURRENTUSER',
+                'LANG="English"',
+                '/DIR="RTools"',
+                '/NOICONS',
+                '/NORESTART',
+            ]
+
+            self.assertEqual(config, config_reference)
+
+    def test_install_not_windows(self):
+        """Try to install on unsupported platform."""
+        if platform.system() == "Windows":
+            return
+
+        with self.assertRaisesRegex(
+            NotImplementedError,
+            r'Download for the C\+\+ toolchain on the current platform has not been implemented:\s*\S+',
+        ):
+            install_cxx_toolchain.main()
+
+    def test_normalize_version(self):
+        """Test supported versions."""
+        if platform.system() != "Windows":
+            return
+        else:
+            for ver in ['4.0', '4', '40']:
+                self.assertEqual(
+                    install_cxx_toolchain.normalize_version(ver), '4.0'
+                )
+
+            for ver in ['3.5', '35']:
+                self.assertEqual(
+                    install_cxx_toolchain.normalize_version(ver), '3.5'
+                )
+
+    def test_toolchain_name(self):
+        """Check toolchain name."""
+        if platform.system() != "Windows":
+            return
+        else:
+            self.assertEqual(
+                install_cxx_toolchain.get_toolchain_name(), "RTools"
+            )
+
+
+if __name__ == '__main__':
+    unittest.main()