Repository: stan-dev/cmdstanpy
Pull request #35 (2019-05-18 21:46:37)

Feature/23 cmdstan path
#### Submission Checklist

- [x] Run unit tests
- [x] Declare copyright holder and open-source license: see below

#### Summary
get location of CmdStan (`cmdstanpy.CMDSTAN_PATH`) from environment var `CMDSTAN` if set,
else use default location `./releases/cmdstan` - the latter is set up via script `make_cmdstan.sh`.

as part of this PR, made `make_cmdstan.sh` more verbose to help catch Travis CI fails due to initial GitHub download timing out.

#### Copyright and Licensing

Please list the copyright holder for the work you are submitting (this will be you or your assignee, such as a university or company):  Columbia University



By submitting this pull request, the copyright holder is agreeing to license the submitted work under the following licenses:
- Code: BSD 3-clause (https://opensource.org/licenses/BSD-3-Clause)




Patch:
diff --git a/cmdstanpy/__init__.py b/cmdstanpy/__init__.py
index 9ffe73d..8e1518b 100644
--- a/cmdstanpy/__init__.py
+++ b/cmdstanpy/__init__.py
@@ -1,11 +1,7 @@
 import atexit
-import os.path
 import shutil
 import tempfile
 
-
-CMDSTAN_PATH = os.path.abspath(os.path.join('.', 'releases', 'cmdstan'))
-TMPDIR = tempfile.mkdtemp()
 STANSUMMARY_STATS = [
     'Mean',
     'MCSE',
@@ -18,6 +14,8 @@ STANSUMMARY_STATS = [
     'R_hat',
 ]
 
+TMPDIR = tempfile.mkdtemp()
+
 
 def cleanup_tmpdir():
     print('deleting tmpfiles dir: {}'.format(TMPDIR))
diff --git a/cmdstanpy/cmds.py b/cmdstanpy/cmds.py
index be819a3..b99f355 100644
--- a/cmdstanpy/cmds.py
+++ b/cmdstanpy/cmds.py
@@ -8,9 +8,9 @@ from multiprocessing import cpu_count
 from multiprocessing.pool import ThreadPool
 from typing import Dict
 
-from cmdstanpy import CMDSTAN_PATH, TMPDIR
+from cmdstanpy import TMPDIR
 from cmdstanpy.lib import Model, StanData, RunSet, SamplerArgs, PosteriorSample
-from cmdstanpy.utils import do_command
+from cmdstanpy.utils import do_command, cmdstan_path
 
 
 def compile_model(
@@ -29,7 +29,7 @@ def compile_model(
     hpp_file = os.path.join(path, hpp_name)
     if overwrite or not os.path.exists(hpp_file):
         print('translating to {}'.format(hpp_file))
-        stanc_path = os.path.join(CMDSTAN_PATH, 'bin', 'stanc')
+        stanc_path = os.path.join(cmdstan_path(), 'bin', 'stanc')
         cmd = [stanc_path, '--o={}'.format(hpp_file), stan_file]
         print('stan to c++: make args {}'.format(cmd))
         do_command(cmd)
@@ -45,7 +45,7 @@ def compile_model(
     cmd = ['make', 'O={}'.format(opt_lvl), exe_file]
     print('compiling c++: make args {}'.format(cmd))
     try:
-        do_command(cmd, CMDSTAN_PATH)
+        do_command(cmd, cmdstan_path())
     except Exception:
         return Model(stan_file)
     return Model(stan_file, exe_file)
diff --git a/cmdstanpy/lib.py b/cmdstanpy/lib.py
index d26b18a..fd3f342 100644
--- a/cmdstanpy/lib.py
+++ b/cmdstanpy/lib.py
@@ -8,9 +8,9 @@ from typing import List, Dict, Tuple
 import numpy as np
 import pandas as pd
 
-from cmdstanpy import CMDSTAN_PATH, TMPDIR
+from cmdstanpy import TMPDIR
 from cmdstanpy.utils import do_command, jsondump, rdump
-from cmdstanpy.utils import check_csv
+from cmdstanpy.utils import check_csv, cmdstan_path
 
 
 class Model(object):
@@ -422,7 +422,7 @@ class PosteriorSample(object):
         Assemble csv tempfile contents into pandasDataFrame.
         """
         names = self.column_names
-        cmd_path = os.path.join(CMDSTAN_PATH, 'bin', 'stansummary')
+        cmd_path = os.path.join(cmdstan_path(), 'bin', 'stansummary')
         tmp_csv_file = 'stansummary-{}-{}-chains-'.format(
             self.model, self.chains)
         fd, tmp_csv_path = tempfile.mkstemp(
@@ -445,7 +445,7 @@ class PosteriorSample(object):
         Run cmdstan/bin/diagnose over all output csv files.
         Echo diagnose stdout/stderr to console.
         """
-        cmd_path = os.path.join(CMDSTAN_PATH, 'bin', 'diagnose')
+        cmd_path = os.path.join(cmdstan_path(), 'bin', 'diagnose')
         csv_files = ' '.join(self.csv_files)
         cmd = '{} {} '.format(cmd_path, csv_files)
         result = do_command(cmd=cmd.split())
diff --git a/cmdstanpy/utils.py b/cmdstanpy/utils.py
index 8fb2eb8..b500683 100644
--- a/cmdstanpy/utils.py
+++ b/cmdstanpy/utils.py
@@ -1,12 +1,50 @@
 """
 Utility functions
 """
+import os
+import os.path
 import json
 import subprocess
 import numpy as np
 from typing import Dict
 
 
+def validate_cmdstan_path(path: str) -> None:
+    """
+    Validate that CmdStan directory exists and binaries have been built.
+    Throws exception if specified path is invalid.
+    """
+    if not os.path.isdir(path):
+        raise ValueError('no such CmdStan directory {}'.format(path))
+    if not os.path.exists(os.path.join(path, 'bin', 'stanc')):
+        raise ValueError(
+            "no CmdStan binaries found, "
+            "do 'make build' in directory {}".format(path)
+        )
+
+
+def set_cmdstan_path(path: str) -> None:
+    """
+    Validate, then set CmdStan directory path.
+    """
+    validate_cmdstan_path(path)
+    os.environ['CMDSTAN'] = path
+
+
+def cmdstan_path() -> str:
+    """
+    Validate, then return CmdStan directory path.
+    """
+    cmdstan_path = ''
+    if 'CMDSTAN' in os.environ:
+        cmdstan_path = os.environ['CMDSTAN']
+    else:
+        cmdstan_path = os.path.abspath(
+            os.path.join('.', 'releases', 'cmdstan'))
+    validate_cmdstan_path(cmdstan_path)
+    return cmdstan_path
+
+
 def do_command(cmd: str, cwd: str = None) -> str:
     """
     Spawn process, print stdout/stderr to console.
@@ -88,6 +126,7 @@ def check_csv(filename: str) -> Dict:
 
 
 def scan_stan_csv(filename: str) -> Dict:
+    """Process stan_csv file line by line."""
     dict = {}
     draws_found = 0
     lineno = 0
@@ -110,8 +149,8 @@ def scan_stan_csv(filename: str) -> Dict:
             if len(line.split(',')) != num_cols:
                 raise ValueError(
                     'file {}, at line {}: bad draw, expecting {} items, '
-                    'found {}'.format(filename, lineno, num_cols,
-                                          len(line.split(',')))
+                    'found {}'.format(
+                        filename, lineno, num_cols, len(line.split(',')))
                 )
             line = fp.readline().strip()
             lineno += 1
@@ -120,6 +159,10 @@ def scan_stan_csv(filename: str) -> Dict:
 
 
 def parse_header(line: str, dict: Dict) -> None:
+    """
+    Parse initial stan_csv file comments lines and
+    save non-default configuration information to dict.
+    """
     if not line.endswith('(Default)'):
         line = line.lstrip(' #\t')
         key_val = line.split('=')
diff --git a/make_cmdstan.sh b/make_cmdstan.sh
index 1c59cab..509b2d3 100755
--- a/make_cmdstan.sh
+++ b/make_cmdstan.sh
@@ -11,7 +11,7 @@ if [[ ! -d releases ]]; then
     echo 'cannot install cmdstan, file "releases" is not a directory'
     exit 1
 fi
-pushd releases
+pushd releases > /dev/null
 
 echo "release dir: `pwd`"
 
@@ -25,19 +25,36 @@ echo "latest cmdstan: $VER"
 CS=cmdstan-${VER}
 if [[ -d $cs && -f ${CS}/bin/stanc && -f ${CS}/examples/bernoulli/bernoulli ]]; then
     echo "cmdstan already installed"
-#    exit 0
+    exit 0
 fi
 
+echo "downloading ${CS}.tar.gz from github"
 curl -s -OL https://github.com/stan-dev/cmdstan/releases/download/v${VER}/${CS}.tar.gz
-echo "download rc code: $?"
+CURL_RC=$?
+if [[ ${CURL_RC} -ne 0 ]]; then
+    echo "github download failed, curl exited with: ${CURL_RC}"
+    exit ${CURL_RC}
+fi
+echo "download complete"
+
+echo "unpacking archive"
 tar xzf ${CS}.tar.gz
-echo "tar rc $?"
+TAR_RC=$?
+if [[ ${TAR_RC} -ne 0 ]]; then
+    echo "corrupt download file ${CS}.tar.gz, tar exited with: ${TAR_RC}"
+    exit ${TAR_RC}
+fi
 
 if [[ -h cmdstan ]]; then
     unlink cmdstan
 fi
 ln -s ${CS} cmdstan
-cd cmdstan
+pushd cmdstan > /dev/null
+echo "building cmdstan binaries"
 make -j2 build examples/bernoulli/bernoulli
 echo "installed ${CS}"
-echo `ls -lFd releases/*`
+
+# cleanup
+pushd -0 > /dev/null
+dirs -c > /dev/null
+echo "all installed versions in releases dir: `ls -Fd releases/* | grep -v @ | grep -v gz`"


Test patch:
diff --git a/test/test_utils.py b/test/test_utils.py
index 10942d2..f4fc692 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -3,28 +3,48 @@ import os.path
 import unittest
 import json
 
-from cmdstanpy import CMDSTAN_PATH, TMPDIR
+from cmdstanpy import TMPDIR
 from cmdstanpy.lib import StanData
-from cmdstanpy.utils import check_csv
+from cmdstanpy.utils import cmdstan_path, set_cmdstan_path, validate_cmdstan_path, check_csv
 
-datafiles_path = os.path.join('test', 'data')
 
+datafiles_path = os.path.join('test', 'data')
 
 rdump = ('''N <- 10
 y <- c(0, 1, 0, 0, 0, 0, 0, 0, 0, 1)
 ''')
 
 
-class ConfigTest(unittest.TestCase):
-    def test_path(self):
+class CmdStanPathTest(unittest.TestCase):
+    def test_default_path(self):
+        abs_rel_path = os.path.abspath(os.path.join('releases', 'cmdstan'))
+        self.assertEqual(abs_rel_path, cmdstan_path())
+
+    def test_set_path(self):
         abs_rel_path = os.path.abspath(os.path.join('releases', 'cmdstan'))
-        self.assertEqual(abs_rel_path, CMDSTAN_PATH)
+        self.assertEqual(abs_rel_path, cmdstan_path())
+        cur_version = os.path.abspath(
+            os.path.join('releases', 'cmdstan-2.19.1'))
+        set_cmdstan_path(cur_version)
+        self.assertEqual(cur_version, cmdstan_path())
+
+    def test_validate_path(self):
+        abs_rel_path = os.path.abspath(os.path.join('releases', 'cmdstan'))
+        validate_cmdstan_path(abs_rel_path)
+        path_foo = os.path.abspath(os.path.join('releases', 'foo'))
+        with self.assertRaisesRegex(ValueError, 'no such CmdStan directory'):
+            validate_cmdstan_path(path_foo)
+        path_test = os.path.abspath('test')
+        with self.assertRaisesRegex(ValueError, 'no CmdStan binaries'):
+            validate_cmdstan_path(path_test)
+
 
 
 class StanDataTest(unittest.TestCase):
     def test_standata_existing(self):
         rdump = os.path.join(datafiles_path, 'bernoulli.data.R')
         standata = StanData(rdump)
+        self.assertEqual(standata.data_file, rdump)
 
     def test_standata_new(self):
         json_file = os.path.join(datafiles_path, 'bernoulli.data.json')