Repository: ebmdatalab/datalab-pandas

Issue:
`bq.cached_read` fails confusingly if parent directory doesn't exist
If the parent of `csv_path` (which is `csv_dir` below) doesn't exist then this line throws an error:
https://github.com/ebmdatalab/datalab-pandas/blob/92099b8c8ea47616545c42a0dc8a394348979103/ebmdatalab/bq.py#L49-L51

Given that this is usually called from a notebook and it's already slightly confusing as to what the current working directory is, this creates even more confusion.

The function should either:
 * Automatically create any parent directories
 * Explicitly check whether `csv_dir` exists and throw a helpful error

Comments:

Patch:
diff --git a/ebmdatalab/bq.py b/ebmdatalab/bq.py
index f6709a9..113bb5c 100644
--- a/ebmdatalab/bq.py
+++ b/ebmdatalab/bq.py
@@ -41,6 +41,7 @@ def cached_read(sql, csv_path=None, use_cache=True, **kwargs):
     if use_cache and already_cached:
         df = pd.read_csv(csv_path)
     else:
+        os.makedirs(csv_dir, exist_ok=True)
         temp_path = os.path.join(
             csv_dir, '.tmp{}.{}'.format(_random_str(8), csv_filename)
         )


Test patch:
diff --git a/ebmdatalab/tests/test_bq.py b/ebmdatalab/tests/test_bq.py
index 6f447bb..01a3684 100644
--- a/ebmdatalab/tests/test_bq.py
+++ b/ebmdatalab/tests/test_bq.py
@@ -2,7 +2,8 @@ from mock import patch
 from ebmdatalab import bq
 from pandas import DataFrame
 import tempfile
-
+import pytest
+import os
 
 def test_fingerprint_sql():
     input_sql = 'select *, "Frob" from x -- comment\n' "where (a >= 4);"
@@ -39,6 +40,46 @@ def test_cached_read(mock_read_gbq):
         # and now with `use_cache` param
         df = bq.cached_read(sql, csv_path=csv_file.name, use_cache=False)
         assert mock_read_gbq.call_count == 2
+        assert False
+
+
+@patch("ebmdatalab.bq.pd.read_gbq")
+def test_cached_read_no_csv_path(mock_read_gbq):
+    mock_read_gbq.return_value = DataFrame([{"a": 3}])
+    sql = "select * from foobar"
+
+    # Test no csv path raises error
+    with tempfile.NamedTemporaryFile() as csv_file:
+        with pytest.raises(AssertionError) as exc_info:
+            df = bq.cached_read(sql, csv_path="")
+
+    assert "You must supply csv_path" in str(exc_info.value)
+
+
+@patch("ebmdatalab.bq.pd.read_gbq")
+def test_cached_read_non_existing_csv_dir_made(mock_read_gbq):
+    mock_read_gbq.return_value = DataFrame([{"a": 3}])
+    sql = "select * from foobar"
+
+    # Make temporary folder to save temporary files in
+    folder = tempfile.TemporaryDirectory()
+
+    with tempfile.NamedTemporaryFile(dir=folder.name) as csv_file:
+        # Test csv_dir exists
+        df = bq.cached_read(sql, csv_path=csv_file.name)
+        assert os.path.exists(folder.name)
+
+        # Delete contents of temporary folder
+        for file in os.listdir(folder.name):
+            os.remove(f"{folder.name}/{file}")
+
+        # Delete temporary folder
+        os.rmdir(folder.name)
+        assert os.path.exists(folder.name) is False
+
+        # Test temporary folder is remade
+        df = bq.cached_read(sql, csv_path=csv_file.name)
+        assert os.path.exists(folder.name)
 
 
 def _check_cached_read(csv_file, mock_read, sql, expected):