Repository: schuderer/mllaunchpad

Issue:
FileDataSink should create missing directories in path
That way, users don't need to worry about creating any missing intermediate directories.

e.g.

```python
ds_dir = os.path.dirname(ds_config["path"])
if not os.path.exists(ds_dir):
    os.makedirs(ds_dir)
```

Discussion:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index f2dea2f..bb18900 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -39,6 +39,9 @@ Unreleased
 * |Feature| Add Spark support (experimental), see ``examples/spark_datasource.py``
   and `issue #145 <https://github.com/schuderer/mllaunchpad/issues/145>`_,
   by `Andreas Schuderer <https://github.com/schuderer>`_.
+* |Enhancement| FileDataSink (`csv`, `euro_csv`, `raw` DataSink types) now attempts to create missing paths if required,
+  `issue #148 <https://github.com/schuderer/mllaunchpad/issues/148>`_,
+  by `Andreas Schuderer <https://github.com/schuderer>`_.
 * |Fixed| Keep generated RAML files free of command line messages,
   `issue #126 <https://github.com/schuderer/mllaunchpad/issues/126>`_,
   by `Andreas Schuderer <https://github.com/schuderer>`_.
diff --git a/mllaunchpad/datasources.py b/mllaunchpad/datasources.py
index d35700c..695b42e 100644
--- a/mllaunchpad/datasources.py
+++ b/mllaunchpad/datasources.py
@@ -104,6 +104,13 @@ def fill_nas(
         return df
 
 
+def ensure_dir_to(file_path):
+    path = os.path.dirname(file_path)
+    if path != "" and not os.path.exists(path):
+        logger.info("Creating missing path to file `{}`.".format(file_path))
+        os.makedirs(path)
+
+
 class SqlDataSource(DataSource):
     """DataSource for RedShift, Postgres, MySQL, SQLite, Oracle, Microsoft SQL (ODBC), and their dialects.
 
@@ -806,6 +813,7 @@ class FileDataSink(DataSink):
         """Write a pandas dataframe to file and optionally the dtypes if included in the configuration.
         The default is not to save the dataframe's row index.
         Configure the DataSink's `options` dict to pass keyword arguments to `my_df.to_csv`.
+        If the directory path leading to the file does not exist, it will be created.
 
         Example::
 
@@ -842,11 +850,14 @@ class FileDataSink(DataSink):
             dtypes_file.loc[
                 dtypes_file["dtypes"] == "datetime64[ns]", "dtypes"
             ] = "datetime"
+            ensure_dir_to(self.dtypes_path)
             dtypes_file.to_csv(self.dtypes_path)
 
         if self.type == "csv":
+            ensure_dir_to(self.path)
             dataframe.to_csv(self.path, **kw_options)
         elif self.type == "euro_csv":
+            ensure_dir_to(self.path)
             dataframe.to_csv(self.path, sep=";", decimal=",", **kw_options)
         else:
             raise TypeError(
@@ -860,6 +871,7 @@ class FileDataSink(DataSink):
         chunksize: Optional[int] = None,
     ) -> None:
         """Write raw (unstructured) data to file.
+        If the directory path leading to the file does not exist, it will be created.
 
         Example::
 
@@ -887,10 +899,12 @@ class FileDataSink(DataSink):
         if self.type == "text_file":
             if "encoding" not in kw_options:
                 kw_options["encoding"] = "utf-8"
+            ensure_dir_to(self.path)
             with open(self.path, "w", **kw_options) as txt_file:
                 raw_str: str = cast(str, raw_data)
                 txt_file.write(raw_str)
         elif self.type == "binary_file":
+            ensure_dir_to(self.path)
             with open(self.path, "wb", **kw_options) as bin_file:
                 raw_bytes: bytes = cast(bytes, raw_data)
                 bin_file.write(raw_bytes)


Test patch:
diff --git a/tests/test_datasources.py b/tests/test_datasources.py
index 3acf145..575035f 100644
--- a/tests/test_datasources.py
+++ b/tests/test_datasources.py
@@ -1,4 +1,5 @@
 # Stdlib imports
+import os
 import sys
 from io import BytesIO
 from unittest import mock
@@ -249,6 +250,34 @@ def test_filedatasink_notimplemented(filedatasink_cfg_and_data):
         mllp_ds.FileDataSink("bla", cfg)
 
 
+@mock.patch("os.makedirs")
+@mock.patch("os.path.exists", return_value=False)
+@mock.patch("pandas.DataFrame.to_csv")
+def test_filedatasink_df_ensure_path(
+    to_csv_mock, exists_mock, makedirs_mock, filedatasink_cfg_and_data
+):
+    cfg, data = filedatasink_cfg_and_data("csv")
+    cfg["path"] = os.path.join("bla/foo", cfg["path"])
+    ds = mllp_ds.FileDataSink("bla", cfg)
+    ds.put_dataframe(data)
+    exists_mock.assert_called_once()
+    makedirs_mock.assert_called_once_with("bla/foo")
+
+
+@mock.patch("os.makedirs")
+@mock.patch("os.path.exists", return_value=True)
+@mock.patch("pandas.DataFrame.to_csv")
+def test_filedatasink_df_ensure_path_noexist(
+    to_csv_mock, exists_mock, makedirs_mock, filedatasink_cfg_and_data
+):
+    cfg, data = filedatasink_cfg_and_data("csv")
+    cfg["path"] = os.path.join("bla/foo", cfg["path"])
+    ds = mllp_ds.FileDataSink("bla", cfg)
+    ds.put_dataframe(data)
+    exists_mock.assert_called_once()
+    makedirs_mock.assert_not_called()
+
+
 # OracleDataSource