Repository: intel/dffml

Issue:
sources: file: Should take a pathlib.Path as filename
Right now if we have a filesource and we pass it a `pathlib.Path` as the filename, it will throw errors.

If you remove the `str` call here the failing test(s) will show you all the things that are wrong.

https://github.com/intel/dffml/blob/41d18ffea176dbec49224eed73910dfca933868f/service/http/tests/test_cli.py#L286-L290

The best way to handle this is probably to do like the models do right now with their directory config parameter. Treat the filename as if it's always a `pathlib.Path`. If the call to instantiate the `FileSource` (or subclass of `FileSource`) passes in a `str`, convert it to a `pathlib.Path` object.

Fix all the code that assumes `config.filename` is going to be a string. Remove calls to `os.path.` where appropriate and replace with `pathlib` alternatives.

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fad2d825a..b81423268 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Classes now use `CONFIG` if it has a default for every field and `config` is `None`
 - Models now dynamically import third party modules.
 - `dffml list records` command prints Records as JSON using `.export()`
+- FileSource takes pathlib.Path as filename
 ### Fixed
 - Race condition in `MemoryRedundancyChecker` when more than 4 possible
   parameter sets for an operation.
diff --git a/dffml/df/types.py b/dffml/df/types.py
index 74e67ed29..539321a88 100644
--- a/dffml/df/types.py
+++ b/dffml/df/types.py
@@ -29,7 +29,27 @@ class DefinitionMissing(Exception):
 
 class Definition(NamedTuple):
     """
-    List[type] is how to specify a list
+    Examples
+    --------
+
+    >>> from dffml import Definition, Input
+    >>> from typing import NamedTuple
+    >>>
+    >>> class Person(NamedTuple):
+    ...   name: str
+    ...   age: int
+    >>>
+    >>> Friend = Definition(name="Friend", primitive="map", spec=Person)
+    >>> Input(value={"name": "Bob", "age": 42}, definition=Friend)
+    Input(value=Person(name='Bob', age=42), definition=Friend)
+    >>>
+    >>> SignUpQueue = Definition(name="SignUpQueue", primitive="array", spec=Person, subspec=True)
+    >>> Input(value=[{"name": "Bob", "age": 42}], definition=SignUpQueue)
+    Input(value=[Person(name='Bob', age=42)], definition=SignUpQueue)
+    >>>
+    >>> AddressBook = Definition(name="AddressBook", primitive="map", spec=Person, subspec=True)
+    >>> Input(value={"bob": {"name": "Bob", "age": 42}}, definition=AddressBook)
+    Input(value={'bob': Person(name='Bob', age=42)}, definition=AddressBook)
     """
 
     name: str
diff --git a/dffml/source/file.py b/dffml/source/file.py
index ffd8519ec..90d72520f 100644
--- a/dffml/source/file.py
+++ b/dffml/source/file.py
@@ -9,6 +9,7 @@ import lzma
 import errno
 import zipfile
 from contextlib import contextmanager
+import pathlib
 
 from ..base import config
 from .source import BaseSource
@@ -35,6 +36,12 @@ class FileSource(BaseSource):
     READMODE_COMPRESSED: str = "rt"
     WRITEMODE_COMPRESSED: str = "wt"
 
+    def __init__(self, config):
+        super().__init__(config)
+
+        if isinstance(getattr(self.config, "filename", None), str):
+            self.config.filename = pathlib.Path(self.config.filename)
+
     async def __aenter__(self) -> "BaseSourceContext":
         await self._open()
         return self
@@ -62,15 +69,16 @@ class FileSource(BaseSource):
                     os.strerror(errno.ENOENT),
                     self.config.filename,
                 )
-        if self.config.filename[::-1].startswith((".gz")[::-1]):
+        if self.config.filename.suffix == ".gz":
             opener = gzip.open(self.config.filename, self.READMODE_COMPRESSED)
-        elif self.config.filename[::-1].startswith((".bz2")[::-1]):
+        elif self.config.filename.suffix == ".bz2":
             opener = bz2.open(self.config.filename, self.READMODE_COMPRESSED)
-        elif self.config.filename[::-1].startswith(
-            (".xz")[::-1]
-        ) or self.config.filename[::-1].startswith((".lzma")[::-1]):
+        elif (
+            self.config.filename.suffix == ".xz"
+            or self.config.filename.suffix == ".lzma"
+        ):
             opener = lzma.open(self.config.filename, self.READMODE_COMPRESSED)
-        elif self.config.filename[::-1].startswith((".zip")[::-1]):
+        elif self.config.filename.suffix == ".zip":
             opener = self.zip_opener_helper()
         else:
             opener = open(self.config.filename, self.READMODE)
@@ -79,21 +87,22 @@ class FileSource(BaseSource):
 
     async def _close(self):
         if self.config.readwrite:
-            if self.config.filename[::-1].startswith((".gz")[::-1]):
+            if self.config.filename.suffix == ".gz":
                 close = gzip.open(
                     self.config.filename, self.WRITEMODE_COMPRESSED
                 )
-            elif self.config.filename[::-1].startswith((".bz2")[::-1]):
+            elif self.config.filename.suffix == ".bz2":
                 close = bz2.open(
                     self.config.filename, self.WRITEMODE_COMPRESSED
                 )
-            elif self.config.filename[::-1].startswith(
-                (".xz")[::-1]
-            ) or self.config.filename[::-1].startswith((".lzma")[::-1]):
+            elif (
+                self.config.filename.suffix == ".xz"
+                or self.config.filename.suffix == ".lzma"
+            ):
                 close = lzma.open(
                     self.config.filename, self.WRITEMODE_COMPRESSED
                 )
-            elif self.config.filename[::-1].startswith((".zip")[::-1]):
+            elif self.config.filename.suffix == ".zip":
                 close = self.zip_closer_helper()
             else:
                 close = open(self.config.filename, "w+")
diff --git a/dffml/util/cli/cmd.py b/dffml/util/cli/cmd.py
index bf86c9466..8028947c9 100644
--- a/dffml/util/cli/cmd.py
+++ b/dffml/util/cli/cmd.py
@@ -12,6 +12,7 @@ from typing import Dict, Any
 from ...record import Record
 from ...feature import Feature
 
+from ..data import export_dict
 from .arg import Arg, parse_unknown
 
 DisplayHelp = "Display help message"
@@ -197,7 +198,7 @@ class CMD(object):
             and result != [CMDOutputOverride]
         ):
             json.dump(
-                result,
+                export_dict(result=result)["result"],
                 sys.stdout,
                 sort_keys=True,
                 indent=4,
diff --git a/dffml/util/data.py b/dffml/util/data.py
index c33b4e570..c285fcf3a 100644
--- a/dffml/util/data.py
+++ b/dffml/util/data.py
@@ -9,6 +9,7 @@ import ast
 import types
 import pydoc
 import inspect
+import dataclasses
 from functools import wraps
 import pathlib
 from typing import Callable
@@ -149,6 +150,8 @@ def export_value(obj, key, value):
         obj[key] = value.export()
     elif hasattr(value, "_asdict"):
         obj[key] = value._asdict()
+    elif dataclasses.is_dataclass(value):
+        obj[key] = export_dict(**dataclasses.asdict(value))
     elif getattr(value, "__module__", None) == "typing":
         obj[key] = STANDARD_TYPES.get(
             str(value).replace("typing.", ""), "generic"
@@ -160,6 +163,8 @@ def export_list(iterable):
         export_value(iterable, i, value)
         if isinstance(value, (dict, types.MappingProxyType)):
             iterable[i] = export_dict(**iterable[i])
+        elif dataclasses.is_dataclass(value):
+            iterable[i] = export_dict(**dataclasses.asdict(value))
         elif isinstance(value, list):
             iterable[i] = export_list(iterable[i])
     return iterable
diff --git a/docs/conf.py b/docs/conf.py
index 253929992..9d6a3f290 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -88,7 +88,7 @@ html_static_path = ["_static"]
 
 
 def setup(app):
-    app.add_javascript("copybutton.js")
+    app.add_js_file("copybutton.js")
 
 
 # -- Extension configuration -------------------------------------------------
diff --git a/model/tensorflow/setup.py b/model/tensorflow/setup.py
index e57d906b4..31700f633 100644
--- a/model/tensorflow/setup.py
+++ b/model/tensorflow/setup.py
@@ -18,7 +18,7 @@ with open(
 with open(os.path.join(self_path, "README.md"), "r", encoding="utf-8") as f:
     readme = f.read()
 
-INSTALL_REQUIRES = ["tensorflow>=2.0.0"] + (
+INSTALL_REQUIRES = ["tensorflow>=2.0.0,<2.2.0"] + (
     ["dffml>=0.3.7"]
     if not any(
         list(
diff --git a/model/tensorflow_hub/setup.py b/model/tensorflow_hub/setup.py
index e31dd47a0..e76b1c89f 100644
--- a/model/tensorflow_hub/setup.py
+++ b/model/tensorflow_hub/setup.py
@@ -20,7 +20,7 @@ with open(os.path.join(self_path, "README.md"), "r", encoding="utf-8") as f:
 
 INSTALL_REQUIRES = (
     [
-        "tensorflow>=2.0.0",
+        "tensorflow>=2.0.0,<2.2.0",
         "tensorflow-hub>=0.6.0",
         "pandas>=0.25.0",
         "numpy>=1.16.4",
diff --git a/model/transformers/setup.py b/model/transformers/setup.py
index 60e7e0efe..dd3e5cea9 100644
--- a/model/transformers/setup.py
+++ b/model/transformers/setup.py
@@ -19,12 +19,12 @@ with open(os.path.join(self_path, "README.md"), "r", encoding="utf-8") as f:
     readme = f.read()
 
 INSTALL_REQUIRES = [
-    "tensorflow>=2.0.0",
+    "tensorflow>=2.0.0,<2.2.0",
     "pandas>=0.25.0",
     "numpy>=1.16.4",
     "seqeval>=0.0.12",
     "fastprogress>=0.2.2",
-    "transformers>=2.5.1",
+    "transformers>=2.5.1,<2.9.0",
 ] + (
     ["dffml>=0.3.7"]
     if not any(


Test patch:
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index d856e3e08..1f81a2040 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -119,6 +119,7 @@ jobs:
         model/transformers=${{ secrets.PYPI_MODEL_TRANSFORMERS }}
         model/scratch=${{ secrets.PYPI_MODEL_SCRATCH }}
         model/scikit=${{ secrets.PYPI_MODEL_SCIKIT }}
+        model/vowpalWabbit=${{ secrets.PYPI_MODEL_VOWPALWABBIT }}
         source/mysql=${{ secrets.PYPI_SOURCE_MYSQL }}
         feature/git=${{ secrets.PYPI_FEATURE_GIT }}
         feature/auth=${{ secrets.PYPI_FEATURE_AUTH }}
diff --git a/service/http/tests/test_cli.py b/service/http/tests/test_cli.py
index 04d0e031f..c56a31d63 100644
--- a/service/http/tests/test_cli.py
+++ b/service/http/tests/test_cli.py
@@ -284,7 +284,7 @@ class TestServer(AsyncTestCase):
         with tempfile.TemporaryDirectory() as tempdir:
             # Source the HTTP API will pre-load
             source = JSONSource(
-                filename=str(pathlib.Path(tempdir, "source.json")),
+                filename=pathlib.Path(tempdir, "source.json"),
                 allowempty=True,
                 readwrite=True,
             )
@@ -302,7 +302,7 @@ class TestServer(AsyncTestCase):
                         "-sources",
                         "mysource=json",
                         "-source-mysource-filename",
-                        source.config.filename,
+                        str(source.config.filename),
                     )
                 )
                 async with self.get(
diff --git a/service/http/tests/test_routes.py b/service/http/tests/test_routes.py
index b80f6fe70..41bb9b94c 100644
--- a/service/http/tests/test_routes.py
+++ b/service/http/tests/test_routes.py
@@ -162,7 +162,7 @@ class TestRoutesConfigure(TestRoutesRunning, AsyncTestCase):
             self.assertEqual(
                 self.cli.app["sources"]["salary"].config,
                 CSVSourceConfig(
-                    filename="dataset.csv",
+                    filename=pathlib.Path("dataset.csv"),
                     tag="untagged",
                     key="key",
                     tagcol="tag",
diff --git a/tests/source/test_file.py b/tests/source/test_file.py
index 41e9fce69..ec5564d96 100644
--- a/tests/source/test_file.py
+++ b/tests/source/test_file.py
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: MIT
 # Copyright (c) 2019 Intel Corporation
+import pathlib
 from unittest.mock import patch, mock_open
 from contextlib import contextmanager
 from typing import AsyncIterator
@@ -125,10 +126,10 @@ class TestFileSource(AsyncTestCase):
             async with FakeFileSource(
                 self.config("testfile", readwrite=False)
             ):
-                m_open.assert_called_once_with("testfile", "r")
+                pass
+            m_open.assert_called_once_with(pathlib.Path("testfile"), "r")
 
     async def test_open_gz(self):
-        source = FakeFileSource("testfile.gz")
         m_open = mock_open()
         with patch("os.path.exists", return_value=True), patch(
             "gzip.open", m_open
@@ -136,7 +137,9 @@ class TestFileSource(AsyncTestCase):
             async with FakeFileSource(
                 self.config("testfile.gz", readwrite=False)
             ):
-                m_open.assert_called_once_with("testfile.gz", "rt")
+                m_open.assert_called_once_with(
+                    pathlib.Path("testfile.gz"), "rt"
+                )
 
     async def test_open_bz2(self):
         m_open = mock_open()
@@ -146,7 +149,9 @@ class TestFileSource(AsyncTestCase):
             async with FakeFileSource(
                 self.config("testfile.bz2", readwrite=False)
             ):
-                m_open.assert_called_once_with("testfile.bz2", "rt")
+                m_open.assert_called_once_with(
+                    pathlib.Path("testfile.bz2"), "rt"
+                )
 
     async def test_open_lzma(self):
         m_open = mock_open()
@@ -156,7 +161,9 @@ class TestFileSource(AsyncTestCase):
             async with FakeFileSource(
                 self.config("testfile.lzma", readwrite=False)
             ):
-                m_open.assert_called_once_with("testfile.lzma", "rt")
+                m_open.assert_called_once_with(
+                    pathlib.Path("testfile.lzma"), "rt"
+                )
 
     async def test_open_xz(self):
         m_open = mock_open()
@@ -166,7 +173,9 @@ class TestFileSource(AsyncTestCase):
             async with FakeFileSource(
                 self.config("testfile.xz", readwrite=False)
             ):
-                m_open.assert_called_once_with("testfile.xz", "rt")
+                m_open.assert_called_once_with(
+                    pathlib.Path("testfile.xz"), "rt"
+                )
 
     async def test_open_zip(self):
         source = FakeFileSource(self.config("testfile.zip", readwrite=False))
@@ -192,7 +201,7 @@ class TestFileSource(AsyncTestCase):
         ):
             async with FakeFileSource(self.config("testfile")):
                 pass
-            m_open.assert_called_once_with("testfile", "w+")
+            m_open.assert_called_once_with(pathlib.Path("testfile"), "w+")
 
     async def test_close_gz(self):
         m_open = mock_open()
@@ -201,7 +210,7 @@ class TestFileSource(AsyncTestCase):
         ):
             async with FakeFileSource(self.config("testfile.gz")):
                 pass
-            m_open.assert_called_once_with("testfile.gz", "wt")
+            m_open.assert_called_once_with(pathlib.Path("testfile.gz"), "wt")
 
     async def test_close_bz2(self):
         m_open = mock_open()
@@ -210,7 +219,7 @@ class TestFileSource(AsyncTestCase):
         ):
             async with FakeFileSource(self.config("testfile.bz2")):
                 pass
-            m_open.assert_called_once_with("testfile.bz2", "wt")
+            m_open.assert_called_once_with(pathlib.Path("testfile.bz2"), "wt")
 
     async def test_close_lzma(self):
         m_open = mock_open()
@@ -219,7 +228,7 @@ class TestFileSource(AsyncTestCase):
         ):
             async with FakeFileSource(self.config("testfile.lzma")):
                 pass
-            m_open.assert_called_once_with("testfile.lzma", "wt")
+            m_open.assert_called_once_with(pathlib.Path("testfile.lzma"), "wt")
 
     async def test_close_xz(self):
         m_open = mock_open()
@@ -228,7 +237,7 @@ class TestFileSource(AsyncTestCase):
         ):
             async with FakeFileSource(self.config("testfile.xz")):
                 pass
-            m_open.assert_called_once_with("testfile.xz", "wt")
+            m_open.assert_called_once_with(pathlib.Path("testfile.xz"), "wt")
 
     async def test_close_zip(self):
         source = FakeFileSource(self.config("testfile.zip"))
diff --git a/tests/test_base.py b/tests/test_base.py
index cc7de64d1..b40530761 100644
--- a/tests/test_base.py
+++ b/tests/test_base.py
@@ -1,5 +1,6 @@
 import unittest
 from typing import List
+import pathlib
 
 from dffml.base import (
     BaseDataFlowFacilitatorObject,
@@ -125,7 +126,9 @@ class TestAutoArgsConfig(unittest.TestCase):
         self.assertEqual(config.label, "unlabeled")
         self.assertFalse(config.readonly)
         self.assertTrue(isinstance(config.source, JSONSource))
-        self.assertEqual(config.source.config.filename, "file.json")
+        self.assertEqual(
+            config.source.config.filename, pathlib.Path("file.json")
+        )
         self.assertEqual(
             config.features,
             Features(
@@ -162,7 +165,9 @@ class TestAutoArgsConfig(unittest.TestCase):
         self.assertEqual(config.label, "default-label")
         self.assertTrue(config.readonly)
         self.assertTrue(isinstance(config.source, CSVSource))
-        self.assertEqual(config.source.config.filename, "file.csv")
+        self.assertEqual(
+            config.source.config.filename, pathlib.Path("file.csv")
+        )
         self.assertEqual(
             config.features,
             Features(