Repository: intel/dffml

Issue:
docs: model: tensorflow: Include examples of how to use from Python API
The model plugins page has command line examples but needs examples from Python as well.

Copy the format of the `examples/quickstart_filenames.py` file and add a `.. code-block:: python` section for each model, keeping the Python example consistent with the command line example for each model. Let's add the python example after the CLI example for each.

For tensorflow, the docstrings of the `Model`s themselves will need to be updated.

Here's a good example from when scikit was updated: https://github.com/intel/dffml/pull/407/commits/a407dc362e4250d143b256aae588af791d90ef08
Also reference this commit https://github.com/intel/dffml/commit/73715ae92446169b22fed2ec38ea2de43cfdad01 to see how we want to make the examples tested.


Discussion:

Resolving patch:
diff --git a/model/tensorflow/dffml_model_tensorflow/dnnr.py b/model/tensorflow/dffml_model_tensorflow/dnnr.py
--- a/model/tensorflow/dffml_model_tensorflow/dnnr.py
+++ b/model/tensorflow/dffml_model_tensorflow/dnnr.py
@@ -195,58 +195,32 @@ class DNNRegressionModel(Model):
       make sure to take a BACKUP of files with same name in the directory
       from where this command is run as it overwrites any existing files.
 
-    .. code-block:: console
-
-        $ cat > train.csv << EOF
-        Feature1,Feature2,TARGET
-        0.93,0.68,3.89
-        0.24,0.42,1.75
-        0.36,0.68,2.75
-        0.53,0.31,2.00
-        0.29,0.25,1.32
-        0.29,0.52,2.14
-        EOF
-        $ cat > test.csv << EOF
-        Feature1,Feature2,TARGET
-        0.57,0.84,3.65
-        0.95,0.19,2.46
-        0.23,0.15,0.93
-        EOF
-        $ dffml train \\
-            -model tfdnnr \\
-            -model-epochs 300 \\
-            -model-steps 2000 \\
-            -model-predict TARGET:float:1 \\
-            -model-hidden 8 16 8 \\
-            -sources s=csv \\
-            -source-filename train.csv \\
-            -model-features \\
-              Feature1:float:1 \\
-              Feature2:float:1 \\
-            -log debug
-        Enabling debug log shows tensorflow losses...
-        $ dffml accuracy \\
-            -model tfdnnr \\
-            -model-predict TARGET:float:1 \\
-            -model-hidden 8 16 8 \\
-            -sources s=csv \\
-            -source-filename test.csv \\
-            -model-features \\
-              Feature1:float:1 \\
-              Feature2:float:1 \\
-            -log critical
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/train_data.sh
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/test_data.sh
+
+    Train the model
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/train.sh
+
+    Assess the accuracy
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/accuracy.sh
+
+    Output
+
+    .. code-block::
+
         0.9468210011
-        $ echo -e 'Feature1,Feature2,TARGET\\n0.21,0.18,0.84\\n' | \\
-          dffml predict all \\
-            -model tfdnnr \\
-            -model-predict TARGET:float:1 \\
-            -model-hidden 8 16 8 \\
-            -sources s=csv \\
-            -source-filename /dev/stdin \\
-            -model-features \\
-              Feature1:float:1 \\
-              Feature2:float:1 \\
-            -log critical
+
+    Make a prediction
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/predict.sh
+
+    Output
+
+    .. code-block:: json
+
         [
             {
                 "extra": {},
@@ -266,6 +240,10 @@ class DNNRegressionModel(Model):
             }
         ]
 
+    Example usage of Tensorflow DNNEstimator model using python API
+
+    .. literalinclude:: /../model/tensorflow/examples/tfdnnr/tfdnnr.py
+
     The ``NaN`` in ``confidence`` is the expected behaviour. (See TODO in
     predict).
 
diff --git a/model/tensorflow/examples/tfdnnr/__init__.py b/model/tensorflow/examples/tfdnnr/__init__.py
new file mode 100644
diff --git a/model/tensorflow/examples/tfdnnr/tfdnnr.py b/model/tensorflow/examples/tfdnnr/tfdnnr.py
new file mode 100644
--- /dev/null
+++ b/model/tensorflow/examples/tfdnnr/tfdnnr.py
@@ -0,0 +1,26 @@
+from dffml import CSVSource, Features, DefFeature
+from dffml.noasync import train, accuracy, predict
+from dffml_model_tensorflow.dnnr import DNNRegressionModel
+
+model = DNNRegressionModel(
+    features=Features(
+        DefFeature("Feature1", float, 1), DefFeature("Feature2", float, 1),
+    ),
+    predict=DefFeature("TARGET", float, 1),
+    epochs=300,
+    steps=2000,
+    hidden=[8, 16, 8],
+)
+
+# Train the model
+train(model, "train.csv")
+
+# Assess accuracy (alternate way of specifying data source)
+print("Accuracy:", accuracy(model, CSVSource(filename="test.csv")))
+
+# Make prediction
+for i, features, prediction in predict(
+    model, {"Feature1": 0.21, "Feature2": 0.18, "TARGET": 0.84},
+):
+    features["TARGET"] = prediction["TARGET"]["value"]
+    print(features)


Test patch:
diff --git a/model/tensorflow/examples/tfdnnr/test_data.sh b/model/tensorflow/examples/tfdnnr/test_data.sh
new file mode 100644
--- /dev/null
+++ b/model/tensorflow/examples/tfdnnr/test_data.sh
@@ -0,0 +1,6 @@
+cat > test.csv << EOF
+Feature1,Feature2,TARGET
+0.57,0.84,3.65
+0.95,0.19,2.46
+0.23,0.15,0.93
+EOF
\ No newline at end of file
diff --git a/model/tensorflow/examples/tfdnnr/test_tfdnnr.py b/model/tensorflow/examples/tfdnnr/test_tfdnnr.py
new file mode 100644
--- /dev/null
+++ b/model/tensorflow/examples/tfdnnr/test_tfdnnr.py
@@ -0,0 +1,59 @@
+import os
+import ast
+import sys
+import json
+import tempfile
+import contextlib
+import subprocess
+import unittest.mock
+
+from dffml.util.os import chdir
+
+
+def sh_filepath(filename):
+    return os.path.join(os.path.dirname(__file__), filename)
+
+
+@contextlib.contextmanager
+def directory_with_csv_files():
+    with tempfile.TemporaryDirectory() as tempdir:
+        with chdir(tempdir):
+            subprocess.check_output(["bash", sh_filepath("train_data.sh")])
+            subprocess.check_output(["bash", sh_filepath("test_data.sh")])
+            yield tempdir
+
+
+class TestExample(unittest.TestCase):
+    def python_test(self, filename):
+        # Path to target file
+        filepath = os.path.join(os.path.dirname(__file__), filename)
+        # Capture output
+        stdout = subprocess.check_output([sys.executable, filepath])
+        lines = stdout.decode().split("\n")
+        # Check the Accuracy
+        self.assertRegex(lines[0], r"Accuracy:  [-+]?\d*\.?\d+|\d+")
+        # Check the TARGET
+        self.assertIsInstance(round(ast.literal_eval(lines[1])["TARGET"]), int)
+
+    def test_python_filenames(self):
+        with directory_with_csv_files() as tempdir:
+            self.python_test("tfdnnr.py")
+
+    def test_shell(self):
+        with directory_with_csv_files() as tempdir:
+            # Run training
+            subprocess.check_output(["bash", sh_filepath("train.sh")])
+            # Check the Accuracy
+            stdout = subprocess.check_output(
+                ["bash", sh_filepath("accuracy.sh")]
+            )
+            self.assertRegex(stdout.decode().strip(), r"[-+]?\d*\.?\d+|\d+")
+            # Make the prediction
+            stdout = subprocess.check_output(
+                ["bash", sh_filepath("predict.sh")]
+            )
+            records = json.loads(stdout.decode())
+            # Check the TARGET
+            self.assertIsInstance(
+                round(records[0]["prediction"]["TARGET"]["value"]), int
+            )