Repository: sql-machine-learning/elasticdl

Issue:
Unify structured and un-structured data using tf.train.Example.
store training data into recordio file using tf.train.Example format as to unify structured data and un-structured data, for example with the TFExampleFile, we can generate recordio file like this:
```
# define feature specification.
feature_spec = [("feature0", tf.string), ("feature1", tf.float32), ("label", tf.int64)]

# write feature data into recordio file.
features = [("feature0", b"abc"), ("feature1", 100.1), ("label", 1)]
tmp_file = tempfile.NamedTemporaryFile(delete=False)
file = TFExampleFile(tmp_file.name, feature_spec, 'w')
file.write(features)

# read feature data from recordio file.
file = TFExampleFile(tmp_file.name, feature_spec, 'r')
features = file.get(0)
```


Discussion:

Resolving patch:
diff --git a/elasticdl/data/tf_example_codec.py b/elasticdl/data/tf_example_codec.py
new file mode 100644
--- /dev/null
+++ b/elasticdl/data/tf_example_codec.py
@@ -0,0 +1,53 @@
+from enum import Enum 
+import tensorflow as tf
+
+class TFExampleCodec(object):
+
+    def __init__(self, feature_schema):
+        self._f_schema = feature_schema
+        self._f_desc = {}
+        self._f_name2type = dict(feature_schema) 
+        for f_name, f_type, in self._f_name2type.items():
+            category = _type_category(f_type)
+            if category == _TypeCategory.STR:
+                self._f_desc[f_name] = tf.FixedLenFeature([], tf.string, default_value='')
+            elif category == _TypeCategory.FLOAT:
+                self._f_desc[f_name] = tf.FixedLenFeature([], f_type, default_value=0.0)
+            elif category == _TypeCategory.INT:
+                self._f_desc[f_name] = tf.FixedLenFeature([], f_type, default_value=0)
+            else:
+                raise ValueError("not supported tensorflow data type.")
+
+    def encode(self, example):
+        f_dict = {}
+        for f_name, f_value in example:
+            f_type = self._f_name2type[f_name]
+            if f_type == tf.string:
+                f_dict[f_name] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[f_value]))
+            elif f_type in (tf.float32, tf.float64):
+                f_dict[f_name] = tf.train.Feature(float_list=tf.train.FloatList(value=[f_value]))
+            elif f_type in (tf.bool, tf.int32, tf.uint32, tf.int64, tf.uint64):
+                f_dict[f_name] = tf.train.Feature(int64_list=tf.train.Int64List(value=[f_value]))
+            else:
+                raise ValueError("not supported tensorflow data type.")
+
+        example = tf.train.Example(features=tf.train.Features(feature=f_dict))
+        return example.SerializeToString()
+
+    def decode(self, raw):
+        return tf.parse_single_example(raw, self._f_desc)
+
+class _TypeCategory(Enum):
+    STR = 1
+    INT = 2
+    FLOAT =3
+
+def _type_category(f_type):
+    if f_type == tf.string:
+        return _TypeCategory.STR 
+    elif f_type.is_floating:
+        return _TypeCategory.FLOAT
+    elif f_type == tf.bool or f_type.is_integer:
+        return _TypeCategory.INT
+    else:
+        return None


Test patch:
diff --git a/elasticdl/data/tf_example_codec_test.py b/elasticdl/data/tf_example_codec_test.py
new file mode 100644
--- /dev/null
+++ b/elasticdl/data/tf_example_codec_test.py
@@ -0,0 +1,43 @@
+import unittest
+import tempfile
+import os
+import tensorflow as tf
+import numpy as np
+from coded_recordio import File
+from tf_example_codec import TFExampleCodec
+
+class TestTFExampleCodec(unittest.TestCase):
+    """ Test tf_example_codec.py
+    """
+
+    def test_encode_and_decode(self):
+        feature_schema = [("f0", tf.string), ("f1", tf.float32), ("label", tf.int64)]
+        example_1 = [("f0", b"abc"), ("f1", 100.1), ("label", 1)]
+        example_2 = [("f0", b"def"), ("f1", 200.1), ("label", 2)]
+        example_3 = [("f0", b"ghi"), ("f1", 300.1), ("label", 3)]
+        examples = [example_1, example_2, example_3]
+        tmp_file = tempfile.NamedTemporaryFile(delete=False)
+
+        # Create the codec for tf.train.Exampel data.
+        codec = TFExampleCodec(feature_schema)
+
+        # Write an encoded RecordIO file.
+        with File(tmp_file.name, "w", encoder=codec.encode) as coded_w:
+            for example in examples:
+                coded_w.write(example)
+
+        # Verify decoded content, with get() interface.
+        with File(tmp_file.name, "r", decoder=codec.decode) as coded_r:
+            with tf.Session() as session:
+                for idx in range(coded_r.count()):
+                    exp = coded_r.get(idx)
+                    expected_exp = examples[idx]
+                    f_0, f_1, label = session.run([exp["f0"], exp["f1"], exp["label"]])
+                    self.assertEqual(f_0, expected_exp[0][1])
+                    self.assertEqual(f_1, np.float32(expected_exp[1][1]))
+                    self.assertEqual(label, np.int64(expected_exp[2][1]))
+
+        os.remove(tmp_file.name)
+
+if __name__ == '__main__':
+    unittest.main()