Repository: OpenMined/PySyft
Pull request #1731 (2018-11-26 17:09:57)

Solving #1673 - adding logic to serde python slice objects.
- Solving #1637 
- Adding _compressed_ test to all serde tests


Patch:
diff --git a/syft/serde.py b/syft/serde.py
index 30a8c7747b..a38ff0e5e0 100644
--- a/syft/serde.py
+++ b/syft/serde.py
@@ -31,7 +31,6 @@ By default, we serialize using msgpack and compress using lz4.
 from typing import Collection
 from typing import Dict
 from typing import Tuple
-from typing import List
 import torch
 import msgpack
 import lz4
@@ -252,7 +251,7 @@ def _detail_collection_set(my_collection: Collection) -> Collection:
     return set(pieces)
 
 
-def _detail_collection_tuple(my_tuple: tuple) -> tuple:
+def _detail_collection_tuple(my_tuple: Tuple) -> Tuple:
     """
     This function is designed to operate in the opposite direction of
     _simplify_collection. It takes a tuple of simple python objects
@@ -332,7 +331,7 @@ def _simplify_range(my_range: range) -> Tuple[int, int, int]:
 
     """
 
-    return [my_range.start, my_range.stop, my_range.step]
+    return (my_range.start, my_range.stop, my_range.step)
 
 
 def _detail_range(my_range_params: Tuple[int, int, int]) -> range:
@@ -358,7 +357,7 @@ def _detail_range(my_range_params: Tuple[int, int, int]) -> range:
 #   numpy array
 
 
-def _simplify_ndarray(my_array: numpy.ndarray) -> Tuple[bin, List, str]:
+def _simplify_ndarray(my_array: numpy.ndarray) -> Tuple[bin, Tuple, str]:
     """
     This function gets the byte representation of the array
         and stores the dtype and shape for reconstruction
@@ -378,10 +377,10 @@ def _simplify_ndarray(my_array: numpy.ndarray) -> Tuple[bin, List, str]:
     arr_shape = my_array.shape
     arr_dtype = my_array.dtype.name
 
-    return [arr_bytes, arr_shape, arr_dtype]
+    return (arr_bytes, arr_shape, arr_dtype)
 
 
-def _detail_ndarray(arr_representation: Tuple[bin, List[int], str]) -> numpy.ndarray:
+def _detail_ndarray(arr_representation: Tuple[bin, Tuple, str]) -> numpy.ndarray:
     """
     This function reconstruct a numpy array from it's byte data, the shape and the dtype
         by first loading the byte data with the appropiate dtype and then reshaping it into the
@@ -406,6 +405,47 @@ def _detail_ndarray(arr_representation: Tuple[bin, List[int], str]) -> numpy.nda
     return res
 
 
+#   slice
+
+
+def _simplify_slice(my_slice: slice) -> Tuple[int, int, int]:
+    """
+    This function creates a list that represents a slice.
+
+    Args:
+        slice: a python slice
+
+    Returns:
+        list: a list holding the start, stop and step values
+
+    Usage:
+
+        slice_representation = _simplify_slice(slice(1,2,3))
+
+    """
+    return (my_slice.start, my_slice.stop, my_slice.step)
+
+
+def _detail_slice(my_slice: Tuple[int, int, int]) -> slice:
+    """
+    This function extracts the start, stop and step from a list.
+
+    Args:
+        list: a list defining the slice parameters [start, stop, step]
+
+    Returns:
+        range: a range object
+
+    Usage:
+        new_range = _detail_range([1, 3, 4])
+
+        assert new_range == range(1, 3, 4)
+
+    """
+
+    return slice(my_slice[0], my_slice[1], my_slice[2])
+
+
 # High Level Simplification Router
 
 
@@ -458,6 +498,7 @@ simplifiers[set] = [3, _simplify_collection]
 simplifiers[dict] = [4, _simplify_dictionary]
 simplifiers[range] = [5, _simplify_range]
 simplifiers[numpy.ndarray] = [6, _simplify_ndarray]
+simplifiers[slice] = [7, _simplify_slice]
 
 
 def _detail(obj: object) -> object:
@@ -489,4 +530,5 @@ detailers = [
     _detail_dictionary,
     _detail_range,
     _detail_ndarray,
+    _detail_slice,
 ]


Test patch:
diff --git a/test/test_serde.py b/test/test_serde.py
index f3d697f47d..7e3acf1163 100644
--- a/test/test_serde.py
+++ b/test/test_serde.py
@@ -48,7 +48,7 @@ class TestSimplify(TestCase):
 
     def test_range_simplify(self):
         input = range(1, 3, 4)
-        target = [5, [1, 3, 4]]
+        target = [5, (1, 3, 4)]
         assert _simplify(input) == target
 
     def test_torch_tensor_simplify(self):
@@ -91,6 +91,25 @@ class TestSerde(TestCase):
         assert (tuple_serialized_deserialized[0] == tensor_one).all()
         assert (tuple_serialized_deserialized[1] == tensor_two).all()
 
+    def test_compressed_tuple(self):
+        # Test with a simple datatype
+        tuple = (1, 2)
+        tuple_serialized = serialize(tuple, compress=True)
+        tuple_serialized_deserialized = deserialize(tuple_serialized, compressed=True)
+        assert tuple == tuple_serialized_deserialized
+
+        # Test with a complex data structure
+        tensor_one = tensor(numpy.random.random((100, 100)))
+        tensor_two = tensor(numpy.random.random((100, 100)))
+        tuple = (tensor_one, tensor_two)
+        tuple_serialized = serialize(tuple, compress=True)
+        tuple_serialized_deserialized = deserialize(tuple_serialized, compressed=True)
+        # `assert tuple_serialized_deserialized == tuple` does not work, therefore it's split
+        # into 3 assertions
+        assert type(tuple_serialized_deserialized) == type(tuple)
+        assert (tuple_serialized_deserialized[0] == tensor_one).all()
+        assert (tuple_serialized_deserialized[1] == tensor_two).all()
+
     def test_bytearray(self):
         bytearr = bytearray("This is a teststring", "utf-8")
         bytearr_serialized = serialize(bytearr, compress=False)
@@ -102,6 +121,17 @@ class TestSerde(TestCase):
         bytearr_serialized_desirialized = deserialize(bytearr_serialized, compressed=False)
         assert bytearr == bytearr_serialized_desirialized
 
+    def test_compressed_bytearray(self):
+        bytearr = bytearray("This is a teststring", "utf-8")
+        bytearr_serialized = serialize(bytearr, compress=True)
+        bytearr_serialized_desirialized = deserialize(bytearr_serialized, compressed=True)
+        assert bytearr == bytearr_serialized_desirialized
+
+        bytearr = bytearray(numpy.random.random((100, 100)))
+        bytearr_serialized = serialize(bytearr, compress=True)
+        bytearr_serialized_desirialized = deserialize(bytearr_serialized, compressed=True)
+        assert bytearr == bytearr_serialized_desirialized
+
     def test_ndarray_serde(self):
         arr = numpy.random.random((100, 100))
         arr_serialized = serialize(arr, compress=False)
@@ -110,6 +140,14 @@ class TestSerde(TestCase):
 
         assert numpy.array_equal(arr, arr_serialized_deserialized)
 
+    def test_compressed_ndarray_serde(self):
+        arr = numpy.random.random((100, 100))
+        arr_serialized = serialize(arr, compress=True)
+
+        arr_serialized_deserialized = deserialize(arr_serialized, compressed=True)
+
+        assert numpy.array_equal(arr, arr_serialized_deserialized)
+
     def test_compress_decompress(self):
         original = msgpack.dumps([1, 2, 3])
         compressed = _compress(original)
@@ -150,6 +188,31 @@ class TestSerde(TestCase):
         assert (dict_serialized_deserialized[0] == tensor_one).all()
         assert (dict_serialized_deserialized[1] == tensor_two).all()
 
+    def test_compressed_dict(self):
+        # Test with integers
+        _dict = {1: 1, 2: 2, 3: 3}
+        dict_serialized = serialize(_dict, compress=True)
+        dict_serialized_deserialized = deserialize(dict_serialized, compressed=True)
+        assert _dict == dict_serialized_deserialized
+
+        # Test with strings
+        _dict = {"one": 1, "two": 2, "three": 3}
+        dict_serialized = serialize(_dict, compress=True)
+        dict_serialized_deserialized = deserialize(dict_serialized, compressed=True)
+        assert _dict == dict_serialized_deserialized
+
+        # Test with a complex data structure
+        tensor_one = tensor(numpy.random.random((100, 100)))
+        tensor_two = tensor(numpy.random.random((100, 100)))
+        _dict = {0: tensor_one, 1: tensor_two}
+        dict_serialized = serialize(_dict, compress=True)
+        dict_serialized_deserialized = deserialize(dict_serialized, compressed=True)
+        # `assert dict_serialized_deserialized == _dict` does not work, therefore it's split
+        # into 3 assertions
+        assert type(dict_serialized_deserialized) == type(_dict)
+        assert (dict_serialized_deserialized[0] == tensor_one).all()
+        assert (dict_serialized_deserialized[1] == tensor_two).all()
+
     def test_range_serde(self):
         _range = range(1, 2, 3)
 
@@ -159,6 +222,15 @@ class TestSerde(TestCase):
 
         assert _range == range_serialized_deserialized
 
+    def test_compressed_range_serde(self):
+        _range = range(1, 2, 3)
+
+        range_serialized = serialize(_range, compress=True)
+
+        range_serialized_deserialized = deserialize(range_serialized, compressed=True)
+
+        assert _range == range_serialized_deserialized
+
     def test_list(self):
         # Test with integers
         _list = [1, 2]
@@ -184,6 +256,31 @@ class TestSerde(TestCase):
         assert (list_serialized_deserialized[0] == tensor_one).all()
         assert (list_serialized_deserialized[1] == tensor_two).all()
 
+    def test_compressed_list(self):
+        # Test with integers
+        _list = [1, 2]
+        list_serialized = serialize(_list, compress=True)
+        list_serialized_deserialized = deserialize(list_serialized, compressed=True)
+        assert _list == list_serialized_deserialized
+
+        # Test with strings
+        _list = ["hello", "world"]
+        list_serialized = serialize(_list, compress=True)
+        list_serialized_deserialized = deserialize(list_serialized, compressed=True)
+        assert _list == list_serialized_deserialized
+
+        # Test with a complex data structure
+        tensor_one = tensor(numpy.random.random((100, 100)))
+        tensor_two = tensor(numpy.random.random((100, 100)))
+        _list = (tensor_one, tensor_two)
+        list_serialized = serialize(_list, compress=True)
+        list_serialized_deserialized = deserialize(list_serialized, compressed=True)
+        # `assert list_serialized_deserialized == _list` does not work, therefore it's split
+        # into 3 assertions
+        assert type(list_serialized_deserialized) == type(_list)
+        assert (list_serialized_deserialized[0] == tensor_one).all()
+        assert (list_serialized_deserialized[1] == tensor_two).all()
+
     def test_set(self):
         # Test with integers
         _set = set([1, 2])
@@ -208,3 +305,62 @@ class TestSerde(TestCase):
         assert type(set_serialized_deserialized) == type(_set)
         assert (set_serialized_deserialized[0] == tensor_one).all()
         assert (set_serialized_deserialized[1] == tensor_two).all()
+
+    def test_compressed_set(self):
+        # Test with integers
+        _set = set([1, 2])
+        set_serialized = serialize(_set, compress=True)
+        set_serialized_deserialized = deserialize(set_serialized, compressed=True)
+        assert _set == set_serialized_deserialized
+
+        # Test with strings
+        _set = set(["hello", "world"])
+        set_serialized = serialize(_set, compress=True)
+        set_serialized_deserialized = deserialize(set_serialized, compressed=True)
+        assert _set == set_serialized_deserialized
+
+        # Test with a complex data structure
+        tensor_one = tensor(numpy.random.random((100, 100)))
+        tensor_two = tensor(numpy.random.random((100, 100)))
+        _set = (tensor_one, tensor_two)
+        set_serialized = serialize(_set, compress=True)
+        set_serialized_deserialized = deserialize(set_serialized, compressed=True)
+        # `assert set_serialized_deserialized == _set` does not work, therefore it's split
+        # into 3 assertions
+        assert type(set_serialized_deserialized) == type(_set)
+        assert (set_serialized_deserialized[0] == tensor_one).all()
+        assert (set_serialized_deserialized[1] == tensor_two).all()
+
+    def test_slice(self):
+        s = slice(0, 100, 2)
+        x = numpy.random.rand(100)
+        s_serialized = serialize(s, compress=False)
+        s_serialized_deserialized = deserialize(s_serialized, compressed=False)
+
+        assert type(s) == type(s_serialized_deserialized)
+        assert (x[s] == x[s_serialized_deserialized]).all()
+
+        s = slice(40, 50)
+        x = numpy.random.rand(100)
+        s_serialized = serialize(s, compress=False)
+        s_serialized_deserialized = deserialize(s_serialized, compressed=False)
+
+        assert type(s) == type(s_serialized_deserialized)
+        assert (x[s] == x[s_serialized_deserialized]).all()
+
+    def test_compressed_slice(self):
+        s = slice(0, 100, 2)
+        x = numpy.random.rand(100)
+        s_serialized = serialize(s, compress=True)
+        s_serialized_deserialized = deserialize(s_serialized, compressed=True)
+
+        assert type(s) == type(s_serialized_deserialized)
+        assert (x[s] == x[s_serialized_deserialized]).all()
+
+        s = slice(40, 50)
+        x = numpy.random.rand(100)
+        s_serialized = serialize(s, compress=True)
+        s_serialized_deserialized = deserialize(s_serialized, compressed=True)
+
+        assert type(s) == type(s_serialized_deserialized)
+        assert (x[s] == x[s_serialized_deserialized]).all()