Repository: OpenMined/PySyft

Issue:
Extract compression from `serde` to separate module
**Is your feature request related to a problem? Please describe.**
Compression and serialization are currently intertwined with each other in the same module.

**Describe the solution you'd like**
In order to pave the way for alternate serialization approaches (e.g. Protobuf), it would be helpful to extract compression out of `serde.py` and into its own module.

**Describe alternatives you've considered**
- _Combine all serialization approaches in the same module._ Seems potentially messy.
- _Put each serialization approach in its own module and duplicate compression._ Would work, but harder to maintain.

**Additional context**
Add any other context or screenshots about the feature request here.



Discussion:

Resolving patch:
diff --git a/syft/serde/__init__.py b/syft/serde/__init__.py
--- a/syft/serde/__init__.py
+++ b/syft/serde/__init__.py
@@ -2,5 +2,5 @@
 from syft.serde.torch_serde import *
 from syft.serde.serde import _simplify
 from syft.serde.serde import _detail
-from syft.serde.serde import _compress
-from syft.serde.serde import _decompress
+from syft.serde.compression import _compress
+from syft.serde.compression import _decompress
diff --git a/syft/serde/compression.py b/syft/serde/compression.py
new file mode 100644
--- /dev/null
+++ b/syft/serde/compression.py
@@ -0,0 +1,128 @@
+"""
+This file exists to provide one common place for all compression methods used in
+simplifying and serializing PySyft objects.
+"""
+
+import lz4
+from lz4 import (  # noqa: F401
+    frame,
+)  # needed as otherwise we will get: module 'lz4' has no attribute 'frame'
+import zstd
+
+from syft.exceptions import CompressionNotFoundException
+
+# COMPRESSION SCHEME INT CODES
+NO_COMPRESSION = 40
+LZ4 = 41
+ZSTD = 42
+scheme_to_bytes = {
+    NO_COMPRESSION: NO_COMPRESSION.to_bytes(1, byteorder="big"),
+    LZ4: LZ4.to_bytes(1, byteorder="big"),
+    ZSTD: ZSTD.to_bytes(1, byteorder="big"),
+}
+
+## SECTION: chosen Compression Algorithm
+
+
+def _apply_compress_scheme(decompressed_input_bin) -> tuple:
+    """
+    Apply the selected compression scheme.
+    By default is used LZ4
+
+    Args:
+        decompressed_input_bin: the binary to be compressed
+    """
+    return apply_lz4_compression(decompressed_input_bin)
+
+
+def apply_lz4_compression(decompressed_input_bin) -> tuple:
+    """
+    Apply LZ4 compression to the input
+
+    Args:
+        decompressed_input_bin: the binary to be compressed
+
+    Returns:
+        a tuple (compressed_result, LZ4)
+    """
+    return lz4.frame.compress(decompressed_input_bin), LZ4
+
+
+def apply_zstd_compression(decompressed_input_bin) -> tuple:
+    """
+    Apply ZSTD compression to the input
+
+    Args:
+        decompressed_input_bin: the binary to be compressed
+
+    Returns:
+        a tuple (compressed_result, ZSTD)
+    """
+
+    return zstd.compress(decompressed_input_bin), ZSTD
+
+
+def apply_no_compression(decompressed_input_bin) -> tuple:
+    """
+    No compression is applied to the input
+
+    Args:
+        decompressed_input_bin: the binary
+
+    Returns:
+        a tuple (the binary, LZ4)
+    """
+
+    return decompressed_input_bin, NO_COMPRESSION
+
+
+def _compress(decompressed_input_bin: bin) -> bin:
+    """
+    This function compresses a binary using the function _apply_compress_scheme
+    if the input has been already compressed in some step, it will return it as it is
+
+    Args:
+        decompressed_input_bin (bin): binary to be compressed
+
+    Returns:
+        bin: a compressed binary
+
+    """
+    compress_stream, compress_scheme = _apply_compress_scheme(decompressed_input_bin)
+    try:
+        z = scheme_to_bytes[compress_scheme] + compress_stream
+        return z
+    except KeyError:
+        raise CompressionNotFoundException(
+            f"Compression scheme not found for compression code: {str(compress_scheme)}"
+        )
+
+
+def _decompress(binary: bin) -> bin:
+    """
+    This function decompresses a binary using the scheme defined in the first byte of the input
+
+    Args:
+        binary (bin): a compressed binary
+
+    Returns:
+        bin: decompressed binary
+
+    """
+
+    # check the 1-byte header to check the compression scheme used
+    compress_scheme = binary[0]
+
+    # remove the 1-byte header from the input stream
+    binary = binary[1:]
+    # 1)  Decompress or return the original stream
+    if compress_scheme == LZ4:
+        return lz4.frame.decompress(binary)
+    elif compress_scheme == ZSTD:
+        return zstd.decompress(binary)
+    elif compress_scheme == NO_COMPRESSION:
+        return binary
+    else:
+        raise CompressionNotFoundException(
+            f"Compression scheme not found for compression code: {str(compress_scheme)}"
+        )
diff --git a/syft/serde/serde.py b/syft/serde/serde.py
--- a/syft/serde/serde.py
+++ b/syft/serde/serde.py
@@ -35,15 +35,11 @@
 from typing import Callable
 
 import inspect
-import lz4
-from lz4 import (  # noqa: F401
-    frame,
-)  # needed as otherwise we will get: module 'lz4' has no attribute 'frame'
 import msgpack
-import zstd
 
 import syft
 from syft import dependency_check
+
 from syft.federated.train_config import TrainConfig
 from syft.frameworks.torch.tensors.decorators.logging import LoggingTensor
 from syft.frameworks.torch.tensors.interpreters.precision import FixedPrecisionTensor
@@ -71,11 +67,11 @@
 from syft.messaging.message import ForceObjectDeleteMessage
 from syft.messaging.message import SearchMessage
 from syft.messaging.message import PlanCommandMessage
+from syft.serde import compression
 from syft.serde.native_serde import MAP_NATIVE_SIMPLIFIERS_AND_DETAILERS
 from syft.workers.abstract import AbstractWorker
 from syft.workers.base import BaseWorker
 
-from syft.exceptions import CompressionNotFoundException
 from syft.exceptions import GetNotPermittedError
 from syft.exceptions import ResponseSignatureError
 
@@ -144,16 +140,6 @@
 # in https://github.com/OpenMined/proto
 EXCEPTION_SIMPLIFIER_AND_DETAILERS = [GetNotPermittedError, ResponseSignatureError]
 
-# COMPRESSION SCHEME INT CODES
-NO_COMPRESSION = 40
-LZ4 = 41
-ZSTD = 42
-scheme_to_bytes = {
-    NO_COMPRESSION: NO_COMPRESSION.to_bytes(1, byteorder="big"),
-    LZ4: LZ4.to_bytes(1, byteorder="big"),
-    ZSTD: ZSTD.to_bytes(1, byteorder="big"),
-}
-
 ## SECTION: High Level Simplification Router
 def _force_full_simplify(worker: AbstractWorker, obj: object) -> object:
     """To force a full simplify generally if the usual _simplify is not suitable.
@@ -300,7 +286,7 @@ def _serialize_msgpack_binary(
     # otherwise we output the compressed stream with header set to '1'
     # even if compressed flag is set to false by the caller we
     # output the input stream as it is with header set to '0'
-    return _compress(binary)
+    return compression._compress(binary)
 
 
 def _serialize_msgpack(
@@ -344,7 +330,7 @@ def _deserialize_msgpack_binary(binary: bin, worker: AbstractWorker = None) -> o
         worker = syft.framework.hook.local_worker
 
     # 1) Decompress the binary if needed
-    binary = _decompress(binary)
+    binary = compression._decompress(binary)
 
     # 2) Deserialize
     # This function converts the binary into the appropriate python
@@ -436,113 +422,6 @@ def deserialize(
     return strategy(binary, worker)
 
 
-## SECTION: chosen Compression Algorithm
-
-
-def _apply_compress_scheme(decompressed_input_bin) -> tuple:
-    """
-    Apply the selected compression scheme.
-    By default is used LZ4
-
-    Args:
-        decompressed_input_bin: the binary to be compressed
-    """
-    return apply_lz4_compression(decompressed_input_bin)
-
-
-def apply_lz4_compression(decompressed_input_bin) -> tuple:
-    """
-    Apply LZ4 compression to the input
-
-    Args:
-        decompressed_input_bin: the binary to be compressed
-
-    Returns:
-        a tuple (compressed_result, LZ4)
-    """
-    return lz4.frame.compress(decompressed_input_bin), LZ4
-
-
-def apply_zstd_compression(decompressed_input_bin) -> tuple:
-    """
-    Apply ZSTD compression to the input
-
-    Args:
-        decompressed_input_bin: the binary to be compressed
-
-    Returns:
-        a tuple (compressed_result, ZSTD)
-    """
-
-    return zstd.compress(decompressed_input_bin), ZSTD
-
-
-def apply_no_compression(decompressed_input_bin) -> tuple:
-    """
-    No compression is applied to the input
-
-    Args:
-        decompressed_input_bin: the binary
-
-    Returns:
-        a tuple (the binary, LZ4)
-    """
-
-    return decompressed_input_bin, NO_COMPRESSION
-
-
-def _compress(decompressed_input_bin: bin) -> bin:
-    """
-    This function compresses a binary using the function _apply_compress_scheme
-    if the input has been already compressed in some step, it will return it as it is
-
-    Args:
-        decompressed_input_bin (bin): binary to be compressed
-
-    Returns:
-        bin: a compressed binary
-
-    """
-    compress_stream, compress_scheme = _apply_compress_scheme(decompressed_input_bin)
-    try:
-        z = scheme_to_bytes[compress_scheme] + compress_stream
-        return z
-    except KeyError:
-        raise CompressionNotFoundException(
-            f"Compression scheme not found for compression code: {str(compress_scheme)}"
-        )
-
-
-def _decompress(binary: bin) -> bin:
-    """
-    This function decompresses a binary using the scheme defined in the first byte of the input
-
-    Args:
-        binary (bin): a compressed binary
-
-    Returns:
-        bin: decompressed binary
-
-    """
-
-    # check the 1-byte header to check the compression scheme used
-    compress_scheme = binary[0]
-
-    # remove the 1-byte header from the input stream
-    binary = binary[1:]
-    # 1)  Decompress or return the original stream
-    if compress_scheme == LZ4:
-        return lz4.frame.decompress(binary)
-    elif compress_scheme == ZSTD:
-        return zstd.decompress(binary)
-    elif compress_scheme == NO_COMPRESSION:
-        return binary
-    else:
-        raise CompressionNotFoundException(
-            f"Compression scheme not found for compression code: {str(compress_scheme)}"
-        )
-
-
 def _simplify(worker: AbstractWorker, obj: object, **kwargs) -> object:
     """
     This function takes an object as input and returns a simple


Test patch:
diff --git a/test/test_serde.py b/test/test_serde.py
--- a/test/test_serde.py
+++ b/test/test_serde.py
@@ -13,6 +13,7 @@
 from syft.frameworks.torch.tensors.interpreters.additive_shared import AdditiveSharingTensor
 from syft.generic.pointers.object_wrapper import ObjectWrapper
 from syft.generic.pointers.pointer_tensor import PointerTensor
+from syft.serde import compression
 from syft.serde import native_serde
 from syft.serde import serde
 from syft.serde import torch_serde
@@ -321,9 +322,9 @@ def test_pointer_tensor_simplify(workers):
 @pytest.mark.parametrize("compress", [True, False])
 def test_torch_Tensor(compress):
     if compress:
-        syft.serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        syft.serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     t = Tensor(numpy.random.random((100, 100)))
     t_serialized = serde.serialize(t)
@@ -340,9 +341,9 @@ def test_torch_Tensor_convenience(compress):
     directly on the tensor itself. This tests to makes sure it
     works correctly."""
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     t = Tensor(numpy.random.random((100, 100)))
     t_serialized = t.serialize()
@@ -354,9 +355,9 @@ def test_torch_Tensor_convenience(compress):
 def test_tuple(compress):
     # Test with a simple datatype
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     tuple = (1, 2)
     tuple_serialized = serde.serialize(tuple)
@@ -379,9 +380,9 @@ def test_tuple(compress):
 @pytest.mark.parametrize("compress", [True, False])
 def test_bytearray(compress):
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     bytearr = bytearray("This is a teststring", "utf-8")
     bytearr_serialized = serde.serialize(bytearr)
@@ -397,9 +398,9 @@ def test_bytearray(compress):
 @pytest.mark.parametrize("compress", [True, False])
 def test_ndarray_serde(compress):
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
     arr = numpy.random.random((100, 100))
     arr_serialized = serde.serialize(arr)
 
@@ -408,30 +409,34 @@ def test_ndarray_serde(compress):
     assert numpy.array_equal(arr, arr_serialized_deserialized)
 
 
-@pytest.mark.parametrize("compress_scheme", [serde.LZ4, serde.ZSTD, serde.NO_COMPRESSION])
+@pytest.mark.parametrize(
+    "compress_scheme", [compression.LZ4, compression.ZSTD, compression.NO_COMPRESSION]
+)
 def test_compress_decompress(compress_scheme):
-    if compress_scheme == serde.LZ4:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
-    elif compress_scheme == serde.ZSTD:
-        serde._apply_compress_scheme = serde.apply_zstd_compression
+    if compress_scheme == compression.LZ4:
+        compression._apply_compress_scheme = compression.apply_lz4_compression
+    elif compress_scheme == compression.ZSTD:
+        compression._apply_compress_scheme = compression.apply_zstd_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     original = msgpack.dumps([1, 2, 3])
-    compressed = serde._compress(original)
-    decompressed = serde._decompress(compressed)
+    compressed = compression._compress(original)
+    decompressed = compression._decompress(compressed)
     assert type(compressed) == bytes
     assert original == decompressed
 
 
-@pytest.mark.parametrize("compress_scheme", [serde.LZ4, serde.ZSTD, serde.NO_COMPRESSION])
+@pytest.mark.parametrize(
+    "compress_scheme", [compression.LZ4, compression.ZSTD, compression.NO_COMPRESSION]
+)
 def test_compressed_serde(compress_scheme):
-    if compress_scheme == serde.LZ4:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
-    elif compress_scheme == serde.ZSTD:
-        serde._apply_compress_scheme = serde.apply_zstd_compression
+    if compress_scheme == compression.LZ4:
+        compression._apply_compress_scheme = compression.apply_lz4_compression
+    elif compress_scheme == compression.ZSTD:
+        compression._apply_compress_scheme = compression.apply_zstd_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     # using numpy.ones because numpy.random.random is not compressed.
     arr = numpy.ones((100, 100))
@@ -446,9 +451,9 @@ def test_compressed_serde(compress_scheme):
 def test_dict(compress):
     # Test with integers
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
     _dict = {1: 1, 2: 2, 3: 3}
     dict_serialized = serde.serialize(_dict)
     dict_serialized_deserialized = serde.deserialize(dict_serialized)
@@ -476,9 +481,9 @@ def test_dict(compress):
 @pytest.mark.parametrize("compress", [True, False])
 def test_range_serde(compress):
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     _range = range(1, 2, 3)
 
@@ -491,9 +496,9 @@ def test_range_serde(compress):
 @pytest.mark.parametrize("compress", [True, False])
 def test_list(compress):
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     # Test with integers
     _list = [1, 2]
@@ -514,9 +519,9 @@ def test_list(compress):
 
     list_serialized = serde.serialize(_list)
     if compress:
-        assert list_serialized[0] == serde.LZ4
+        assert list_serialized[0] == compression.LZ4
     else:
-        assert list_serialized[0] == serde.NO_COMPRESSION
+        assert list_serialized[0] == compression.NO_COMPRESSION
 
     list_serialized_deserialized = serde.deserialize(list_serialized)
     # `assert list_serialized_deserialized == _list` does not work, therefore it's split
@@ -529,9 +534,9 @@ def test_list(compress):
 @pytest.mark.parametrize("compress", [True, False])
 def test_set(compress):
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     # Test with integers
     _set = set([1, 2])
@@ -553,9 +558,9 @@ def test_set(compress):
 
     set_serialized = serde.serialize(_set)
     if compress:
-        assert set_serialized[0] == serde.LZ4
+        assert set_serialized[0] == compression.LZ4
     else:
-        assert set_serialized[0] == serde.NO_COMPRESSION
+        assert set_serialized[0] == compression.NO_COMPRESSION
 
     set_serialized_deserialized = serde.deserialize(set_serialized)
     # `assert set_serialized_deserialized == _set` does not work, therefore it's split
@@ -568,9 +573,9 @@ def test_set(compress):
 @pytest.mark.parametrize("compress", [True, False])
 def test_slice(compress):
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     s = slice(0, 100, 2)
     x = numpy.random.rand(100)
@@ -592,9 +597,9 @@ def test_slice(compress):
 @pytest.mark.parametrize("compress", [True, False])
 def test_float(compress):
     if compress:
-        serde._apply_compress_scheme = serde.apply_lz4_compression
+        compression._apply_compress_scheme = compression.apply_lz4_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     x = 0.5
     y = 1.5
@@ -612,36 +617,38 @@ def test_float(compress):
 @pytest.mark.parametrize(
     "compress, compress_scheme",
     [
-        (True, serde.LZ4),
-        (False, serde.LZ4),
-        (True, serde.ZSTD),
-        (False, serde.ZSTD),
-        (True, serde.NO_COMPRESSION),
-        (False, serde.NO_COMPRESSION),
+        (True, compression.LZ4),
+        (False, compression.LZ4),
+        (True, compression.ZSTD),
+        (False, compression.ZSTD),
+        (True, compression.NO_COMPRESSION),
+        (False, compression.NO_COMPRESSION),
     ],
 )
 def test_hooked_tensor(compress, compress_scheme):
     if compress:
-        if compress_scheme == serde.LZ4:
-            serde._apply_compress_scheme = serde.apply_lz4_compression
-        elif compress_scheme == serde.ZSTD:
-            serde._apply_compress_scheme = serde.apply_zstd_compression
+        if compress_scheme == compression.LZ4:
+            compression._apply_compress_scheme = compression.apply_lz4_compression
+        elif compress_scheme == compression.ZSTD:
+            compression._apply_compress_scheme = compression.apply_zstd_compression
         else:
-            serde._apply_compress_scheme = serde.apply_no_compression
+            compression._apply_compress_scheme = compression.apply_no_compression
     else:
-        serde._apply_compress_scheme = serde.apply_no_compression
+        compression._apply_compress_scheme = compression.apply_no_compression
 
     t = Tensor(numpy.ones((100, 100)))
     t_serialized = serde.serialize(t)
     assert (
-        t_serialized[0] == compress_scheme if compress else t_serialized[0] == serde.NO_COMPRESSION
+        t_serialized[0] == compress_scheme
+        if compress
+        else t_serialized[0] == compression.NO_COMPRESSION
     )
     t_serialized_deserialized = serde.deserialize(t_serialized)
     assert (t == t_serialized_deserialized).all()
 
 
 def test_pointer_tensor(hook, workers):
-    serde._apply_compress_scheme = serde.apply_no_compression
+    compression._apply_compress_scheme = compression.apply_no_compression
     t = PointerTensor(
         id=1000, location=workers["alice"], owner=workers["alice"], id_at_location=12345
     )
@@ -663,7 +670,7 @@ def test_pointer_tensor_detail(id):
 
 
 def test_numpy_tensor_serde():
-    serde._apply_compress_scheme = serde.apply_lz4_compression
+    compression._apply_compress_scheme = compression.apply_lz4_compression
 
     serde._serialize_tensor = syft.serde.numpy_tensor_serializer
     serde._deserialize_tensor = syft.serde.numpy_tensor_deserializer
@@ -671,7 +678,7 @@ def test_numpy_tensor_serde():
     tensor = torch.tensor(numpy.ones((10, 10)), requires_grad=False)
 
     tensor_serialized = serde.serialize(tensor)
-    assert tensor_serialized[0] != serde.NO_COMPRESSION
+    assert tensor_serialized[0] != compression.NO_COMPRESSION
     tensor_deserialized = serde.deserialize(tensor_serialized)
 
     # Back to Pytorch serializer