Repository: fatiando/pooch

Issue:
Add decompression processors
**Description of the desired feature**

Many times, downloaded files will be compressed to save space in storage and during download. But it's sometimes a hassle to decompress them every time before reading the file. In the case of netCDF grids, decompressing into a temporary file makes it impossible to have `xarray` lazily load the grid. 

#59 introduced processor hooks and #72 introduced the `Unzip` processor. Following this logic, it would be good to have a `Decompress` processor that decompresses the downloaded file. It can guess the compression method from the file name or be given it explicitly as an `__init__` argument. 

The class would look something like this: 

```python
class Decompress:
    def __init__(self, method="auto"):
        self.method = method
    def __call__(self, fname, action, pooch):
        ...
        return decompressed
```

Comments:

Patch:
diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml
index 869388e..606ed7f 100644
--- a/.azure-pipelines.yml
+++ b/.azure-pipelines.yml
@@ -90,6 +90,7 @@ jobs:
         python.version: '2.7'
         PYTHON: '2.7'
         CONDA_REQUIREMENTS: 'requirements27.txt'
+        CONDA_INSTALL_EXTRA: 'codecov backports.lzma bz2file'
 
   steps:
 
@@ -171,6 +172,7 @@ jobs:
         python.version: '2.7'
         PYTHON: '2.7'
         CONDA_REQUIREMENTS: 'requirements27.txt'
+        CONDA_INSTALL_EXTRA: 'codecov backports.lzma bz2file'
 
   steps:
 
diff --git a/.travis.yml b/.travis.yml
index b804aed..d306f53 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -49,6 +49,7 @@ matrix:
           env:
               - PYTHON=2.7
               - CONDA_REQUIREMENTS=requirements27.txt
+              - CONDA_INSTALL_EXTRA="codecov backports.lzma bz2file"
 
 # Setup the build environment
 before_install:
diff --git a/doc/api/index.rst b/doc/api/index.rst
index 0c5a4d2..2f05190 100644
--- a/doc/api/index.rst
+++ b/doc/api/index.rst
@@ -43,6 +43,7 @@ Processors
 
    Unzip
    Untar
+   Decompress
 
 Miscellaneous
 -------------
diff --git a/doc/install.rst b/doc/install.rst
index 419256c..a13ec63 100644
--- a/doc/install.rst
+++ b/doc/install.rst
@@ -34,6 +34,12 @@ The following are only required for Python 2.7:
 * `pathlib <https://pypi.org/project/pathlib/>`__
 * `backports.tempfile <https://pypi.org/project/backports.tempfile/>`__
 
+If using :class:`pooch.Decompress` in Python 2.7, the following extra packages might
+need to be installed (they will not be automatically installed when installing Pooch):
+
+* `backports.lzma <https://pypi.org/project/backports.lzma/>`__
+* `bz2file <https://pypi.org/project/bz2file/>`__
+
 
 Installing with conda
 ---------------------
diff --git a/doc/usage.rst b/doc/usage.rst
index 7c099ba..2285562 100644
--- a/doc/usage.rst
+++ b/doc/usage.rst
@@ -200,6 +200,8 @@ takes a ``processor`` argument that allows us to specify a function that is exec
 post-download and before returning the local file path. The processor also lets us
 overwrite the file name returned by :meth:`pooch.Pooch.fetch`.
 
+See the :ref:`api` for a list of all available post-processing hooks.
+
 For example, let's say our data file is zipped and we want to store an unzipped copy of
 it and read that instead. We can do this with a post-processing hook that unzips the
 file and returns the path to the unzipped file instead of the original zip archive:
diff --git a/pooch/__init__.py b/pooch/__init__.py
index 514b635..2232b12 100644
--- a/pooch/__init__.py
+++ b/pooch/__init__.py
@@ -4,7 +4,7 @@ from . import version
 from .core import Pooch, create
 from .utils import os_cache, file_hash, make_registry, check_version
 from .downloaders import HTTPDownloader
-from .processors import Unzip, Untar
+from .processors import Unzip, Untar, Decompress
 
 
 def test(doctest=True, verbose=True, coverage=False):
diff --git a/pooch/processors.py b/pooch/processors.py
index efa2a1a..eff758a 100644
--- a/pooch/processors.py
+++ b/pooch/processors.py
@@ -3,11 +3,30 @@
 Post-processing hooks for Pooch.fetch
 """
 import os
-
+import sys
+import gzip
+import shutil
 from zipfile import ZipFile
 from tarfile import TarFile
 from warnings import warn
 
+# Try getting the 2.7 backports of lzma and bz2. Can be deleted when dropping 2.7
+if sys.version_info[0] < 3:
+    try:
+        import bz2file as bz2
+    except ImportError:
+        bz2 = None
+else:
+    import bz2
+try:
+    import lzma
+except ImportError:
+    try:
+        from backports import lzma
+    except ImportError:
+        # Make this an optional dependency
+        lzma = None
+
 
 class ExtractorProcessor:  # pylint: disable=too-few-public-methods
     """
@@ -24,6 +43,7 @@ class ExtractorProcessor:  # pylint: disable=too-few-public-methods
         If None, will unpack all files in the archive. Otherwise, *members*
         must be a list of file names to unpack from the archive. Only these
         files will be unpacked.
+
     """
 
     suffix = None  # String appended to unpacked archive. To be implemented in subclass
@@ -92,13 +112,13 @@ class Unzip(ExtractorProcessor):  # pylint: disable=too-few-public-methods
 
     The output folder is ``{fname}.unzip``.
 
-
     Parameters
     ----------
     members : list or None
         If None, will unpack all files in the zip archive. Otherwise, *members* must be
         a list of file names to unpack from the archive. Only these files will be
         unpacked.
+
     """
 
     suffix = ".unzip"
@@ -174,3 +194,110 @@ class Untar(ExtractorProcessor):  # pylint: disable=too-few-public-methods
                             output.write(data_file.read())
                     finally:
                         data_file.close()
+
+
+class Decompress:  # pylint: disable=too-few-public-methods
+    """
+    Processor that decompress a file and returns the decompressed version.
+
+    Use with :meth:`pooch.Pooch.fetch` to decompress a downloaded data file so that it
+    can be easily opened. Useful for data files that take a long time to decompress
+    (exchanging disk space for speed).
+
+    The output file is ``{fname}.decomp``.
+
+    Supported decompression methods are LZMA (``.xz``), bzip2 (``.bz2``), and gzip
+    (``.gz``).
+
+    File names with the standard extensions (see above) can use ``method="auto"`` to
+    automatically determine the compression method. This can be overwritten by setting
+    the *method* argument.
+
+    .. warning::
+
+        With **Python 2.7**, methods "lzma"/"xz" and "bzip2" require extra dependencies
+        to be installed: ``backports.lzma`` for "lzma" and ``bz2file`` for "bzip2".
+
+    Parameters
+    ----------
+    method : str
+        Name of the compression method. Can be "auto", "lzma", "xz", "bzip2", or "gzip".
+
+    """
+
+    modules = {"lzma": lzma, "xz": lzma, "gzip": gzip, "bzip2": bz2}
+
+    def __init__(self, method="auto"):
+        self.method = method
+
+    def __call__(self, fname, action, pooch):
+        """
+        Decompress the given file.
+
+        The output file will be ``fname`` with ``.decomp`` appended to it.
+
+        Parameters
+        ----------
+        fname : str
+            Full path of the compressed file in local storage.
+        action : str
+            Indicates what action was taken by :meth:`pooch.Pooch.fetch`. One of:
+            * ``"download"``: The file didn't exist locally and was downloaded
+            * ``"update"``: The local file was outdated and was re-download
+            * ``"fetch"``: The file exists and is updated so it wasn't downloaded
+        pooch : :class:`pooch.Pooch`
+            The instance of :class:`pooch.Pooch` that is calling this.
+
+        Returns
+        -------
+        fname : str
+            The full path to the decompressed file.
+        """
+        decompressed = fname + ".decomp"
+        if action in ("update", "download") or not os.path.exists(decompressed):
+            warn(
+                "Decompressing '{}' to '{}' using method '{}'.".format(
+                    fname, decompressed, self.method
+                )
+            )
+            module = self._compression_module(fname)
+            with open(decompressed, "w+b") as output:
+                with module.open(fname) as compressed:
+                    shutil.copyfileobj(compressed, output)
+        return decompressed
+
+    def _compression_module(self, fname):
+        """
+        Get the Python compression module compatible with fname and the chosen method.
+
+        If the *method* attribute is "auto", will select a method based on the
+        extension. If no recognized extension is in the file name, will raise a
+        ValueError.
+        """
+        method = self.method
+        if method == "auto":
+            ext = os.path.splitext(fname)[-1]
+            valid_methods = {".xz": "lzma", ".gz": "gzip", ".bz2": "bzip2"}
+            if ext not in valid_methods:
+                raise ValueError(
+                    "Unrecognized extension '{}'. Must be one of '{}'.".format(
+                        ext, list(valid_methods.keys())
+                    )
+                )
+            method = valid_methods[ext]
+        if method not in self.modules:
+            raise ValueError(
+                "Invalid compression method '{}'. Must be one of '{}'.".format(
+                    method, list(self.modules.keys())
+                )
+            )
+        # Check for Python 2.7 extra dependencies
+        if method in ["lzma", "xz"] and self.modules["lzma"] is None:
+            raise ValueError(
+                "LZMA/xz support requires the 'backports.lzma' package in Python 2.7"
+            )
+        if method == "bzip2" and self.modules["bzip2"] is None:
+            raise ValueError(
+                "bzip2 support requires the 'bz2file' package in Python 2.7"
+            )
+        return self.modules[method]


Test patch:
diff --git a/pooch/tests/data/registry-custom-url.txt b/pooch/tests/data/registry-custom-url.txt
index 97be99b..63f432e 100644
--- a/pooch/tests/data/registry-custom-url.txt
+++ b/pooch/tests/data/registry-custom-url.txt
@@ -4,3 +4,6 @@ store.zip 0498d2a001e71051bbd2acd2346f38da7cbd345a633cb7bf0f8a20938714b51a
 tiny-data.txt baee0894dba14b12085eacb204284b97e362f4f3e5a5807693cc90ef415c1b2d https://some-site/tiny-data.txt
 tiny-data.tar.gz 41503f083814f43a01a8e9a30c28d7a9fe96839a99727a7fdd0acf7cd5bab63b
 store.tar.gz 088c7f4e0f1859b1c769bb6065de24376f366374817ede8691a6ac2e49f29511
+tiny-data.txt.bz2 753663687a4040c90c8578061867d1df623e6aa8011c870a5dbd88ee3c82e306
+tiny-data.txt.gz 2e2da6161291657617c32192dba95635706af80c6e7335750812907b58fd4b52
+tiny-data.txt.xz 99dcb5c32a6e916344bacb4badcbc2f2b6ee196977d1d8187610c21e7e607765
diff --git a/pooch/tests/data/registry.txt b/pooch/tests/data/registry.txt
index 8a7dcef..5795633 100644
--- a/pooch/tests/data/registry.txt
+++ b/pooch/tests/data/registry.txt
@@ -4,3 +4,6 @@ tiny-data.zip 0d49e94f07bc1866ec57e7fd1b93a351fba36842ec9b13dd50bf94e8dfa35cbb
 store.zip 0498d2a001e71051bbd2acd2346f38da7cbd345a633cb7bf0f8a20938714b51a
 tiny-data.tar.gz 41503f083814f43a01a8e9a30c28d7a9fe96839a99727a7fdd0acf7cd5bab63b
 store.tar.gz 088c7f4e0f1859b1c769bb6065de24376f366374817ede8691a6ac2e49f29511
+tiny-data.txt.bz2 753663687a4040c90c8578061867d1df623e6aa8011c870a5dbd88ee3c82e306
+tiny-data.txt.gz 2e2da6161291657617c32192dba95635706af80c6e7335750812907b58fd4b52
+tiny-data.txt.xz 99dcb5c32a6e916344bacb4badcbc2f2b6ee196977d1d8187610c21e7e607765
diff --git a/pooch/tests/test_processors.py b/pooch/tests/test_processors.py
index c2dae3f..acbcc19 100644
--- a/pooch/tests/test_processors.py
+++ b/pooch/tests/test_processors.py
@@ -12,7 +12,7 @@ import warnings
 import pytest
 
 from .. import Pooch
-from ..processors import Unzip, Untar, ExtractorProcessor
+from ..processors import Unzip, Untar, ExtractorProcessor, Decompress
 
 from .utils import pooch_test_url, pooch_test_registry, check_tiny_data
 
@@ -21,6 +21,66 @@ REGISTRY = pooch_test_registry()
 BASEURL = pooch_test_url()
 
 
+@pytest.mark.parametrize(
+    "method,ext",
+    [("auto", "xz"), ("lzma", "xz"), ("xz", "xz"), ("bzip2", "bz2"), ("gzip", "gz")],
+    ids=["auto", "lzma", "xz", "bz2", "gz"],
+)
+def test_decompress(method, ext):
+    "Check that decompression after download works for all formats"
+    processor = Decompress(method=method)
+    with TemporaryDirectory() as local_store:
+        path = Path(local_store)
+        true_path = str(path / ".".join(["tiny-data.txt", ext, "decomp"]))
+        # Setup a pooch in a temp dir
+        pup = Pooch(path=path, base_url=BASEURL, registry=REGISTRY)
+        # Check the warnings when downloading and from the processor
+        with warnings.catch_warnings(record=True) as warn:
+            fname = pup.fetch("tiny-data.txt." + ext, processor=processor)
+            assert len(warn) == 2
+            assert all(issubclass(w.category, UserWarning) for w in warn)
+            assert str(warn[-2].message).split()[0] == "Downloading"
+            assert str(warn[-1].message).startswith("Decompressing")
+            assert method in str(warn[-1].message)
+        assert fname == true_path
+        check_tiny_data(fname)
+        # Check that processor doesn't execute when not downloading
+        with warnings.catch_warnings(record=True) as warn:
+            fname = pup.fetch("tiny-data.txt." + ext, processor=processor)
+            assert not warn
+        assert fname == true_path
+        check_tiny_data(fname)
+
+
+def test_decompress_fails():
+    "Should fail if method='auto' and no extension is given in the file name"
+    with TemporaryDirectory() as local_store:
+        path = Path(local_store)
+        pup = Pooch(path=path, base_url=BASEURL, registry=REGISTRY)
+        with pytest.raises(ValueError) as exception:
+            with warnings.catch_warnings():
+                pup.fetch("tiny-data.txt", processor=Decompress(method="auto"))
+        assert exception.value.args[0].startswith("Unrecognized extension '.txt'")
+        # Should also fail for a bad method name
+        with pytest.raises(ValueError) as exception:
+            with warnings.catch_warnings():
+                pup.fetch("tiny-data.txt", processor=Decompress(method="bla"))
+        assert exception.value.args[0].startswith("Invalid compression method 'bla'")
+
+
+@pytest.mark.parametrize(
+    "method,ext", [("lzma", "xz"), ("bzip2", "bz2")], ids=["lzma", "bz2"]
+)
+def test_decompress_27_missing_dependencies(method, ext):
+    "Raises an exception when missing extra dependencies for 2.7"
+    decompress = Decompress(method=method)
+    decompress.modules[method] = None
+    with pytest.raises(ValueError) as exception:
+        with warnings.catch_warnings():
+            decompress(fname="meh.txt." + ext, action="download", pooch=None)
+    assert method in exception.value.args[0]
+
+
 def test_extractprocessor_fails():
     "The base class should be used and should fail when passed to fecth"
     with TemporaryDirectory() as local_store:
diff --git a/pooch/tests/utils.py b/pooch/tests/utils.py
index 9ea7f33..65e6c3a 100644
--- a/pooch/tests/utils.py
+++ b/pooch/tests/utils.py
@@ -57,5 +57,8 @@ def pooch_test_registry():
         "store.zip": "0498d2a001e71051bbd2acd2346f38da7cbd345a633cb7bf0f8a20938714b51a",
         "tiny-data.tar.gz": "41503f083814f43a01a8e9a30c28d7a9fe96839a99727a7fdd0acf7cd5bab63b",
         "store.tar.gz": "088c7f4e0f1859b1c769bb6065de24376f366374817ede8691a6ac2e49f29511",
+        "tiny-data.txt.bz2": "753663687a4040c90c8578061867d1df623e6aa8011c870a5dbd88ee3c82e306",
+        "tiny-data.txt.gz": "2e2da6161291657617c32192dba95635706af80c6e7335750812907b58fd4b52",
+        "tiny-data.txt.xz": "99dcb5c32a6e916344bacb4badcbc2f2b6ee196977d1d8187610c21e7e607765",
     }
     return registry