Repository: PyFilesystem/pyfilesystem2
Pull request #16 (2017-01-16 20:37:58)

Add support for tar (TarFS) and tests
This PR contains edits to support Tar archives (through python **tarfile** module).

I tried to be consistent with the ZipFS implementation, although since tar archives are much more Linux-friendly, there was no need for the mirrored MemoryFS for ReadTarFS.

For the tar formats, supported compressions are 'gz', 'bz2', and 'xz' for Python3 only. When a TarFS is created, the compression is guessed from the file extension, but if needed it can be given with the `compression` argument as well.

I also added tests, based on the zipfs tests, and tried to stay as clear as possible. Tell me if you have any questions. 


Patch:
diff --git a/fs/compress.py b/fs/compress.py
index 9afa9fe..09353d7 100644
--- a/fs/compress.py
+++ b/fs/compress.py
@@ -11,10 +11,13 @@ from __future__ import unicode_literals
 from datetime import datetime
 import time
 import zipfile
+import tarfile
 
 import six
 
+from .enums import ResourceType
 from .path import relpath
+from .time import datetime_to_epoch
 from .errors import NoSysPath
 from .walk import Walker
 
@@ -93,3 +96,88 @@ def write_zip(src_fs,
                     # Write from a file which is (presumably)
                     # more memory efficient
                     _zip.write(sys_path, zip_name)
+
+
+
+def write_tar(src_fs,
+              file,
+              compression=None,
+              encoding="utf-8",
+              walker=None):
+    """
+    Write the contents of a filesystem to a zip file.
+
+    :param file: Destination file, may be a file name or an open file
+        object.
+    :type file: str or file-like.
+    :param compression: Compression to use.
+    :type compression: str
+    :param encoding: The encoding to use for filenames. The default is
+        ``"utf-8"``.
+    :type encoding: str
+    :param walker: A :class:`~fs.walk.Walker` instance, or None to use
+        default walker. You can use this to specify which files you
+        want to compress.
+    :type walker: Walker or None
+
+    """
+
+    type_map = {
+        ResourceType.block_special_file: tarfile.BLKTYPE,
+        ResourceType.character: tarfile.CHRTYPE,
+        ResourceType.directory: tarfile.DIRTYPE,
+        ResourceType.fifo: tarfile.FIFOTYPE,
+        ResourceType.file: tarfile.REGTYPE,
+        ResourceType.socket: tarfile.AREGTYPE,   # no type for socket
+        ResourceType.symlink: tarfile.SYMTYPE,
+        ResourceType.unknown: tarfile.AREGTYPE,  # no type for unknown
+    }
+
+    mode = 'w:{}'.format(compression or '')
+    try:
+        _tar = tarfile.open(fileobj=file, mode=mode)
+    except (TypeError, AttributeError):
+        _tar = tarfile.open(file, mode=mode)
+
+    walker = walker or Walker()
+    with _tar:
+        gen_walk = walker.info(src_fs, namespaces=["details", "stat", "access"])
+        for path, info in gen_walk:
+            # Tar names must be relative
+            tar_name = relpath(path)
+            if not six.PY3:
+                # Python2 expects bytes filenames
+                tar_name = tar_name.encode(encoding, 'replace')
+
+            tar_info = tarfile.TarInfo(tar_name)
+
+            if info.has_namespace('stat'):
+                mtime = info.get('stat', 'st_mtime', None)\
+                                    or time.time()
+            else:
+                mtime = info.modified or time.time()
+
+            if isinstance(mtime, datetime):
+                mtime = datetime_to_epoch(mtime)
+            if isinstance(mtime, float):
+                mtime = int(mtime)
+            tar_info.mtime = mtime
+
+            for tarattr,infoattr in {'uid':'uid',
+                                     'gid':'gid',
+                                     'uname':'user',
+                                     'gname':'group'}.items():
+                if getattr(info, infoattr) is not None:
+                    setattr(tar_info, tarattr,
+                            getattr(info, infoattr))
+
+            tar_info.mode = getattr(info.permissions, 'mode', 420)
+
+            if info.is_dir:
+                tar_info.type = tarfile.DIRTYPE
+                _tar.addfile(tar_info)
+            else:
+                tar_info.type = type_map.get(info.type, tarfile.REGTYPE)
+                tar_info.size = info.size
+                with src_fs.openbin(path) as bin_file:
+                    _tar.addfile(tar_info, bin_file)
diff --git a/fs/opener.py b/fs/opener.py
index 2ac4d18..ae3e04a 100644
--- a/fs/opener.py
+++ b/fs/opener.py
@@ -344,6 +344,18 @@ class ZipOpener(Opener):
         )
         return zip_fs
 
+@registry.install
+class TarOpener(Opener):
+    protocols = ['tar']
+
+    def open_fs(self, fs_url, parse_result, writeable, create, cwd):
+        from .tarfs import TarFS
+        tar_fs = TarFS(
+            parse_result.resource,
+            write=create
+        )
+        return tar_fs
+
 
 @registry.install
 class FTPOpener(Opener):
diff --git a/fs/tarfs.py b/fs/tarfs.py
new file mode 100644
index 0000000..7be1a24
--- /dev/null
+++ b/fs/tarfs.py
@@ -0,0 +1,344 @@
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import tarfile
+import six
+
+from . import errors
+from .base import FS
+from .compress import write_tar
+from .enums import ResourceType
+from .info import Info
+from .iotools import RawWrapper
+from .opener import open_fs
+from .path import dirname, normpath, relpath, basename
+from .wrapfs import WrapFS
+from .permissions import Permissions
+
+
+class TarFS(WrapFS):
+    """
+    Read and write tar files.
+
+    There are two ways to open a TarFS for the use cases of reading
+    a tar file, and creating a new one.
+
+    If you open the TarFS with  ``write`` set to ``False`` (the
+    default), then the filesystem will be a read only filesystem which
+    maps to the files and directories within the tar file. Files are
+    decompressed on the fly when you open them.
+
+    Here's how you might extract and print a readme from a tar file::
+
+        with TarFS('foo.tar.gz') as tar_fs:
+            readme = tar_fs.gettext('readme.txt')
+
+    If you open the TarFS with ``write`` set to ``True``, then the TarFS
+    will be a empty temporary filesystem. Any files / directories you
+    create in the TarFS will be written in to a tar file when the TarFS
+    is closed. The compression is set from the new file name but may be
+    set manually with the ``compression`` argument.
+
+    Here's how you might write a new tar file containing a readme.txt
+    file::
+
+        with TarFS('foo.tar.xz', write=True) as new_tar:
+            new_tar.settext(
+                'readme.txt',
+                'This tar file was written by PyFilesystem'
+            )
+
+    :param file: An OS filename, or a open file object.
+    :type file: str or file
+    :param write: Set to ``True`` to write a new tar file, or ``False``
+        to read an existing tar file.
+    :type write: bool
+    :param compression:  Compression to use (one of the formats
+        supported by ``tarfile``: ``xz``, ``gz``, ``bz2``, or None).
+    :type compression: str
+    :param temp_fs: An opener string for the temporary filesystem
+        used to store data prior to tarring.
+    :type temp_fs: str
+
+    """
+
+    _compression_formats = {
+        #FMT    #UNIX      #MSDOS
+        'xz':  ('.tar.xz',  '.txz'),
+        'bz2': ('.tar.bz2', '.tbz'),
+        'gz':  ('.tar.gz',  '.tgz'),
+    }
+
+    def __new__(cls,
+                file,
+                write=False,
+                compression=None,
+                encoding="utf-8",
+                temp_fs="temp://__tartemp__"):
+
+        filename = str(getattr(file, 'name', file))
+
+        if write and compression is None:
+            compression = None
+            for comp, extensions in six.iteritems(cls._compression_formats):
+                if filename.endswith(extensions):
+                    compression = comp
+                    break
+
+        if write:
+            return WriteTarFS(file,
+                              compression=compression,
+                              encoding=encoding,
+                              temp_fs=temp_fs)
+        # else:
+        return ReadTarFS(file, encoding=encoding)
+
+
+@six.python_2_unicode_compatible
+class WriteTarFS(WrapFS):
+    """A writable tar file."""
+
+    def __init__(self,
+                 file,
+                 compression=None,
+                 encoding="utf-8",
+                 temp_fs="temp://__tartemp__"):
+        self._file = file
+        self.compression = compression
+        self.encoding = encoding
+        self._temp_fs_url = temp_fs
+        self._temp_fs = open_fs(temp_fs)
+        self._meta = self._temp_fs.getmeta().copy()
+        super(WriteTarFS, self).__init__(self._temp_fs)
+
+    def __repr__(self):
+        t = "WriteTarFS({!r}, compression={!r}, encoding={!r}, temp_fs={!r})"
+        return t.format(
+            self._file,
+            self.compression,
+            self.encoding,
+            self._temp_fs_url
+        )
+
+    def __str__(self):
+        return "<TarFS-write '{}'>".format(self._file)
+
+    def delegate_path(self, path):
+        return self._temp_fs, path
+
+    def delegate_fs(self):
+        return self._temp_fs
+
+    def close(self):
+        if not self.isclosed():
+            try:
+                self.write_tar()
+            finally:
+                self._temp_fs.close()
+        super(WriteTarFS, self).close()
+
+    def write_tar(self, file=None, compression=None, encoding=None):
+        """
+        Write tar to a file.
+
+        ..note ::
+            This is called automatically when the TarFS is closed.
+
+        :param file: Destination file, may be a file name or an open
+            file object.
+        :type file: str or file-like
+        :param compression: Compression to use (one of the constants
+            defined in the tarfile module in the stdlib).
+
+        """
+        if not self.isclosed():
+            write_tar(
+                self._temp_fs,
+                file or self._file,
+                compression=compression or self.compression,
+                encoding=encoding or self.encoding
+            )
+
+
+@six.python_2_unicode_compatible
+class ReadTarFS(FS):
+    """A readable tar file."""
+
+    _meta = {
+        'case_insensitive': True,
+        'network': False,
+        'read_only': True,
+        'supports_rename': False,
+        'thread_safe': True,
+        'unicode_paths': True,
+        'virtual': False,
+    }
+
+    _typemap = type_map = {
+        tarfile.BLKTYPE: ResourceType.block_special_file,
+        tarfile.CHRTYPE: ResourceType.character,
+        tarfile.DIRTYPE: ResourceType.directory,
+        tarfile.FIFOTYPE: ResourceType.fifo,
+        tarfile.REGTYPE: ResourceType.file,
+        tarfile.AREGTYPE: ResourceType.file,
+        tarfile.SYMTYPE: ResourceType.symlink,
+        tarfile.CONTTYPE: ResourceType.file,
+        tarfile.LNKTYPE: ResourceType.symlink,
+    }
+
+    def __init__(self, file, encoding='utf-8'):
+        super(ReadTarFS, self).__init__()
+        self._file = file
+        self.encoding = encoding
+        try:
+            self._tar = tarfile.open(fileobj=file, mode='r')
+        except (TypeError, AttributeError):
+            self._tar = tarfile.open(file, mode='r')
+        self._directory_fs = None
+
+    def __repr__(self):
+        return "ReadTarFS({!r})".format(self._file)
+
+    def __str__(self):
+        return "<TarFS '{}'>".format(self._file)
+
+    def getinfo(self, path, namespaces=None):
+        self.check()
+        namespaces = namespaces or ()
+        path = relpath(normpath(path))
+        if not path:
+            raw_info = {
+                "basic":
+                {
+                    "name": "",
+                    "is_dir": True,
+                },
+                "details":
+                {
+                    "type": int(ResourceType.directory)
+                }
+            }
+        else:
+            try:
+                member = self._tar.getmember(path)
+            except KeyError:
+                raise errors.ResourceNotFound(path)
+
+
+            raw_tar_info = member.get_info(
+                *([self.encoding, None]
+                if six.PY2 else [])
+            )
+
+            raw_tar_info.update({
+                k.replace('is', 'is_'):getattr(member, k)()
+                for k in dir(member)
+                if k.startswith('is')
+            })
+            raw_info = {
+                "basic":
+                {
+                    "name": basename(member.name),
+                    "is_dir": member.isdir(),
+                },
+                "details":
+                {
+                    "size": member.size,
+                    "type": int(self.type_map[member.type]),
+                    "modified": member.mtime,
+                },
+                "access": {
+                    "gid": member.gid,
+                    "group": member.gname,
+                    "permissions":\
+                        Permissions.create(member.mode).dump(),
+                    "uid": member.uid,
+                    "user": member.uname,
+                },
+                "tar": raw_tar_info,
+            }
+        return Info(raw_info)
+
+    def setinfo(self, path, info):
+        self.check()
+        raise errors.ResourceReadOnly(path)
+
+    def listdir(self, path):
+        self.check()
+        path = relpath(path)
+        if path:
+            try:
+                member = self._tar.getmember(path)
+            except KeyError:
+                six.raise_from(errors.ResourceNotFound(path), None)
+            else:
+                if not member.isdir():
+                    six.raise_from(errors.DirectoryExpected(path), None)
+
+        return [
+            basename(member.name)
+            for member in self._tar
+            if dirname(member.path)==path
+        ]
+
+    def makedir(self, path, permissions=None, recreate=False):
+        self.check()
+        raise errors.ResourceReadOnly(path)
+
+    def openbin(self, path, mode="r", buffering=-1, **kwargs):
+        self.check()
+        path = relpath(normpath(path))
+
+        if 'w' in mode or '+' in mode or 'a' in mode:
+            raise errors.ResourceReadOnly(path)
+
+        try:
+            member = self._tar.getmember(path)
+        except KeyError:
+            six.raise_from(errors.ResourceNotFound(path), None)
+
+        if not member.isfile():
+            raise errors.FileExpected(path)
+
+        rw = RawWrapper(self._tar.extractfile(member))
+
+        if six.PY2: # Patch nonexistent file.flush in Python2
+            def _flush():
+                pass
+            rw.flush = _flush
+
+        return rw
+
+    def remove(self, path):
+        self.check()
+        raise errors.ResourceReadOnly(path)
+
+    def removedir(self, path):
+        self.check()
+        raise errors.ResourceReadOnly(path)
+
+    def close(self):
+        super(ReadTarFS, self).close()
+        self._tar.close()
+
+    def isclosed(self):
+        return self._tar.closed
+
+
+if __name__ == "__main__":  # pragma: nocover
+    from fs.tree import render
+    from fs.opener import open_fs
+
+    with TarFS('tests.tar') as tar_fs:
+        print(tar_fs.listdir('/'))
+        print(tar_fs.listdir('/tests/'))
+        print(tar_fs.gettext('tests/ttt/settings.ini'))
+        render(tar_fs)
+        print(tar_fs)
+        print(repr(tar_fs))
+
+    with TarFS("TarFS.tar", write=True) as tar_fs:
+        tar_fs.makedirs('foo/bar')
+        tar_fs.settext('foo/bar/baz.txt', 'Hello, World')
+        print(tar_fs)
+        print(repr(tar_fs))
diff --git a/fs/zipfs.py b/fs/zipfs.py
index 970eeab..1967680 100644
--- a/fs/zipfs.py
+++ b/fs/zipfs.py
@@ -271,8 +271,10 @@ class ReadZipFS(FS):
         if 'w' in mode or '+' in mode or 'a' in mode:
             raise errors.ResourceReadOnly(path)
 
-        if not self._directory.isfile(path):
+        if not self._directory.exists(path):
             raise errors.ResourceNotFound(path)
+        elif self._directory.isdir(path):
+            raise errors.FileExpected(path)
 
         zip_name = self._path_to_zip_name(path)
         bin_file = self._zip.open(zip_name, 'r')
@@ -316,4 +318,3 @@ if __name__ == "__main__":  # pragma: nocover
         zip_fs.settext('foo/bar/baz.txt', 'Hello, World')
         print(zip_fs)
         print(repr(zip_fs))
-


Test patch:
diff --git a/fs/test.py b/fs/test.py
index 8665783..a6237f4 100644
--- a/fs/test.py
+++ b/fs/test.py
@@ -720,6 +720,10 @@ class FSTestCases(object):
             f.write(text)
         self.assert_bytes('foo/hello', text)
 
+        # Test FileExpected raised
+        with self.assertRaises(errors.FileExpected):
+            self.fs.openbin('foo') #directory
+
         # Open from missing dir
         with self.assertRaises(errors.ResourceNotFound):
             self.fs.openbin('/foo/bar/test.txt')
diff --git a/tests/test_archives.py b/tests/test_archives.py
index daec857..5bfb3ca 100644
--- a/tests/test_archives.py
+++ b/tests/test_archives.py
@@ -74,6 +74,10 @@ class ArchiveTestCases(object):
             sorted(self.source_fs.listdir('/')),
             sorted(self.fs.listdir('/'))
         )
+        with self.assertRaises(errors.DirectoryExpected):
+            self.fs.listdir('top.txt')
+        with self.assertRaises(errors.ResourceNotFound):
+            self.fs.listdir('nothere')
 
     def test_open(self):
         with self.fs.open('top.txt') as f:
@@ -90,6 +94,9 @@ class ArchiveTestCases(object):
         with self.assertRaises(errors.ResourceNotFound):
             with self.fs.open('nothere.txt') as f:
                 pass
+        with self.assertRaises(errors.FileExpected):
+            with self.fs.open('foo') as f:
+                pass
 
     def test_gets(self):
         self.assertEqual(
diff --git a/tests/test_opener.py b/tests/test_opener.py
index 4809e82..b573b3b 100644
--- a/tests/test_opener.py
+++ b/tests/test_opener.py
@@ -91,6 +91,20 @@ class TestRegistry(unittest.TestCase):
         finally:
             os.remove(zip_name)
 
+    def test_open_tarfs(self):
+        fh, tar_name = tempfile.mkstemp(suffix='.tar.gz')
+        os.close(fh)
+        try:
+            # Test creating tar
+            with opener.open_fs('tar://' + tar_name, create=True) as make_tar:
+                self.assertEqual(make_tar.compression, 'gz')
+                make_tar.settext('foo.txt', 'foofoo')
+            # Test opening tar
+            with opener.open_fs('tar://' + tar_name) as tar_fs:
+                self.assertEqual(tar_fs.gettext('foo.txt'), 'foofoo')
+        finally:
+            os.remove(tar_name)
+
     def test_open_fs(self):
         mem_fs = opener.open_fs("mem://")
         mem_fs_2 = opener.open_fs(mem_fs)
diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py
new file mode 100644
index 0000000..9fdbdf9
--- /dev/null
+++ b/tests/test_tarfs.py
@@ -0,0 +1,156 @@
+from __future__ import unicode_literals
+
+import os
+import six
+import gzip
+import tarfile
+import tempfile
+import unittest
+
+from fs import tarfs
+from fs import errors
+from fs.compress import write_tar
+from fs.opener import open_fs
+from fs.test import FSTestCases
+
+from .test_archives import ArchiveTestCases
+
+
+class TestWriteTarFS(FSTestCases, unittest.TestCase):
+    """
+    Test TarFS implementation.
+
+    When writing, a TarFS is essentially a TempFS.
+    """
+
+    def make_fs(self):
+        fh, _tar_file = tempfile.mkstemp()
+        os.close(fh)
+        fs = tarfs.TarFS(_tar_file, write=True)
+        fs._tar_file = _tar_file
+        return fs
+
+    def destroy_fs(self, fs):
+        fs.close()
+        del fs._tar_file
+
+class TestWriteTarFSToFileobj(FSTestCases, unittest.TestCase):
+    """
+    Test TarFS implementation.
+
+    When writing, a TarFS is essentially a TempFS.
+
+    """
+
+    def make_fs(self):
+        _tar_file = six.BytesIO()
+        fs = tarfs.TarFS(_tar_file, write=True)
+        fs._tar_file = _tar_file
+        return fs
+
+    def destroy_fs(self, fs):
+        fs.close()
+        del fs._tar_file
+
+class TestWriteGZippedTarFS(FSTestCases, unittest.TestCase):
+
+    def make_fs(self):
+        fh, _tar_file = tempfile.mkstemp()
+        os.close(fh)
+        fs = tarfs.TarFS(_tar_file, write=True, compression="gz")
+        fs._tar_file = _tar_file
+        return fs
+
+    def destroy_fs(self, fs):
+        fs.close()
+        del fs._tar_file
+
+    def assert_is_bzip(self):
+        try:
+            tarfile.open(fs._tar_file, 'r:gz')
+        except tarfile.ReadError:
+            self.fail("{} is not a valid gz archive".format(fs._tar_file))
+        for other_comps in ['xz', 'bz2', '']:
+            with self.assertRaises(tarfile.ReadError):
+                tarfile.open(fs._tar_file,
+                             'r:{}'.format(other_comps))
+
+@unittest.skipIf(six.PY2, "Python2 does not support LZMA")
+class TestWriteXZippedTarFS(FSTestCases, unittest.TestCase):
+
+    def make_fs(self):
+        fh, _tar_file = tempfile.mkstemp()
+        os.close(fh)
+        fs = tarfs.TarFS(_tar_file, write=True, compression="xz")
+        fs._tar_file = _tar_file
+        return fs
+
+    def destroy_fs(self, fs):
+        fs.close()
+        del fs._tar_file
+
+    def assert_is_xz(self):
+        try:
+            tarfile.open(fs._tar_file, 'r:xz')
+        except tarfile.ReadError:
+            self.fail("{} is not a valid xz archive".format(fs._tar_file))
+        for other_comps in ['bz2', 'gz', '']:
+            with self.assertRaises(tarfile.ReadError):
+                tarfile.open(fs._tar_file,
+                             'r:{}'.format(other_comps))
+
+class TestWriteBZippedTarFS(FSTestCases, unittest.TestCase):
+
+    def make_fs(self):
+        fh, _tar_file = tempfile.mkstemp()
+        os.close(fh)
+        fs = tarfs.TarFS(_tar_file, write=True, compression="gz")
+        fs._tar_file = _tar_file
+        return fs
+
+    def destroy_fs(self, fs):
+        fs.close()
+        del fs._tar_file
+
+    def assert_is_bzip(self):
+        try:
+            tarfile.open(fs._tar_file, 'r:bz2')
+        except tarfile.ReadError:
+            self.fail("{} is not a valid bz2 archive".format(fs._tar_file))
+        for other_comps in ['xz', 'gz', '']:
+            with self.assertRaises(tarfile.ReadError):
+                tarfile.open(fs._tar_file,
+                             'r:{}'.format(other_comps))
+
+class TestReadTarFS(ArchiveTestCases, unittest.TestCase):
+    """
+    Test Reading tar files.
+
+    """
+    def compress(self, fs):
+        fh, self._temp_path = tempfile.mkstemp()
+        os.close(fh)
+        write_tar(fs, self._temp_path)
+
+    def load_archive(self):
+        return tarfs.TarFS(self._temp_path)
+
+    def remove_archive(self):
+        os.remove(self._temp_path)
+
+    def test_read_from_fileobject(self):
+        try:
+            tarfs.TarFS(open(self._temp_path, 'rb'))
+        except:
+            self.fail("Couldn't open tarfs from fileobject")
+
+    def test_read_from_filename(self):
+        try:
+            tarfs.TarFS(self._temp_path)
+        except:
+            self.fail("Couldn't open tarfs from fileobject")
+
+class TestReadTarFSMem(TestReadTarFS):
+
+    def make_source_fs(self):
+        return open_fs('mem://')