Repository: azavea/raster-vision

Issue:
Unit Tests for `filesystem`
![screenshot from 2018-10-11 14-31-49](https://user-images.githubusercontent.com/11281373/46825966-d216d180-cd62-11e8-8a8f-91109b94c81f.png)



Discussion:

Resolving patch:
diff --git a/rastervision/filesystem/filesystem.py b/rastervision/filesystem/filesystem.py
--- a/rastervision/filesystem/filesystem.py
+++ b/rastervision/filesystem/filesystem.py
@@ -29,32 +29,32 @@ def matches_uri(uri: str, mode: str) -> bool:
 
         Mode can be 'r' or 'w'
         """
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def file_exists(uri: str) -> bool:
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def read_str(uri: str) -> str:
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def read_bytes(uri: str) -> bytes:
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def write_str(uri: str, data: str) -> None:
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def write_bytes(uri: str, data: bytes) -> None:
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
@@ -67,7 +67,7 @@ def sync_to_dir(src_dir_uri: str, dest_dir_uri: str,
            - dest_dir_uri: A destination that can be sync to by this FileSystem.
            - delete: True if the destination should be deleted first. Defaults to False.
         """
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
@@ -81,22 +81,22 @@ def sync_from_dir(src_dir_uri: str,
            - dest_dir_uri: A destination that can be sync to by this FileSystem.
            - delete: True if the destination should be deleted first. Defaults to False.
         """
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def copy_to(src_path: str, dst_uri: str) -> None:
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def copy_from(uri: str, path: str) -> None:
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def local_path(uri: str, download_dir: str) -> None:
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
@@ -104,9 +104,9 @@ def last_modified(uri: str) -> datetime:
         """Returns the last modified  date in UTC of this URI,
         or None if this FileSystem does not support this operation.
         """
-        pass
+        pass  # pragma: no cover
 
     @staticmethod
     @abstractmethod
     def list_paths(uri, ext=None):
-        pass
+        pass  # pragma: no cover
diff --git a/rastervision/filesystem/http_filesystem.py b/rastervision/filesystem/http_filesystem.py
--- a/rastervision/filesystem/http_filesystem.py
+++ b/rastervision/filesystem/http_filesystem.py
@@ -12,11 +12,8 @@
 class HttpFileSystem(FileSystem):
     @staticmethod
     def matches_uri(uri: str, mode: str) -> bool:
-        if mode == 'r':
-            parsed_uri = urlparse(uri)
-            return parsed_uri.scheme in ['http', 'https']
-        else:
-            return False
+        parsed_uri = urlparse(uri)
+        return parsed_uri.scheme in ['http', 'https']
 
     @staticmethod
     def file_exists(uri: str) -> bool:
@@ -25,7 +22,7 @@ def file_exists(uri: str) -> bool:
             if response.getcode() == 200:
                 return int(response.headers['content-length']) > 0
             else:
-                return False
+                return False  # pragma: no cover
         except urllib.error.URLError:
             return False
 
@@ -68,7 +65,7 @@ def copy_from(uri: str, path: str) -> None:
             with open(path, 'wb') as out_file:
                 try:
                     shutil.copyfileobj(response, out_file)
-                except Exception:
+                except Exception:  # pragma: no cover
                     raise NotReadableError('Could not read {}'.format(uri))
 
     @staticmethod
@@ -83,5 +80,5 @@ def last_modified(uri: str) -> datetime:
         return None
 
     @staticmethod
-    def list_paths(uri, suffix=None):
+    def list_paths(uri, suffix=None):  # pragma: no cover
         raise NotImplementedError()
diff --git a/rastervision/filesystem/s3_filesystem.py b/rastervision/filesystem/s3_filesystem.py
--- a/rastervision/filesystem/s3_filesystem.py
+++ b/rastervision/filesystem/s3_filesystem.py
@@ -132,7 +132,7 @@ def write_bytes(uri: str, data: bytes) -> None:
     @staticmethod
     def sync_from_dir(src_dir_uri: str,
                       dest_dir_uri: str,
-                      delete: bool = False) -> None:
+                      delete: bool = False) -> None:  # pragma: no cover
         command = ['aws', 's3', 'sync', src_dir_uri, dest_dir_uri]
         if delete:
             command.append('--delete')
@@ -140,7 +140,7 @@ def sync_from_dir(src_dir_uri: str,
 
     @staticmethod
     def sync_to_dir(src_dir_uri: str, dest_dir_uri: str,
-                    delete: bool = False) -> None:
+                    delete: bool = False) -> None:  # pragma: no cover
         command = ['aws', 's3', 'sync', src_dir_uri, dest_dir_uri]
         if delete:
             command.append('--delete')
@@ -189,7 +189,7 @@ def last_modified(uri: str) -> datetime:
         return head_data['LastModified']
 
     @staticmethod
-    def list_paths(uri, ext=None):
+    def list_paths(uri, ext=''):
         parsed_uri = urlparse(uri)
         bucket = parsed_uri.netloc
         prefix = os.path.join(parsed_uri.path[1:])


Test patch:
diff --git a/tests/utils/test_files.py b/tests/utils/test_files.py
--- a/tests/utils/test_files.py
+++ b/tests/utils/test_files.py
@@ -1,23 +1,22 @@
 import os
 import unittest
 import json
+import datetime
 
 import boto3
 from moto import mock_s3
 
 import rastervision as rv
-from rastervision.utils.files import (file_to_str, str_to_file,
-                                      download_if_needed, upload_or_copy,
-                                      load_json_config, ProtobufParseException,
-                                      make_dir, get_local_path, file_exists)
+from rastervision.utils.files import (
+    file_to_str, str_to_file, download_if_needed, upload_or_copy,
+    load_json_config, ProtobufParseException, make_dir, get_local_path,
+    file_exists, sync_from_dir, sync_to_dir)
 from rastervision.filesystem import (NotReadableError, NotWritableError)
+from rastervision.filesystem.filesystem import FileSystem
 from rastervision.protos.task_pb2 import TaskConfig as TaskConfigMsg
 from rastervision.rv_config import RVConfig
 
-
-class TestMakeDir(unittest.TestCase):
-    def setUp(self):
-        self.lorem = """ Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
+LOREM = """ Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
         eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut
         enim ad minim veniam, quis nostrud exercitation ullamco
         laboris nisi ut aliquip ex ea commodo consequat. Duis aute
@@ -26,6 +25,11 @@ def setUp(self):
         cupidatat non proident, sunt in culpa qui officia deserunt
         mollit anim id est laborum.  """
 
+
+class TestMakeDir(unittest.TestCase):
+    def setUp(self):
+        self.lorem = LOREM
+
         # Mock S3 bucket
         self.mock_s3 = mock_s3()
         self.mock_s3.start()
@@ -79,16 +83,6 @@ def test_file_exists_s3_false(self):
         s3_path = 's3://{}/hello.txt'.format(self.bucket_name)
         self.assertFalse(file_exists(s3_path))
 
-    def test_file_exists_http_true(self):
-        http_path = ('https://raw.githubusercontent.com/tensorflow/models/'
-                     '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md')
-        self.assertTrue(file_exists(http_path))
-
-    def test_file_exists_http_false(self):
-        http_path = ('https://raw.githubusercontent.com/tensorflow/models/'
-                     '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/XXX')
-        self.assertFalse(file_exists(http_path))
-
     def test_check_empty(self):
         path = os.path.join(self.temp_dir.name, 'hello', 'hello.txt')
         dir = os.path.dirname(path)
@@ -294,5 +288,214 @@ def test_invalid_json(self):
             load_json_config(self.file_path, TaskConfigMsg())
 
 
+class TestS3Misc(unittest.TestCase):
+    def setUp(self):
+        self.lorem = LOREM
+
+        # Mock S3 bucket
+        self.mock_s3 = mock_s3()
+        self.mock_s3.start()
+        self.s3 = boto3.client('s3')
+        self.bucket_name = 'mock_bucket'
+        self.s3.create_bucket(Bucket=self.bucket_name)
+
+        # Temporary directory
+        self.temp_dir = RVConfig.get_tmp_dir()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+        self.mock_s3.stop()
+
+    def test_last_modified_s3(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum1.txt')
+        s3_path = 's3://{}/lorem1.txt'.format(self.bucket_name)
+        directory = os.path.dirname(path)
+        make_dir(directory, check_empty=False)
+
+        fs = FileSystem.get_file_system(s3_path, 'r')
+
+        with open(path, 'w+') as file:
+            file.write(self.lorem)
+        upload_or_copy(path, s3_path)
+        stamp = fs.last_modified(s3_path)
+
+        self.assertTrue(isinstance(stamp, datetime.datetime))
+
+    def test_list_paths_s3(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        s3_path = 's3://{}/xxx/lorem.txt'.format(self.bucket_name)
+        s3_directory = 's3://{}/xxx/'.format(self.bucket_name)
+        directory = os.path.dirname(path)
+        make_dir(directory, check_empty=False)
+
+        fs = FileSystem.get_file_system(s3_path, 'r')
+
+        with open(path, 'w+') as file:
+            file.write(self.lorem)
+        upload_or_copy(path, s3_path)
+
+        self.assertEqual(len(fs.list_paths(s3_directory)), 1)
+
+
+class TestLocalMisc(unittest.TestCase):
+    def setUp(self):
+        self.lorem = LOREM
+        self.temp_dir = RVConfig.get_tmp_dir()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
+    def test_bytes_local(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        directory = os.path.dirname(path)
+        make_dir(directory, check_empty=False)
+
+        expected = bytes([0x00, 0x01, 0x02])
+        fs = FileSystem.get_file_system(path, 'r')
+
+        fs.write_bytes(path, expected)
+        actual = fs.read_bytes(path)
+
+        self.assertEqual(actual, expected)
+
+    def test_bytes_local_false(self):
+        path = os.path.join(self.temp_dir.name, 'xxx')
+        fs = FileSystem.get_file_system(path, 'r')
+        self.assertRaises(NotReadableError, lambda: fs.read_bytes(path))
+
+    def test_sync_from_dir_local(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        src = os.path.dirname(path)
+        dst = os.path.join(self.temp_dir.name, 'xxx')
+        make_dir(src, check_empty=False)
+        make_dir(dst, check_empty=False)
+
+        fs = FileSystem.get_file_system(path, 'r')
+        fs.write_bytes(path, bytes([0x00, 0x01]))
+        sync_from_dir(src, dst, delete=True)
+
+        self.assertEqual(len(fs.list_paths(dst)), 1)
+
+    def test_sync_from_dir_noop_local(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        src = os.path.join(self.temp_dir.name, 'lorem')
+        make_dir(src, check_empty=False)
+
+        fs = FileSystem.get_file_system(src, 'r')
+        fs.write_bytes(path, bytes([0x00, 0x01]))
+        sync_from_dir(src, src, delete=True)
+
+        self.assertEqual(len(fs.list_paths(src)), 1)
+
+    def test_sync_to_dir_local(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        src = os.path.dirname(path)
+        dst = os.path.join(self.temp_dir.name, 'xxx')
+        make_dir(src, check_empty=False)
+        make_dir(dst, check_empty=False)
+
+        fs = FileSystem.get_file_system(path, 'r')
+        fs.write_bytes(path, bytes([0x00, 0x01]))
+        sync_to_dir(src, dst, delete=True)
+
+        self.assertEqual(len(fs.list_paths(dst)), 1)
+
+    def test_copy_to_local(self):
+        path1 = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        path2 = os.path.join(self.temp_dir.name, 'yyy', 'ipsum.txt')
+        dir1 = os.path.dirname(path1)
+        dir2 = os.path.dirname(path2)
+        make_dir(dir1, check_empty=False)
+        make_dir(dir2, check_empty=False)
+
+        with open(path1, 'w+') as file:
+            file.write(self.lorem)
+
+        fs = FileSystem.get_file_system(path1, 'r')
+        upload_or_copy(path1, path2)
+        self.assertEqual(len(fs.list_paths(dir2)), 1)
+
+    def test_last_modified(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum1.txt')
+        directory = os.path.dirname(path)
+        make_dir(directory, check_empty=False)
+
+        fs = FileSystem.get_file_system(path, 'r')
+
+        with open(path, 'w+') as file:
+            file.write(self.lorem)
+        stamp = fs.last_modified(path)
+
+        self.assertTrue(isinstance(stamp, datetime.datetime))
+
+
+class TestHttpMisc(unittest.TestCase):
+    def setUp(self):
+        self.lorem = LOREM
+        self.temp_dir = RVConfig.get_tmp_dir()
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
+    def test_file_exists_http_true(self):
+        http_path = ('https://raw.githubusercontent.com/tensorflow/models/'
+                     '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md')
+        self.assertTrue(file_exists(http_path))
+
+    def test_file_exists_http_false(self):
+        http_path = ('https://raw.githubusercontent.com/tensorflow/models/'
+                     '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/XXX')
+        self.assertFalse(file_exists(http_path))
+
+    def test_write_str_http(self):
+        self.assertRaises(NotWritableError,
+                          lambda: str_to_file('xxx', 'http://localhost/'))
+
+    def test_sync_to_http(self):
+        src = self.temp_dir.name
+        dst = 'http://localhost/'
+        self.assertRaises(NotWritableError, lambda: sync_to_dir(src, dst))
+
+    def test_sync_from_http(self):
+        src = 'http://localhost/'
+        dst = self.temp_dir.name
+        self.assertRaises(NotReadableError, lambda: sync_from_dir(src, dst))
+
+    def test_copy_to_http(self):
+        path = os.path.join(self.temp_dir.name, 'lorem', 'ipsum.txt')
+        dst = 'http://localhost/'
+        directory = os.path.dirname(path)
+        make_dir(directory, check_empty=False)
+
+        with open(path, 'w+') as file:
+            file.write(self.lorem)
+
+        self.assertRaises(NotWritableError, lambda: upload_or_copy(path, dst))
+        os.remove(path)
+
+    def test_copy_from_http(self):
+        http_path = ('https://raw.githubusercontent.com/tensorflow/models/'
+                     '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md')
+        expected = os.path.join(
+            self.temp_dir.name, 'http', 'raw.githubusercontent.com',
+            'tensorflow/models',
+            '17fa52864bfc7a7444a8b921d8a8eb1669e14ebd/README.md')
+        download_if_needed(http_path, self.temp_dir.name)
+
+        self.assertTrue(file_exists(expected))
+        os.remove(expected)
+
+    def test_last_modified_http(self):
+        uri = 'http://localhost/'
+        fs = FileSystem.get_file_system(uri, 'r')
+        self.assertEqual(fs.last_modified(uri), None)
+
+    def test_write_bytes_http(self):
+        uri = 'http://localhost/'
+        fs = FileSystem.get_file_system(uri, 'r')
+        self.assertRaises(NotWritableError,
+                          lambda: fs.write_bytes(uri, bytes([0x00, 0x01])))
+
+
 if __name__ == '__main__':
     unittest.main()