Repository: rasterio/rasterio

Issue:
MemoryFile and seek
# Expected behavior and actual behavior.

I am trying to download a file from AWS to a MemoryFile using boto3 (Sentinel 2 JP2). I would like to use `client.download_fileobj()` because I have finer control over the download : 

https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-example-download-file.html

## Steps to reproduce the problem.

```
import rasterio.io
import boto3

with rasterio.io.MemoryFile() as memfile:
    boto3.client('s3').download_fileobj(
        Fileobj=memfile,
        Bucket='sentinel-s2-l1c',
        Key='tiles/15/R/TN/2019/3/28/0/B02.jp2',
        ExtraArgs={'RequestPayer': 'requester'},
        Config=None)
```
Raises:

```
ValueError: seek position past end of file: 8388608
```

This works fine with io.BytesIO or the GCP storage client `blob.download_to_file()`.
There is a difference of behavior regarding the seek on positions w.r.t. BytesIO (the problem is probably linked to the multithreading in the AWS case?).

## Operating system
Ubuntu 18

## Rasterio version and provenance
rasterio-1.1.4-cp38-cp38-manylinux1_x86_64.whl



Discussion:
@thomascoquet thank you for the report. MemoryFile doesn't support multi-part writing. I'd like to see if we can, will report back here.
@thomascoquet I've done it in the past but in a slightly different way
https://github.com/developmentseed/cogeo-watchbot-light/blob/master/app/translator.py#L22-L26

```python
def _download_obj(bucket: str, key: str) -> BytesIO:
    buff = BytesIO()
    client.download_fileobj(bucket, key, buff)
    buff.seek(0)
    return buff

with rasterio.open(_download_obj("bucket", "key")) as src:
```


@vincentsarago I ended up doing something similar with BytesIO, thanks for the reply!
Though if you use TransferConfig https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig to disable concurrency, download_fileobj should (untested) work and then you can remove the configuration when we fix this in rasterio.
Confirmed, no error from this.

```python
import rasterio.io
import boto3
from boto3.s3.transfer import TransferConfig


with rasterio.io.MemoryFile() as memfile:
    boto3.client('s3').download_fileobj(
        Fileobj=memfile,
        Bucket='sentinel-s2-l1c',
        Key='tiles/15/R/TN/2019/3/28/0/B02.jp2',
        ExtraArgs={'RequestPayer': 'requester'},
        Config=TransferConfig(use_threads=False))
```
Leaving this as a note to myself: https://github.com/boto/boto3/issues/1304.

Resolving patch:
diff --git a/rasterio/io.py b/rasterio/io.py
--- a/rasterio/io.py
+++ b/rasterio/io.py
@@ -85,7 +85,7 @@ class MemoryFile(MemoryFileBase):
      'width': 791}
 
     """
-    def __init__(self, file_or_bytes=None, filename=None, ext=''):
+    def __init__(self, file_or_bytes=None, dirname=None, filename=None, ext=''):
         """Create a new file in memory
 
         Parameters
@@ -102,7 +102,7 @@ def __init__(self, file_or_bytes=None, filename=None, ext=''):
         MemoryFile
         """
         super(MemoryFile, self).__init__(
-            file_or_bytes=file_or_bytes, filename=filename, ext=ext)
+            file_or_bytes=file_or_bytes, dirname=dirname, filename=filename, ext=ext)
 
     @ensure_env
     def open(self, driver=None, width=None, height=None, count=None, crs=None,
@@ -125,7 +125,7 @@ def open(self, driver=None, width=None, height=None, count=None, crs=None,
 
         if self.closed:
             raise IOError("I/O operation on closed file.")
-        if self.exists():
+        if len(self) > 0:
             log.debug("VSI path: {}".format(mempath.path))
             return DatasetReader(mempath, driver=driver, sharing=sharing, **kwargs)
         else:


Test patch:
diff --git a/tests/test_memoryfile.py b/tests/test_memoryfile.py
--- a/tests/test_memoryfile.py
+++ b/tests/test_memoryfile.py
@@ -49,6 +49,13 @@ def rgb_data_and_profile(path_rgb_byte_tif):
     return data, profile
 
 
+def test_initial_empty():
+    with MemoryFile() as memfile:
+        assert len(memfile) == 0
+        assert len(memfile.getbuffer()) == 0
+        assert memfile.tell() == 0
+
+
 def test_initial_not_bytes():
     """Creating a MemoryFile from not bytes fails."""
     with pytest.raises(TypeError):
@@ -116,6 +123,21 @@ def test_non_initial_bytes_in_two(rgb_file_bytes):
             assert src.read().shape == (3, 718, 791)
 
 
+def test_non_initial_bytes_in_two_reverse(rgb_file_bytes):
+    """MemoryFile contents can be read from bytes in two steps, tail first, and opened.
+    Demonstrates fix of #1926."""
+    with MemoryFile() as memfile:
+        memfile.seek(600000)
+        assert memfile.write(rgb_file_bytes[600000:]) == len(rgb_file_bytes) - 600000
+        memfile.seek(0)
+        assert memfile.write(rgb_file_bytes[:600000]) == 600000
+        with memfile.open() as src:
+            assert src.driver == "GTiff"
+            assert src.count == 3
+            assert src.dtypes == ("uint8", "uint8", "uint8")
+            assert src.read().shape == (3, 718, 791)
+
+
 def test_no_initial_bytes(rgb_data_and_profile):
     """An empty MemoryFile can be opened and written into."""
     data, profile = rgb_data_and_profile
@@ -264,10 +286,10 @@ def test_memfile_copyfiles(path_rgb_msk_byte_tif):
     """Multiple files can be copied to a MemoryFile using copyfiles"""
     with rasterio.open(path_rgb_msk_byte_tif) as src:
         src_basename = os.path.basename(src.name)
-        with MemoryFile(filename=src_basename) as memfile:
+        with MemoryFile(dirname="foo", filename=src_basename) as memfile:
             copyfiles(src.name, memfile.name)
             with memfile.open() as rgb2:
-                assert sorted(rgb2.files) == sorted(['/vsimem/{}'.format(src_basename), '/vsimem/{}.msk'.format(src_basename)])
+                assert sorted(rgb2.files) == sorted(['/vsimem/foo/{}'.format(src_basename), '/vsimem/foo/{}.msk'.format(src_basename)])
 
 
 def test_multi_memfile(path_rgb_msk_byte_tif):
@@ -277,9 +299,9 @@ def test_multi_memfile(path_rgb_msk_byte_tif):
     with open(path_rgb_msk_byte_tif + '.msk', 'rb') as msk_fp:
         msk_bytes = msk_fp.read()
 
-    with MemoryFile(tif_bytes, filename='foo.tif') as tifmemfile, MemoryFile(msk_bytes, filename='foo.tif.msk') as mskmemfile:
+    with MemoryFile(tif_bytes, dirname="bar", filename='foo.tif') as tifmemfile, MemoryFile(msk_bytes, dirname="bar", filename='foo.tif.msk') as mskmemfile:
         with tifmemfile.open() as src:
-            assert sorted(src.files) == sorted(['/vsimem/foo.tif', '/vsimem/foo.tif.msk'])
+            assert sorted(os.path.basename(fn) for fn in src.files) == sorted(['foo.tif', 'foo.tif.msk'])
             assert src.mask_flag_enums == ([MaskFlags.per_dataset],) * 3