Repository: mapbox/mapbox-sdk-py
Pull request #97 (2015-12-14 22:38:28)

accept file objects instead of paths
This is a breaking change to the API (affects both `.upload` and `.stage` methods of the service)

Working but needs
* [x] doctest changes
* [x] test with other file handlers like StringIO
* [x] changelog
* [x] apply to CLI and kick the tires a bit




Patch:
diff --git a/CHANGES b/CHANGES
index 1dd9353..b0c1223 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,8 @@
+0.6.0 (TBD)
+------------------
+- The Upload API methods `upload` and `stage` take an open file object
+  instead of a file path. Client code will need to be updated.
+
 0.5.0 (2015-12-08)
 ------------------
 - We've vendorized the `polyline` module to work around its out of date `six`
diff --git a/docs/uploads.md b/docs/uploads.md
index 412c960..60484d7 100644
--- a/docs/uploads.md
+++ b/docs/uploads.md
@@ -35,11 +35,13 @@ In the example below, we use a string defined in a test fixture.
 >>> from time import sleep
 >>> from random import randint
 >>> mapid = getfixture('uploads_dest_id') # 'uploads-test'
->>> upload_resp = service.upload('tests/twopoints.geojson', mapid)
+>>> with open('tests/twopoints.geojson', 'r') as src:
+...     upload_resp = service.upload(src, mapid)
 >>> if upload_resp.status_code == 409:
 ...     for i in range(5):
 ...         sleep(5)
-...         upload_resp = service.upload('tests/twopoints.geojson', mapid)
+...         with open('tests/twopoints.geojson', 'r') as src:
+...             upload_resp = service.upload(src, mapid)
 ...         if upload_resp.status_code != 409:
 ...             break
 
diff --git a/mapbox/errors.py b/mapbox/errors.py
index 659072d..90977d3 100644
--- a/mapbox/errors.py
+++ b/mapbox/errors.py
@@ -39,5 +39,5 @@ class InvalidParameterError(ValidationError):
     pass
 
 
-class FileIOError(ValidationError):
+class InvalidFileError(ValidationError):
     pass
diff --git a/mapbox/services/uploads.py b/mapbox/services/uploads.py
index 8761623..8798e7c 100644
--- a/mapbox/services/uploads.py
+++ b/mapbox/services/uploads.py
@@ -1,11 +1,9 @@
 # mapbox
-import os
-
 from boto3.session import Session as boto3_session
 from uritemplate import URITemplate
 
 from .base import Service
-from mapbox import errors
+from mapbox.errors import InvalidFileError
 
 
 class Uploader(Service):
@@ -16,7 +14,7 @@ class Uploader(Service):
         from mapbox import Uploader
 
         u = Uploader()
-        url = u.stage('test.tif')
+        url = u.stage(open('test.tif'))
         job = u.create(url, 'test1').json()
 
         assert job in u.list().json()
@@ -45,11 +43,16 @@ class Uploader(Service):
                 404: "Token does not have upload scope"})
         return resp
 
-    def stage(self, filepath, creds=None):
+    def stage(self, fileobj, creds=None):
         """Stages the user's file on S3
         If creds are not provided, temporary credientials will be generated
         Returns the URL to the staged resource.
         """
+        if not hasattr(fileobj, 'read'):
+            raise InvalidFileError(
+                "Object `{0}` has no .read method, "
+                "a file-like object is required".format(fileobj))
+
         if not creds:
             res = self._get_credentials()
             creds = res.json()
@@ -61,11 +64,7 @@ class Uploader(Service):
             region_name="us-east-1")
 
         s3 = session.resource('s3')
-        if not os.path.exists(filepath):
-            raise errors.FileIOError(
-                "{0} does not exist".format(filepath))
-        with open(filepath, 'rb') as data:
-            res = s3.Object(creds['bucket'], creds['key']).put(Body=data)
+        res = s3.Object(creds['bucket'], creds['key']).put(Body=fileobj)
 
         return creds['url']
 
@@ -144,10 +143,10 @@ class Uploader(Service):
         self.handle_http_error(resp)
         return resp
 
-    def upload(self, filepath, tileset, name=None):
-        """High level function to upload a local file to mapbox tileset
+    def upload(self, fileobj, tileset, name=None):
+        """High level function to upload a file object to mapbox tileset
         Effectively replicates the upload functionality using the HTML form
         Returns a response object where the json() is a dict with upload metadata
         """
-        url = self.stage(filepath)
+        url = self.stage(fileobj)
         return self.create(url, tileset, name=name)


Test patch:
diff --git a/tests/test_upload.py b/tests/test_upload.py
index eedb66d..1b350bb 100644
--- a/tests/test_upload.py
+++ b/tests/test_upload.py
@@ -200,7 +200,8 @@ def test_stage(monkeypatch):
         body=query_body, status=200,
         content_type='application/json')
 
-    stage_url = mapbox.Uploader(access_token=access_token).stage('tests/moors.json')
+    with open('tests/moors.json', 'r') as src:
+        stage_url = mapbox.Uploader(access_token=access_token).stage(src)
     assert stage_url.startswith("https://tilestream-tilesets-production.s3.amazonaws.com/_pending")
 
 
@@ -233,14 +234,20 @@ def test_upload(monkeypatch):
         body=upload_response_body, status=201,
         content_type='application/json')
 
-    res = mapbox.Uploader(access_token=access_token).upload(
-        'tests/moors.json', 'test1')
+    with open('tests/moors.json', 'r') as src:
+        res = mapbox.Uploader(access_token=access_token).upload(src, 'test1')
+
     assert res.status_code == 201
     job = res.json()
     assert job['tileset'] == "{0}.test1".format(username)
 
+
 @responses.activate
-def test_upload_doesnotexist():
+def test_upload_error(monkeypatch):
+    """Upload a file and create a tileset, fails with 409"""
+
+    monkeypatch.setattr(mapbox.services.uploads, 'boto3_session', MockSession)
+
     # Credentials
     query_body = """
        {{"key": "_pending/{username}/key.test",
@@ -257,6 +264,21 @@ def test_upload_doesnotexist():
         body=query_body, status=200,
         content_type='application/json')
 
-    service = mapbox.Uploader(access_token=access_token)
-    with pytest.raises(mapbox.errors.ValidationError):
-        service.upload('tests/doesnotexist.gml', 'test-dne')
+    responses.add(
+        responses.POST,
+        'https://api.mapbox.com/uploads/v1/{0}?access_token={1}'.format(username, access_token),
+        match_querystring=True,
+        body="", status=409,
+        content_type='application/json')
+
+    with open('tests/moors.json', 'r') as src:
+        res = mapbox.Uploader(access_token=access_token).upload(src, 'test1')
+
+    assert res.status_code == 409
+
+
+def test_invalid_fileobj():
+    """Must be file object, not path"""
+    with pytest.raises(mapbox.errors.InvalidFileError):
+        mapbox.Uploader(access_token=access_token).upload(
+            'tests/moors.json', 'test1')