Repository: mapbox/mapbox-sdk-py
Pull request #148 (2017-01-18 15:09:42)

Add a S3 upload callback and require boto3>=1.4
For version 0.11.0. This can support a progress bar for mapbox-upload from mapbox-cli-py (https://github.com/mapbox/mapbox-cli-py/pull/86).


Patch:
diff --git a/CHANGES b/CHANGES
index efa1af7..3fcb802 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,8 @@
+Next
+----
+- Require boto3>=1.4 to get `upload_fileobj()`.
+- Add a callback function for AWS SDK uploads to `uploads.stage()` (#147).
+
 0.10.1 (2016-11-22)
 -------------------
 - New requirement: depend on uritemplate>=2.0.
diff --git a/docs/geocoding.md b/docs/geocoding.md
index b4dc2c5..7dc1641 100644
--- a/docs/geocoding.md
+++ b/docs/geocoding.md
@@ -88,7 +88,7 @@ collection, sorted by relevance to the query.
 >>> first['type'] == 'Feature'
 True
 >>> sorted(first.keys())
-['bbox', 'center', 'context', 'geometry', 'id', 'place_name', 'properties', 'relevance', 'text', 'type']
+[u'bbox', u'center', u'context', u'geometry', u'id', u'place_name', u'place_type', u'properties', u'relevance', u'text', u'type']
 
 ```
 
diff --git a/mapbox/__init__.py b/mapbox/__init__.py
index 6b1a679..e73f3ff 100644
--- a/mapbox/__init__.py
+++ b/mapbox/__init__.py
@@ -1,5 +1,5 @@
 # mapbox
-__version__ = "0.10.1"
+__version__ = "0.11.0"
 
 from .services.datasets import Datasets
 from .services.directions import Directions
diff --git a/mapbox/services/uploads.py b/mapbox/services/uploads.py
index b4844e4..3e765a0 100644
--- a/mapbox/services/uploads.py
+++ b/mapbox/services/uploads.py
@@ -42,7 +42,7 @@ class Uploader(Service):
                 404: "Token does not have upload scope"})
         return resp
 
-    def stage(self, fileobj, creds=None):
+    def stage(self, fileobj, creds=None, callback=None):
         """Stages the user's file on S3
         If creds are not provided, temporary credientials will be generated
         Returns the URL to the staged resource.
@@ -63,12 +63,9 @@ class Uploader(Service):
             region_name="us-east-1")
 
         s3 = session.resource('s3')
-        # We'll use Bucket.upload_file() for actual files.
-        # Progress reporting for the CLI is a TODO.
-        if hasattr(fileobj, 'name') and os.path.exists(fileobj.name):
-            s3.Bucket(creds['bucket']).upload_file(fileobj.name, creds['key'])
-        else:
-            res = s3.Object(creds['bucket'], creds['key']).put(Body=fileobj)
+
+        bucket = s3.Bucket(creds['bucket'])
+        bucket.upload_fileobj(fileobj, creds['key'], Callback=callback)
 
         return creds['url']
 
@@ -150,10 +147,10 @@ class Uploader(Service):
         self.handle_http_error(resp)
         return resp
 
-    def upload(self, fileobj, tileset, name=None, patch=False):
+    def upload(self, fileobj, tileset, name=None, patch=False, callback=None):
         """High level function to upload a file object to mapbox tileset
         Effectively replicates the upload functionality using the HTML form
         Returns a response object where the json() is a dict with upload metadata
         """
-        url = self.stage(fileobj)
+        url = self.stage(fileobj, callback=callback)
         return self.create(url, tileset, name=name, patch=patch)
diff --git a/setup.py b/setup.py
index 6e4d935..0247e92 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ setup(name='mapbox',
       include_package_data=True,
       zip_safe=False,
       install_requires=[
-          'boto3',
+          'boto3>=1.4',
           'cachecontrol',
           'click',
           'click-plugins',
@@ -38,8 +38,7 @@ setup(name='mapbox',
           'iso3166',
           'requests',
           'polyline>=1.3.1',
-          'uritemplate>=2.0',
-      ],
+          'uritemplate>=2.0'],
       extras_require={
           'test': [
               'coveralls', 'pytest>=2.8.3', 'pytest-cov', 'responses', 'tox']})


Test patch:
diff --git a/tests/test_upload.py b/tests/test_upload.py
index a643a64..c498dd1 100644
--- a/tests/test_upload.py
+++ b/tests/test_upload.py
@@ -182,11 +182,18 @@ class MockSession(object):
         self.bucket = bucket
         return self
 
-    def upload_file(self, filename, key):
-        assert self.bucket
-        self.filename = filename
+    def upload_fileobj(self, data, key, Callback=None):
+        self.data = data
         self.key = key
-        return True
+        self.Callback = Callback
+
+        bytes_read = data.read(8192)
+        if bytes_read and self.Callback:
+            self.Callback(len(bytes_read))
+        while bytes_read:
+            bytes_read = data.read(8192)
+            if bytes_read and self.Callback:
+                self.Callback(len(bytes_read))
 
 
 @responses.activate
@@ -274,8 +281,11 @@ def test_upload(monkeypatch):
         body=upload_response_body, status=201,
         content_type='application/json')
 
+    def print_cb(num_bytes):
+        print("{0} bytes uploaded".format(num_bytes))
+
     with open('tests/moors.json', 'r') as src:
-        res = mapbox.Uploader(access_token=access_token).upload(src, 'test1')
+        res = mapbox.Uploader(access_token=access_token).upload(src, 'test1', callback=print_cb)
 
     assert res.status_code == 201
     job = res.json()