Repository: piskvorky/smart_open
Pull request #63 (2016-02-24 00:25:24)

Passing extra parameters when writing to S3 keys (#35)
Passing along extra parameters to `initiate_multipart_upload` when
writing S3 keys directly. This is necessary, for example, to allow
server-side encryption to be used through the `encrypt_key` parameter.


Patch:
diff --git a/.travis.yml b/.travis.yml
index 025aabb..07af80a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,4 @@
 language: python
-notifications:
-  email: false
 python:
 - "2.7"
 - "2.6"
@@ -9,6 +7,7 @@ python:
 - "3.5"
 install:
 - if [[ $TRAVIS_PYTHON_VERSION == 2.6 ]]; then pip install ordereddict; fi
-- pip install httpretty==0.8.10 requests==2.8.1 mock moto responses
+- pip install mock moto responses
 - python setup.py install
+- pip freeze
 script: python -W ignore setup.py test
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index add920d..b1debcb 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,7 +1,17 @@
+* 1.3.4,
+
+  - Allow passing boto `host` to smart_open (PR #71, @robcowie)
+
+* 1.3.3,  
+
+  - Allow passing boto `profile_name` to smart_open (PR #68, @robcowie)
+  - Write an empty key to S3 even if nothing is written to S3OpenWrite (PR #61, @petedmarsh)
+
 * 1.3.2, 3rd January 2016
 
   - Bug fix release to enable 'wb+' file mode (PR #50)
 
+
 * 1.3.1, 18th December 2015
 
   - Disable multiprocessing if unavailable. Allows to run on Google Compute Engine. (PR #41, @nikicc)
diff --git a/README.rst b/README.rst
index 92fe595..978c5ed 100644
--- a/README.rst
+++ b/README.rst
@@ -81,6 +81,19 @@ For more info (S3 credentials in URI, minimum S3 part size...) and full method s
   >>> import smart_open
   >>> help(smart_open.smart_open_lib)
 
+S3-Specific Options
+-------------------
+
+There are a few optional keyword arguments that are useful only for S3 access.
+
+.. code-block:: python
+
+  >>> smart_open.smart_open('s3://', host='s3.amazonaws.com')
+  >>> smart_open.smart_open('s3://', profile_name='my-profile')
+
+
+These are both passed to `boto.s3_connect()` as keyword arguments.
+
 Why?
 ----
 
diff --git a/setup.py b/setup.py
index 47f749a..daaab90 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,7 @@ def read(fname):
 
 setup(
     name = 'smart_open',
-    version = '1.3.2',
+    version = '1.3.3',
     description = 'Utils for streaming large files (S3, HDFS, gzip, bz2...)',
     long_description = read('README.rst'),
 
@@ -45,9 +45,8 @@ setup(
 
     install_requires=[
         'boto >= 2.32',
-        'httpretty==0.8.10',
         'bz2file',
-        'requests<=2.8.1',
+        'requests',
     ],
 
     test_suite="smart_open.tests",
@@ -56,12 +55,13 @@ setup(
         'Development Status :: 4 - Beta',
         'Environment :: Console',
         'Intended Audience :: Developers',
-        'License :: Public Domain',
+        'License :: OSI Approved :: MIT License',
         'Operating System :: OS Independent',
         'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
         'Topic :: System :: Distributed Computing',
         'Topic :: Database :: Front-Ends',
     ],
diff --git a/smart_open/smart_open_lib.py b/smart_open/smart_open_lib.py
index 35c71cd..cd06d47 100644
--- a/smart_open/smart_open_lib.py
+++ b/smart_open/smart_open_lib.py
@@ -118,13 +118,25 @@ def smart_open(uri, mode="rb", **kw):
             # compression, if any, is determined by the filename extension (.gz, .bz2)
             return file_smart_open(parsed_uri.uri_path, mode)
         elif parsed_uri.scheme in ("s3", "s3n"):
-            s3_connection = boto.connect_s3(aws_access_key_id=parsed_uri.access_id, aws_secret_access_key=parsed_uri.access_secret)
+            # Get an S3 host. It is required for sigv4 operations.
+            host = kw.pop('host', None)
+            if not host:
+                host = boto.config.get('s3', 'host', 's3.amazonaws.com')
+
+            # For credential order of precedence see
+            # http://boto.cloudhackers.com/en/latest/boto_config_tut.html#credentials
+            s3_connection = boto.connect_s3(
+                aws_access_key_id=parsed_uri.access_id,
+                host=host,
+                aws_secret_access_key=parsed_uri.access_secret,
+                profile_name=kw.pop('profile_name', None))
+
             bucket = s3_connection.get_bucket(parsed_uri.bucket_id)
             if mode in ('r', 'rb'):
                 key = bucket.get_key(parsed_uri.key_id)
                 if key is None:
                     raise KeyError(parsed_uri.key_id)
-                return S3OpenRead(key, **kw)
+                return S3OpenRead(key)
             elif mode in ('w', 'wb'):
                 key = bucket.get_key(parsed_uri.key_id, validate=False)
                 if key is None:
@@ -152,7 +164,7 @@ def smart_open(uri, mode="rb", **kw):
         if mode in ('r', 'rb'):
             return S3OpenRead(uri)
         elif mode in ('w', 'wb'):
-            return S3OpenWrite(uri)
+            return S3OpenWrite(uri, **kw)
     elif hasattr(uri, 'read'):
         # simply pass-through if already a file-like
         return uri
@@ -492,9 +504,11 @@ class S3OpenWrite(object):
         else:
             # AWS complains with "The XML you provided was not well-formed or did not validate against our published schema"
             # when the input is completely empty => abort the upload, no file created
-            # TODO: or create the empty file some other way?
             logger.info("empty input, ignoring multipart upload")
             self.outkey.bucket.cancel_multipart_upload(self.mp.key_name, self.mp.id)
+            # So, instead, create an empty file like this
+            logger.info("setting an empty value for the key")
+            self.outkey.set_contents_from_string('')
 
     def __enter__(self):
         return self


Test patch:
diff --git a/smart_open/tests/test_smart_open.py b/smart_open/tests/test_smart_open.py
index d1f18e9..3054f69 100644
--- a/smart_open/tests/test_smart_open.py
+++ b/smart_open/tests/test_smart_open.py
@@ -135,15 +135,23 @@ class SmartOpenReadTest(unittest.TestCase):
     @mock.patch('smart_open.smart_open_lib.s3_iter_lines')
     def test_s3_boto(self, mock_s3_iter_lines, mock_boto):
         """Is S3 line iterator called correctly?"""
+        # Configure the mock boto.config.get to return default host
+        smart_open.smart_open_lib.boto.config.get.return_value = 's3.amazonaws.com'
+
         # no credentials
         smart_open_object = smart_open.smart_open("s3://mybucket/mykey")
         smart_open_object.__iter__()
-        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None)
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name=None, host='s3.amazonaws.com')
 
         # with credential
         smart_open_object = smart_open.smart_open("s3://access_id:access_secret@mybucket/mykey")
         smart_open_object.__iter__()
-        mock_boto.connect_s3.assert_called_with(aws_access_key_id="access_id", aws_secret_access_key="access_secret")
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id="access_id", aws_secret_access_key="access_secret", profile_name=None, host='s3.amazonaws.com')
+
+        # with credential profile
+        smart_open_object = smart_open.smart_open("s3://mybucket/mykey", profile_name="my_credentials")
+        smart_open_object.__iter__()
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name="my_credentials", host='s3.amazonaws.com')
 
         # lookup bucket, key; call s3_iter_lines
         smart_open_object = smart_open.smart_open("s3://access_id:access_secret@mybucket/mykey")
@@ -152,6 +160,10 @@ class SmartOpenReadTest(unittest.TestCase):
         mock_boto.connect_s3().get_bucket().get_key.assert_called_with("mykey")
         self.assertTrue(mock_s3_iter_lines.called)
 
+        # with user-specified host
+        smart_open_object = smart_open.smart_open("s3://access_id:access_secret@mybucket/mykey", host='aa.domain.com')
+        smart_open_object.__iter__()
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id="access_id", aws_secret_access_key="access_secret", profile_name=None, host='aa.domain.com')
 
     @mock_s3
     def test_s3_iter_moto(self):
@@ -310,14 +322,16 @@ class SmartOpenTest(unittest.TestCase):
         smart_open.smart_open("file:///some/file.txt", "w+")
         mock_file.assert_called_with("/some/file.txt", "w+")
 
-
     @mock.patch('smart_open.smart_open_lib.boto')
     @mock.patch('smart_open.smart_open_lib.S3OpenWrite')
     def test_s3_mode_mock(self, mock_write, mock_boto):
         """Are s3:// open modes passed correctly?"""
+        # Configure the mock boto.config.get to return default host
+        smart_open.smart_open_lib.boto.config.get.return_value = 's3.amazonaws.com'
+
         # correct write mode, correct s3 URI
         smart_open.smart_open("s3://mybucket/mykey", "w")
-        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None)
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name=None, host='s3.amazonaws.com')
         mock_boto.connect_s3().lookup.return_value = True
         mock_boto.connect_s3().get_bucket.assert_called_with("mybucket")
         self.assertTrue(mock_write.called)
@@ -433,6 +447,27 @@ class S3OpenWriteTest(unittest.TestCase):
         self.assertEqual(output, [b"testtest\n", b"test"])
 
 
+    @mock_s3
+    def test_write_04(self):
+        """Does writing no data cause key with an empty value to be created?"""
+        # fake connection, bucket and key
+        conn = boto.connect_s3()
+        conn.create_bucket("mybucket")
+        mybucket = conn.get_bucket("mybucket")
+        mykey = boto.s3.key.Key()
+        mykey.name = "testkey"
+        mykey.bucket = mybucket
+
+        smart_open_write = smart_open.S3OpenWrite(mykey)
+        with smart_open_write as fin:
+            pass
+
+        # read back the same key and check its content
+        output = list(smart_open.smart_open("s3://mybucket/testkey"))
+
+        self.assertEqual(output, [])
+
+
 class WebHdfsWriteTest(unittest.TestCase):
     """
     Test writing into webhdfs files.