Repository: piskvorky/smart_open
Pull request #71 (2016-04-22 13:45:27)

Allow optional S3 host in smart_open kargs.
Boto requires a `host` argument for sigv4 api calls. If not provided it raises a `boto.s3.connection.HostRequiredError` exception.

To solve this a `host` keyword argument is now supported and is passed to boto when creating the S3 connection. If it is not supplied, a default is derived from boto config.

```python
smart_open('s3://bucket/prefix/')
smart_open('s3://bucket/prefix/', host='s3.amazonaws.com')
smart_open('s3://bucket/prefix/', host='s3.eu-central-1.amazonaws.com')
```

tmylk: Thanks for the PR. Just a PEP8 correction and ready to merge.
robcowie: Done. Slipped in a little README update as well. Let me know if you're happy with it.

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 8608d14..b1debcb 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,7 @@
+* 1.3.4,
+
+  - Allow passing boto `host` to smart_open (PR #71, @robcowie)
+
 * 1.3.3,  
 
   - Allow passing boto `profile_name` to smart_open (PR #68, @robcowie)
diff --git a/README.rst b/README.rst
index 92fe595..978c5ed 100644
--- a/README.rst
+++ b/README.rst
@@ -81,6 +81,19 @@ For more info (S3 credentials in URI, minimum S3 part size...) and full method s
   >>> import smart_open
   >>> help(smart_open.smart_open_lib)
 
+S3-Specific Options
+-------------------
+
+There are a few optional keyword arguments that are useful only for S3 access.
+
+.. code-block:: python
+
+  >>> smart_open.smart_open('s3://', host='s3.amazonaws.com')
+  >>> smart_open.smart_open('s3://', profile_name='my-profile')
+
+
+These are both passed to `boto.s3_connect()` as keyword arguments.
+
 Why?
 ----
 
diff --git a/setup.py b/setup.py
index c773a85..2808ff7 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,7 @@ def read(fname):
 
 setup(
     name = 'smart_open',
-    version = '1.3.2',
+    version = '1.3.3',
     description = 'Utils for streaming large files (S3, HDFS, gzip, bz2...)',
     long_description = read('README.rst'),
 
diff --git a/smart_open/smart_open_lib.py b/smart_open/smart_open_lib.py
index 426b4dc..22bb6cd 100644
--- a/smart_open/smart_open_lib.py
+++ b/smart_open/smart_open_lib.py
@@ -118,10 +118,16 @@ def smart_open(uri, mode="rb", **kw):
             # compression, if any, is determined by the filename extension (.gz, .bz2)
             return file_smart_open(parsed_uri.uri_path, mode)
         elif parsed_uri.scheme in ("s3", "s3n"):
+            # Get an S3 host. It is required for sigv4 operations.
+            host = kw.pop('host', None)
+            if not host:
+                host = boto.config.get('s3', 'host', 's3.amazonaws.com')
+
             # For credential order of precedence see
             # http://boto.cloudhackers.com/en/latest/boto_config_tut.html#credentials
             s3_connection = boto.connect_s3(
                 aws_access_key_id=parsed_uri.access_id,
+                host=host,
                 aws_secret_access_key=parsed_uri.access_secret,
                 profile_name=kw.pop('profile_name', None))
 


Test patch:
diff --git a/smart_open/tests/test_smart_open.py b/smart_open/tests/test_smart_open.py
index b3caa19..3054f69 100644
--- a/smart_open/tests/test_smart_open.py
+++ b/smart_open/tests/test_smart_open.py
@@ -135,20 +135,23 @@ class SmartOpenReadTest(unittest.TestCase):
     @mock.patch('smart_open.smart_open_lib.s3_iter_lines')
     def test_s3_boto(self, mock_s3_iter_lines, mock_boto):
         """Is S3 line iterator called correctly?"""
+        # Configure the mock boto.config.get to return default host
+        smart_open.smart_open_lib.boto.config.get.return_value = 's3.amazonaws.com'
+
         # no credentials
         smart_open_object = smart_open.smart_open("s3://mybucket/mykey")
         smart_open_object.__iter__()
-        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name=None)
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name=None, host='s3.amazonaws.com')
 
         # with credential
         smart_open_object = smart_open.smart_open("s3://access_id:access_secret@mybucket/mykey")
         smart_open_object.__iter__()
-        mock_boto.connect_s3.assert_called_with(aws_access_key_id="access_id", aws_secret_access_key="access_secret", profile_name=None)
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id="access_id", aws_secret_access_key="access_secret", profile_name=None, host='s3.amazonaws.com')
 
         # with credential profile
         smart_open_object = smart_open.smart_open("s3://mybucket/mykey", profile_name="my_credentials")
         smart_open_object.__iter__()
-        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name="my_credentials")
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name="my_credentials", host='s3.amazonaws.com')
 
         # lookup bucket, key; call s3_iter_lines
         smart_open_object = smart_open.smart_open("s3://access_id:access_secret@mybucket/mykey")
@@ -157,6 +160,10 @@ class SmartOpenReadTest(unittest.TestCase):
         mock_boto.connect_s3().get_bucket().get_key.assert_called_with("mykey")
         self.assertTrue(mock_s3_iter_lines.called)
 
+        # with user-specified host
+        smart_open_object = smart_open.smart_open("s3://access_id:access_secret@mybucket/mykey", host='aa.domain.com')
+        smart_open_object.__iter__()
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id="access_id", aws_secret_access_key="access_secret", profile_name=None, host='aa.domain.com')
 
     @mock_s3
     def test_s3_iter_moto(self):
@@ -315,14 +322,16 @@ class SmartOpenTest(unittest.TestCase):
         smart_open.smart_open("file:///some/file.txt", "w+")
         mock_file.assert_called_with("/some/file.txt", "w+")
 
-
     @mock.patch('smart_open.smart_open_lib.boto')
     @mock.patch('smart_open.smart_open_lib.S3OpenWrite')
     def test_s3_mode_mock(self, mock_write, mock_boto):
         """Are s3:// open modes passed correctly?"""
+        # Configure the mock boto.config.get to return default host
+        smart_open.smart_open_lib.boto.config.get.return_value = 's3.amazonaws.com'
+
         # correct write mode, correct s3 URI
         smart_open.smart_open("s3://mybucket/mykey", "w")
-        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name=None)
+        mock_boto.connect_s3.assert_called_with(aws_access_key_id=None, aws_secret_access_key=None, profile_name=None, host='s3.amazonaws.com')
         mock_boto.connect_s3().lookup.return_value = True
         mock_boto.connect_s3().get_bucket.assert_called_with("mybucket")
         self.assertTrue(mock_write.called)