Repository: boto/boto

Issue:
SNS can't publish non-ascii message
According to documentation, SNS publish method accepts a UTF-8 encoded message, but the true is that it fails:

``` pycon
>>> from boto import connect_sns
>>> connect_sns().publish('foo', u'\xf1'.encode('utf8'))
---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-2-9b367f4dfce9> in <module>()
----> 1 connect_sns().publish('foo', u'\xf1'.encode('utf8'))

/home/daniel/src/boto/boto/sns/connection.pyc in publish(self, topic, message, subject, target_arn, message_structure)
    257         if message_structure is not None:
    258             params['MessageStructure'] = message_structure
--> 259         return self._make_request('Publish', params, '/', 'POST')
    260 
    261     def subscribe(self, topic, protocol, endpoint):

/home/daniel/src/boto/boto/sns/connection.pyc in _make_request(self, action, params, path, verb)
    724         params['ContentType'] = 'JSON'
    725         response = self.make_request(action=action, verb=verb,
--> 726                                      path=path, params=params)
    727         body = response.read()
    728         boto.log.debug(body)

/home/daniel/src/boto/boto/connection.pyc in make_request(self, action, params, path, verb)
   1071         if self.APIVersion:
   1072             http_request.params['Version'] = self.APIVersion
-> 1073         return self._mexe(http_request)
   1074 
   1075     def build_list_params(self, params, items, label):

/home/daniel/src/boto/boto/connection.pyc in _mexe(self, request, sender, override_num_retries, retry_handler)
    895                 # we now re-sign each request before it is retried
    896                 boto.log.debug('Token: %s' % self.provider.security_token)
--> 897                 request.authorize(connection=self)
    898                 # Only force header for non-s3 connections, because s3 uses
    899                 # an older signing method + bucket resource URLs that include

/home/daniel/src/boto/boto/connection.pyc in authorize(self, connection, **kwargs)
    376                 self.headers[key] = urllib.quote_plus(val.encode('utf-8'), safe)
    377 
--> 378         connection._auth_handler.add_auth(self, **kwargs)
    379 
    380         self.headers['User-Agent'] = UserAgent

/home/daniel/src/boto/boto/auth.pyc in add_auth(self, req, **kwargs)
    488         if self._provider.security_token:
    489             req.headers['X-Amz-Security-Token'] = self._provider.security_token
--> 490         qs = self.query_string(req)
    491         if qs and req.method == 'POST':
    492             # Stash request parameters into post body

/home/daniel/src/boto/boto/auth.pyc in query_string(self, http_request)
    330         pairs = []
    331         for pname in parameter_names:
--> 332             pval = str(http_request.params[pname]).encode('utf-8')
    333             pairs.append(urllib.quote(pname, safe='') + '=' +
    334                          urllib.quote(pval, safe='-_~'))

UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 0: ordinal not in range(128)
```

SNS can't publish non-ascii message
According to documentation, SNS publish method accepts a UTF-8 encoded message, but the true is that it fails:

``` pycon
>>> from boto import connect_sns
>>> connect_sns().publish('foo', u'\xf1'.encode('utf8'))
---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-2-9b367f4dfce9> in <module>()
----> 1 connect_sns().publish('foo', u'\xf1'.encode('utf8'))

/home/daniel/src/boto/boto/sns/connection.pyc in publish(self, topic, message, subject, target_arn, message_structure)
    257         if message_structure is not None:
    258             params['MessageStructure'] = message_structure
--> 259         return self._make_request('Publish', params, '/', 'POST')
    260 
    261     def subscribe(self, topic, protocol, endpoint):

/home/daniel/src/boto/boto/sns/connection.pyc in _make_request(self, action, params, path, verb)
    724         params['ContentType'] = 'JSON'
    725         response = self.make_request(action=action, verb=verb,
--> 726                                      path=path, params=params)
    727         body = response.read()
    728         boto.log.debug(body)

/home/daniel/src/boto/boto/connection.pyc in make_request(self, action, params, path, verb)
   1071         if self.APIVersion:
   1072             http_request.params['Version'] = self.APIVersion
-> 1073         return self._mexe(http_request)
   1074 
   1075     def build_list_params(self, params, items, label):

/home/daniel/src/boto/boto/connection.pyc in _mexe(self, request, sender, override_num_retries, retry_handler)
    895                 # we now re-sign each request before it is retried
    896                 boto.log.debug('Token: %s' % self.provider.security_token)
--> 897                 request.authorize(connection=self)
    898                 # Only force header for non-s3 connections, because s3 uses
    899                 # an older signing method + bucket resource URLs that include

/home/daniel/src/boto/boto/connection.pyc in authorize(self, connection, **kwargs)
    376                 self.headers[key] = urllib.quote_plus(val.encode('utf-8'), safe)
    377 
--> 378         connection._auth_handler.add_auth(self, **kwargs)
    379 
    380         self.headers['User-Agent'] = UserAgent

/home/daniel/src/boto/boto/auth.pyc in add_auth(self, req, **kwargs)
    488         if self._provider.security_token:
    489             req.headers['X-Amz-Security-Token'] = self._provider.security_token
--> 490         qs = self.query_string(req)
    491         if qs and req.method == 'POST':
    492             # Stash request parameters into post body

/home/daniel/src/boto/boto/auth.pyc in query_string(self, http_request)
    330         pairs = []
    331         for pname in parameter_names:
--> 332             pval = str(http_request.params[pname]).encode('utf-8')
    333             pairs.append(urllib.quote(pname, safe='') + '=' +
    334                          urllib.quote(pval, safe='-_~'))

UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 0: ordinal not in range(128)
```



Discussion:
I can reproduce it on 2.23.0 and develop branch.

I can reproduce it on 2.23.0 and develop branch.


Resolving patch:
diff --git a/boto/auth.py b/boto/auth.py
--- a/boto/auth.py
+++ b/boto/auth.py
@@ -329,7 +329,7 @@ def query_string(self, http_request):
         parameter_names = sorted(http_request.params.keys())
         pairs = []
         for pname in parameter_names:
-            pval = str(http_request.params[pname]).encode('utf-8')
+            pval = boto.utils.get_utf8_value(http_request.params[pname])
             pairs.append(urllib.quote(pname, safe='') + '=' +
                          urllib.quote(pval, safe='-_~'))
         return '&'.join(pairs)
@@ -341,7 +341,7 @@ def canonical_query_string(self, http_request):
             return ""
         l = []
         for param in sorted(http_request.params):
-            value = str(http_request.params[param])
+            value = boto.utils.get_utf8_value(http_request.params[param])
             l.append('%s=%s' % (urllib.quote(param, safe='-_.~'),
                                 urllib.quote(value, safe='-_.~')))
         return '&'.join(l)


Test patch:
diff --git a/tests/unit/auth/test_sigv4.py b/tests/unit/auth/test_sigv4.py
--- a/tests/unit/auth/test_sigv4.py
+++ b/tests/unit/auth/test_sigv4.py
@@ -64,6 +64,18 @@ def test_canonical_query_string(self):
         query_string = auth.canonical_query_string(request)
         self.assertEqual(query_string, 'Foo.1=aaa&Foo.10=zzz')
 
+    def test_query_string(self):
+        auth = HmacAuthV4Handler('sns.us-east-1.amazonaws.com',
+                                 Mock(), self.provider)
+        params = {
+            'Message': u'We \u2665 utf-8'.encode('utf-8'),
+        }
+        request = HTTPRequest(
+            'POST', 'https', 'sns.us-east-1.amazonaws.com', 443,
+            '/', None, params, {}, '')
+        query_string = auth.query_string(request)
+        self.assertEqual(query_string, 'Message=We%20%E2%99%A5%20utf-8')
+
     def test_canonical_uri(self):
         auth = HmacAuthV4Handler('glacier.us-east-1.amazonaws.com',
                                  Mock(), self.provider)
diff --git a/tests/unit/sns/test_connection.py b/tests/unit/sns/test_connection.py
--- a/tests/unit/sns/test_connection.py
+++ b/tests/unit/sns/test_connection.py
@@ -225,6 +225,17 @@ def test_publish_with_json(self):
             'MessageStructure': 'json',
         }, ignore_params_values=['Version', 'ContentType'])
 
+    def test_publish_with_utf8_message(self):
+        self.set_http_response(status_code=200)
+        subject = message = u'We \u2665 utf-8'.encode('utf-8')
+        self.service_connection.publish('topic', message, subject)
+        self.assert_request_parameters({
+            'Action': 'Publish',
+            'TopicArn': 'topic',
+            'Subject': subject,
+            'Message': message,
+        }, ignore_params_values=['Version', 'ContentType'])
+
 
 if __name__ == '__main__':
     unittest.main()