Repository: StackStorm/st2

Issue:
Bug Report - Unicode exception when executing core.http with special character on the body
##### SUMMARY

When running _core.http_ with method post and a body with special character (e.G {"msg": "Áñçó"}), result on a unicode exception like the output below:

```python
'ascii' codec can't encode characters in position 14-15: ordinal not in range(128)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2actions/container/base.py", line 127, in _do_run
    result = jsonify.try_loads(result)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/http_runner/http_runner.py", line 102, in run
    result = client.run()
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/http_runner/http_runner.py", line 226, in run
    verify=self.verify
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/api.py", line 58, in request
    return session.request(method=method, url=url, **kwargs)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/sessions.py", line 518, in request
    resp = self.send(prep, **send_kwargs)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/sessions.py", line 639, in send
    r = adapter.send(request, **kwargs)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/adapters.py", line 438, in send
    timeout=timeout
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 600, in urlopen
    chunked=chunked)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 356, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/lib/python2.7/httplib.py", line 1021, in request
    self._send_request(method, url, body, headers)
  File "/usr/lib/python2.7/httplib.py", line 1055, in _send_request
    self.endheaders(body)
  File "/usr/lib/python2.7/httplib.py", line 1017, in endheaders
    self._send_output(message_body)
  File "/usr/lib/python2.7/httplib.py", line 868, in _send_output
    self.send(message_body)
  File "/usr/lib/python2.7/httplib.py", line 840, in send
    self.sock.sendall(data)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/eventlet/greenio/base.py", line 401, in sendall
    tail = self.send(data, flags)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/eventlet/greenio/base.py", line 395, in send
    return self._send_loop(self.fd.send, data, flags)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/eventlet/greenio/base.py", line 382, in _send_loop
    return send_method(data, *args)
```

##### ISSUE TYPE
 - Bug Report

<!--- If you are looking for help, check forum.stackstorm.com. -->

##### STACKSTORM VERSION
st2 2.10.3, on Python 2.7.6


##### OS / ENVIRONMENT / INSTALL METHOD
Docker installation 


##### STEPS TO REPRODUCE
<!--- For bugs, show how to reproduce the problem, using a minimal test-case.
For new features, show how the feature would be used. -->

<!--- Paste example actions and workflows between quotes below -->
**http Action**

```yaml
---
      name: "Send Message"
            ref: "core.http"
            pack: "default"
            parameters:
                url: "https://webhook.site/c3f497fd-f228-43f9-bbf4-62a061ebbe12"
                body: '{ "msg": "Áóñçì" }'
                headers: {Content-Type: application/json; charset=utf-8}
                method: "POST"
```

##### EXPECTED RESULTS
<!--- What did you expect to happen when running the steps above? -->
Send the post and get 200 return code

##### ACTUAL RESULTS
<!--- What happened? What output did you get? -->
The unicode exception below:

```python
'ascii' codec can't encode characters in position 14-15: ordinal not in range(128)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/st2actions/container/base.py", line 127, in _do_run
    result = jsonify.try_loads(result)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/http_runner/http_runner.py", line 102, in run
    result = client.run()
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/http_runner/http_runner.py", line 226, in run
    verify=self.verify
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/api.py", line 58, in request
    return session.request(method=method, url=url, **kwargs)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/sessions.py", line 518, in request
    resp = self.send(prep, **send_kwargs)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/sessions.py", line 639, in send
    r = adapter.send(request, **kwargs)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/adapters.py", line 438, in send
    timeout=timeout
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 600, in urlopen
    chunked=chunked)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.py", line 356, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/usr/lib/python2.7/httplib.py", line 1021, in request
    self._send_request(method, url, body, headers)
  File "/usr/lib/python2.7/httplib.py", line 1055, in _send_request
    self.endheaders(body)
  File "/usr/lib/python2.7/httplib.py", line 1017, in endheaders
    self._send_output(message_body)
  File "/usr/lib/python2.7/httplib.py", line 868, in _send_output
    self.send(message_body)
  File "/usr/lib/python2.7/httplib.py", line 840, in send
    self.sock.sendall(data)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/eventlet/greenio/base.py", line 401, in sendall
    tail = self.send(data, flags)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/eventlet/greenio/base.py", line 395, in send
    return self._send_loop(self.fd.send, data, flags)
  File "/opt/stackstorm/st2/local/lib/python2.7/site-packages/eventlet/greenio/base.py", line 382, in _send_loop
    return send_method(data, *args)
```


Discussion:
Does the API endpoint return unicode data?

Since calling ``core.http`` action directly with unicode body (outside of the context of a rule) seems to work fine.

```bash
st2 run core.http url="https://postman-echo.com/post" headers='{"Content-Type": "application/json; charset=utf-8"}' method=POST body='{ "msg": "Áóñçì" }'
.
id: 5c8a9bbe07612906bede857b
status: succeeded
parameters: 
  body: "{ "msg": "Áóñçì" }"
  headers:
    Content-Type: application/json; charset=utf-8
  method: POST
  url: https://postman-echo.com/post
result: 
  body:
    args: {}
    data: "{ "msg": "Áóñç"
    files: {}
    form: {}
    headers:
      accept: '*/*'
      accept-encoding: gzip, deflate
      content-length: '18'
      content-type: application/json; charset=utf-8
      host: postman-echo.com
      user-agent: st2/v3.0dev
      x-forwarded-port: '443'
      x-forwarded-proto: https
      x-stanley-action: http
    json: null
    url: https://postman-echo.com/post
  headers:
    Connection: keep-alive
    Content-Encoding: gzip
    Content-Length: '263'
    Content-Type: application/json; charset=utf-8
    Date: Thu, 14 Mar 2019 18:25:51 GMT
    ETag: W/"179-mfBxydVGBkn5kpRrr5TvrdUEC9o"
    Server: nginx
    Vary: Accept-Encoding
    set-cookie: sails.sid=s%3AFEQTev5vV5rD2VdQCw5ifhpOlszFH3Ph.F2Vd%2B5ArKyf6qrcMrjyizS4ABxrgygsqUpDsUzZiPew; Path=/; HttpOnly
  parsed: true
  status_code: 200
```

Please also include more log lines - including the lines before `'ascii' codec can't encode characters in position 14-15: ordinal not in range(128)` since it looks like there is some context missing.

Also, is this action part of a workflow?
I've tested today this problem via action chain, and I receive a broken json.
When I send ```{ "token": "589919693", "rid": "589919693", "msg": "áçéíì" }``` I receive ```{ "token": "589919693", "rid": "589919693", "msg": "áçéí```
Missing the character of message, " and }.

Manual test via core.http from web interface, same error of topic:
```'ascii' codec can't encode characters in position 14-15: ordinal not in range(128)```
Please provide the whole action chain definition file which can be used to reproduce this issue.
With the same test via CLI but to another platform to get webhooks, the json still is broken.

```bash
st2 run core.http url="https://webhook.site/c3f497fd-f228-43f9-bbf4-62a061ebbe12" headers='{"Content-Type": "application/json; charset=utf-8"}' method=POST body='{ "msg": "Áóñçì" }'
```
```yaml
id: 5c8ba433d23ae600ffbed15c
status: succeeded
parameters: 
  body: "{ "msg": "Áóñçì" }"
  headers:
    Content-Type: application/json; charset=utf-8
  method: POST
  url: https://webhook.site/c3f497fd-f228-43f9-bbf4-62a061ebbe12
result: 
  body: ''
  headers:
    Cache-Control: no-cache, private
    Content-Encoding: gzip
    Content-Type: text/plain; charset=UTF-8
    Date: Fri, 15 Mar 2019 13:10:12 GMT
    Server: nginx/1.10.3
    Transfer-Encoding: chunked
    Vary: Accept-Encoding
    X-RateLimit-Limit: '30'
    X-RateLimit-Remaining: '29'
    X-Request-Id: ea6daede-f24b-43b3-9600-8563977ce2ba
    X-Token-Id: c3f497fd-f228-43f9-bbf4-62a061ebbe12
  parsed: false
  status_code: 200
```
Response:
```json
{ "msg": "Áóñç
```
As requested:
```yaml
---
    chain:
        -
            name: "Register User"
            ref: "core.http"
            parameters:
                url: "http://rocketchat:3000/api/v1/livechat/visitor"
                body: '{ "visitor": { "name": "{{name}}", "token": "{{token}}" }}'
                method: "POST"
            on-success: "Create User Room"
        -
            name: "Create User Room"
            ref: "core.http"
            parameters:
                url: "http://rocketchat:3000/api/v1/livechat/room?token={{token}}&rid={{token}}"
                method: "GET"
            on-success: "Send Message"
            on-failure: "Fail Get Room"
        -
            name: "Fail Get Room"
            ref: "telegram.send_message"
            parameters:
                chat_id: "{{token}}"
                message: "Estamos fora do horário de atendimento."
        -
            name: "Send Message"
            ref: "core.http"
            parameters:
                #url: "http://rocketchat:3000/api/v1/livechat/message"
                url: "https://webhook.site/c3f497fd-f228-43f9-bbf4-62a061ebbe12"
                body: '{ "token": "{{token}}", "rid": "{{token}}", "msg": "{{message}}" }'
                headers: {Content-Type: application/json; charset=utf-8, Accept-Encoding: gzip, deflate}
                method: "POST"
    default: "Register User"
```
Thanks.

I was finally able to reproduce it with "https://httpbin.org/post" test endpoint. I will look into a fix.

Resolving patch:
diff --git a/contrib/runners/http_runner/http_runner/http_runner.py b/contrib/runners/http_runner/http_runner/http_runner.py
--- a/contrib/runners/http_runner/http_runner/http_runner.py
+++ b/contrib/runners/http_runner/http_runner/http_runner.py
@@ -211,6 +211,10 @@ def run(self):
             if self.username or self.password:
                 self.auth = HTTPBasicAuth(self.username, self.password)
 
+            # Ensure data is bytes since that what request expects
+            if isinstance(data, six.text_type):
+                data = data.encode('utf-8')
+
             resp = requests.request(
                 self.method,
                 self.url,


Test patch:
diff --git a/contrib/runners/http_runner/tests/unit/test_http_runner.py b/contrib/runners/http_runner/tests/unit/test_http_runner.py
--- a/contrib/runners/http_runner/tests/unit/test_http_runner.py
+++ b/contrib/runners/http_runner/tests/unit/test_http_runner.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Licensed to the StackStorm, Inc ('StackStorm') under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -14,6 +15,8 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
+import six
 import mock
 import unittest2
 
@@ -21,6 +24,12 @@
 import st2tests.config as tests_config
 
 
+if six.PY2:
+    EXPECTED_DATA = ''
+else:
+    EXPECTED_DATA = b''
+
+
 class MockResult(object):
     close = mock.Mock()
 
@@ -101,9 +110,14 @@ def test_https_verify(self, mock_requests):
 
         self.assertTrue(client.verify)
 
+        if six.PY2:
+            data = ''
+        else:
+            data = b''
+
         mock_requests.request.assert_called_with(
             'GET', url, allow_redirects=False, auth=None, cookies=None,
-            data='', files=None, headers={}, params=None, proxies=None,
+            data=data, files=None, headers={}, params=None, proxies=None,
             timeout=60, verify=True)
 
     @mock.patch('http_runner.http_runner.requests')
@@ -123,7 +137,7 @@ def test_https_verify_false(self, mock_requests):
 
         mock_requests.request.assert_called_with(
             'GET', url, allow_redirects=False, auth=None, cookies=None,
-            data='', files=None, headers={}, params=None, proxies=None,
+            data=EXPECTED_DATA, files=None, headers={}, params=None, proxies=None,
             timeout=60, verify=False)
 
     @mock.patch('http_runner.http_runner.requests')
@@ -145,5 +159,57 @@ def test_https_auth_basic(self, mock_requests):
 
         mock_requests.request.assert_called_once_with(
             'GET', url, allow_redirects=False, auth=client.auth, cookies=None,
-            data='', files=None, headers={}, params=None, proxies=None,
+            data=EXPECTED_DATA, files=None, headers={}, params=None, proxies=None,
             timeout=60, verify=False)
+
+    @mock.patch('http_runner.http_runner.requests')
+    def test_http_unicode_body_data(self, mock_requests):
+        url = 'http://127.0.0.1:8888'
+        method = 'POST'
+        mock_result = MockResult()
+
+        # 1. String data
+        headers = {}
+        body = 'žžžžž'
+        client = HTTPClient(url=url, method=method, headers=headers, body=body, timeout=0.1)
+
+        mock_result.text = '{"foo": "bar"}'
+        mock_result.headers = {'Content-Type': 'application/json'}
+        mock_result.status_code = 200
+        mock_requests.request.return_value = mock_result
+
+        result = client.run()
+        self.assertEqual(result['status_code'], 200)
+
+        call_kwargs = mock_requests.request.call_args_list[0][1]
+
+        expected_data = u'žžžžž'.encode('utf-8')
+        self.assertEqual(call_kwargs['data'], expected_data)
+
+        # 1. Object / JSON data
+        body = {
+            'foo': u'ažž'
+        }
+        headers = {
+            'Content-Type': 'application/json; charset=utf-8'
+        }
+        client = HTTPClient(url=url, method=method, headers=headers, body=body, timeout=0.1)
+
+        mock_result.text = '{"foo": "bar"}'
+        mock_result.headers = {'Content-Type': 'application/json'}
+        mock_result.status_code = 200
+        mock_requests.request.return_value = mock_result
+
+        result = client.run()
+        self.assertEqual(result['status_code'], 200)
+
+        call_kwargs = mock_requests.request.call_args_list[1][1]
+
+        if six.PY2:
+            expected_data = {
+                'foo': u'a\u017e\u017e'
+            }
+        else:
+            expected_data = body
+
+        self.assertEqual(call_kwargs['data'], expected_data)