Repository: googleapis/google-cloud-python

Issue:
BigQuery upload_from_file unicode file-like must be opened in binary-mode if it's more than RESUMABLE_UPLOAD_THRESHOLD, otherwise str-mode.
# Steps to reproduce

``` python
import sys
from gcloud.bigquery import Client, SchemaField
from gcloud.bigquery.job import CreateDisposition, WriteDisposition

csv_filename = 'sandwiches.csv'

if len(sys.argv) > 1:
    csv_filename = sys.argv[1]

bq = Client()

ds = bq.dataset('test_unicode')
ds.location = 'EU'


def test_unicode_upload(filename, mode):

    if not ds.exists():
        print('Creating dataset: {}'.format(ds.name))
        ds.create()

    fields = [
        SchemaField('name', 'STRING'),
        SchemaField('main_ingredient', 'STRING'),
    ]

    table = ds.table('sandwiches', fields)

    print('Uploading CSV: {}, mode={!r}'.format(csv_filename, mode))
    table.upload_from_file(
        open(filename, mode),
        encoding='UTF-8',
        source_format='CSV',
        write_disposition=WriteDisposition.WRITE_TRUNCATE,
        create_disposition=CreateDisposition.CREATE_IF_NEEDED)

test_unicode_upload(csv_filename, 'r') # Works

test_unicode_upload(csv_filename, 'rb')
# Fails in http.client.HTTPConnection()._send_request
#
# /usr/lib/python3.4/http/client.py in _send_request(self, method, url,
# body, headers)
#    1178         if isinstance(body, str):
#    1179             # RFC 2616 Section 3.7.1 says that text default has a
#    1180             # default charset of iso-8859-1.
# -> 1181             body = body.encode('iso-8859-1')
#    1182         self.endheaders(body)
#
# UnicodeEncodeError: 'latin-1' codec can't encode characters in position
#649-650: ordinal not in range(256)
```
## `sandwiches.csv`

``` csv
name,main_ingredient
Räksmörgås,Räkor
Baguette,Bröd
```
# Expected behavior

When i send in a binary-mode file-like I expect upload_from_file to pass the data through to BigQuery as-is, and that the BigQuery load job will decode it for me using `encoding=`.
# Enviroment

``` console
$ python --version
Python 3.4.3+
$ pip freeze | egrep 'httplib2|gcloud'
gcloud==0.13.0
httplib2==0.9.2

```



Discussion:
It seems the binary file object is just getting [passed off](https://github.com/GoogleCloudPlatform/gcloud-python/blob/6563cb7096ded06a638dcb277bc4baca42333bd3/gcloud/bigquery/table.py#L860) to our [vendored in](https://github.com/GoogleCloudPlatform/gcloud-python/blob/6563cb7096ded06a638dcb277bc4baca42333bd3/gcloud/bigquery/table.py#L30) streaming package; not quite sure how it handles the encoding / what the right answer is.

Thanks a lot for reporting!

If it helps, I stepped into the debugger and found that the characters get mangled before being sent:

``` python
ipdb> print(body)
--===============4443280759287164299==
Content-Type: application/json
MIME-Version: 1.0

{"configuration": {"load": {"createDisposition": "CREATE_IF_NEEDED", "sourceFormat": "CSV", "writeDisposition": "WRITE_TRUNCATE", "schema": {"fields": [{"mode": "NULLABLE", "name": "name", "type": "STRING"}, {"mode": "NULLABLE", "name": "main_ingredient", "type": "STRING"}]}, "encoding": "UTF-8", "destinationTable": {"tableId": "sandwiches", "datasetId": "test_unicode", "projectId": "PROJECT"}}}}
--===============4443280759287164299==
Content-Type: application/octet-stream
MIME-Version: 1.0
Content-Transfer-Encoding: binary

name,main_ingredient
R��ksm��rg��s,R��kor
Baguette,Br��d

--===============4443280759287164299==--
```

e.g.

``` python
ipdb> body[-78:-43]
'R��ksm��rg��s,R��kor\nBaguette,Br��d'
ipdb> body[-77]
'�'
ipdb> type(body[-77])
<class 'str'>
ipdb> body[-77].encode('utf-8')
b'\xef\xbf\xbd'
ipdb> body[-76]
'�'
ipdb> body[-76].encode('utf-8')
b'\xef\xbf\xbd'
```

@dhermes  It seem that the file must be read in binary-mode when the upload strategy is `RESUMABLE_UPLOAD`, and in str-mode otherwise.

Here's a new script to reproduce

``` python
import os
import sys
from itertools import product

import math
from gcloud.streaming.transfer import RESUMABLE_UPLOAD_THRESHOLD
from gcloud.bigquery import Client, SchemaField
from gcloud.bigquery.job import CreateDisposition, WriteDisposition

csv_filename = 'sandwiches.csv'
large_csv_filename = 'large-sandwiches.csv'

if len(sys.argv) > 1:
    csv_filename = sys.argv[1]

bq = Client()

ds = bq.dataset('test_unicode')
ds.location = 'EU'


def create_large_sandwiches(in_path, out_path):
    target_size = RESUMABLE_UPLOAD_THRESHOLD

    # Assume we get all the content
    in_fd = open(in_path, 'rb')
    in_bytes = in_fd.read()  # type: bytes

    in_fd.seek(0)
    lines = in_fd.readlines()
    last_line = lines[-1]

    repeat = math.ceil((target_size - len(in_bytes)) / len(last_line))

    open(out_path, 'wb').writelines(last_line for _ in range(0, repeat))


def test_unicode_upload(filename, mode):
    if not ds.exists():
        print('Creating dataset: {}'.format(ds.name))
        ds.create()

    fields = [
        SchemaField('name', 'STRING'),
        SchemaField('main_ingredient', 'STRING'),
    ]

    table = ds.table('sandwiches', fields)

    stat = os.stat(filename)

    _log.info('Uploading CSV: filename=%r, mode=%r, st_size=%d',
              filename, mode, stat.st_size)
    table.upload_from_file(
        open(filename, mode),
        encoding='UTF-8',
        source_format='CSV',
        write_disposition=WriteDisposition.WRITE_TRUNCATE,
        create_disposition=CreateDisposition.CREATE_IF_NEEDED)


if __name__ == '__main__':
    import logging
    logging.basicConfig(level=logging.INFO)
    _log = logging.getLogger(__name__)

    create_large_sandwiches(csv_filename, large_csv_filename)

    for args in product([csv_filename, large_csv_filename], ['r', 'rb']):
        try:
            test_unicode_upload(*args)
        except:
            _log.exception('Upload failed')
```

@tseaver Can you PTAL? I assume you have more familiarity with `gcloud.streaming` / `apitools` than I do.

Logically, the stream should be opened in binary mode:  the back-end [requires](https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.encoding) that it be encoded as either UTF-8 or ISO-8859-1, and assumes UTF-8 if no value is passed.

For multipart uploads (those smaller than `RESUMABLE_UPLOAD_THRESHOLD`), our [vendored-in fork](https://github.com/GoogleCloudPlatform/gcloud-python/blob/023f31676f2122a11c9b32e8f9ceaf0090bbba0b/gcloud/streaming/transfer.py#L846-L850) handles the stream precisely as [`apitools` does](https://github.com/craigcitro/apitools/blob/5bc3a0e540b8ca5c6dda473292eb78f40607bcf6/apitools/base/py/transfer.py#L752-L756).

@craigcitro, have you seen issues with multipart uploads blowing up when passed non-ASCII streams opened in binary mode?

I have not -- but @thobrla is much more likely to have run into this kind of thing. Travis, does this sound familiar?

Not familiar - gsutil opens all files as binary.

@dhermes if you've got an easy repro on hand -- can you set a breakpoint where @tseaver pointed in `transfer.py` and look at `self.stream` and `http_request`? eg is the bad translation happening in that chunk of transfer, or is something getting mismatched earlier?

@thobrla does gsutil do anything to force small uploads to use the resumable strategy?  Or does it ever trigger the "mulitpart" upload (metadata + binary file)?

Whether we use resumable upload is configurable by a byte threshold, and we use multipart if it is below that threshold (the default is 8MiB).  You can see our code for initiating the multipart upload with apitools [here](https://github.com/GoogleCloudPlatform/gsutil/blob/f3ae660f32d5e20cff4b3ad2b71def8a6ea64be4/gslib/gcs_json_api.py#L1179).  The file we send is opened in 'rb' mode.

@thobrla [gsutil doesn't seem to run on Python 3](https://github.com/GoogleCloudPlatform/gsutil/blob/master/setup.py#L154-L157), so it might be unaffected by this bug.

@tseaver I think that the use of `six.StringIO` in [`Upload._configure_multiport_request`](https://github.com/GoogleCloudPlatform/gcloud-python/blob/master/gcloud/streaming/transfer.py#L852-L869) might be close to the root cause of this issue.
- PY2: `six.StringIO == six.BytesIO == StringIO.StringIO`
- PY3: `six.StringIO == io.StringIO != io.BytesIO`[1](https://github.com/kelp404/six/blob/master/six.py#L595-L630).

@joar Thanks for the detective work!

@joar indeed, nice work!

that code came in [right here](https://github.com/google/apitools/commit/0ad6a8248841e4c5f496793225a332d2cc4a8b14); @dhermes I always trust you to know more about py3+strings than I do, so tell me if there's a better way for us to do that. 😉 

@craigcitro @dhermes I've made a public domain tool that I believe fixes this problem: https://gist.github.com/joar/ce5e54c0befd5ff24c312fcd62caab10. If this doesn't get me around the CLA, then it has at least taught me how to use sed.

It could also be useful to put some guards around the `Upload._stream` variable to make sure people running Python 3 aren't giving it "Text" IO.

@joar I'd definitely call this a bug in the transfer code, which I'd like to fix -- I just don't use python3 in my day-to-day, so I'm no expert on fixing these things.

From the gist, it looks like the right thing to do here is detect py3, and use the right wrapping class from `email_generator`?

Correct, and additionally use `six.BytesIO` on both PY2,3 to hold the multipart stream. The rest of the sed script is to make `Request.loggable_body` work.


Resolving patch:
diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py
--- a/gcloud/bigquery/table.py
+++ b/gcloud/bigquery/table.py
@@ -742,7 +742,7 @@ def upload_from_file(self,
         - ``text/csv``.
 
         :type file_obj: file
-        :param file_obj: A file handle open for reading.
+        :param file_obj: A file handle opened in binary mode for reading.
 
         :type source_format: str
         :param source_format: one of 'CSV' or 'NEWLINE_DELIMITED_JSON'.
@@ -809,8 +809,9 @@ def upload_from_file(self,
         :rtype: :class:`gcloud.bigquery.jobs.LoadTableFromStorageJob`
         :returns: the job instance used to load the data (e.g., for
                   querying status)
-        :raises: :class:`ValueError` if size is not passed in and can not be
-                 determined
+        :raises: :class:`ValueError` if ``size`` is not passed in and can not
+                 be determined, or if the ``file_obj`` can be detected to be
+                 a file opened in text mode.
         """
         client = self._require_client(client)
         connection = client.connection
@@ -820,6 +821,12 @@ def upload_from_file(self,
         if rewind:
             file_obj.seek(0, os.SEEK_SET)
 
+        mode = getattr(file_obj, 'mode', None)
+        if mode is not None and mode != 'rb':
+            raise ValueError(
+                "Cannot upload files opened in text mode:  use "
+                "open(filename, mode='rb')")
+
         # Get the basic stats about the file.
         total_bytes = size
         if total_bytes is None:
diff --git a/gcloud/streaming/transfer.py b/gcloud/streaming/transfer.py
--- a/gcloud/streaming/transfer.py
+++ b/gcloud/streaming/transfer.py
@@ -9,6 +9,7 @@
 import six
 from six.moves import http_client
 
+from gcloud._helpers import _to_bytes
 from gcloud.streaming.buffered_stream import BufferedStream
 from gcloud.streaming.exceptions import CommunicationError
 from gcloud.streaming.exceptions import HttpError
@@ -849,13 +850,13 @@ def _configure_multipart_request(self, http_request):
         msg.set_payload(self.stream.read())
         msg_root.attach(msg)
 
-        # NOTE: We encode the body, but can't use
-        #       `email.message.Message.as_string` because it prepends
-        #       `> ` to `From ` lines.
-        # NOTE: We must use six.StringIO() instead of io.StringIO() since the
-        #       `email` library uses cStringIO in Py2 and io.StringIO in Py3.
-        stream = six.StringIO()
-        generator = email_generator.Generator(stream, mangle_from_=False)
+        # NOTE: generate multipart message as bytes, not text
+        stream = six.BytesIO()
+        if six.PY3:  # pragma: NO COVER  Python3
+            generator_class = email_generator.BytesGenerator
+        else:
+            generator_class = email_generator.Generator
+        generator = generator_class(stream, mangle_from_=False)
         generator.flatten(msg_root, unixfrom=False)
         http_request.body = stream.getvalue()
 
@@ -863,10 +864,11 @@ def _configure_multipart_request(self, http_request):
         http_request.headers['content-type'] = (
             'multipart/related; boundary="%s"' % multipart_boundary)
 
-        body_components = http_request.body.split(multipart_boundary)
-        headers, _, _ = body_components[-2].partition('\n\n')
-        body_components[-2] = '\n\n'.join([headers, '<media body>\n\n--'])
-        http_request.loggable_body = multipart_boundary.join(body_components)
+        boundary_bytes = _to_bytes(multipart_boundary)
+        body_components = http_request.body.split(boundary_bytes)
+        headers, _, _ = body_components[-2].partition(b'\n\n')
+        body_components[-2] = b'\n\n'.join([headers, b'<media body>\n\n--'])
+        http_request.loggable_body = boundary_bytes.join(body_components)
 
     def _configure_resumable_request(self, http_request):
         """Helper for 'configure_request': set up resumable request."""


Test patch:
diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py
--- a/gcloud/bigquery/test_table.py
+++ b/gcloud/bigquery/test_table.py
@@ -1289,6 +1289,19 @@ def _row_data(row):
         self.assertEqual(req['path'], '/%s' % PATH)
         self.assertEqual(req['data'], SENT)
 
+    def test_upload_from_file_text_mode_file_failure(self):
+
+        class TextModeFile(object):
+            mode = 'r'
+
+        conn = _Connection()
+        client = _Client(project=self.PROJECT, connection=conn)
+        dataset = _Dataset(client)
+        file_obj = TextModeFile()
+        table = self._makeOne(self.TABLE_NAME, dataset=dataset)
+        with self.assertRaises(ValueError):
+            table.upload_from_file(file_obj, 'CSV', size=1234)
+
     def test_upload_from_file_size_failure(self):
         conn = _Connection()
         client = _Client(project=self.PROJECT, connection=conn)
@@ -1351,12 +1364,14 @@ def _upload_from_file_helper(self, **kw):
         return conn.http._requested, PATH, BODY
 
     def test_upload_from_file_w_bound_client_multipart(self):
-        from email.parser import Parser
         import json
         from six.moves.urllib.parse import parse_qsl
         from six.moves.urllib.parse import urlsplit
+        from gcloud._helpers import _to_bytes
+        from gcloud.streaming.test_transfer import _email_chunk_parser
 
         requested, PATH, BODY = self._upload_from_file_helper()
+        parse_chunk = _email_chunk_parser()
 
         self.assertEqual(len(requested), 1)
         req = requested[0]
@@ -1369,18 +1384,17 @@ def test_upload_from_file_w_bound_client_multipart(self):
         self.assertEqual(dict(parse_qsl(qs)),
                          {'uploadType': 'multipart'})
 
-        parser = Parser()
         ctype, boundary = [x.strip()
                            for x in req['headers']['content-type'].split(';')]
         self.assertEqual(ctype, 'multipart/related')
         self.assertTrue(boundary.startswith('boundary="=='))
         self.assertTrue(boundary.endswith('=="'))
 
-        divider = '--' + boundary[len('boundary="'):-1]
+        divider = b'--' + _to_bytes(boundary[len('boundary="'):-1])
         chunks = req['body'].split(divider)[1:-1]  # discard prolog / epilog
         self.assertEqual(len(chunks), 2)
 
-        text_msg = parser.parsestr(chunks[0].strip())
+        text_msg = parse_chunk(chunks[0].strip())
         self.assertEqual(dict(text_msg._headers),
                          {'Content-Type': 'application/json',
                           'MIME-Version': '1.0'})
@@ -1394,7 +1408,7 @@ def test_upload_from_file_w_bound_client_multipart(self):
         self.assertEqual(load_config['destinationTable'], DESTINATION_TABLE)
         self.assertEqual(load_config['sourceFormat'], 'CSV')
 
-        app_msg = parser.parsestr(chunks[1].strip())
+        app_msg = parse_chunk(chunks[1].strip())
         self.assertEqual(dict(app_msg._headers),
                          {'Content-Type': 'application/octet-stream',
                           'Content-Transfer-Encoding': 'binary',
diff --git a/gcloud/streaming/test_transfer.py b/gcloud/streaming/test_transfer.py
--- a/gcloud/streaming/test_transfer.py
+++ b/gcloud/streaming/test_transfer.py
@@ -1029,7 +1029,7 @@ def test_configure_request_w_simple_wo_body(self):
         self.assertEqual(request.loggable_body, '<media body>')
 
     def test_configure_request_w_simple_w_body(self):
-        from email.parser import Parser
+        from gcloud._helpers import _to_bytes
         from gcloud.streaming.transfer import SIMPLE_UPLOAD
         CONTENT = b'CONTENT'
         BODY = b'BODY'
@@ -1045,7 +1045,6 @@ def test_configure_request_w_simple_w_body(self):
         self.assertEqual(url_builder.query_params, {'uploadType': 'multipart'})
         self.assertEqual(url_builder.relative_path, config.simple_path)
 
-        parser = Parser()
         self.assertEqual(list(request.headers), ['content-type'])
         ctype, boundary = [x.strip()
                            for x in request.headers['content-type'].split(';')]
@@ -1053,23 +1052,24 @@ def test_configure_request_w_simple_w_body(self):
         self.assertTrue(boundary.startswith('boundary="=='))
         self.assertTrue(boundary.endswith('=="'))
 
-        divider = '--' + boundary[len('boundary="'):-1]
+        divider = b'--' + _to_bytes(boundary[len('boundary="'):-1])
         chunks = request.body.split(divider)[1:-1]  # discard prolog / epilog
         self.assertEqual(len(chunks), 2)
 
-        text_msg = parser.parsestr(chunks[0].strip())
+        parse_chunk = _email_chunk_parser()
+        text_msg = parse_chunk(chunks[0].strip())
         self.assertEqual(dict(text_msg._headers),
                          {'Content-Type': 'text/plain',
                           'MIME-Version': '1.0'})
         self.assertEqual(text_msg._payload, BODY.decode('ascii'))
 
-        app_msg = parser.parsestr(chunks[1].strip())
+        app_msg = parse_chunk(chunks[1].strip())
         self.assertEqual(dict(app_msg._headers),
                          {'Content-Type': self.MIME_TYPE,
                           'Content-Transfer-Encoding': 'binary',
                           'MIME-Version': '1.0'})
         self.assertEqual(app_msg._payload, CONTENT.decode('ascii'))
-        self.assertTrue('<media body>' in request.loggable_body)
+        self.assertTrue(b'<media body>' in request.loggable_body)
 
     def test_configure_request_w_resumable_wo_total_size(self):
         from gcloud.streaming.transfer import RESUMABLE_UPLOAD
@@ -1784,6 +1784,18 @@ def test__send_chunk_w_total_size_stream_exhausted(self):
         self.assertEqual(end, SIZE)
 
 
+def _email_chunk_parser():
+    import six
+    if six.PY3:  # pragma: NO COVER  Python3
+        from email.parser import BytesParser
+        parser = BytesParser()
+        return parser.parsebytes
+    else:
+        from email.parser import Parser
+        parser = Parser()
+        return parser.parsestr
+
+
 class _Dummy(object):
     def __init__(self, **kw):
         self.__dict__.update(kw)