Repository: googleapis/google-cloud-python

Issue:
Add support for uploading table data via multipart/resumable requests
See:
- https://cloud.google.com/bigquery/loading-data-post-request#multipart
- https://cloud.google.com/bigquery/loading-data-post-request#resumable



Discussion:
Reminder: After implementation, update the usage doc

Now incorporates (closed) #1262.


Resolving patch:
diff --git a/gcloud/bigquery/client.py b/gcloud/bigquery/client.py
--- a/gcloud/bigquery/client.py
+++ b/gcloud/bigquery/client.py
@@ -102,20 +102,18 @@ def dataset(self, dataset_name):
         """
         return Dataset(dataset_name, client=self)
 
-    def _job_from_resource(self, resource):
+    def job_from_resource(self, resource):
         """Detect correct job type from resource and instantiate.
 
-        Helper for :meth:`list_jobs`.
-
         :type resource: dict
         :param resource: one job resource from API response
 
         :rtype; One of:
                 :class:`gcloud.bigquery.job.LoadTableFromStorageJob`,
-                :class:`gcloud.bigquery.job.CopyJob,
-                :class:`gcloud.bigquery.job.ExtractTableToStorageJob,
-                :class:`gcloud.bigquery.job.QueryJob,
-                :class:`gcloud.bigquery.job.RunSyncQueryJob
+                :class:`gcloud.bigquery.job.CopyJob`,
+                :class:`gcloud.bigquery.job.ExtractTableToStorageJob`,
+                :class:`gcloud.bigquery.job.QueryJob`,
+                :class:`gcloud.bigquery.job.RunSyncQueryJob`
         :returns: the job instance, constructed via the resource
         """
         config = resource['configuration']
@@ -176,8 +174,7 @@ def list_jobs(self, max_results=None, page_token=None, all_users=None,
         path = '/projects/%s/jobs' % (self.project,)
         resp = self.connection.api_request(method='GET', path=path,
                                            query_params=params)
-        jobs = [self._job_from_resource(resource)
-                for resource in resp['jobs']]
+        jobs = [self.job_from_resource(resource) for resource in resp['jobs']]
         return jobs, resp.get('nextPageToken')
 
     def load_table_from_storage(self, job_name, destination, *source_uris):
diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py
--- a/gcloud/bigquery/table.py
+++ b/gcloud/bigquery/table.py
@@ -15,6 +15,8 @@
 """Define API Datasets."""
 
 import datetime
+import json
+import os
 
 import six
 
@@ -22,6 +24,10 @@
 from gcloud._helpers import _microseconds_from_datetime
 from gcloud._helpers import _millis_from_datetime
 from gcloud.exceptions import NotFound
+from gcloud.streaming.http_wrapper import Request
+from gcloud.streaming.http_wrapper import make_api_request
+from gcloud.streaming.transfer import RESUMABLE_UPLOAD
+from gcloud.streaming.transfer import Upload
 from gcloud.bigquery._helpers import _rows_from_json
 
 
@@ -693,6 +699,225 @@ def insert_data(self,
 
         return errors
 
+    def upload_from_file(self,  # pylint: disable=R0913,R0914
+                         file_obj,
+                         source_format,
+                         rewind=False,
+                         size=None,
+                         num_retries=6,
+                         allow_jagged_rows=None,
+                         allow_quoted_newlines=None,
+                         create_disposition=None,
+                         encoding=None,
+                         field_delimiter=None,
+                         ignore_unknown_values=None,
+                         max_bad_records=None,
+                         quote_character=None,
+                         skip_leading_rows=None,
+                         write_disposition=None,
+                         client=None):
+        """Upload the contents of this table from a file-like object.
+
+        The content type of the upload will either be
+        - The value passed in to the function (if any)
+        - ``text/csv``.
+
+        :type file_obj: file
+        :param file_obj: A file handle open for reading.
+
+        :type source_format: string
+        :param source_format: one of 'CSV' or 'NEWLINE_DELIMITED_JSON'.
+                              job configuration option; see
+                              :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type rewind: boolean
+        :param rewind: If True, seek to the beginning of the file handle before
+                       writing the file to Cloud Storage.
+
+        :type size: int
+        :param size: The number of bytes to read from the file handle.
+                     If not provided, we'll try to guess the size using
+                     :func:`os.fstat`. (If the file handle is not from the
+                     filesystem this won't be possible.)
+
+        :type num_retries: integer
+        :param num_retries: Number of upload retries. Defaults to 6.
+
+        :type allow_jagged_rows: boolean
+        :param allow_jagged_rows: job configuration option;  see
+                                  :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type allow_quoted_newlines: boolean
+        :param allow_quoted_newlines: job configuration option; see
+                                      :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type create_disposition: string
+        :param create_disposition: job configuration option; see
+                                   :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type encoding: string
+        :param encoding: job configuration option; see
+                         :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type field_delimiter: string
+        :param field_delimiter: job configuration option; see
+                                :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type ignore_unknown_values: boolean
+        :param ignore_unknown_values: job configuration option; see
+                                      :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type max_bad_records: integer
+        :param max_bad_records: job configuration option; see
+                                :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type quote_character: string
+        :param quote_character: job configuration option; see
+                                :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type skip_leading_rows: integer
+        :param skip_leading_rows: job configuration option; see
+                                  :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type write_disposition: string
+        :param write_disposition: job configuration option; see
+                                  :meth:`gcloud.bigquery.job.LoadJob`
+
+        :type client: :class:`gcloud.storage.client.Client` or ``NoneType``
+        :param client: Optional. The client to use.  If not passed, falls back
+                       to the ``client`` stored on the current dataset.
+
+        :rtype: :class:`gcloud.bigquery.jobs.LoadTableFromStorageJob`
+        :returns: the job instance used to load the data (e.g., for
+                  querying status)
+        :raises: :class:`ValueError` if size is not passed in and can not be
+                 determined
+        """
+        client = self._require_client(client)
+        connection = client.connection
+        content_type = 'application/octet-stream'
+
+        # Rewind the file if desired.
+        if rewind:
+            file_obj.seek(0, os.SEEK_SET)
+
+        # Get the basic stats about the file.
+        total_bytes = size
+        if total_bytes is None:
+            if hasattr(file_obj, 'fileno'):
+                total_bytes = os.fstat(file_obj.fileno()).st_size
+            else:
+                raise ValueError('total bytes could not be determined. Please '
+                                 'pass an explicit size.')
+        headers = {
+            'Accept': 'application/json',
+            'Accept-Encoding': 'gzip, deflate',
+            'User-Agent': connection.USER_AGENT,
+            'content-type': 'application/json',
+        }
+
+        metadata = {
+            'configuration': {
+                'load': {
+                    'sourceFormat': source_format,
+                    'schema': {
+                        'fields': _build_schema_resource(self._schema),
+                    },
+                    'destinationTable': {
+                        'projectId': self._dataset.project,
+                        'datasetId': self._dataset.name,
+                        'tableId': self.name,
+                    }
+                }
+            }
+        }
+
+        _configure_job_metadata(metadata, allow_jagged_rows,
+                                allow_quoted_newlines, create_disposition,
+                                encoding, field_delimiter,
+                                ignore_unknown_values, max_bad_records,
+                                quote_character, skip_leading_rows,
+                                write_disposition)
+
+        upload = Upload(file_obj, content_type, total_bytes,
+                        auto_transfer=False)
+
+        url_builder = _UrlBuilder()
+        upload_config = _UploadConfig()
+
+        # Base URL may change once we know simple vs. resumable.
+        base_url = connection.API_BASE_URL + '/upload'
+        path = '/projects/%s/jobs' % (self._dataset.project,)
+        upload_url = connection.build_api_url(api_base_url=base_url, path=path)
+
+        # Use apitools 'Upload' facility.
+        request = Request(upload_url, 'POST', headers,
+                          body=json.dumps(metadata))
+
+        upload.configure_request(upload_config, request, url_builder)
+        query_params = url_builder.query_params
+        base_url = connection.API_BASE_URL + '/upload'
+        request.url = connection.build_api_url(api_base_url=base_url,
+                                               path=path,
+                                               query_params=query_params)
+        upload.initialize_upload(request, connection.http)
+
+        if upload.strategy == RESUMABLE_UPLOAD:
+            http_response = upload.stream_file(use_chunks=True)
+        else:
+            http_response = make_api_request(connection.http, request,
+                                             retries=num_retries)
+        response_content = http_response.content
+        if not isinstance(response_content,
+                          six.string_types):  # pragma: NO COVER  Python3
+            response_content = response_content.decode('utf-8')
+        return client.job_from_resource(json.loads(response_content))
+
+
+def _configure_job_metadata(metadata,  # pylint: disable=R0913
+                            allow_jagged_rows,
+                            allow_quoted_newlines,
+                            create_disposition,
+                            encoding,
+                            field_delimiter,
+                            ignore_unknown_values,
+                            max_bad_records,
+                            quote_character,
+                            skip_leading_rows,
+                            write_disposition):
+    """Helper for :meth:`Table.upload_from_file`."""
+    load_config = metadata['configuration']['load']
+
+    if allow_jagged_rows is not None:
+        load_config['allowJaggedRows'] = allow_jagged_rows
+
+    if allow_quoted_newlines is not None:
+        load_config['allowQuotedNewlines'] = allow_quoted_newlines
+
+    if create_disposition is not None:
+        load_config['createDisposition'] = create_disposition
+
+    if encoding is not None:
+        load_config['encoding'] = encoding
+
+    if field_delimiter is not None:
+        load_config['fieldDelimiter'] = field_delimiter
+
+    if ignore_unknown_values is not None:
+        load_config['ignoreUnknownValues'] = ignore_unknown_values
+
+    if max_bad_records is not None:
+        load_config['maxBadRecords'] = max_bad_records
+
+    if quote_character is not None:
+        load_config['quote'] = quote_character
+
+    if skip_leading_rows is not None:
+        load_config['skipLeadingRows'] = skip_leading_rows
+
+    if write_disposition is not None:
+        load_config['writeDisposition'] = write_disposition
+
 
 def _parse_schema_resource(info):
     """Parse a resource fragment into a schema field.
@@ -739,3 +964,20 @@ def _build_schema_resource(fields):
             info['fields'] = _build_schema_resource(field.fields)
         infos.append(info)
     return infos
+
+
+class _UploadConfig(object):
+    """Faux message FBO apitools' 'configure_request'."""
+    accept = ['*/*']
+    max_size = None
+    resumable_multipart = True
+    resumable_path = u'/upload/bigquery/v2/projects/{project}/jobs'
+    simple_multipart = True
+    simple_path = u'/upload/bigquery/v2/projects/{project}/jobs'
+
+
+class _UrlBuilder(object):
+    """Faux builder FBO apitools' 'configure_request'"""
+    def __init__(self):
+        self.query_params = {}
+        self._relative_path = ''


Test patch:
diff --git a/gcloud/bigquery/test_client.py b/gcloud/bigquery/test_client.py
--- a/gcloud/bigquery/test_client.py
+++ b/gcloud/bigquery/test_client.py
@@ -128,12 +128,12 @@ def test_dataset(self):
         self.assertEqual(dataset.name, DATASET)
         self.assertTrue(dataset._client is client)
 
-    def test__job_from_resource_unknown_type(self):
+    def test_job_from_resource_unknown_type(self):
         PROJECT = 'PROJECT'
         creds = _Credentials()
         client = self._makeOne(PROJECT, creds)
         with self.assertRaises(ValueError):
-            client._job_from_resource({'configuration': {'nonesuch': {}}})
+            client.job_from_resource({'configuration': {'nonesuch': {}}})
 
     def test_list_jobs_defaults(self):
         from gcloud.bigquery.job import LoadTableFromStorageJob
diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py
--- a/gcloud/bigquery/test_table.py
+++ b/gcloud/bigquery/test_table.py
@@ -1,3 +1,4 @@
+# pylint: disable=too-many-lines
 # Copyright 2015 Google Inc. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -1286,6 +1287,208 @@ def _row_data(row):
         self.assertEqual(req['path'], '/%s' % PATH)
         self.assertEqual(req['data'], SENT)
 
+    def test_upload_from_file_size_failure(self):
+        conn = _Connection()
+        client = _Client(project=self.PROJECT, connection=conn)
+        dataset = _Dataset(client)
+        file_obj = object()
+        table = self._makeOne(self.TABLE_NAME, dataset=dataset)
+        with self.assertRaises(ValueError):
+            table.upload_from_file(file_obj, 'CSV', size=None)
+
+    def _upload_from_file_helper(self, **kw):
+        import csv
+        import datetime
+        from six.moves.http_client import OK
+        from gcloud._helpers import UTC
+        from gcloud._testing import _NamedTemporaryFile
+        from gcloud.bigquery.table import SchemaField
+
+        WHEN_TS = 1437767599.006
+        WHEN = datetime.datetime.utcfromtimestamp(WHEN_TS).replace(
+            tzinfo=UTC)
+        PATH = 'projects/%s/jobs' % (self.PROJECT,)
+        response = {'status': OK}
+        conn = _Connection(
+            (response, b'{}'),
+        )
+        client = _Client(project=self.PROJECT, connection=conn)
+        expected_job = object()
+        if 'client' in kw:
+            kw['client']._job = expected_job
+        else:
+            client._job = expected_job
+        dataset = _Dataset(client)
+        full_name = SchemaField('full_name', 'STRING', mode='REQUIRED')
+        age = SchemaField('age', 'INTEGER', mode='REQUIRED')
+        joined = SchemaField('joined', 'TIMESTAMP', mode='NULLABLE')
+        table = self._makeOne(self.TABLE_NAME, dataset=dataset,
+                              schema=[full_name, age, joined])
+        ROWS = [
+            ('Phred Phlyntstone', 32, WHEN),
+            ('Bharney Rhubble', 33, WHEN + datetime.timedelta(seconds=1)),
+            ('Wylma Phlyntstone', 29, WHEN + datetime.timedelta(seconds=2)),
+            ('Bhettye Rhubble', 27, None),
+        ]
+
+        with _NamedTemporaryFile() as temp:
+            with open(temp.name, 'w') as file_obj:
+                writer = csv.writer(file_obj)
+                writer.writerow(('full_name', 'age', 'joined'))
+                writer.writerows(ROWS)
+
+            with open(temp.name, 'rb') as file_obj:
+                BODY = file_obj.read()
+                explicit_size = kw.pop('_explicit_size', False)
+                if explicit_size:
+                    kw['size'] = len(BODY)
+                job = table.upload_from_file(
+                    file_obj, 'CSV', rewind=True, **kw)
+
+        self.assertTrue(job is expected_job)
+        return conn.http._requested, PATH, BODY
+
+    def test_upload_from_file_w_bound_client_multipart(self):
+        from email.parser import Parser
+        import json
+        from six.moves.urllib.parse import parse_qsl
+        from six.moves.urllib.parse import urlsplit
+
+        requested, PATH, BODY = self._upload_from_file_helper()
+
+        self.assertEqual(len(requested), 1)
+        req = requested[0]
+        self.assertEqual(req['method'], 'POST')
+        uri = req['uri']
+        scheme, netloc, path, qs, _ = urlsplit(uri)
+        self.assertEqual(scheme, 'http')
+        self.assertEqual(netloc, 'example.com')
+        self.assertEqual(path, '/%s' % PATH)
+        self.assertEqual(dict(parse_qsl(qs)),
+                         {'uploadType': 'multipart'})
+
+        parser = Parser()
+        ctype, boundary = [x.strip()
+                           for x in req['headers']['content-type'].split(';')]
+        self.assertEqual(ctype, 'multipart/related')
+        self.assertTrue(boundary.startswith('boundary="=='))
+        self.assertTrue(boundary.endswith('=="'))
+
+        divider = '--' + boundary[len('boundary="'):-1]
+        chunks = req['body'].split(divider)[1:-1]  # discard prolog / epilog
+        self.assertEqual(len(chunks), 2)
+
+        text_msg = parser.parsestr(chunks[0].strip())
+        self.assertEqual(dict(text_msg._headers),
+                         {'Content-Type': 'application/json',
+                          'MIME-Version': '1.0'})
+        metadata = json.loads(text_msg._payload)
+        load_config = metadata['configuration']['load']
+        DESTINATION_TABLE = {
+            'projectId': self.PROJECT,
+            'datasetId': self.DS_NAME,
+            'tableId': self.TABLE_NAME,
+        }
+        self.assertEqual(load_config['destinationTable'], DESTINATION_TABLE)
+        self.assertEqual(load_config['sourceFormat'], 'CSV')
+
+        app_msg = parser.parsestr(chunks[1].strip())
+        self.assertEqual(dict(app_msg._headers),
+                         {'Content-Type': 'application/octet-stream',
+                          'Content-Transfer-Encoding': 'binary',
+                          'MIME-Version': '1.0'})
+        body = BODY.decode('ascii')
+        body_lines = [line.strip() for line in body.splitlines()]
+        payload_lines = app_msg._payload.splitlines()
+        self.assertEqual(payload_lines, body_lines)
+
+    # pylint: disable=R0915
+    def test_upload_from_file_w_explicit_client_resumable(self):
+        import json
+        from six.moves.http_client import OK
+        from six.moves.urllib.parse import parse_qsl
+        from six.moves.urllib.parse import urlsplit
+        from gcloud._testing import _Monkey
+        from gcloud.bigquery import table as MUT
+
+        UPLOAD_PATH = 'https://example.com/upload/test'
+        initial_response = {'status': OK, 'location': UPLOAD_PATH}
+        upload_response = {'status': OK}
+        conn = _Connection(
+            (initial_response, b'{}'),
+            (upload_response, b'{}'),
+        )
+        client = _Client(project=self.PROJECT, connection=conn)
+
+        class _UploadConfig(object):
+            accept = ['*/*']
+            max_size = None
+            resumable_multipart = True
+            resumable_path = u'/upload/bigquery/v2/projects/{project}/jobs'
+            simple_multipart = True
+            simple_path = u''  # force resumable
+
+        with _Monkey(MUT, _UploadConfig=_UploadConfig):
+            orig_requested, PATH, BODY = self._upload_from_file_helper(
+                allow_jagged_rows=False,
+                allow_quoted_newlines=False,
+                create_disposition='CREATE_IF_NEEDED',
+                encoding='utf8',
+                field_delimiter=',',
+                ignore_unknown_values=False,
+                max_bad_records=0,
+                quote_character='"',
+                skip_leading_rows=1,
+                write_disposition='WRITE_APPEND',
+                client=client,
+                _explicit_size=True)
+
+        self.assertEqual(len(orig_requested), 0)
+
+        requested = conn.http._requested
+        self.assertEqual(len(requested), 2)
+        req = requested[0]
+        self.assertEqual(req['method'], 'POST')
+        uri = req['uri']
+        scheme, netloc, path, qs, _ = urlsplit(uri)
+        self.assertEqual(scheme, 'http')
+        self.assertEqual(netloc, 'example.com')
+        self.assertEqual(path, '/%s' % PATH)
+        self.assertEqual(dict(parse_qsl(qs)),
+                         {'uploadType': 'resumable'})
+
+        self.assertEqual(req['headers']['content-type'], 'application/json')
+        metadata = json.loads(req['body'])
+        load_config = metadata['configuration']['load']
+        DESTINATION_TABLE = {
+            'projectId': self.PROJECT,
+            'datasetId': self.DS_NAME,
+            'tableId': self.TABLE_NAME,
+        }
+        self.assertEqual(load_config['destinationTable'], DESTINATION_TABLE)
+        self.assertEqual(load_config['sourceFormat'], 'CSV')
+        self.assertEqual(load_config['allowJaggedRows'], False)
+        self.assertEqual(load_config['allowQuotedNewlines'], False)
+        self.assertEqual(load_config['createDisposition'], 'CREATE_IF_NEEDED')
+        self.assertEqual(load_config['encoding'], 'utf8')
+        self.assertEqual(load_config['fieldDelimiter'], ',')
+        self.assertEqual(load_config['ignoreUnknownValues'], False)
+        self.assertEqual(load_config['maxBadRecords'], 0)
+        self.assertEqual(load_config['quote'], '"')
+        self.assertEqual(load_config['skipLeadingRows'], 1)
+        self.assertEqual(load_config['writeDisposition'], 'WRITE_APPEND')
+
+        req = requested[1]
+        self.assertEqual(req['method'], 'PUT')
+        self.assertEqual(req['uri'], UPLOAD_PATH)
+        headers = req['headers']
+        length = len(BODY)
+        self.assertEqual(headers['Content-Type'], 'application/octet-stream')
+        self.assertEqual(headers['Content-Range'],
+                         'bytes 0-%d/%d' % (length - 1, length))
+        self.assertEqual(headers['content-length'], '%d' % (length,))
+        self.assertEqual(req['body'], BODY)
+
 
 class Test_parse_schema_resource(unittest2.TestCase, _SchemaBase):
 
@@ -1401,6 +1604,9 @@ def __init__(self, project='project', connection=None):
         self.project = project
         self.connection = connection
 
+    def job_from_resource(self, resource):  # pylint: disable=W0613
+        return self._job
+
 
 class _Dataset(object):
 
@@ -1418,12 +1624,38 @@ def project(self):
         return self._client.project
 
 
-class _Connection(object):
+class _Responder(object):
 
     def __init__(self, *responses):
-        self._responses = responses
+        self._responses = responses[:]
         self._requested = []
 
+    def _respond(self, **kw):
+        self._requested.append(kw)
+        response, self._responses = self._responses[0], self._responses[1:]
+        return response
+
+
+class _HTTP(_Responder):
+
+    connections = {}  # For google-apitools debugging.
+
+    def request(self, uri, method, headers, body, **kw):
+        if hasattr(body, 'read'):
+            body = body.read()
+        return self._respond(uri=uri, method=method, headers=headers,
+                             body=body, **kw)
+
+
+class _Connection(_Responder):
+
+    API_BASE_URL = 'http://example.com'
+    USER_AGENT = 'testing 1.2.3'
+
+    def __init__(self, *responses):
+        super(_Connection, self).__init__(*responses)
+        self.http = _HTTP(*responses)
+
     def api_request(self, **kw):
         from gcloud.exceptions import NotFound
         self._requested.append(kw)
@@ -1434,3 +1666,13 @@ def api_request(self, **kw):
             raise NotFound('miss')
         else:
             return response
+
+    def build_api_url(self, path, query_params=None,
+                      api_base_url=API_BASE_URL):
+        from six.moves.urllib.parse import urlencode
+        from six.moves.urllib.parse import urlsplit
+        from six.moves.urllib.parse import urlunsplit
+        # Mimic the build_api_url interface.
+        qs = urlencode(query_params or {})
+        scheme, netloc, _, _, _ = urlsplit(api_base_url)
+        return urlunsplit((scheme, netloc, path, qs, ''))