Repository: googleapis/python-bigquery

Issue:
google-cloud-bigquery not compatible with pandas-gbq error
I have a compatibility issue with the new version of google-cloud-bigquery. When I try to install pandas `pip install --upgrade google-cloud-bigquery[bqstorage,pandas]` I get this error:

`ERROR: pandas-gbq 0.13.3 has requirement google-cloud-bigquery[bqstorage,pandas]<2.0.0dev,>=1.11.1, but you'll have google-cloud-bigquery 2.0.0 which is incompatible.`

As the results I cannot use pandas-gbq's functionalities such as `to_dataframe` etc.

Will appreciate your suggestions.

#### Environment details

  - OS type and version: MAC
  - Python version: Python 3.7.4
  - pip version: pip 19.3.1
  - `google-cloud-bigquery` version: 1.28.0



Discussion:
I recently released `pandas-gbq` version 0.14.0, which is compatible with `google-cloud-bigquery` 2.0.
I upgraded `pandas-gbq` to version 0.14.0 and it resolved the error I mentioned above when upgrading `google-cloud-bigquery` BUT I still have a problem writing BiqQuery results into a pandas DataFrame. So my assumption that it was caused by `pandas-gbq` is probably wrong.

I also seem to have another problem with PyYAML when upgrading `google-cloud-bigquery`:

`ERROR: awscli 1.16.277 has requirement PyYAML<5.2,>=3.10; python_version != "2.6" and python_version != "3.3", but you'll have pyyaml 5.3.1 which is incompatible.
Installing collected packages: pyyaml, libcst, google-cloud-bigquery-storage, google-cloud-bigquery
  Found existing installation: PyYAML 5.1.2
ERROR: Cannot uninstall 'PyYAML'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.`

It appears that the PyYAML conflict is due to `libscst`. I'll do some investigating as to why that dependency is needed.
I checked with the team and `libcst` is only needed for https://github.com/googleapis/python-bigquery/blob/master/scripts/fixup_bigquery_v2_keywords.py I'll follow-up with a PR to remove this dependency as that script isn't even relevant for this library.

Resolving patch:
diff --git a/scripts/fixup_bigquery_v2_keywords.py b/scripts/fixup_bigquery_v2_keywords.py
deleted file mode 100644
--- a/scripts/fixup_bigquery_v2_keywords.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2020 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import argparse
-import os
-import libcst as cst
-import pathlib
-import sys
-from typing import (Any, Callable, Dict, List, Sequence, Tuple)
-
-
-def partition(
-    predicate: Callable[[Any], bool],
-    iterator: Sequence[Any]
-) -> Tuple[List[Any], List[Any]]:
-    """A stable, out-of-place partition."""
-    results = ([], [])
-
-    for i in iterator:
-        results[int(predicate(i))].append(i)
-
-    # Returns trueList, falseList
-    return results[1], results[0]
-
-
-class bigqueryCallTransformer(cst.CSTTransformer):
-    CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata')
-    METHOD_TO_PARAMS: Dict[str, Tuple[str]] = {
-    'delete_model': ('project_id', 'dataset_id', 'model_id', ),
-    'get_model': ('project_id', 'dataset_id', 'model_id', ),
-    'list_models': ('project_id', 'dataset_id', 'max_results', 'page_token', ),
-    'patch_model': ('project_id', 'dataset_id', 'model_id', 'model', ),
-
-    }
-
-    def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode:
-        try:
-            key = original.func.attr.value
-            kword_params = self.METHOD_TO_PARAMS[key]
-        except (AttributeError, KeyError):
-            # Either not a method from the API or too convoluted to be sure.
-            return updated
-
-        # If the existing code is valid, keyword args come after positional args.
-        # Therefore, all positional args must map to the first parameters.
-        args, kwargs = partition(lambda a: not bool(a.keyword), updated.args)
-        if any(k.keyword.value == "request" for k in kwargs):
-            # We've already fixed this file, don't fix it again.
-            return updated
-
-        kwargs, ctrl_kwargs = partition(
-            lambda a: not a.keyword.value in self.CTRL_PARAMS,
-            kwargs
-        )
-
-        args, ctrl_args = args[:len(kword_params)], args[len(kword_params):]
-        ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl))
-                           for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS))
-
-        request_arg = cst.Arg(
-            value=cst.Dict([
-                cst.DictElement(
-                    cst.SimpleString("'{}'".format(name)),
-                    cst.Element(value=arg.value)
-                )
-                # Note: the args + kwargs looks silly, but keep in mind that
-                # the control parameters had to be stripped out, and that
-                # those could have been passed positionally or by keyword.
-                for name, arg in zip(kword_params, args + kwargs)]),
-            keyword=cst.Name("request")
-        )
-
-        return updated.with_changes(
-            args=[request_arg] + ctrl_kwargs
-        )
-
-
-def fix_files(
-    in_dir: pathlib.Path,
-    out_dir: pathlib.Path,
-    *,
-    transformer=bigqueryCallTransformer(),
-):
-    """Duplicate the input dir to the output dir, fixing file method calls.
-
-    Preconditions:
-    * in_dir is a real directory
-    * out_dir is a real, empty directory
-    """
-    pyfile_gen = (
-        pathlib.Path(os.path.join(root, f))
-        for root, _, files in os.walk(in_dir)
-        for f in files if os.path.splitext(f)[1] == ".py"
-    )
-
-    for fpath in pyfile_gen:
-        with open(fpath, 'r') as f:
-            src = f.read()
-
-        # Parse the code and insert method call fixes.
-        tree = cst.parse_module(src)
-        updated = tree.visit(transformer)
-
-        # Create the path and directory structure for the new file.
-        updated_path = out_dir.joinpath(fpath.relative_to(in_dir))
-        updated_path.parent.mkdir(parents=True, exist_ok=True)
-
-        # Generate the updated source file at the corresponding path.
-        with open(updated_path, 'w') as f:
-            f.write(updated.code)
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description="""Fix up source that uses the bigquery client library.
-
-The existing sources are NOT overwritten but are copied to output_dir with changes made.
-
-Note: This tool operates at a best-effort level at converting positional
-      parameters in client method calls to keyword based parameters.
-      Cases where it WILL FAIL include
-      A) * or ** expansion in a method call.
-      B) Calls via function or method alias (includes free function calls)
-      C) Indirect or dispatched calls (e.g. the method is looked up dynamically)
-
-      These all constitute false negatives. The tool will also detect false
-      positives when an API method shares a name with another method.
-""")
-    parser.add_argument(
-        '-d',
-        '--input-directory',
-        required=True,
-        dest='input_dir',
-        help='the input directory to walk for python files to fix up',
-    )
-    parser.add_argument(
-        '-o',
-        '--output-directory',
-        required=True,
-        dest='output_dir',
-        help='the directory to output files fixed via un-flattening',
-    )
-    args = parser.parse_args()
-    input_dir = pathlib.Path(args.input_dir)
-    output_dir = pathlib.Path(args.output_dir)
-    if not input_dir.is_dir():
-        print(
-            f"input directory '{input_dir}' does not exist or is not a directory",
-            file=sys.stderr,
-        )
-        sys.exit(-1)
-
-    if not output_dir.is_dir():
-        print(
-            f"output directory '{output_dir}' does not exist or is not a directory",
-            file=sys.stderr,
-        )
-        sys.exit(-1)
-
-    if os.listdir(output_dir):
-        print(
-            f"output directory '{output_dir}' is not empty",
-            file=sys.stderr,
-        )
-        sys.exit(-1)
-
-    fix_files(input_dir, output_dir)
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,6 @@
 dependencies = [
     "google-api-core[grpc] >= 1.22.2, < 2.0.0dev",
     "proto-plus >= 1.10.0",
-    "libcst >= 0.2.5",
     "google-cloud-core >= 1.4.1, < 2.0dev",
     "google-resumable-media >= 0.6.0, < 2.0dev",
     "six >=1.13.0,< 2.0.0dev",
@@ -125,7 +124,6 @@
     install_requires=dependencies,
     extras_require=extras,
     python_requires=">=3.6",
-    scripts=["scripts/fixup_bigquery_v2_keywords.py"],
     include_package_data=True,
     zip_safe=False,
 )
diff --git a/synth.py b/synth.py
--- a/synth.py
+++ b/synth.py
@@ -36,6 +36,7 @@
         "README.rst",
         "noxfile.py",
         "setup.py",
+        "scripts/fixup_bigquery_v2_keywords.py",
         library / f"google/cloud/bigquery/__init__.py",
         library / f"google/cloud/bigquery/py.typed",
         # There are no public API endpoints for the generated ModelServiceClient,


Test patch: