Repository: ibis-project/ibis

Issue:
SUPP: Move all CI to Azure pipelines
The main win here is reduced CI times.

[Build insights from CircleCI](https://circleci.com/build-insights/gh/ibis-project) shows a median queue time of 30 minutes for builds of `ibis-project/master`. I believe we can reduce that to close to zero (once we drop support for Python 3.5) due to the fact that Azure pipelines gives public projects 10 free concurrent build workers.

I've already got [a build](https://dev.azure.com/cpcloud/ibis/_build/results?buildId=511) going that implements the changes to run all of our currently running CircleCI pipelines using Azure.





Discussion:

Resolving patch:
diff --git a/ci/datamgr.py b/ci/datamgr.py
--- a/ci/datamgr.py
+++ b/ci/datamgr.py
@@ -15,8 +15,6 @@
 from plumbum import local
 from toolz import dissoc
 
-import ibis
-
 SCRIPT_DIR = Path(__file__).parent.absolute()
 DATA_DIR_NAME = 'ibis-testing-data'
 DATA_DIR = Path(
@@ -26,7 +24,30 @@
 TEST_TABLES = ['functional_alltypes', 'diamonds', 'batting', 'awards_players']
 
 
-logger = ibis.util.get_logger('datamgr')
+def get_logger(name, level=None, format=None, propagate=False):
+    logging.basicConfig()
+    handler = logging.StreamHandler()
+
+    if format is None:
+        format = (
+            '%(relativeCreated)6d '
+            '%(name)-20s '
+            '%(levelname)-8s '
+            '%(threadName)-25s '
+            '%(message)s'
+        )
+    handler.setFormatter(logging.Formatter(fmt=format))
+    logger = logging.getLogger(name)
+    logger.propagate = propagate
+    logger.setLevel(
+        level
+        or getattr(logging, os.environ.get('LOGLEVEL', 'WARNING').upper())
+    )
+    logger.addHandler(handler)
+    return logger
+
+
+logger = get_logger(Path(__file__).with_suffix('').name)
 
 
 def recreate_database(driver, params, **kwargs):
diff --git a/ci/impalamgr.py b/ci/impalamgr.py
--- a/ci/impalamgr.py
+++ b/ci/impalamgr.py
@@ -23,7 +23,8 @@
 )
 
 
-logger = ibis.util.get_logger('impalamgr')
+logger = ibis.util.get_logger(Path(__file__).with_suffix('').name)
+
 
 ENV = IbisTestEnv()
 


Test patch:
diff --git a/ibis/bigquery/tests/conftest.py b/ibis/bigquery/tests/conftest.py
--- a/ibis/bigquery/tests/conftest.py
+++ b/ibis/bigquery/tests/conftest.py
@@ -10,6 +10,7 @@
 
 def connect(project_id, dataset_id):
     ga = pytest.importorskip('google.auth')
+    service_account = pytest.importorskip('google.oauth2.service_account')
     google_application_credentials = os.environ.get(
         "GOOGLE_APPLICATION_CREDENTIALS", None
     )
@@ -34,8 +35,13 @@ def connect(project_id, dataset_id):
         'No BigQuery credentials found using project_id={}, '
         'dataset_id={}. Skipping BigQuery tests.'
     ).format(project_id, dataset_id)
+    credentials = service_account.Credentials.from_service_account_file(
+        google_application_credentials
+    )
     try:
-        return ibis.bigquery.connect(project_id, dataset_id)
+        return ibis.bigquery.connect(
+            project_id, dataset_id, credentials=credentials
+        )
     except ga.exceptions.DefaultCredentialsError:
         pytest.skip(skip_message)
 
diff --git a/ibis/tests/all/conftest.py b/ibis/tests/all/conftest.py
--- a/ibis/tests/all/conftest.py
+++ b/ibis/tests/all/conftest.py
@@ -108,11 +108,7 @@ def pytest_pyfunc_call(pyfuncitem):
         assert isinstance(backend, Backend), "backend has type {!r}".format(
             type(backend).__name__
         )
-        pytest.xfail(
-            reason='{}: {}'.format(
-                type(backend).__name__, e
-            )
-        )
+        pytest.xfail(reason='{}: {}'.format(type(backend).__name__, e))
 
 
 pytestmark = pytest.mark.backend
@@ -188,3 +184,4 @@ def geo_df(geo):
     # Currently geo is implemented just for MapD
     if geo is not None:
         return geo.execute(limit=None)
+    return None
diff --git a/ibis/tests/test_version.py b/ibis/tests/test_version.py
--- a/ibis/tests/test_version.py
+++ b/ibis/tests/test_version.py
@@ -8,8 +8,8 @@
 
 
 @pytest.mark.skipif(
-    bool(os.environ.get("CIRCLECI", os.environ.get("AZURECI", None))),
-    reason="Testing import time on Circle CI is flaky due to machine variance",
+    bool(os.environ.get("AZURECI")),
+    reason="Testing import time on CI is flaky due to machine variance",
 )
 def test_import_time():
     pb = pytest.importorskip("plumbum")