Repository: dask/dask

Issue:
test_s3.py failing on Travis
`dask/bytes/tests/test_s3.py` is failing on the Python 3.6 Travis build. Here's the traceback:

```python
============================= test session starts ==============================
platform linux -- Python 3.6.6, pytest-3.10.0, py-1.7.0, pluggy-0.8.0 -- /home/travis/miniconda/envs/test-environment/bin/python
cachedir: .pytest_cache
rootdir: /home/travis/build/dask/dask, inifile: setup.cfg
plugins: xdist-1.24.1, forked-0.2
collected 6561 items / 1 errors / 5 skipped                                    
==================================== ERRORS ====================================
_________________ ERROR collecting dask/bytes/tests/test_s3.py _________________
dask/bytes/tests/test_s3.py:10: in <module>
    moto = pytest.importorskip('moto')
../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/__init__.py:8: in <module>
    from .acm import mock_acm  # flake8: noqa
../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/acm/__init__.py:2: in <module>
    from .models import acm_backends
../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/acm/models.py:7: in <module>
    from moto.ec2 import ec2_backends
../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/ec2/__init__.py:2: in <module>
    from .models import ec2_backends
../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/ec2/models.py:13: in <module>
    import boto.ec2
../../../miniconda/envs/test-environment/lib/python3.6/site-packages/boto/__init__.py:1216: in <module>
    boto.plugin.load_plugins(config)
E   AttributeError: module 'boto' has no attribute 'plugin'
```

See https://travis-ci.org/dask/dask/jobs/463980937



Discussion:
Boto3 appears to have been updated such that moto doesn't work with latest. Suggest pinning to previous version.

On December 5, 2018 11:13:46 PM EST, James Bourbeau <notifications@github.com> wrote:
>`dask/bytes/tests/test_s3.py` is failing on the Python 3.6 Travis
>build. Here's the traceback:
>
>```python
>============================= test session starts
>==============================
>platform linux -- Python 3.6.6, pytest-3.10.0, py-1.7.0, pluggy-0.8.0
>-- /home/travis/miniconda/envs/test-environment/bin/python
>cachedir: .pytest_cache
>rootdir: /home/travis/build/dask/dask, inifile: setup.cfg
>plugins: xdist-1.24.1, forked-0.2
>collected 6561 items / 1 errors / 5 skipped                            
>       
>==================================== ERRORS
>====================================
>_________________ ERROR collecting dask/bytes/tests/test_s3.py
>_________________
>dask/bytes/tests/test_s3.py:10: in <module>
>    moto = pytest.importorskip('moto')
>../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/__init__.py:8:
>in <module>
>    from .acm import mock_acm  # flake8: noqa
>../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/acm/__init__.py:2:
>in <module>
>    from .models import acm_backends
>../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/acm/models.py:7:
>in <module>
>    from moto.ec2 import ec2_backends
>../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/ec2/__init__.py:2:
>in <module>
>    from .models import ec2_backends
>../../../miniconda/envs/test-environment/lib/python3.6/site-packages/moto/ec2/models.py:13:
>in <module>
>    import boto.ec2
>../../../miniconda/envs/test-environment/lib/python3.6/site-packages/boto/__init__.py:1216:
>in <module>
>    boto.plugin.load_plugins(config)
>E   AttributeError: module 'boto' has no attribute 'plugin'
>```
>
>See https://travis-ci.org/dask/dask/jobs/463980937
>
>
>-- 
>You are receiving this because you are subscribed to this thread.
>Reply to this email directly or view it on GitHub:
>https://github.com/dask/dask/issues/4273

-- 
Sent from my Android device with K-9 Mail. Please excuse my brevity.
There does appear to be a genuine bug also, affecting test_s3::test_open_files_write - `open_files` with `mode='wb'` and a list of paths returns only one `OpenFile`. Looking into it now.

Resolving patch:
diff --git a/dask/bytes/core.py b/dask/bytes/core.py
--- a/dask/bytes/core.py
+++ b/dask/bytes/core.py
@@ -295,6 +295,7 @@ def expand_paths_if_needed(paths, mode, num, fs, name_function):
         raise ValueError("When writing data, only one filename mask can be specified.")
     for curr_path in paths:
         if '*' in curr_path:
+            glob = True
             if 'w' in mode:
                 # expand using name_function
                 expanded_paths.extend(_expand_paths(curr_path, name_function, num))
@@ -302,9 +303,10 @@ def expand_paths_if_needed(paths, mode, num, fs, name_function):
                 # expand using glob
                 expanded_paths.extend(fs.glob(curr_path))
         else:
+            glob = False
             expanded_paths.append(curr_path)
     # if we generated more paths that asked for, trim the list
-    if 'w' in mode and len(expanded_paths) > num:
+    if 'w' in mode and len(expanded_paths) > num and glob:
         expanded_paths = expanded_paths[:num]
     return expanded_paths
 


Test patch:
diff --git a/dask/bytes/tests/test_s3.py b/dask/bytes/tests/test_s3.py
--- a/dask/bytes/tests/test_s3.py
+++ b/dask/bytes/tests/test_s3.py
@@ -1,9 +1,12 @@
 from __future__ import print_function, division, absolute_import
 
+import io
 import sys
+import os
 from contextlib import contextmanager
 
 import pytest
+import numpy as np
 
 s3fs = pytest.importorskip('s3fs')
 boto3 = pytest.importorskip('boto3')
@@ -32,43 +35,142 @@
                                    b'{"amount": 800, "name": "Dennis"}\n')}
 
 
-@pytest.yield_fixture
-def s3():
-    # writable local S3 system
-    import moto
+@contextmanager
+def ensure_safe_environment_variables():
+    """
+    Get a context manager to safely set environment variables
+    All changes will be undone on close, hence environment variables set
+    within this contextmanager will neither persist nor change global state.
+    """
+    saved_environ = dict(os.environ)
     try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(saved_environ)
+
+
+@pytest.fixture()
+def s3():
+    with ensure_safe_environment_variables():
+        # temporary workaround as moto fails for botocore >= 1.11 otherwise,
+        # see https://github.com/spulec/moto/issues/1924 & 1952
+        os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
+        os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
+
+        # writable local S3 system
+        import moto
         with moto.mock_s3():
             client = boto3.client('s3')
             client.create_bucket(Bucket=test_bucket_name, ACL='public-read-write')
             for f, data in files.items():
                 client.put_object(Bucket=test_bucket_name, Key=f, Body=data)
             yield s3fs.S3FileSystem(anon=True)
-    finally:
-        httpretty.HTTPretty.disable()
-        httpretty.HTTPretty.reset()
+
+            httpretty.HTTPretty.disable()
+            httpretty.HTTPretty.reset()
 
 
 @contextmanager
 def s3_context(bucket, files):
-    m = moto.mock_s3()
-    m.start()
+    with ensure_safe_environment_variables():
+        # temporary workaround as moto fails for botocore >= 1.11 otherwise,
+        # see https://github.com/spulec/moto/issues/1924 & 1952
+        os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
+        os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
+
+        with moto.mock_s3():
+            client = boto3.client('s3')
+            client.create_bucket(Bucket=bucket, ACL='public-read-write')
+            for f, data in files.items():
+                client.put_object(Bucket=bucket, Key=f, Body=data)
+
+            yield DaskS3FileSystem(anon=True)
+
+            for f, data in files.items():
+                try:
+                    client.delete_object(Bucket=bucket, Key=f, Body=data)
+                except Exception:
+                    pass
+                finally:
+                    httpretty.HTTPretty.disable()
+                    httpretty.HTTPretty.reset()
+
+
+@pytest.fixture()
+@pytest.mark.slow
+def s3_with_yellow_tripdata(s3):
+    """
+    Fixture with sample yellowtrip CSVs loaded into S3.
+
+    Provides the following CSVs:
+
+    * s3://test/nyc-taxi/2015/yellow_tripdata_2015-01.csv
+    * s3://test/nyc-taxi/2014/yellow_tripdata_2015-mm.csv
+      for mm from 01 - 12.
+    """
+    pd = pytest.importorskip('pandas')
+
+    data = {'VendorID': {0: 2, 1: 1, 2: 1, 3: 1, 4: 1},
+            'tpep_pickup_datetime': {0: '2015-01-15 19:05:39',
+                                     1: '2015-01-10 20:33:38',
+                                     2: '2015-01-10 20:33:38',
+                                     3: '2015-01-10 20:33:39',
+                                     4: '2015-01-10 20:33:39'},
+            'tpep_dropoff_datetime': {0: '2015-01-15 19:23:42',
+                                      1: '2015-01-10 20:53:28',
+                                      2: '2015-01-10 20:43:41',
+                                      3: '2015-01-10 20:35:31',
+                                      4: '2015-01-10 20:52:58'},
+            'passenger_count': {0: 1, 1: 1, 2: 1, 3: 1, 4: 1},
+            'trip_distance': {0: 1.59, 1: 3.3, 2: 1.8, 3: 0.5, 4: 3.0},
+            'pickup_longitude': {0: -73.993896484375,
+                                 1: -74.00164794921875,
+                                 2: -73.96334075927734,
+                                 3: -74.00908660888672,
+                                 4: -73.97117614746094},
+            'pickup_latitude': {0: 40.7501106262207,
+                                1: 40.7242431640625,
+                                2: 40.80278778076172,
+                                3: 40.71381759643555,
+                                4: 40.762428283691406},
+            'RateCodeID': {0: 1, 1: 1, 2: 1, 3: 1, 4: 1},
+            'store_and_fwd_flag': {0: 'N', 1: 'N', 2: 'N', 3: 'N', 4: 'N'},
+            'dropoff_longitude': {0: -73.97478485107422,
+                                  1: -73.99441528320312,
+                                  2: -73.95182037353516,
+                                  3: -74.00432586669923,
+                                  4: -74.00418090820312},
+            'dropoff_latitude': {0: 40.75061798095703,
+                                 1: 40.75910949707031,
+                                 2: 40.82441329956055,
+                                 3: 40.71998596191406,
+                                 4: 40.742652893066406},
+            'payment_type': {0: 1, 1: 1, 2: 2, 3: 2, 4: 2},
+            'fare_amount': {0: 12.0, 1: 14.5, 2: 9.5, 3: 3.5, 4: 15.0},
+            'extra': {0: 1.0, 1: 0.5, 2: 0.5, 3: 0.5, 4: 0.5},
+            'mta_tax': {0: 0.5, 1: 0.5, 2: 0.5, 3: 0.5, 4: 0.5},
+            'tip_amount': {0: 3.25, 1: 2.0, 2: 0.0, 3: 0.0, 4: 0.0},
+            'tolls_amount': {0: 0.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0},
+            'improvement_surcharge': {0: 0.3, 1: 0.3, 2: 0.3, 3: 0.3, 4: 0.3},
+            'total_amount': {0: 17.05, 1: 17.8, 2: 10.8, 3: 4.8, 4: 16.3}}
+    sample = pd.DataFrame(data)
+    df = sample.take(np.arange(5).repeat(10000))
+    file = io.BytesIO()
+    sfile = io.TextIOWrapper(file)
+    df.to_csv(sfile, index=False)
+
+    key = "nyc-taxi/2015/yellow_tripdata_2015-01.csv"
     client = boto3.client('s3')
-    client.create_bucket(Bucket=bucket, ACL='public-read-write')
-    for f, data in files.items():
-        client.put_object(Bucket=bucket, Key=f, Body=data)
-
-    yield DaskS3FileSystem(anon=True)
-
-    for f, data in files.items():
-        try:
-            client.delete_object(Bucket=bucket, Key=f, Body=data)
-        except Exception:
-            pass
-        finally:
-            m.stop()
-            httpretty = pytest.importorskip('httpretty')
-            httpretty.HTTPretty.disable()
-            httpretty.HTTPretty.reset()
+    client.put_object(Bucket=test_bucket_name, Key=key, Body=file)
+    key = "nyc-taxi/2014/yellow_tripdata_2014-{:0>2d}.csv"
+
+    for i in range(1, 13):
+        file.seek(0)
+        client.put_object(Bucket=test_bucket_name,
+                          Key=key.format(i),
+                          Body=file)
+    yield
 
 
 def test_get_s3():
@@ -135,14 +237,15 @@ def test_read_bytes_blocksize_none(s3):
     assert sum(map(len, values)) == len(files)
 
 
-@pytest.mark.slow
-@pytest.mark.network
-def test_read_bytes_blocksize_on_large_data():
-    _, L = read_bytes('s3://dask-data/nyc-taxi/2015/yellow_tripdata_2015-01.csv',
-                      blocksize=None, anon=True)
+def test_read_bytes_blocksize_on_large_data(s3_with_yellow_tripdata):
+    _, L = read_bytes(
+        's3://{}/nyc-taxi/2015/yellow_tripdata_2015-01.csv'.format(
+            test_bucket_name),
+        blocksize=None, anon=True)
     assert len(L) == 1
 
-    _, L = read_bytes('s3://dask-data/nyc-taxi/2014/*.csv', blocksize=None, anon=True)
+    _, L = read_bytes('s3://{}/nyc-taxi/2014/*.csv'.format(test_bucket_name),
+                      blocksize=None, anon=True)
     assert len(L) == 12