Repository: vaexio/vaex

Issue:
[BUG-REPORT] Vaex cannot open more than 816 parquet files on s3
**Description**

I try to open many parquet partitions. Vaex opens first 816, then fails with the following error:

> ERROR:MainThread:vaex:error opening 's3://my/path/part-00817-db9aee77-5b44-4571-857b-ea2879609450.c000.snappy.parquet'
> Traceback (most recent call last):
>   File "/home/myuser/p/VS_Code_Projects/SkillExtractor/test_vaex_multiprocessing.py", line 32, in <module>
>     df = vaex.concat([df, vaex.open(parquet_file)])
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/__init__.py", line 211, in open
>     ds = vaex.dataset.open(path, fs_options=fs_options, *args, **kwargs)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/dataset.py", line 42, in open
>     return opener.open(path, *args, **kwargs)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/arrow/opener.py", line 29, in open
>     return open_parquet(path, *args, **kwargs)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/vaex/arrow/dataset.py", line 175, in open_parquet
>     arrow_ds = pyarrow.dataset.dataset(path, filesystem=file_system)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py", line 671, in dataset
>     return _filesystem_dataset(source, **kwargs)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py", line 428, in _filesystem_dataset
>     fs, paths_or_selector = _ensure_single_source(source, filesystem)
>   File "/home/myuser/anaconda3/lib/python3.8/site-packages/pyarrow/dataset.py", line 395, in _ensure_single_source
>     file_info = filesystem.get_file_info([path])[0]
>   File "pyarrow/_fs.pyx", line 434, in pyarrow._fs.FileSystem.get_file_info
>   File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
>   File "pyarrow/error.pxi", line 99, in pyarrow.lib.check_status
> OSError: When getting information for key 'demand/positions-20211123/place_country_code=US/part-00817-db9aee77-5b44-4571-857b-ea2879609450.c000.snappy.parquet' in bucket 'tn-datafeed-dev': AWS Error [code 99]: curlCode: 6, Couldn't resolve host name

Tried to open in two different ways, in both cases the same error.

Way 1:

`df = vaex.open("s3://some/path/*.parquet")`

Way 2:

```
s3 = boto3.resource('s3')
my_bucket = s3.Bucket('my_bucket_name')

parquet_files = []
for my_bucket_object in my_bucket.objects.filter(Prefix='some/prefix'):
    if my_bucket_object.key.endswith(".parquet"):
        parquet_files.append("s3://" + my_bucket._name + "/" + my_bucket_object.key)

df = None
for parquet_file in parquet_files:
    if not df:
        df = vaex.open(parquet_file)
    else:
        try:
            df = vaex.concat([df, vaex.open(parquet_file)])
        except Exception as e:
            print(f"WARNING: The file {parquet_file} could not be read!")
            print(e)
```

Everything is fine until it reaches 816 files open. I tried to start with parquet # 817 and it works until another certain number of files is open.

**Software information**
 - Vaex version (`import vaex; vaex.__version__)`: {'vaex-core': '4.0.0a5'}
 - Vaex was installed via: pip
 - OS: AWS Linux v2





Discussion:
Are you on EC2 and in the same region as the bucket? (that might help me reproduce it)

Resolving patch:
diff --git a/packages/vaex-core/vaex/file/s3arrow.py b/packages/vaex-core/vaex/file/s3arrow.py
--- a/packages/vaex-core/vaex/file/s3arrow.py
+++ b/packages/vaex-core/vaex/file/s3arrow.py
@@ -7,6 +7,11 @@
 
 from . import split_options, FileProxy, split_scheme
 from .cache import FileSystemHandlerCached
+from ..cache import fingerprint
+
+
+region_cache = {}
+fs_arrow_cache = {}
 
 
 def glob(path, fs_options={}):
@@ -15,21 +20,36 @@ def glob(path, fs_options={}):
 
 
 def parse(path, fs_options, for_arrow=False):
+    # Remove this line for testing purposes to fake not having s3 support
+    # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')
     path, fs_options = split_options(path, fs_options)
     path = path.replace('arrow+s3://', 's3://')
-    scheme, _ = split_scheme(path)
+    fullpath = path
+    scheme, path = split_scheme(path)
     assert scheme == 's3'
+
     # anon is for backwards compatibility
     fs_options['anonymous'] = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])
     fs_options = patch_profile(fs_options)
     use_cache = fs_options.pop('cache', 'true') in [True, 'true', 'True', '1']
+    bucket = path.split('/')[0]
     if 'region' not in fs_options:
-        # we use this to get the default region
-        file_system, path = pa.fs.FileSystem.from_uri(path)
-        # Remove this line for testing purposes to fake not having s3 support
-        # raise pyarrow.lib.ArrowNotImplementedError('FOR TESTING')
-        fs_options['region'] = file_system.region
-    fs = pa.fs.S3FileSystem(**fs_options)
+        # cache region
+        if bucket not in region_cache:
+            # we use this to get the default region
+            file_system, _ = pa.fs.FileSystem.from_uri(fullpath)
+            region = file_system.region
+            region_cache[bucket] = region
+        else:
+            region = region_cache[bucket]
+        fs_options['region'] = region
+    # bucket and options make up a unique key
+    key = fingerprint(bucket, fs_options)
+    if key not in fs_arrow_cache:
+        fs = pa.fs.S3FileSystem(**fs_options)
+        fs_arrow_cache[key] = fs
+    else:
+        fs = fs_arrow_cache[key]
     if use_cache:
         fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)
         if for_arrow:
diff --git a/packages/vaex-core/vaex/file/s3fs.py b/packages/vaex-core/vaex/file/s3fs.py
--- a/packages/vaex-core/vaex/file/s3fs.py
+++ b/packages/vaex-core/vaex/file/s3fs.py
@@ -6,6 +6,10 @@
 from . import split_options, split_scheme
 from .cache import FileSystemHandlerCached
 from .s3 import patch_profile
+from ..cache import fingerprint
+
+
+fs_cache = {}
 
 
 def translate_options(fs_options):
@@ -70,7 +74,14 @@ def parse(path, fs_options, for_arrow=False):
     anon = (fs_options.pop('anon', None) in [True, 'true', 'True', '1']) or (fs_options.pop('anonymous', None) in [True, 'true', 'True', '1'])
     fs_options = patch_profile(fs_options)
     fs_options = translate_options(fs_options)
-    s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)
+
+    bucket = path.split('/')[0]
+    key = fingerprint(bucket, fs_options)
+    if key not in fs_cache:
+        s3 = s3fs.S3FileSystem(anon=anon, default_fill_cache=False, **fs_options)
+        fs_cache[key] = s3
+    else:
+        s3 = fs_cache[key]
     fs = pyarrow.fs.FSSpecHandler(s3)
     if use_cache:
         fs = FileSystemHandlerCached(fs, scheme='s3', for_arrow=for_arrow)


Test patch: