Repository: dask/dask

Issue:
Dask Dataframe to_parquet S3 repeated append broken
**What happened**:

When using `ddf.to_parquet(path='s3://...', append=True)` repeatedly on an existing path, parquet partition and metadata files seem to be partly overwritten. I believe this might be an issue with the s3 filesystem cache.

**What you expected to happen**:

I expect to be able to append repeatedly to a parquet file on S3 and be able to consistently read back the same rows that have been written. 

**Minimal Complete Verifiable Example**:

```python
import fsspec                                                                                                                                                                                                                                                                     
import pandas as pd                                                                                                                                                                                                                                                               
import dask.dataframe as dd                                                                                                                                                                                                                                                       
from dask.distributed import Client, LocalCluster                                                                                                                                                                                                                                 
import sys                                                                                                                                                                                                                                                                        
import shutil                                                                                                                                                                                                                                                                     
                                                                                                                                                                                                                                                                                  
                                                                                                                                                                                                                                                                                  
def run_test(path, invalidate_cache=False, invalidate_cache_before_verify=True):                                                                                                                                                                                                  
    is_s3 = path.startswith("s3")                                                                                                                                                                                                                                                 
    count = 20                                                                                                                                                                                                                                                                    
    for i in range(count):                                                                                                                                                                                                                                                        
        df = pd.DataFrame.from_dict({"a": [i], "b": [i + count]})                                                                                                                                                                                                                 
        df = df.set_index("a")                                                                                                                                                                                                                                                    
        df = dd.from_pandas(df, npartitions=1)                                                                                                                                                                                                                                    
        append = True if i else False                                                                                                                                                                                                                                             
        df.to_parquet(path=path, append=append, engine="pyarrow")                                                                                                                                                                                                                 
        if is_s3 and invalidate_cache:                                                                                                                                                                                                                                            
            fsspec.filesystem("s3").invalidate_cache()                                                                                                                                                                                                                            
                                                                                                                                                                                                                                                                                  
    if is_s3 and invalidate_cache_before_verify:                                                                                                                                                                                                                                  
        fsspec.filesystem("s3").invalidate_cache()                                                                                                                                                                                                                                
    written = dd.read_parquet(path)                                                                                                                                                                                                                                               
    print("want", count, "got", len(written), "npartitions", written.npartitions)                                                                                                                                                                                                 
                                                                                                                                                                                                                                                                                  
                                                                                                                                                                                                                                                                                  
def cleanup(path):                                                                                                                                                                                                                                                                
    if path.startwith("s3"):                                                                                                                                                                                                                                                      
        fs = fsspec.filesystem("s3")                                                                                                                                                                                                                                              
        fs.delete(path, recursive=True)                                                                                                                                                                                                                                           
    else:                                                                                                                                                                                                                                                                         
        shutil.rmtree(path)                                                                                                                                                                                                                                                       
                                                                                                                                                                                                                                                                                  
                                                                                                                                                                                                                                                                                  
def main():                                                                                                                                                                                                                                                                       
    path = "test_arrow.parquet"                                                                                                                                                                                                                                                   
    if len(sys.argv) == 2:                                                                                                                                                                                                                                                        
        path = sys.argv[1]                                                                                                                                                                                                                                                        
                                                                                                                                                                                                                                                                                  
    # LocalCluster if not s3 to avoid worker path issues                                                                                                                                                                                                                          
    if not path.startswith("s3"):                                                                                                                                                                                                                                                 
        c = Client(LocalCluster())                                                                                                                                                                                                                                                
    else:                                                                                                                                                                                                                                                                         
        c = Client()                                                                                                                                                                                                                                                              
                                                                                                                                                                                                                                                                                  
    print(                                                                                                                                                                                                                                                                        
        "Test without invalidating s3 cache between partitions, don't invalidate before verify\n"                                                                                                                                                                                 
    )                                                                                                                                                                                                                                                                             
    try:                                                                                                                                                                                                                                                                          
        run_test(path, invalidate_cache=False, invalidate_cache_before_verify=False)                                                                                                                                                                                              
    except Exception as e:                                                                                                                                                                                                                                                        
        print(e)                                                                                                                                                                                                                                                                  
                                                                                                                                                                                                                                                                                  
    cleanup(path)                                                                                                                                                                                                                                                                 
                                                                                                                                                                                                                                                                                  
    print("\nTest without invalidating s3 cache\n")                                                                                                                                                                                                                               
    run_test(path, invalidate_cache=False, invalidate_cache_before_verify=True)                                                                                                                                                                                                   
    cleanup(path)                                                                                                                                                                                                                                                                 
                                                                                                                                                                                                                                                                                  
    print("\nTest invalidating s3 cache\n")                                                                                                                                                                                                                                       
    run_test(path, invalidate_cache=True)                                                                                                                                                                                                                                         
    cleanup(path)                                                                                                                                                                                                                                                                 
                                                                                                                                                                                                                                                                                  
                                                                                                                                                                                                                                                                                  
if __name__ == "__main__":                                                                                                                                                                                                                                                        
    main() 
```

Expected:

```
python basic_append.py "s3://<bucketname>/test_parquet"
Test without invalidating s3 cache between partitions, don't invalidate before verify                                                                                                                                                                                             
                                                                                                                                                                                                                                                                                  
want 20 got 20 npartitions 20                                                                                                                                                                                                                                                     
                                                                                                                                                                                                                                                                                  
Test without invalidating s3 cache                                                                                                                                                                                                                                                
                                                                                                                                                                                                                                                                                  
want 20 got 20 npartitions 20                                                                                                                                                                                                                                                     
                                                                                                                                                                                                                                                                                  
Test invalidating s3 cache                                                                                                                                                                                                                                                        
                                                                                                                                                                                                                                                                                  
want 20 got 20 npartitions 20 
```

Actual:

```
python basic_append.py "s3://<bucketname>/test_parquet"
Test without invalidating s3 cache between partitions, don't invalidate before verify

[Errno 16] The remote file corresponding to filename <bucket_name>/test_parquet/_metadata and Etag "<etag>" no longer exists.

Test without invalidating s3 cache

want 20 got 1 npartitions 1

Test invalidating s3 cache

want 20 got 20 npartitions 20
```

**Anything else we need to know?**:

I believe this may be related to use of caching behavior in the s3 filesystem cache, given the exception thrown when immediately reading back a parquet file written to S3.

@gforsyth mentioned @martindurant might be interested.

**Environment**:

- Dask version: 2021.07.1
- Python version: 3.9
- Operating System: Ubuntu 20.04
- Install method (conda, pip, source): conda



Discussion:
Would you mind also running this without a client, only in-process threads? 
@martindurant It works correctly using only in-process threads.
Do you mind setting the "s3fs" logger to DEBUG just for the line `df.to_parquet(path=path, append=append, engine="pyarrow") ` ? 
It shouldn't be too surprising that when files get written in another process, the file-cache gets out of date. I guess when we write the _metadata file last, this works out ok; and the worker processes are fine too, since they will have written to the same set of directories and invalidated those entries.

So, in short, it would be OK to explicitly call invalidate using the path and storage options passed to to_parquet. I'd be surprised if this situation isn't possible elsewhere - but we've had no reports.

Resolving patch:
diff --git a/dask/dataframe/io/parquet/core.py b/dask/dataframe/io/parquet/core.py
--- a/dask/dataframe/io/parquet/core.py
+++ b/dask/dataframe/io/parquet/core.py
@@ -871,8 +871,15 @@ def to_parquet(
     # Convert data_write + dsk to computable collection
     graph = HighLevelGraph.from_collections(final_name, dsk, dependencies=(data_write,))
     out = Scalar(graph, final_name, "")
+
     if compute:
-        return out.compute(**compute_kwargs)
+        out = out.compute(**compute_kwargs)
+
+    # Invalidate the filesystem listing cache for the output path after write.
+    # We do this before returning, even if `compute=False`. This helps ensure
+    # that reading files that were just written succeeds.
+    fs.invalidate_cache(path)
+
     return out
 
 


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -4,6 +4,7 @@
 import sys
 import warnings
 from decimal import Decimal
+from unittest.mock import MagicMock
 
 import numpy as np
 import pandas as pd
@@ -1611,6 +1612,19 @@ def test_to_parquet_lazy(tmpdir, scheduler, engine):
     assert_eq(ddf, ddf2, check_divisions=False)
 
 
+@PYARROW_MARK
+@pytest.mark.parametrize("compute", [False, True])
+def test_to_parquet_calls_invalidate_cache(tmpdir, monkeypatch, compute):
+    from fsspec.implementations.local import LocalFileSystem
+
+    invalidate_cache = MagicMock()
+    monkeypatch.setattr(LocalFileSystem, "invalidate_cache", invalidate_cache)
+    ddf.to_parquet(tmpdir, compute=compute, engine="pyarrow")
+    path = LocalFileSystem._strip_protocol(str(tmpdir))
+    assert invalidate_cache.called
+    assert invalidate_cache.call_args.args[0] == path
+
+
 @FASTPARQUET_MARK
 def test_timestamp96(tmpdir):
     fn = str(tmpdir)