stas00 (CONTRIBUTOR) commented on 2021-02-27 19:38:23:
OK, it definitely leads to a race condition in how it's used right now. Here is how you can reproduce it - by injecting a random  sleep time different for each process before the locks are acquired. 
```
--- a/src/datasets/metric.py
+++ b/src/datasets/metric.py
@@ -348,6 +348,16 @@ class Metric(MetricInfoMixin):

         elif self.process_id == 0:
             # Let's acquire a lock on each node files to be sure they are finished writing
+
+            import time
+            import random
+            import os
+            pid = os.getpid()
+            random.seed(pid)
+            secs = random.randint(1, 15)
+            time.sleep(secs)
+            print(f"sleeping {secs}")
+
             file_paths, filelocks = self._get_all_cache_files()

             # Read the predictions and references
@@ -385,7 +395,10 @@ class Metric(MetricInfoMixin):

         if predictions is not None:
             self.add_batch(predictions=predictions, references=references)
+        print("FINALIZE START")
+
         self._finalize()
+        print("FINALIZE END")

         self.cache_file_name = None
         self.filelock = None
```

then run with 2 procs: `python -m torch.distributed.launch --nproc_per_node=2`
```
export BS=16; rm -r output_dir; PYTHONPATH=src USE_TF=0 CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 examples/seq2seq/run_seq2seq.py --model_name_or_path t5-small --output_dir output_dir --adam_eps 1e-06 --do_eval --do_train --do_predict --evaluation_strategy=steps  --label_smoothing 0.1 --learning_rate 3e-5 --logging_first_step --logging_steps 1000 --max_source_length 128 --max_target_length 128 --num_train_epochs 1 --overwrite_output_dir --per_device_eval_batch_size $BS --per_device_train_batch_size $BS --predict_with_generate --eval_steps 25000  --sortish_sampler --task translation_en_to_ro  --val_max_target_length 128 --warmup_steps 500 --max_train_samples 10 --max_val_samples 10 --max_test_samples 10  --dataset_name wmt16 --dataset_config ro-en --source_prefix "translate English to Romanian: "
```

```
***** Running Evaluation *****
  Num examples = 10
  Batch size = 16
  0%|                                                                                                                                      | 0/1 [00:00<?, ?it/s]FINALIZE START
FINALIZE START
sleeping 11
FINALIZE END
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:11<00:00, 11.06s/it]
sleeping 11
Traceback (most recent call last):
  File "/mnt/nvme1/code/huggingface/datasets-master/src/datasets/metric.py", line 368, in _finalize
    self.data = Dataset(**reader.read_files([{"filename": f} for f in file_paths]))
  File "/mnt/nvme1/code/huggingface/datasets-master/src/datasets/arrow_reader.py", line 236, in read_files
    pa_table = self._read_files(files, in_memory=in_memory)
  File "/mnt/nvme1/code/huggingface/datasets-master/src/datasets/arrow_reader.py", line 171, in _read_files
    pa_table: pa.Table = self._get_dataset_from_filename(f_dict, in_memory=in_memory)
  File "/mnt/nvme1/code/huggingface/datasets-master/src/datasets/arrow_reader.py", line 302, in _get_dataset_from_filename
    pa_table = ArrowReader.read_table(filename, in_memory=in_memory)
  File "/mnt/nvme1/code/huggingface/datasets-master/src/datasets/arrow_reader.py", line 322, in read_table
    stream = stream_from(filename)
  File "pyarrow/io.pxi", line 782, in pyarrow.lib.memory_map
  File "pyarrow/io.pxi", line 743, in pyarrow.lib.MemoryMappedFile._open
  File "pyarrow/error.pxi", line 122, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 97, in pyarrow.lib.check_status
FileNotFoundError: [Errno 2] Failed to open local file '/home/stas/.cache/huggingface/metrics/sacrebleu/default/default_experiment-1-0.arrow'. Detail: [errno 2] No such file or directory

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "examples/seq2seq/run_seq2seq.py", line 645, in <module>
    main()
  File "examples/seq2seq/run_seq2seq.py", line 601, in main
    metrics = trainer.evaluate(
  File "/mnt/nvme1/code/huggingface/transformers-mp-pp/src/transformers/trainer_seq2seq.py", line 74, in evaluate
    return super().evaluate(eval_dataset, ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix)
  File "/mnt/nvme1/code/huggingface/transformers-mp-pp/src/transformers/trainer.py", line 1703, in evaluate
    output = self.prediction_loop(
  File "/mnt/nvme1/code/huggingface/transformers-mp-pp/src/transformers/trainer.py", line 1876, in prediction_loop
    metrics = self.compute_metrics(EvalPrediction(predictions=preds, label_ids=label_ids))
  File "examples/seq2seq/run_seq2seq.py", line 556, in compute_metrics
    result = metric.compute(predictions=decoded_preds, references=decoded_labels)
  File "/mnt/nvme1/code/huggingface/datasets-master/src/datasets/metric.py", line 402, in compute
    self._finalize()
  File "/mnt/nvme1/code/huggingface/datasets-master/src/datasets/metric.py", line 370, in _finalize
    raise ValueError(
ValueError: Error in finalize: another metric instance is already using the local cache file. Please specify an experiment_id to avoid colision between distributed metric instances.
```