TypeError: can't pickle _thread.lock objects
Author: abs-xyz
Pull request: no
Labels: name: bug

### System Info

```shell
- `transformers` version: 4.19.4
- Platform: Linux-5.4.188+-x86_64-with-Ubuntu-18.04-bionic
- Python version: 3.7.13
- Huggingface_hub version: 0.7.0
- PyTorch version (GPU?): 1.11.0+cu113 (False)
- Tensorflow version (GPU?): 2.8.2 (False)

[Google Colab]
```


### Who can help?

@amogkam

### Information

- [ ] My own modified scripts
- [ ] The official example scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction

Note: this is related to [this closed issue](https://github.com/huggingface/transformers/issues/11249).

This is the code I'm using:
```
args = TrainingArguments(
    f"{model_name}-hyperp-{task}",
    evaluation_strategy = "epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    num_train_epochs=3,
    weight_decay=0.01,
    skip_memory_metrics=True,       # https://github.com/huggingface/transformers/issues/12177 [picking error]
)

trainer = Trainer(
    model_init=model_init,             # function to initialize model (using 'from_pretrained')
    args=args,
    train_dataset = tokenized_datasets["train"],
    eval_dataset = tokenized_datasets["validation"],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)

trainer.hyperparameter_search(
    hp_space=lambda _: tune_config,
    backend="ray",
    n_trials=10,
    resources_per_trial={"cpu": 1, "gpu": 0},
    scheduler=scheduler,
    keep_checkpoints_num=1,
    checkpoint_score_attr="training_iteration",
    progress_reporter=reporter,
    local_dir="/ray_results/",
    name="tune_transformer_pbt",
    log_to_file=True,
)
```

Error:
```
TypeError                                 Traceback (most recent call last)
[<ipython-input-50-3716493001d6>](https://localhost:8080/#) in <module>()
     33     local_dir="/ray_results/",
     34     name="tune_transformer_pbt",
---> 35     log_to_file=True,
     36 )
     37 

[/usr/local/lib/python3.7/dist-packages/transformers/trainer.py](https://localhost:8080/#) in hyperparameter_search(self, hp_space, compute_objective, n_trials, direction, backend, hp_name, **kwargs)
   2083             HPSearchBackend.WANDB: run_hp_search_wandb,
   2084         }
-> 2085         best_run = backend_dict[backend](self, n_trials, direction, **kwargs)
   2086 
   2087         self.hp_search_backend = None

[/usr/local/lib/python3.7/dist-packages/transformers/integrations.py](https://localhost:8080/#) in run_hp_search_ray(trainer, n_trials, direction, **kwargs)
    296         config=trainer.hp_space(None),
    297         num_samples=n_trials,
--> 298         **kwargs,
    299     )
    300     best_trial = analysis.get_best_trial(metric="objective", mode=direction[:3])

[/usr/local/lib/python3.7/dist-packages/ray/tune/tune.py](https://localhost:8080/#) in run(run_or_experiment, name, metric, mode, stop, time_budget_s, config, resources_per_trial, num_samples, local_dir, search_alg, scheduler, keep_checkpoints_num, checkpoint_score_attr, checkpoint_freq, checkpoint_at_end, verbose, progress_reporter, log_to_file, trial_name_creator, trial_dirname_creator, sync_config, export_formats, max_failures, fail_fast, restore, server_port, resume, reuse_actors, trial_executor, raise_on_failed_trial, callbacks, max_concurrent_trials, _experiment_checkpoint_dir, loggers, _remote)
    363 
    364     if not trial_executor or isinstance(trial_executor, RayTrialExecutor):
--> 365         _ray_auto_init()
    366 
    367     if _remote:

[/usr/local/lib/python3.7/dist-packages/ray/tune/tune.py](https://localhost:8080/#) in _ray_auto_init()
    876             "call `ray.init(...)` before `tune.run`."
    877         )
--> 878         ray.init()

[/usr/local/lib/python3.7/dist-packages/ray/_private/client_mode_hook.py](https://localhost:8080/#) in wrapper(*args, **kwargs)
    103             if func.__name__ != "init" or is_client_mode_enabled_by_default:
    104                 return getattr(ray, func.__name__)(*args, **kwargs)
--> 105         return func(*args, **kwargs)
    106 
    107     return wrapper

[/usr/local/lib/python3.7/dist-packages/ray/worker.py](https://localhost:8080/#) in init(address, num_cpus, num_gpus, resources, object_store_memory, local_mode, ignore_reinit_error, include_dashboard, dashboard_host, dashboard_port, job_config, configure_logging, logging_level, logging_format, log_to_driver, namespace, runtime_env, storage, _enable_object_reconstruction, _redis_max_memory, _plasma_directory, _node_ip_address, _driver_object_store_memory, _memory, _redis_password, _temp_dir, _metrics_export_port, _system_config, _tracing_startup_hook, _node_name, **kwargs)
   1120 
   1121     for hook in _post_init_hooks:
-> 1122         hook()
   1123 
   1124     node_id = global_worker.core_worker.get_current_node_id()

[/usr/local/lib/python3.7/dist-packages/ray/tune/registry.py](https://localhost:8080/#) in flush(self)
    230                 self.references[k] = v
    231             else:
--> 232                 self.references[k] = ray.put(v)
    233         self.to_flush.clear()

[/usr/local/lib/python3.7/dist-packages/ray/_private/client_mode_hook.py](https://localhost:8080/#) in wrapper(*args, **kwargs)
    103             if func.__name__ != "init" or is_client_mode_enabled_by_default:
    104                 return getattr(ray, func.__name__)(*args, **kwargs)
--> 105         return func(*args, **kwargs)
    106 
    107     return wrapper

[/usr/local/lib/python3.7/dist-packages/ray/worker.py](https://localhost:8080/#) in put(value, _owner)
   1892     with profiling.profile("ray.put"):
   1893         try:
-> 1894             object_ref = worker.put_object(value, owner_address=serialize_owner_address)
   1895         except ObjectStoreFullError:
   1896             logger.info(

[/usr/local/lib/python3.7/dist-packages/ray/worker.py](https://localhost:8080/#) in put_object(self, value, object_ref, owner_address)
    305             ), "Local Mode does not support inserting with an ObjectRef"
    306 
--> 307         serialized_value = self.get_serialization_context().serialize(value)
    308         # This *must* be the first place that we construct this python
    309         # ObjectRef because an entry with 0 local references is created when

[/usr/local/lib/python3.7/dist-packages/ray/serialization.py](https://localhost:8080/#) in serialize(self, value)
    419             return RawSerializedObject(value)
    420         else:
--> 421             return self._serialize_to_msgpack(value)

[/usr/local/lib/python3.7/dist-packages/ray/serialization.py](https://localhost:8080/#) in _serialize_to_msgpack(self, value)
    398             metadata = ray_constants.OBJECT_METADATA_TYPE_PYTHON
    399             pickle5_serialized_object = self._serialize_to_pickle5(
--> 400                 metadata, python_objects
    401             )
    402         else:

[/usr/local/lib/python3.7/dist-packages/ray/serialization.py](https://localhost:8080/#) in _serialize_to_pickle5(self, metadata, value)
    359         except Exception as e:
    360             self.get_and_clear_contained_object_refs()
--> 361             raise e
    362         finally:
    363             self.set_out_of_band_serialization()

[/usr/local/lib/python3.7/dist-packages/ray/serialization.py](https://localhost:8080/#) in _serialize_to_pickle5(self, metadata, value)
    355             self.set_in_band_serialization()
    356             inband = pickle.dumps(
--> 357                 value, protocol=5, buffer_callback=writer.buffer_callback
    358             )
    359         except Exception as e:

[/usr/local/lib/python3.7/dist-packages/ray/cloudpickle/cloudpickle_fast.py](https://localhost:8080/#) in dumps(obj, protocol, buffer_callback)
     71                 file, protocol=protocol, buffer_callback=buffer_callback
     72             )
---> 73             cp.dump(obj)
     74             return file.getvalue()
     75 

[/usr/local/lib/python3.7/dist-packages/ray/cloudpickle/cloudpickle_fast.py](https://localhost:8080/#) in dump(self, obj)
    618     def dump(self, obj):
    619         try:
--> 620             return Pickler.dump(self, obj)
    621         except RuntimeError as e:
    622             if "recursion" in e.args[0]:

TypeError: can't pickle _thread.lock objects
```

I've already added `skip_memory_metrics=True` in the `TrainingArguments`.

### Expected behavior

```shell
Expecting to not give this error.
```


Comments:
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) are likely to be ignored.

Hello! Facing the same issue with the following system info: 
```
datasets==2.5.1
huggingface-hub==0.10.0
multidict==6.0.2
multiprocess==0.70.13
numpy==1.23.3
tokenizers==0.12.1
torch==1.9.0+cu111
torchaudio==0.9.0
tqdm==4.64.1
transformers==4.22.2
```

This issue is only triggered when I keep load_best_model_at_end as True (I am not doing any hyperparameter search): Training code and stack trace are: 

### Training Code with Trigger 
```
training_args = TrainingArguments(
    output_dir=f'../asr/models_src_raw/{args.lang}',
    overwrite_output_dir = True, 
    group_by_length=True,
    per_device_train_batch_size=16,
    gradient_accumulation_steps=2,
    evaluation_strategy="steps",
    num_train_epochs=80,
    gradient_checkpointing=True,
    fp16=True,
    save_steps=10,
    eval_steps=10,
    logging_steps=10,
    learning_rate=3e-4,
    warmup_steps=300,
    save_total_limit=1,
    load_best_model_at_end = True, 
    metric_for_best_model = wer_metric, 
    skip_memory_metrics = True
    )

    trainer = Trainer(
        model=model,
        data_collator=data_collator,
        args=training_args,
        compute_metrics=compute_metrics,
        train_dataset=train,
        eval_dataset =test,
        tokenizer=processor.feature_extractor,
    )

    trainer.train()

```
(If I remove the load_best_model_at_end, Works smoothly) 

Hi, with transformers 4.26.1 on Sage maker I am still having this error: TypeError: cannot pickle '_thread.lock' object.

def hp_space(trial):
return {
"learning_rate": trial.suggest_float("learning_rate", 1e-5, 1e-3, log=True),
"num_train_epochs": trial.suggest_int("num_train_epochs", 1, 10),
"seed": trial.suggest_int("seed", 1, 40),
"per_device_train_batch_size": trial.suggest_categorical("per_device_train_batch_size", [16, 32, 64]),
"weight_decay": trial.suggest_float("weight_decay", 1e-3, 1e-1, log=True),
}

best_run = trainer.hyperparameter_search(n_trials=20, direction="minimize", hp_space=hp_space)