Object of type 'BitsAndBytesConfig' is not JSON serializable
Author: karthsu
Pull request: no

### System Info

- `transformers` version: 4.30.0                                                                                                                                             
- Platform: Linux-5.15.0-69-generic-x86_64-with-glibc2.29                                                                                                                    
- Python version: 3.8.10                                                                                                                                                     
- Huggingface_hub version: 0.15.1                                                                                                                                            
- Safetensors version: 0.3.1                                                                                                                                                 
- PyTorch version (GPU?): 1.13.1 (True)                                                                                                                                      
- Tensorflow version (GPU?): not installed (NA)                                                                                                                              
- Flax version (CPU?/GPU?/TPU?): not installed (NA)                                                                                                                          
- Jax version: not installed                                                                                                                                                 
- JaxLib version: not installed                                                                                                                                              
- Using GPU in script?: Yes                                                                                                                                        
- Using distributed or parallel set-up in script?: No

### Who can help?

@sgugger @younesbelkada 

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [X] My own task or dataset (give details below)

### Reproduction

This is the script Im using:

```
import pandas as pd
import os
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, BitsAndBytesConfig
import torch
from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training, TaskType, prepare_model_for_kbit_training
from transformers import DataCollatorForSeq2Seq
import evaluate
import nltk
import numpy as np
from nltk.tokenize import sent_tokenize
from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments
from datasets import Dataset, DatasetDict
import argparse
import pickle
import json

parser = argparse.ArgumentParser(description='Options')
parser.add_argument('--dataset_dir', default='data', type=str, help="folder in which the dataset is stored")
parser.add_argument('--output_dir', default="lora-instructcodet5p", type=str, help="output directory for the model")
parser.add_argument('--results_dir', default="results", type=str, help="where the results should be stored")
args = parser.parse_args()

nltk.download("punkt")
tokenized_dataset = DatasetDict.load_from_disk(args.dataset_dir)
# Metric
metric = evaluate.load("rouge")
pad_tok = 50256
token_id="Salesforce/instructcodet5p-16b"

tokenizer = AutoTokenizer.from_pretrained(token_id)
# helper function to postprocess text
def postprocess_text(preds, labels):
    preds = [pred.strip() for pred in preds]
    labels = [label.strip() for label in labels]

    # rougeLSum expects newline after each sentence
    preds = ["\n".join(sent_tokenize(pred)) for pred in preds]
    labels = ["\n".join(sent_tokenize(label)) for label in labels]

    return preds, labels

def compute_metrics(eval_preds):
    preds, labels = eval_preds
    if isinstance(preds, tuple):
        preds = preds[0]
    for idx in range(len(preds)):
        for idx2 in range(len(preds[idx])):
            if preds[idx][idx2]==-100:
                preds[idx][idx2] = 50256
    decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
    # Replace -100 in the labels as we can't decode them.
    labels = np.where(labels != pad_tok, labels, tokenizer.pad_token_id)
    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

    # Some simple post-processing
    decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)

    result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)
    result = {k: round(v * 100, 4) for k, v in result.items()}
    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]
    result["gen_len"] = np.mean(prediction_lens)
    return result
def get_dict(predicts):
    d = {}
    for num in range(len(tokenized_dataset['test'])):
        pred = tokenizer.decode([n for n in predicts[0][num] if n!=50256 and n!=-100])[1:]
        d[num+1] = {'Question':tokenizer.decode([n for n in tokenized_dataset['test'][num]['input_ids'] if n!=50256]),
                    'Ground truth solution':tokenizer.decode([n for n in tokenized_dataset['test'][num]['labels'] if n!=50256]),
                   'Prediction': pred if pred else None}
    return d

def find_all_linear_names(model):
    cls = torch.nn.Linear
    lora_module_names = set()
    for name, module in model.named_modules():
        if isinstance(module, cls):
            names = name.split('.')
            lora_module_names.add(names[0] if len(names) == 1 else names[-1])

    if 'lm_head' in lora_module_names:
        lora_module_names.remove('lm_head')
    return list(lora_module_names)


def main():
    device = 'cuda'

    # huggingface hub model id
    model_id="instructcodet5p-16b"
    if not os.path.exists(model_id):
        model_id=token_id
    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_use_double_quant=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=torch.bfloat16
    )
    # load model from the hub
    model = AutoModelForSeq2SeqLM.from_pretrained(model_id,
                                              # torch_dtype=torch.bfloat16,
                                              low_cpu_mem_usage=True,
                                              trust_remote_code=True, decoder_start_token_id=1, pad_token_id=pad_tok, device_map="auto", quantization_config=bnb_config)
    modules = find_all_linear_names(model)
    # Define LoRA Config
    lora_config = LoraConfig(
     r=8,
     lora_alpha=32,
     target_modules=modules,
     lora_dropout=0.05,
     bias="none",
     task_type=TaskType.SEQ_2_SEQ_LM
    )
    # prepare int-8 model for training
    model = prepare_model_for_kbit_training(model, False)

    # add LoRA adaptor
    model = get_peft_model(model, lora_config)

    model.print_trainable_parameters()

    # we want to ignore tokenizer pad token in the loss
    label_pad_token_id = pad_tok

    # Data collator
    data_collator = DataCollatorForSeq2Seq(
        tokenizer,
        model=model,
        label_pad_token_id=label_pad_token_id,
        pad_to_multiple_of=8
    )
    output_dir=args.output_dir

    training_args = Seq2SeqTrainingArguments(
        output_dir=output_dir,
        per_device_train_batch_size=1,
        per_device_eval_batch_size=1,
        predict_with_generate=True,
        weight_decay=0.05,
        # warmup_steps=200,
        fp16=False, # Overflows with fp16
        learning_rate=1e-3,
        num_train_epochs=5,
        # logging & evaluation strategies
        logging_dir=f"{output_dir}/logs",
        logging_strategy="epoch",
        # logging_steps=500,
        evaluation_strategy="epoch",
        save_strategy="epoch",
        save_total_limit=20,
        # load_best_model_at_end=True,
        # metric_for_best_model="overall_f1",
        # push to hub parameters
        report_to="tensorboard",
        push_to_hub=False,
        generation_max_length=200,
        optim="paged_adamw_8bit"
    )

    # Create Trainer instance
    trainer = Seq2SeqTrainer(
        model=model,
        args=training_args,
        data_collator=data_collator,
        train_dataset=tokenized_dataset["train"],
        eval_dataset=tokenized_dataset["validation"],
        compute_metrics=compute_metrics,
    )

    # train model
    trainer.train()
    # Save our LoRA model & tokenizer results
    
    predicts = trainer.predict(tokenized_dataset['test'], max_length=200)
    with open('predicts.pkl', 'wb') as file:
        pickle.dump(predicts, file)
    d = get_dict(predicts)
    for num in d:
        print("Question:\n%s"%(d[num]['Question']))
        print('Ground Truth Solution:\n')
        print(d[num]['Ground truth solution'])
        print()
        print('Prediction:\n')
        print(d[num]['Prediction'])
        print()
    
    peft_model_id=args.results_dir
    trainer.model.save_pretrained(peft_model_id)
    tokenizer.save_pretrained(peft_model_id)
    # if you want to save the base model to call
    # trainer.model.base_model.save_pretrained(peft_model_id)
    with open('generations.json', "w") as json_file:
        json.dump(d, json_file)
    #Evaluate on test data
    # trainer.evaluate()

if __name__ == '__main__':
    main()
```

### Expected behavior

I'm trying to use QLoRA for fine-tuning on a Seq2Seq Task using [InstructCodeT5+](https://huggingface.co/Salesforce/instructcodet5p-16b) guided by this example [notebook](https://colab.research.google.com/drive/1VoYNfYDKcKRQRor98Zbf2-9VQTtGJ24k?usp=sharing#scrollTo=jq0nX33BmfaC). 

I am getting the following error:

```
Traceback (most recent call last):
  File "training.py", line 242, in <module>
    main()
  File "training.py", line 215, in main
    trainer.train()
  File "/home/ubuntu/.local/lib/python3.8/site-packages/transformers/trainer.py", line 1645, in train
    return inner_training_loop(
  File "/home/ubuntu/.local/lib/python3.8/site-packages/transformers/trainer.py", line 1853, in _inner_training_loop
    self.control = self.callback_handler.on_train_begin(args, self.state, self.control)
  File "/home/ubuntu/.local/lib/python3.8/site-packages/transformers/trainer_callback.py", line 353, in on_train_begin
    return self.call_event("on_train_begin", args, state, control)
  File "/home/ubuntu/.local/lib/python3.8/site-packages/transformers/trainer_callback.py", line 397, in call_event
    result = getattr(callback, event)(
  File "/home/ubuntu/.local/lib/python3.8/site-packages/transformers/integrations.py", line 640, in on_train_begin
    model_config_json = model.config.to_json_string()
  File "/home/ubuntu/.local/lib/python3.8/site-packages/transformers/configuration_utils.py", line 836, in to_json_string
    return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"
  File "/usr/lib/python3.8/json/__init__.py", line 234, in dumps
    return cls(
  File "/usr/lib/python3.8/json/encoder.py", line 201, in encode
    chunks = list(chunks)
  File "/usr/lib/python3.8/json/encoder.py", line 431, in _iterencode
    yield from _iterencode_dict(o, _current_indent_level)
  File "/usr/lib/python3.8/json/encoder.py", line 405, in _iterencode_dict
    yield from chunks
  File "/usr/lib/python3.8/json/encoder.py", line 438, in _iterencode
    o = _default(o)
  File "/usr/lib/python3.8/json/encoder.py", line 179, in default
    raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type BitsAndBytesConfig is not JSON serializable
```

Expecting the model to run and train as per the example notebook referenced above. Any help is appreciated!

Comments:
Thanks for reporting, see the comment here: https://github.com/huggingface/transformers/pull/24094#pullrequestreview-1471475968 
That suggestion should solve the issue

@younesbelkada, I am running a finetuning of Llama2 7b and I needed to use the **gradient_checkpointing=True** in the **TrainingArguments** to handle CUDA out of memory.
Adding this configuration caused the error **TypeError: Object of type BitsAndBytesConfig is not JSON serializable** when **save_checkpoint**.

#24137 not solved this issue.

Follow bellow my log:

`
INFO:    fuse: warning: library too old, some operations may not work

==========
== CUDA ==
==========

CUDA Version 11.8.0

Container image Copyright (c) 2016-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.

This container image and its contents are governed by the NVIDIA Deep Learning Container License.
By pulling and using the container, you accept the terms and conditions of this license:
https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license

A copy of this license is made available in this container at /NGC-DL-CONTAINER-LICENSE for your convenience.


Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:29<00:29, 29.73s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:39<00:00, 17.96s/it]
Loading checkpoint shards: 100%|██████████| 2/2 [00:39<00:00, 19.73s/it]
{
  "_name_or_path": "/scratch/LLM/LLAMA2/Llama-2-7b-chat-hf",
  "architectures": [
    "LlamaForCausalLM"
  ],
  "attention_bias": false,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "hidden_act": "silu",
  "hidden_size": 4096,
  "initializer_range": 0.02,
  "intermediate_size": 11008,
  "max_position_embeddings": 4096,
  "model_type": "llama",
  "num_attention_heads": 32,
  "num_hidden_layers": 32,
  "num_key_value_heads": 32,
  "pretraining_tp": 1,
  "quantization_config": {
    "bnb_4bit_compute_dtype": "bfloat16",
    "bnb_4bit_quant_type": "nf4",
    "bnb_4bit_use_double_quant": true,
    "llm_int8_enable_fp32_cpu_offload": false,
    "llm_int8_has_fp16_weight": false,
    "llm_int8_skip_modules": null,
    "llm_int8_threshold": 6.0,
    "load_in_4bit": true,
    "load_in_8bit": false,
    "quant_method": "bitsandbytes"
  },
  "rms_norm_eps": 1e-06,
  "rope_scaling": null,
  "rope_theta": 10000.0,
  "tie_word_embeddings": false,
  "torch_dtype": "float16",
  "transformers_version": "4.35.0.dev0",
  "use_cache": false,
  "vocab_size": 32000
}


  0%|          | 0/750 [00:00<?, ?it/s]You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.
/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.
  warnings.warn(

  0%|          | 1/750 [00:07<1:33:01,  7.45s/it]
  0%|          | 2/750 [00:10<1:03:41,  5.11s/it]
  0%|          | 3/750 [00:14<53:38,  4.31s/it]  
  1%|          | 4/750 [00:17<48:40,  3.92s/it]
  1%|          | 5/750 [00:20<44:54,  3.62s/it]
  1%|          | 6/750 [00:23<42:28,  3.43s/it]
  1%|          | 7/750 [00:26<40:52,  3.30s/it]
  1%|          | 8/750 [00:29<39:40,  3.21s/it]
  1%|          | 9/750 [00:32<38:46,  3.14s/it]
  1%|▏         | 10/750 [00:35<38:06,  3.09s/it]
  1%|▏         | 11/750 [00:38<37:39,  3.06s/it]
  2%|▏         | 12/750 [00:41<37:18,  3.03s/it]
  2%|▏         | 13/750 [00:44<37:00,  3.01s/it]
  2%|▏         | 14/750 [00:47<36:44,  2.99s/it]
  2%|▏         | 15/750 [00:50<36:28,  2.98s/it]
  2%|▏         | 16/750 [00:53<36:15,  2.96s/it]
  2%|▏         | 17/750 [00:56<36:03,  2.95s/it]
  2%|▏         | 18/750 [00:59<35:52,  2.94s/it]
  3%|▎         | 19/750 [01:02<35:37,  2.92s/it]
  3%|▎         | 20/750 [01:05<35:03,  2.88s/it]
  3%|▎         | 21/750 [01:07<34:07,  2.81s/it]
  3%|▎         | 22/750 [01:10<32:55,  2.71s/it]
  3%|▎         | 23/750 [01:12<32:05,  2.65s/it]
  3%|▎         | 24/750 [01:15<31:28,  2.60s/it]
  3%|▎         | 25/750 [01:17<30:59,  2.57s/it]
                                                
{'loss': 1.7484, 'learning_rate': 0.0002, 'epoch': 0.1}

  3%|▎         | 25/750 [01:17<30:59,  2.57s/it]Traceback (most recent call last):
  File "/home/andre/ondemand/data/sys/myjobs/projects/default/4/finetuning.py", line 92, in <module>
    configured_trainer.train()
  File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 1506, in train
    return inner_training_loop(
  File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 1869, in _inner_training_loop
    self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)
  File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 2224, in _maybe_log_save_evaluate
    self._save_checkpoint(model, trial, metrics=metrics)
  File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 2281, in _save_checkpoint
    self.save_model(output_dir, _internal_call=True)
  File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 2768, in save_model
    self._save(output_dir)
  File "/usr/local/lib/python3.10/dist-packages/transformers/trainer.py", line 2831, in _save
    self.tokenizer.save_pretrained(output_dir)
  File "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py", line 2445, in save_pretrained
    out_str = json.dumps(tokenizer_config, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
  File "/usr/lib/python3.10/json/__init__.py", line 238, in dumps
    **kw).encode(obj)
  File "/usr/lib/python3.10/json/encoder.py", line 201, in encode
    chunks = list(chunks)
  File "/usr/lib/python3.10/json/encoder.py", line 431, in _iterencode
    yield from _iterencode_dict(o, _current_indent_level)
  File "/usr/lib/python3.10/json/encoder.py", line 405, in _iterencode_dict
    yield from chunks
  File "/usr/lib/python3.10/json/encoder.py", line 438, in _iterencode
    o = _default(o)
  File "/usr/lib/python3.10/json/encoder.py", line 179, in default
    raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type BitsAndBytesConfig is not JSON serializable
`

Hi @andreducfer 
Thanks for the ping, can you open a new ticket for that issue with ideally a simple reproducer of the issue?