Repository: huggingface/optimum

Issue:
ORTModel.save_pretrained doesn't save model_type key in its config.json
### System Info

```shell
optimum==1.4.0
transformers==4.22.1
onnx==1.12.0
onnxruntime==1.12.1

Python 3.9.13
```


### Who can help?

@lewtun 
@michaelbenayoun 


### Information

- [X] The official example scripts
- [ ] My own modified scripts

### Tasks

- [X] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction

```
from optimum.onnxruntime import ORTModelForQuestionAnswering
from transformers import AutoTokenizer

# Load a model from transformers and export it to ONNX
ort_model = ORTModelForQuestionAnswering.from_pretrained(MODEL_PATH, from_transformers=True)
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)

ONNX_PATH = f"{MODEL_PATH}/onnx"

# Save the onnx model and tokenizer
ort_model.save_pretrained(ONNX_PATH)
tokenizer.save_pretrained(ONNX_PATH)


from optimum.onnxruntime import ORTOptimizer

optimizer = ORTOptimizer.from_pretrained(ONNX_PATH) # <- This fails

```
### Expected behavior

Loading the ORT model for the optimizer with `ORTOptimizer.from_pretrained(ONNX_PATH)` fails with:
`ValueError: Unrecognized model in ... Should have a model_type key in its config.json, or contain one of the following strings in its name: ...`

The `config.json` generated when converting to Onnx doesn't have a `model_type` key.
Unlike the `config.json` of the 🤗  Transformers model: `"model_type": "xlm-roberta"`; There only seems to be a `"name": "XLMRoberta"` key.

But the optimizer and quantizer classes expect a `model_type` key when loading the ort_model.

I was expecting the `.save_pretrained` method of a `ORTModel` class to also save the `model_type` key from the 🤗 Transformers `config.json` model.



Discussion:

Resolving patch:
diff --git a/optimum/modeling_base.py b/optimum/modeling_base.py
--- a/optimum/modeling_base.py
+++ b/optimum/modeling_base.py
@@ -208,12 +208,11 @@ def from_pretrained(
             model_id, revision = model_id.split("@")
 
         if os.path.isdir(model_id) and CONFIG_NAME in os.listdir(model_id):
-            config_file = os.path.join(model_id, CONFIG_NAME)
+            config = AutoConfig.from_pretrained(os.path.join(model_id, CONFIG_NAME))
         else:
             try:
-                config_file = hf_hub_download(
-                    repo_id=model_id,
-                    filename=CONFIG_NAME,
+                config = AutoConfig.from_pretrained(
+                    pretrained_model_name_or_path=model_id,
                     revision=revision,
                     cache_dir=cache_dir,
                     force_download=force_download,
@@ -221,11 +220,9 @@ def from_pretrained(
                 )
             except requests.exceptions.RequestException:
                 logger.warning("config.json NOT FOUND in HuggingFace Hub")
-                config_file = None
+                config = None
 
-        if config_file is not None:
-            with open(config_file, "r", encoding="utf-8") as f:
-                config = json.load(f)
+        if config is not None:
             model_kwargs.update({"config": config})
 
         if from_transformers:
diff --git a/optimum/onnxruntime/modeling_ort.py b/optimum/onnxruntime/modeling_ort.py
--- a/optimum/onnxruntime/modeling_ort.py
+++ b/optimum/onnxruntime/modeling_ort.py
@@ -107,7 +107,7 @@ class ORTModel(OptimizedModel):
     base_model_prefix = "onnx_model"
     auto_model_class = AutoModel
 
-    def __init__(self, model: ort.InferenceSession = None, config=None, **kwargs):
+    def __init__(self, model: ort.InferenceSession = None, config: PretrainedConfig = None, **kwargs):
         self.model = model
         self.config = config
         self.model_save_dir = kwargs.get("model_save_dir", None)
@@ -298,11 +298,10 @@ def _from_pretrained(
                 kwargs will be passed to the model during initialization
         """
         local_files_only = kwargs.pop("local_files_only", False)
-        config_dict = kwargs.pop("config", {})
+        config = kwargs.pop("config", {})
         model_file_name = file_name if file_name is not None else ONNX_WEIGHTS_NAME
         # load model from local directory
         if os.path.isdir(model_id):
-            config = PretrainedConfig.from_dict(config_dict)
             model = ORTModel.load_model(os.path.join(model_id, model_file_name), **kwargs)
             kwargs["model_save_dir"] = Path(model_id)
             kwargs["latest_model_name"] = model_file_name
@@ -321,7 +320,6 @@ def _from_pretrained(
             kwargs["model_save_dir"] = Path(model_cache_path).parent
             kwargs["latest_model_name"] = Path(model_cache_path).name
             model = ORTModel.load_model(model_cache_path, **kwargs)
-            config = PretrainedConfig.from_dict(config_dict)
 
         return cls(model=model, config=config, **kwargs)
 
@@ -386,7 +384,7 @@ def _from_transformers(
             opset=onnx_config.default_onnx_opset,
             output=save_dir.joinpath(ONNX_WEIGHTS_NAME),
         )
-        kwargs["config"] = model.config.__dict__
+        kwargs["config"] = model.config
         # 3. load normal model
         return cls._from_pretrained(save_dir.as_posix(), **kwargs)
 
diff --git a/optimum/onnxruntime/modeling_seq2seq.py b/optimum/onnxruntime/modeling_seq2seq.py
--- a/optimum/onnxruntime/modeling_seq2seq.py
+++ b/optimum/onnxruntime/modeling_seq2seq.py
@@ -20,7 +20,7 @@
 
 import torch
 import transformers
-from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer, PretrainedConfig
+from transformers import AutoConfig, AutoModelForSeq2SeqLM, AutoTokenizer
 from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward, default_cache_path
 from transformers.generation_utils import GenerationMixin
 from transformers.modeling_outputs import BaseModelOutput, Seq2SeqLMOutput
@@ -342,8 +342,7 @@ def _from_pretrained(
         """
         use_cache = kwargs.pop("use_cache", True)
         local_files_only = kwargs.pop("local_files_only", False)
-        config_dict = kwargs.pop("config", {})
-        config = PretrainedConfig.from_dict(config_dict)
+        config = kwargs.pop("config", {})
         encoder_file_name = encoder_file_name or ONNX_ENCODER_NAME
         decoder_file_name = decoder_file_name or ONNX_DECODER_NAME
         decoder_with_past_file_name = decoder_with_past_file_name or ONNX_DECODER_WITH_PAST_NAME
@@ -477,7 +476,7 @@ def _from_transformers(
                 output=save_dir.joinpath(ONNX_DECODER_WITH_PAST_NAME),
             )
 
-        kwargs["config"] = model.config.__dict__
+        kwargs["config"] = model.config
         return cls._from_pretrained(save_dir, **kwargs)
 
     def to(self, device: Union[torch.device, str, int]):


Test patch: