Issue loading models saved with older version
Author: valbarriere
Pull request: no

### System Info

- `transformers` version: 4.28.1
- Platform: Linux-3.10.0-1160.80.1.el7.x86_64-x86_64-with-debian-bullseye-sid
- Python version: 3.7.10
- Huggingface_hub version: 0.15.1
- Safetensors version: not installed
- PyTorch version (GPU?): 1.10.0+cu102 (True)
- Tensorflow version (GPU?): 2.4.1 (True)
- Flax version (CPU?/GPU?/TPU?): not installed (NA)
- Jax version: not installed
- JaxLib version: not installed
- Using GPU in script?: <fill in>
- Using distributed or parallel set-up in script?: <fill in

### Who can help?

@ArthurZucker and @younesbelkada

### Information

- [ ] The official example scripts
- [X] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [X] My own task or dataset (give details below)

### Reproduction

1. Save a model with transformers 3.5.1
2. Load it with transformers 4.28.1 

Here's my code to create the model, before loading it and obtaining the error. Do not take into account all the inputs variable of the function 

```
def compile_and_return_model(model_name, nb_classes, learning_rate, 
	proxies=None, MAX_SEQ_LEN=128, use_token_type_ids_roberta=True, 
							 verbose=False):
	"""
	Create the deep learning model, compile it and return it
	
	TODO: input_img_dim is hard-coded, but this value should come from the loaded dataset !!  
	"""
	### The models from the transformers library  
	
	MODELS_TF_dict = {'bert-transformers' : (TFBertModel, BertTokenizer),
				 'xlm-roberta-transformers' : (TFXLMRobertaModel, XLMRobertaTokenizer),
				 'roberta-transformers' : (TFRobertaModel, RobertaTokenizer),
				# TODO: t5 does not work right now, even with pytorch.... 
				 't5-transformers' : (TFT5Model, T5Tokenizer), # 't5-11b' is supposed the best model at the end of 04/2020
				 'electra-transformers' : (TFElectraModel, ElectraTokenizer), # TODO: do the tests with Electra model 
				  'camembert-transformers' : (TFCamembertModel, CamembertTokenizer), 
				 'automodel' : (TFAutoModel, AutoTokenizer)
				 # 'longformer-transformers' : (TFLongformerModel, LongformerTokenizer), # TODO: do the tests with Longformer model 
			 }

	### Model compiling 

	model_gen, tok_gen = (TFXLMRobertaModel, XLMRobertaTokenizer)
	
	str_model_to_load, str_tokenizer_to_load = return_model_tokenize_to_load(model_name)

	model_to_load = 'jplu/tf-xlm-roberta-base'
	str_tokenizer_to_load = 'jplu/tf-xlm-roberta-base'

	tokenizer = tok_gen.from_pretrained(str_tokenizer_to_load, proxies=proxies, cache_dir = CACHE_DIR)
	from_pt = False
	model_tf_transformer = model_gen.from_pretrained(str_model_to_load, proxies=proxies, cache_dir = CACHE_DIR, from_pt=from_pt)
		
	if ('roberta' in model_name) and use_token_type_ids_roberta:
		nb_sentences = use_token_type_ids_roberta + 1
		print('Adding a special token_type_embeddings layer to RoBERTa, of dimension {}'.format(nb_sentences))
		model_tf_transformer.config.type_vocab_size = nb_sentences 
		from transformers import __version__ as tv
		if int(tv.split('.')[0]) > 3:
			model_tf_transformer.roberta.embeddings.token_type_embeddings = tf.Variable(np.random.normal(0.0,model_tf_transformer.config.initializer_range, (model_tf_transformer.config.type_vocab_size, model_tf_transformer.config.hidden_size)).astype('float32'))
		else:
			model_tf_transformer.roberta.embeddings.token_type_embeddings = tf.keras.layers.Embedding(2, model_tf_transformer.config.hidden_size, embeddings_initializer = tf.keras.initializers.RandomNormal(mean=0.0, stddev=model_tf_transformer.config.initializer_range))
          
		tokenizer.create_token_type_ids = True
		
	#input layers
	# MAX_SEQ_LEN=128
	input_word_ids = tf.keras.layers.Input(shape=(MAX_SEQ_LEN,), dtype=tf.int32, name="input_word_ids")
	input_mask = tf.keras.layers.Input(shape=(MAX_SEQ_LEN,), dtype=tf.int32, name="input_mask")
	segment_ids = tf.keras.layers.Input(shape=(MAX_SEQ_LEN,), dtype=tf.int32, name="segment_ids")

	sequence_output = model_tf_transformer({'input_ids' : input_word_ids, 'attention_mask' : input_mask, 'token_type_ids' : segment_ids})[0]

	### SAME config as in the class RobertaClassificationHead of `transformers` --> take `sequence_output[:, 0, :]`
	# dropout value from https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-config.json
	hidden_size_output = 768
	x = sequence_output[:, 0, :] # take 
 token (equiv. to [CLS])
	x = tf.keras.layers.Dropout(0.1)(x)
	x = tf.keras.layers.Dense(hidden_size_output, activation="tanh", name="hidden-layer1")(x)
	x = tf.keras.layers.Dropout(0.1)(x)

	out = tf.keras.layers.Dense(nb_classes, name="dense_output")(x)
	from_logits = True

	model = tf.keras.models.Model(
		inputs=[input_word_ids, input_mask, segment_ids], 
		outputs=out	 
	)

	loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=from_logits)
	metric = tf.keras.metrics.SparseCategoricalAccuracy('acc')

	model.compile(optimizer=optimizer, loss=loss, metrics=[metric])

	if verbose:
		model.summary(line_length=150)
	
	return model, tokenizer
```

### Expected behavior

Hi,  

I am using transformers 4.28.1 and I have an issue loading a model I saved with transformers 3.5.1

It seems that some layers are not loaded in the same order and it is messing with the general loading process... 

```
WARNING:tensorflow:Skipping loading of weights for layer tfxlm_roberta_model due to mismatch in shape ((2, 768) vs (768,)).
WARNING:tensorflow:Skipping loading of weights for layer tfxlm_roberta_model due to mismatch in shape ((768,) vs (2, 768)).
Loading already trained model... 
Traceback (most recent call last):
  File "../Transformers_MM_class.py", line 1257, in <module>
    dict_lab=dict_lab, path_corpus=path_corpus_test, list_countries=['France', 'Ukraine'], n_duplicates=100)
  File "/home/barriva/Valentin_code/biases_calculation_huggingfacehub.py", line 216, in calculate_sentiment_bias
    model, X_text, y = prepare_data_and_model(modelFilePath, model, input_data_File, dict_lab, path_corpus)
  File "/home/barriva/Valentin_code/biases_calculation_huggingfacehub.py", line 123, in prepare_data_and_model
    model.load_weights(modelFilePath)
  File "/jeodpp_envs/bdap-gpu-txt-tf2.4.1/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 2234, in load_weights
    hdf5_format.load_weights_from_hdf5_group(f, self.layers)
  File "/jeodpp_envs/bdap-gpu-txt-tf2.4.1/lib/python3.7/site-packages/tensorflow/python/keras/saving/hdf5_format.py", line 710, in load_weights_from_hdf5_group
    K.batch_set_value(weight_value_tuples)
  File "/jeodpp_envs/bdap-gpu-txt-tf2.4.1/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
    return target(*args, **kwargs)
  File "/jeodpp_envs/bdap-gpu-txt-tf2.4.1/lib/python3.7/site-packages/tensorflow/python/keras/backend.py", line 3706, in batch_set_value
    x.assign(np.asarray(value, dtype=dtype(x)))
  File "/jeodpp_envs/bdap-gpu-txt-tf2.4.1/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py", line 891, in assign
    (tensor_name, self._shape, value_tensor.shape))
ValueError: Cannot assign to variable Variable:0 due to variable shape (2, 768) and value shape (768,) are incompatible
```

Comments:
Hey! You are jumping between two majors releases where backward compatibility is usually broken. So this is expected

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md) are likely to be ignored.