RuntimeError: Error while creating shape using tf-xlm-roberta-large
Author: msafi04
Pull request: no
Labels: name: wontfix

I get the following runtime error after the 2nd fold.

this is my model:
maxlen = 50
`with strategy.scope():
        #bert_encoder = TFBertModel.from_pretrained(model_name)
        base_model = TFAutoModel.from_pretrained(model_name)
        input_word_ids = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_word_ids")
        input_mask = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_mask")
        input_type_ids = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_type_ids")

        embedding = base_model([input_word_ids, input_mask, input_type_ids])[0]
        output = tf.keras.layers.Dense(3, activation = 'softmax')(embedding[:, 0, :])

        model = tf.keras.Model(inputs = [input_word_ids, input_mask, input_type_ids], outputs = output)
        model.compile(tf.keras.optimizers.Adam(lr = 1e-5), loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])

`
And the traceback below:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-27-a45ce90453f2> in <module>
     22 
     23     K.clear_session()
---> 24     model = build_model(maxlen, model_name)
     25     checkpoint = tf.keras.callbacks.ModelCheckpoint(
     26                 'XLMRoberta_fold-%i.h5'%fold, monitor = 'val_loss', verbose = 1, save_best_only = True,

<ipython-input-23-9faa2e5f1d9b> in build_model(maxlen, model_name)
      2     with strategy.scope():
      3         #bert_encoder = TFBertModel.from_pretrained(model_name)
----> 4         base_model = TFAutoModel.from_pretrained(model_name)
      5         input_word_ids = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_word_ids")
      6         input_mask = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_mask")

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_auto.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    421         for config_class, model_class in TF_MODEL_MAPPING.items():
    422             if isinstance(config, config_class):
--> 423                 return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
    424         raise ValueError(
    425             "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_utils.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    482             return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, allow_missing_keys=True)
    483 
--> 484         model(model.dummy_inputs, training=False)  # build the network with dummy inputs
    485 
    486         assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
    966           with base_layer_utils.autocast_context_manager(
    967               self._compute_dtype):
--> 968             outputs = self.call(cast_inputs, *args, **kwargs)
    969           self._handle_activity_regularization(inputs, outputs)
    970           self._set_mask_metadata(inputs, outputs, input_masks)

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_roberta.py in call(self, inputs, **kwargs)
    229             heads.
    230         """
--> 231         outputs = self.roberta(inputs, **kwargs)
    232         return outputs
    233 

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
    966           with base_layer_utils.autocast_context_manager(
    967               self._compute_dtype):
--> 968             outputs = self.call(cast_inputs, *args, **kwargs)
    969           self._handle_activity_regularization(inputs, outputs)
    970           self._set_mask_metadata(inputs, outputs, input_masks)

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_bert.py in call(self, inputs, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, output_attentions, output_hidden_states, training)
    604             # head_mask = tf.constant([0] * self.num_hidden_layers)
    605 
--> 606         embedding_output = self.embeddings([input_ids, position_ids, token_type_ids, inputs_embeds], training=training)
    607         encoder_outputs = self.encoder(
    608             [embedding_output, extended_attention_mask, head_mask, output_attentions, output_hidden_states],

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
    962         # Eager execution on data tensors.
    963         with backend.name_scope(self._name_scope()):
--> 964           self._maybe_build(inputs)
    965           cast_inputs = self._maybe_cast_inputs(inputs)
    966           with base_layer_utils.autocast_context_manager(

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in _maybe_build(self, inputs)
   2414         # operations.
   2415         with tf_utils.maybe_init_scope(self):
-> 2416           self.build(input_shapes)  # pylint:disable=not-callable
   2417       # We must set also ensure that the layer is marked as built, and the build
   2418       # shape is stored since user defined build functions may not be calling

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_bert.py in build(self, input_shape)
    144                 "weight",
    145                 shape=[self.vocab_size, self.hidden_size],
--> 146                 initializer=get_initializer(self.initializer_range),
    147             )
    148         super().build(input_shape)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in add_weight(self, name, shape, dtype, initializer, regularizer, trainable, constraint, partitioner, use_resource, synchronization, aggregation, **kwargs)
    575         synchronization=synchronization,
    576         aggregation=aggregation,
--> 577         caching_device=caching_device)
    578     if regularizer is not None:
    579       # TODO(fchollet): in the future, this should be handled at the

/opt/conda/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py in _add_variable_with_custom_getter(self, name, shape, dtype, initializer, getter, overwrite, **kwargs_for_getter)
    741         dtype=dtype,
    742         initializer=initializer,
--> 743         **kwargs_for_getter)
    744 
    745     # If we set an initializer and the variable processed it, tracking will not

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer_utils.py in make_variable(name, shape, dtype, initializer, trainable, caching_device, validate_shape, constraint, use_resource, collections, synchronization, aggregation, partitioner)
    139       synchronization=synchronization,
    140       aggregation=aggregation,
--> 141       shape=variable_shape if variable_shape else None)
    142 
    143 

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in __call__(cls, *args, **kwargs)
    257   def __call__(cls, *args, **kwargs):
    258     if cls is VariableV1:
--> 259       return cls._variable_v1_call(*args, **kwargs)
    260     elif cls is Variable:
    261       return cls._variable_v2_call(*args, **kwargs)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in _variable_v1_call(cls, initial_value, trainable, collections, validate_shape, caching_device, name, variable_def, dtype, expected_shape, import_scope, constraint, use_resource, synchronization, aggregation, shape)
    218         synchronization=synchronization,
    219         aggregation=aggregation,
--> 220         shape=shape)
    221 
    222   def _variable_v2_call(cls,

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in getter(**kwargs)
     64 
     65   def getter(**kwargs):
---> 66     return captured_getter(captured_previous, **kwargs)
     67 
     68   return getter

/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py in creator_with_resource_vars(next_creator, **kwargs)
   1765         kwargs["initial_value"] = kwargs["initial_value"].wrapped_value
   1766 
-> 1767       return self._create_variable(next_creator, **kwargs)
   1768 
   1769     def distributed_getter(getter, *args, **kwargs):

/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py in _create_variable(self, next_creator, **kwargs)
    670                                            tpu_values.TPUMirroredVariable,
    671                                            tpu_values.TPUSyncOnReadVariable,
--> 672                                            **kwargs)
    673 
    674   def _reduce_to(self, reduce_op, value, destinations, experimental_hints):

/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/values.py in create_mirrored_variable(strategy, real_mirrored_creator, mirrored_cls, sync_on_read_cls, **kwargs)
    692   # here.
    693   with tape.stop_recording():
--> 694     value_list = real_mirrored_creator(**kwargs)
    695     var_cls = sync_on_read_cls if is_sync_on_read else mirrored_cls
    696     result = var_cls(strategy, value_list, aggregation)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py in _real_mirrored_creator(**kwargs)
    660 
    661           with context.device_policy(context.DEVICE_PLACEMENT_SILENT):
--> 662             v = next_creator(**kwargs)
    663 
    664           assert not isinstance(v, tpu_values.TPUMirroredVariable)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in <lambda>(**kwargs)
    196                         shape=None):
    197     """Call on Variable class. Useful to force the signature."""
--> 198     previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
    199     for _, getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
    200       previous_getter = _make_getter(getter, previous_getter)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py in default_variable_creator(next_creator, **kwargs)
   2596         synchronization=synchronization,
   2597         aggregation=aggregation,
-> 2598         shape=shape)
   2599   else:
   2600     return variables.RefVariable(

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in __call__(cls, *args, **kwargs)
    261       return cls._variable_v2_call(*args, **kwargs)
    262     else:
--> 263       return super(VariableMetaclass, cls).__call__(*args, **kwargs)
    264 
    265 

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py in __init__(self, initial_value, trainable, collections, validate_shape, caching_device, name, dtype, variable_def, import_scope, constraint, distribute_strategy, synchronization, aggregation, shape)
   1432           aggregation=aggregation,
   1433           shape=shape,
-> 1434           distribute_strategy=distribute_strategy)
   1435 
   1436   def _init_from_args(self,

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py in _init_from_args(self, initial_value, trainable, collections, caching_device, name, dtype, constraint, synchronization, aggregation, distribute_strategy, shape)
   1568                 name="initial_value", dtype=dtype)
   1569           if shape is not None:
-> 1570             if not initial_value.shape.is_compatible_with(shape):
   1571               raise ValueError(
1572                   "The initial value's shape (%s) is not compatible with "

/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py in shape(self)
   1063         # `_tensor_shape` is declared and defined in the definition of
   1064         # `EagerTensor`, in C.
-> 1065         self._tensor_shape = tensor_shape.TensorShape(self._shape_tuple())
   1066       except core._NotOkStatusException as e:
   1067         six.raise_from(core._status_to_exception(e.code, e.message), None)

RuntimeError: Error while creating shape

Comments:
Facing the same issue on on GCP + TPU, same TF and TPU versions. 

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.