Trying to made a keras model with transformer layers defined in hf-transformers, keep running into `AttributeError: Tensor.op is meaningless when eager execution is enabled, when trying to make a keras model`
Author: Santosh-Gupta
Pull request: no
Labels: name: wontfix

# ❓ Questions & Help


## Details

I am trying to build models using huggingface transformer layers. However, I keep running into `AttributeError: Tensor.op is meaningless when eager execution is enabled.`

I don't want to disable eager execution, as I heard it interfere's with some of keras' other functionality (feel free to correct if this is wrong). 

Here's the abridged code for one attempt (full code here https://colab.research.google.com/drive/1pnFDEQB4EuxNM1pSgbWJNKD2208dIIN0?usp=sharing )

```
from transformers.modeling_tf_bert import TFBertLayer

class TFBertEncoderAlter(tf.keras.layers.Layer):
    def __init__(self, config, **kwargs):
        super().__init__(**kwargs)
        self.output_hidden_states = config.output_hidden_states
        self.layer = [TFBertLayer(config, name="layer_._{}".format(i)) for i in range(config.num_hidden_layers)]

    def call(self, inputs, training=False):
        hidden_states, attention_mask, output_attentions = inputs

        all_hidden_states = ()
        all_attentions = ()
        for i, layer_module in enumerate(self.layer):
            if self.output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_outputs = layer_module(
                [hidden_states, attention_mask, output_attentions], training=training
            )
            hidden_states = layer_outputs[0]

            if cast_bool_to_primitive(output_attentions) is True:
                all_attentions = all_attentions + (layer_outputs[1],)

        # Add last layer
        if self.output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        outputs = (hidden_states,)
        if self.output_hidden_states:
            outputs = outputs + (all_hidden_states,)
        if cast_bool_to_primitive(output_attentions) is True:
            outputs = outputs + (all_attentions,)
        return outputs  # outputs, (hidden states), (attentions)

P_trans11 = TFBertEncoderAlter(config, name='Encoder')

inputHiddenVals = tf.keras.Input(shape=[None, None], dtype=tf.float32, name='input_Q',
                                batch_size=None) 

P_outputs = P_trans11((outt, None, None))
modelNew = tf.keras.Model(inputHiddenVals,P_outputs)

```

Here is the output 

```
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-35-8cd9393cb573> in <module>()
      5 
      6 P_outputs = P_trans11((outt, None, None))
----> 7 modelNew = tf.keras.Model(inputHiddenVals,P_outputs)

6 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in __init__(self, *args, **kwargs)
    165 
    166   def __init__(self, *args, **kwargs):
--> 167     super(Model, self).__init__(*args, **kwargs)
    168     _keras_api_gauge.get_cell('model').set(True)
    169     # Model must be created under scope of DistStrat it will be trained with.

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in __init__(self, *args, **kwargs)
    171         'inputs' in kwargs and 'outputs' in kwargs):
    172       # Graph network
--> 173       self._init_graph_network(*args, **kwargs)
    174     else:
    175       # Subclassed network

/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
    454     self._self_setattr_tracking = False  # pylint: disable=protected-access
    455     try:
--> 456       result = method(self, *args, **kwargs)
    457     finally:
    458       self._self_setattr_tracking = previous_value  # pylint: disable=protected-access

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in _init_graph_network(self, inputs, outputs, name, **kwargs)
    252 
    253     if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs):
--> 254       base_layer_utils.create_keras_history(self._nested_outputs)
    255 
    256     self._base_init(name=name, **kwargs)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py in create_keras_history(tensors)
    184     keras_tensors: The Tensors found that came from a Keras Layer.
    185   """
--> 186   _, created_layers = _create_keras_history_helper(tensors, set(), [])
    187   return created_layers
    188 

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
    210     if getattr(tensor, '_keras_history', None) is not None:
    211       continue
--> 212     op = tensor.op  # The Op that created this Tensor.
    213     if op not in processed_ops:
    214       if op.type.startswith('Sparse'):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in op(self)
   1111   def op(self):
   1112     raise AttributeError(
-> 1113         "Tensor.op is meaningless when eager execution is enabled.")
   1114 
   1115   @property

AttributeError: Tensor.op is meaningless when eager execution is enabled.
```


Here is a different approach ( full code here https://colab.research.google.com/drive/1bieigPh98l9POzT3Tdz8DWDN_nh18YG1?usp=sharing )

```
from transformers.modeling_tf_bert import TFBertEncoder, TFBertMainLayer, TFBertLayer
from transformers.modeling_tf_utils import (
    get_initializer,
    keras_serializable,
    shape_list,
)
from transformers.configuration_bert import BertConfig


class TFBertEncoder0(tf.keras.layers.Layer):
    def __init__(self, config, **kwargs):
        super().__init__(**kwargs)
        self.output_attentions = config.output_attentions
        self.output_hidden_states = config.output_hidden_states
        self.layer = [TFBertLayer(config, name="layer_._{}".format(i)) for i in range(config.num_hidden_layers)]

    def call(self, inputs, training=False):
        hidden_states, attention_mask, head_mask = inputs

        all_hidden_states = ()
        all_attentions = ()
        for i, layer_module in enumerate(self.layer):
            if self.output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_outputs = layer_module([hidden_states, attention_mask, head_mask], training=training)
            hidden_states = layer_outputs[0]

            if self.output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        # Add last layer
        if self.output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        outputs = (hidden_states,)
        if self.output_hidden_states:
            outputs = outputs + (all_hidden_states,)
        if self.output_attentions:
            outputs = outputs + (all_attentions,)
        return outputs  # outputs, (hidden states), (attentions)

@keras_serializable
class TFBertMainLayerAlter4(tf.keras.layers.Layer):
    config_class = BertConfig

    def __init__(self, config, **kwargs):
        super().__init__(**kwargs)
        self.num_hidden_layers = config.num_hidden_layers
        self.initializer_range = config.initializer_range
        self.output_attentions = config.output_attentions

        self.encoder = TFBertEncoder0(config, name="encoder")

    def _prune_heads(self, heads_to_prune):
        """ Prunes heads of the model.
            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
            See base class PreTrainedModel
        """
        raise NotImplementedError

    def call(
        self,
        inputs,
        training=False,
    ):

        encoder_outputs = self.encoder(
            [inputs, None, None], training=training
        )

        sequence_output = encoder_outputs[0]

        outputs = (sequence_output,) + encoder_outputs[
            1:
        ]  # add hidden_states and attentions if they are here
        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)

P_trans11 = TFBertMainLayerAlter4(config3, name="roberta")

inputHiddenVals = tf.keras.Input(shape=[None, None], dtype=tf.float32, name='input_Q',
                                batch_size=None) 

P_outputs = P_trans11(outt)
modelNew = tf.keras.Model(inputHiddenVals,P_outputs)
```

Again, same result

```
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-43-79b3b73e5f5f> in <module>()
      5 
      6 P_outputs = P_trans11(outt)
----> 7 modelNew = tf.keras.Model(inputHiddenVals,P_outputs)

6 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in __init__(self, *args, **kwargs)
    165 
    166   def __init__(self, *args, **kwargs):
--> 167     super(Model, self).__init__(*args, **kwargs)
    168     _keras_api_gauge.get_cell('model').set(True)
    169     # Model must be created under scope of DistStrat it will be trained with.

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in __init__(self, *args, **kwargs)
    171         'inputs' in kwargs and 'outputs' in kwargs):
    172       # Graph network
--> 173       self._init_graph_network(*args, **kwargs)
    174     else:
    175       # Subclassed network

/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
    454     self._self_setattr_tracking = False  # pylint: disable=protected-access
    455     try:
--> 456       result = method(self, *args, **kwargs)
    457     finally:
    458       self._self_setattr_tracking = previous_value  # pylint: disable=protected-access

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/network.py in _init_graph_network(self, inputs, outputs, name, **kwargs)
    252 
    253     if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs):
--> 254       base_layer_utils.create_keras_history(self._nested_outputs)
    255 
    256     self._base_init(name=name, **kwargs)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py in create_keras_history(tensors)
    184     keras_tensors: The Tensors found that came from a Keras Layer.
    185   """
--> 186   _, created_layers = _create_keras_history_helper(tensors, set(), [])
    187   return created_layers
    188 

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
    210     if getattr(tensor, '_keras_history', None) is not None:
    211       continue
--> 212     op = tensor.op  # The Op that created this Tensor.
    213     if op not in processed_ops:
    214       if op.type.startswith('Sparse'):

/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in op(self)
   1111   def op(self):
   1112     raise AttributeError(
-> 1113         "Tensor.op is meaningless when eager execution is enabled.")
   1114 
   1115   @property

AttributeError: Tensor.op is meaningless when eager execution is enabled.
```

Here's another attempt (full code here https://colab.research.google.com/drive/1UVJ7XSx0vXpgApe6E7ECVb9LNSJN_D9-?usp=sharing )

```
from transformers.modeling_tf_bert import TFBertLayer

l1 = TFBertLayer(config)
l2 = TFBertLayer(config)

inputHiddenVals = tf.keras.Input(shape=[None, None], dtype=tf.float32, name='input_Q',
                                batch_size=None) 

P_outputs = l1((outt, None, None))[0]
P_outputs2 = l2((outt, None, None))[0]

modelNew = tf.keras.Model(inputHiddenVals,P_outputs2)
```

Again, same results. 

**A link to original question on Stack Overflow**:

https://stackoverflow.com/questions/62504903/attributeerror-tensor-op-is-meaningless-when-eager-execution-is-enabled-when-t

Comments:
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.