GPT2 Heatmap Error: 'Parameter' object has no attribute 'get_shape'
Author: BigSalmon2
Pull request: no

```
import numpy as np
import tensorflow as tf

def compute_textual_saliency(model, embedding_matrix, tokenizer, text):
    token_ids = tokenizer.encode(text, add_special_tokens=True)
    vocab_size = embedding_matrix.get_shape()[0]

    heatmap_data = []

    for masked_token_index in range(len(token_ids)):
        # print(f'processing token {masked_token_index + 1} / {len(token_ids)}')

        if masked_token_index == 0:
            heatmap_data.append({
                'token': '[CLR]',
                'meta': ['', '', ''],
                'heat': [1] + [0] * (len(token_ids) - 1)
            })
        elif masked_token_index == len(token_ids) - 1:
            heatmap_data.append({
                'token': ' ',
                'format': True
            })
            heatmap_data.append({
                'token': '[SEP]',
                'meta': ['', '', ''],
                'heat': [0] * (len(token_ids) - 1) + [1]
            })
        else:
            # Get the actual token
            target_token = tokenizer.convert_ids_to_tokens(
                token_ids[masked_token_index])

            if target_token[0:2] == '##':
                target_token = target_token[2:]
            else:
                heatmap_data.append({
                    'token': ' ',
                    'format': True
                })

            # integers are not differentable, so use a one-hot encoding
            # of the intput
            token_ids_tensor = tf.constant([
                token_ids[0:masked_token_index] +
                [tokenizer.mask_token_id] +
                token_ids[masked_token_index + 1:]
            ], dtype='int32')
            token_ids_tensor_one_hot = tf.one_hot(token_ids_tensor, vocab_size)

            # To select, the correct output witch is what the importance
            # measure targets, create a masking tensor. tf.gather_nd could also
            # be used, but this is easier.
            output_mask = np.zeros((1, len(token_ids), vocab_size))
            output_mask[0, masked_token_index, token_ids[masked_token_index]] = 1
            output_mask_tensor = tf.constant(output_mask, dtype='float32')

            # Compute gradient of the logits of the correct target, w.r.t. the
            # input
            with tf.GradientTape(watch_accessed_variables=False) as tape:
                tape.watch(token_ids_tensor_one_hot)
                inputs_embeds = tf.matmul(token_ids_tensor_one_hot,embedding_matrix)
                predict, = model({"inputs_embeds": inputs_embeds})
                predict_mask_correct_token = tf.reduce_sum(predict * output_mask_tensor)

            # Get the top-3 predictions
            (_, top_3_indices) = tf.math.top_k(predict[0, masked_token_index, :], 3)
            top_3_predicted_tokens = tokenizer.convert_ids_to_tokens(top_3_indices)

            # compute the connectivity
            connectivity_non_normalized = tf.norm(
                tape.gradient(predict_mask_correct_token, token_ids_tensor_one_hot),
                axis=2)
            connectivity_tensor = (
                connectivity_non_normalized /
                tf.reduce_max(connectivity_non_normalized)
            )
            connectivity = connectivity_tensor[0].numpy().tolist()

            heatmap_data.append({
                'token': target_token,
                'meta': top_3_predicted_tokens,
                'heat': connectivity
            })

    return heatmap_data
```

```
text = ("context the formal study of grammar is an important part of education"
        " from a young age through advanced learning though the rules taught"
        " in schools are not a grammar in the sense most linguists use")

from transformers import TFDistilBertForMaskedLM, DistilBertTokenizer
dbert_model = TFDistilBertForMaskedLM.from_pretrained('distilbert-base-uncased')
dbert_tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')
dbert_embmat = dbert_model.distilbert.embeddings.word_embeddings


from transformers import AutoTokenizer, AutoModelWithLMHead
gpt2_model = AutoModelWithLMHead.from_pretrained("gpt2")
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
gpt2_embmat = gpt2_model.transformer.wte.weight


from textualheatmap import TextualHeatmap
heatmap = TextualHeatmap(facet_titles = ['BERT', 'Distil BERT'], show_meta=True)
heatmap.set_data([
    compute_textual_saliency(dbert_model, dbert_embmat, dbert_tokenizer, text),
    compute_textual_saliency(gpt2_model, gpt2_embmat, gpt2_tokenizer, text)
])
```

`AttributeError: 'Parameter' object has no attribute 'get_shape'`

Comments:
As the error says, `get_shape` does not exist in PyTorch parameters. Use `[...].shape` instead of `[...].get_shape()`.