Martijn van Beers commited on
Commit
66d5973
1 Parent(s): 265622b

Make the custom model work with IG

Browse files

IG doesn't create tensors that have `requires_grad`, so check for
that before adding the hook

Files changed (1) hide show
  1. lib/roberta2.py +2 -1
lib/roberta2.py CHANGED
@@ -280,7 +280,8 @@ class RobertaSelfAttention(nn.Module):
280
  attention_probs = nn.functional.softmax(attention_scores, dim=-1)
281
 
282
  self.save_attn(attention_probs)
283
- attention_probs.register_hook(self.save_attn_gradients)
 
284
 
285
  # This is actually dropping out entire tokens to attend to, which might
286
  # seem a bit unusual, but is taken from the original Transformer paper.
 
280
  attention_probs = nn.functional.softmax(attention_scores, dim=-1)
281
 
282
  self.save_attn(attention_probs)
283
+ if attention_probs.requires_grad:
284
+ attention_probs.register_hook(self.save_attn_gradients)
285
 
286
  # This is actually dropping out entire tokens to attend to, which might
287
  # seem a bit unusual, but is taken from the original Transformer paper.