Repository: huggingface/transformers (Python)
Date: 2022-04-18 15:46:00+00:00
Task type: Bug Fix

Issue:
Missing activation Function
I think the sigmoid / softmax activation function is missing here

https://github.com/huggingface/transformers/blob/31ec2cb2badfbdd4c1ac9c6c9b8a74e974984206/src/transformers/models/roberta/modeling_tf_roberta.py#L1299


Issue discussion:

Solution patch:
diff --git a/src/transformers/activations.py b/src/transformers/activations.py
--- a/src/transformers/activations.py
+++ b/src/transformers/activations.py
@@ -152,19 +152,19 @@ def forward(self, input: Tensor) -> Tensor:
 
 
 ACT2FN = {
-    "relu": nn.ReLU(),
-    "silu": SiLUActivation(),
-    "swish": SiLUActivation(),
     "gelu": GELUActivation(),
-    "tanh": nn.Tanh(),
-    "gelu_python": GELUActivation(use_gelu_python=True),
-    "gelu_new": NewGELUActivation(),
-    "gelu_fast": FastGELUActivation(),
-    "quick_gelu": QuickGELUActivation(),
     "gelu_10": ClippedGELUActivation(-10, 10),
-    "mish": MishActivation(),
+    "gelu_fast": FastGELUActivation(),
+    "gelu_new": NewGELUActivation(),
+    "gelu_python": GELUActivation(use_gelu_python=True),
     "linear": LinearActivation(),
+    "mish": MishActivation(),
+    "quick_gelu": QuickGELUActivation(),
+    "relu": nn.ReLU(),
     "sigmoid": nn.Sigmoid(),
+    "silu": SiLUActivation(),
+    "swish": SiLUActivation(),
+    "tanh": nn.Tanh(),
 }
 
 
diff --git a/src/transformers/activations_tf.py b/src/transformers/activations_tf.py
--- a/src/transformers/activations_tf.py
+++ b/src/transformers/activations_tf.py
@@ -113,16 +113,17 @@ def approximate_gelu_wrap(x):
 
 ACT2FN = {
     "gelu": gelu,
-    "relu": tf.keras.activations.relu,
-    "swish": tf.keras.activations.swish,
-    "silu": tf.keras.activations.swish,
+    "gelu_10": gelu_10,
+    "gelu_fast": gelu_fast,
     "gelu_new": gelu_new,
+    "glu": glu,
     "mish": mish,
-    "tanh": tf.keras.activations.tanh,
-    "gelu_fast": gelu_fast,
     "quick_gelu": quick_gelu,
-    "gelu_10": gelu_10,
-    "glu": glu,
+    "relu": tf.keras.activations.relu,
+    "sigmoid": tf.keras.activations.sigmoid,
+    "silu": tf.keras.activations.swish,
+    "swish": tf.keras.activations.swish,
+    "tanh": tf.keras.activations.tanh,
 }
 
 


Test patch:
diff --git a/tests/utils/test_activations.py b/tests/utils/test_activations.py
--- a/tests/utils/test_activations.py
+++ b/tests/utils/test_activations.py
@@ -46,18 +46,19 @@ def test_gelu_10(self):
         self.assertTrue(torch.allclose(y_gelu * clipped_mask, y_gelu_10 * clipped_mask))
 
     def test_get_activation(self):
-        get_activation("swish")
-        get_activation("silu")
-        get_activation("relu")
-        get_activation("tanh")
-        get_activation("gelu_new")
+        get_activation("gelu")
+        get_activation("gelu_10")
         get_activation("gelu_fast")
+        get_activation("gelu_new")
         get_activation("gelu_python")
-        get_activation("gelu_10")
-        get_activation("quick_gelu")
-        get_activation("mish")
         get_activation("linear")
+        get_activation("mish")
+        get_activation("quick_gelu")
+        get_activation("relu")
         get_activation("sigmoid")
+        get_activation("silu")
+        get_activation("swish")
+        get_activation("tanh")
         with self.assertRaises(KeyError):
             get_activation("bogus")
         with self.assertRaises(KeyError):
diff --git a/tests/utils/test_activations_tf.py b/tests/utils/test_activations_tf.py
--- a/tests/utils/test_activations_tf.py
+++ b/tests/utils/test_activations_tf.py
@@ -42,17 +42,18 @@ def test_gelu_10(self):
         self.assertTrue(np.allclose(y_gelu * clipped_mask, y_gelu_10 * clipped_mask))
 
     def test_get_activation(self):
-        get_tf_activation("swish")
-        get_tf_activation("silu")
         get_tf_activation("gelu")
-        get_tf_activation("relu")
-        get_tf_activation("tanh")
-        get_tf_activation("gelu_new")
-        get_tf_activation("gelu_fast")
         get_tf_activation("gelu_10")
+        get_tf_activation("gelu_fast")
+        get_tf_activation("gelu_new")
+        get_tf_activation("glu")
         get_tf_activation("mish")
         get_tf_activation("quick_gelu")
-        get_tf_activation("glu")
+        get_tf_activation("relu")
+        get_tf_activation("sigmoid")
+        get_tf_activation("silu")
+        get_tf_activation("swish")
+        get_tf_activation("tanh")
         with self.assertRaises(KeyError):
             get_tf_activation("bogus")
         with self.assertRaises(KeyError):