Repository: keras-team/keras (Python)
Date: 2023-11-30 01:14:54+00:00
Task type: Feature

Issue:
Add HardSwish activation
HardSwish has been supported by TFLite for quite some time, but it is still missing in Keras.

I believe adding this activation would be beneficial for those working on INT8 quantized models.
I already have a working implementation and can submit the PR if it sounds good.

References that use HardSwish:
- [MobileNetV3](https://arxiv.org/abs/1905.02244)
- [LeViT](https://arxiv.org/abs/2104.01136)

To get .tflite
```python
import tensorflow as tf

from keras import layers
from keras import models
from keras.layers.activations import HardSwish

inputs = layers.Input(shape=[224, 224, 3])
outputs = HardSwish()(inputs)

model = models.Model(inputs=inputs, outputs=outputs)
model.summary()

converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
tflite_model = converter.convert()
with open("model.tflite", "wb") as f:
    f.write(tflite_model)

```
In netron visualization:
![hard_swish](https://github.com/keras-team/keras/assets/20734616/7b498b7a-4023-4b79-a9c9-461f778fb998)

The PR:
https://github.com/james77777778/keras/tree/add-hardswish


Issue discussion:

Solution patch:
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
--- a/.github/workflows/actions.yml
+++ b/.github/workflows/actions.yml
@@ -53,7 +53,7 @@ jobs:
       - name: Test applications with pytest
         if: ${{ steps.filter.outputs.applications == 'true' }}
         run: |
-          pytest keras/applications --cov=keras.applications
+          pytest keras/applications --cov=keras/applications
           coverage xml --include='keras/applications/*' -o apps-coverage.xml
       - name: Codecov keras.applications
         if: ${{ steps.filter.outputs.applications == 'true' }}
diff --git a/keras/activations/__init__.py b/keras/activations/__init__.py
--- a/keras/activations/__init__.py
+++ b/keras/activations/__init__.py
@@ -4,6 +4,7 @@
 from keras.activations.activations import exponential
 from keras.activations.activations import gelu
 from keras.activations.activations import hard_sigmoid
+from keras.activations.activations import hard_swish
 from keras.activations.activations import leaky_relu
 from keras.activations.activations import linear
 from keras.activations.activations import log_softmax
@@ -36,6 +37,7 @@
     sigmoid,
     exponential,
     hard_sigmoid,
+    hard_swish,
     linear,
     mish,
     log_softmax,
diff --git a/keras/activations/activations.py b/keras/activations/activations.py
--- a/keras/activations/activations.py
+++ b/keras/activations/activations.py
@@ -374,6 +374,29 @@ def hard_sigmoid(x):
     return ops.hard_sigmoid(x)
 
 
+@keras_export("keras.activations.hard_swish")
+def hard_swish(x):
+    """Hard swish activation function.
+
+    The hard swish activation is defined as:
+
+    - `0` if `if x < -3`
+    - `x` if `x > 3`
+    - `x * (x + 3) / 6` if `-3 <= x <= 3`
+
+    It's a faster, piecewise linear approximation of the swish activation.
+
+    Args:
+        x: Input tensor.
+
+    Reference:
+
+    - [A Howard, 2019](https://arxiv.org/abs/1905.02244)
+    """
+    x = backend.convert_to_tensor(x)
+    return x * ops.relu6(x + 3.0) * (1.0 / 6.0)
+
+
 @keras_export("keras.activations.linear")
 def linear(x):
     """Linear activation function (pass-through).
diff --git a/keras/applications/mobilenet_v3.py b/keras/applications/mobilenet_v3.py
--- a/keras/applications/mobilenet_v3.py
+++ b/keras/applications/mobilenet_v3.py
@@ -540,7 +540,7 @@ def hard_sigmoid(x):
 
 
 def hard_swish(x):
-    return layers.Multiply()([x, hard_sigmoid(x)])
+    return layers.Activation("hard_swish")(x)
 
 
 # This function is taken from the original tf repo.


Test patch:
diff --git a/keras/activations/activations_test.py b/keras/activations/activations_test.py
--- a/keras/activations/activations_test.py
+++ b/keras/activations/activations_test.py
@@ -40,6 +40,10 @@ def _ref_hard_sigmoid(x):
     return z
 
 
+def _ref_hard_swish(x):
+    return x * np.minimum(np.maximum(0.0, x + 3.0), 6.0) * (1.0 / 6.0)
+
+
 def _ref_sigmoid(x):
     if x >= 0:
         return 1 / (1 + np.exp(-x))
@@ -333,6 +337,39 @@ def test_hard_sigmoid(self):
             result_positive_above_1, expected_positive_above_1, rtol=1e-05
         )
 
+    def test_hard_swish(self):
+        # Basic test for random values between -3 and 3
+        x = np.random.uniform(-3, 3, (2, 5)).astype("float32")
+        result = activations.hard_swish(x[np.newaxis, :])[0]
+        expected = np.vectorize(_ref_hard_swish)(x)
+        self.assertAllClose(result, expected, rtol=1e-05)
+
+        # Test with 1D array
+        x_1d = np.random.uniform(-10, 10, 5).astype("float32")
+        result_1d = activations.hard_swish(x_1d)
+        expected_1d = np.vectorize(_ref_hard_swish)(x_1d)
+        self.assertAllClose(result_1d, expected_1d, rtol=1e-05)
+
+        # Test with 3D array
+        x_3d = np.random.uniform(-10, 10, (3, 3, 3)).astype("float32")
+        result_3d = activations.hard_swish(x_3d)
+        expected_3d = np.vectorize(_ref_hard_swish)(x_3d)
+        self.assertAllClose(result_3d, expected_3d, rtol=1e-05)
+
+        # Test with strictly positive values much larger than 3
+        x_positive_above_3 = np.random.uniform(5, 10, (2, 5)).astype("float32")
+        result_positive_above_3 = activations.hard_swish(x_positive_above_3)
+        expected_positive_above_3 = x_positive_above_3
+        self.assertAllClose(
+            result_positive_above_3, expected_positive_above_3, rtol=1e-05
+        )
+
+        # Test with strictly negative values much smaller than -3
+        x_negatives = np.random.uniform(-10, -5, (2, 5)).astype("float32")
+        result = activations.hard_swish(x_negatives)
+        expected_zeros = np.zeros_like(x_negatives)
+        self.assertAllClose(result, expected_zeros, rtol=1e-05)
+
     def test_relu_negative_slope(self):
         # Define the input tensor
         x = np.array([-10, -5, 0.0, 5, 10])
diff --git a/keras/applications/applications_test.py b/keras/applications/applications_test.py
--- a/keras/applications/applications_test.py
+++ b/keras/applications/applications_test.py
@@ -179,10 +179,21 @@ def test_application_notop_variable_input_channels(
     @parameterized.named_parameters(test_parameters)
     @pytest.mark.skipif(PIL is None, reason="Requires PIL.")
     def test_application_base(self, app, _, app_module, image_data_format):
+        import tensorflow as tf
+
         if app == nasnet.NASNetMobile and backend.backend() == "torch":
             self.skipTest(
                 "NASNetMobile pretrained incorrect with torch backend."
             )
+        if (
+            image_data_format == "channels_first"
+            and len(tf.config.list_physical_devices("GPU")) == 0
+            and backend.backend() == "tensorflow"
+        ):
+            self.skipTest(
+                "Conv2D doesn't support channels_first using CPU with "
+                "tensorflow backend"
+            )
         self.skip_if_invalid_image_data_format_for_model(app, image_data_format)
         backend.set_image_data_format(image_data_format)
 
diff --git a/keras/applications/imagenet_utils_test.py b/keras/applications/imagenet_utils_test.py
--- a/keras/applications/imagenet_utils_test.py
+++ b/keras/applications/imagenet_utils_test.py
@@ -3,6 +3,7 @@
 from absl.testing import parameterized
 
 import keras
+from keras import backend
 from keras import testing
 from keras.applications import imagenet_utils as utils
 from keras.mixed_precision import set_dtype_policy
@@ -53,8 +54,8 @@ def test_preprocess_input(self):
         for mode in ["torch", "tf"]:
             x = np.random.uniform(0, 255, (2, 10, 10, 3))
             xint = x.astype("int")
-            x2 = utils.preprocess_input(x, mode=mode)
-            xint2 = utils.preprocess_input(xint)
+            x2 = utils.preprocess_input(x, "channels_last", mode=mode)
+            xint2 = utils.preprocess_input(xint, "channels_last")
             self.assertAllClose(x, x2)
             self.assertNotEqual(xint.astype("float").max(), xint2.max())
 
@@ -64,7 +65,7 @@ def test_preprocess_input(self):
         x2 = utils.preprocess_input(
             x, data_format="channels_last", mode="caffe"
         )
-        xint2 = utils.preprocess_input(xint)
+        xint2 = utils.preprocess_input(xint, data_format="channels_last")
         self.assertAllClose(x, x2[..., ::-1])
         self.assertNotEqual(xint.astype("float").max(), xint2.max())
 
@@ -77,8 +78,12 @@ def test_preprocess_input(self):
     )
     @pytest.mark.requires_trainable_backend
     def test_preprocess_input_symbolic(self, mode):
+        backend_data_format = backend.image_data_format()
         # Test image batch
-        x = np.random.uniform(0, 255, (2, 10, 10, 3))
+        if backend_data_format == "channels_last":
+            x = np.random.uniform(0, 255, (2, 10, 10, 3))
+        elif backend_data_format == "channels_first":
+            x = np.random.uniform(0, 255, (2, 3, 10, 10))
         inputs = keras.layers.Input(shape=x.shape[1:])
         outputs = keras.layers.Lambda(
             lambda x: utils.preprocess_input(x, mode=mode),
@@ -87,6 +92,8 @@ def test_preprocess_input_symbolic(self, mode):
         model = keras.Model(inputs, outputs)
         self.assertEqual(model.predict(x).shape, x.shape)
 
+        x = np.random.uniform(0, 255, (2, 10, 10, 3))
+        inputs = keras.layers.Input(shape=x.shape[1:])
         outputs1 = keras.layers.Lambda(
             lambda x: utils.preprocess_input(x, "channels_last", mode=mode),
             output_shape=x.shape[1:],
@@ -104,7 +111,10 @@ def test_preprocess_input_symbolic(self, mode):
         self.assertAllClose(out1, out2.transpose(0, 2, 3, 1))
 
         # Test single image
-        x = np.random.uniform(0, 255, (10, 10, 3))
+        if backend_data_format == "channels_last":
+            x = np.random.uniform(0, 255, (10, 10, 3))
+        elif backend_data_format == "channels_first":
+            x = np.random.uniform(0, 255, (3, 10, 10))
         inputs = keras.layers.Input(shape=x.shape)
         outputs = keras.layers.Lambda(
             lambda x: utils.preprocess_input(x, mode=mode), output_shape=x.shape
@@ -112,6 +122,8 @@ def test_preprocess_input_symbolic(self, mode):
         model = keras.Model(inputs, outputs)
         self.assertEqual(model.predict(x[np.newaxis])[0].shape, x.shape)
 
+        x = np.random.uniform(0, 255, (10, 10, 3))
+        inputs = keras.layers.Input(shape=x.shape)
         outputs1 = keras.layers.Lambda(
             lambda x: utils.preprocess_input(x, "channels_last", mode=mode),
             output_shape=x.shape,