Repository: keras-team/keras

Issue:
Scoring methods in Luong-style attention
TensorFlow version (you are using): 2.7.0
Are you willing to contribute it (Yes/No) : Yes

**Describe the feature and the current behavior/state**.

Describe the feature clearly here. Be sure to convey here why the requested feature is needed. Any brief description about the use-case would help.
Luong-style attention attention use three types of scoring methods, namely dot, general and concat. This can be found in the 3rd page of the [original paper](https://arxiv.org/pdf/1508.04025.pdf) and explained [here](https://stackoverflow.com/a/44454253). 
![tiQkz](https://user-images.githubusercontent.com/56781123/148428296-f88efa8d-5d59-4099-9ee1-cca4410385c4.png)

Right now `concat` method for scoring is not implemented and only [dot and general](https://github.com/keras-team/keras/blob/master/keras/layers/dense_attention.py#L339) has been implemented in the layer. An use case can be found [here](https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/seq2seq-translation.ipynb). 


**Will this change the current api? How?**
Yes. An extra parameter in the Attention layer to specify the scoring method. 

**Who will benefit from this feature?**
Anyone using Luong-style attention layer can opt for concat scoring method.




Discussion:

Resolving patch:
diff --git a/keras/layers/dense_attention.py b/keras/layers/dense_attention.py
--- a/keras/layers/dense_attention.py
+++ b/keras/layers/dense_attention.py
@@ -242,6 +242,10 @@ class Attention(BaseDenseAttention):
       Defaults to `False`.
     dropout: Float between 0 and 1. Fraction of the units to drop for the
       attention scores. Defaults to 0.0.
+    score_mode: Function to use to compute attention scores, one of 
+      `{"dot", "concat"}`. `"dot"` refers to the dot product between the query
+      and key vectors. `"concat"` refers to the hyperbolic tangent of the 
+      concatenation of the query and key vectors.
 
   Call Args:
 
@@ -319,12 +323,19 @@ class Attention(BaseDenseAttention):
   ```
   """
 
-  def __init__(self, use_scale=False, **kwargs):
+  def __init__(self, use_scale=False, score_mode='dot', **kwargs):
     super(Attention, self).__init__(**kwargs)
     self.use_scale = use_scale
+    self.score_mode= score_mode
+    if self.score_mode not in ['dot', 'concat']:
+      raise ValueError(
+                f"Received: score_mode={score_mode}. Acceptable values "
+                "are: ['dot', 'concat']"
+            )
 
   def build(self, input_shape):
-    """Creates scale variable if use_scale==True."""
+    """Creates scale variable if use_scale==True and
+      v parameter if score_mode==concat"""
     if self.use_scale:
       self.scale = self.add_weight(
           name='scale',
@@ -334,6 +345,15 @@ def build(self, input_shape):
           trainable=True)
     else:
       self.scale = None
+    if self.score_mode == 'concat':
+      self.concat_score_weight = self.add_weight(
+          name='concat_score_weight',
+          shape=(),
+          initializer='ones',
+          dtype=self.dtype,
+          trainable=True)
+    else:
+      self.concat_score_weight = None
     super(Attention, self).build(input_shape)
 
   def _calculate_scores(self, query, key):
@@ -345,13 +365,28 @@ def _calculate_scores(self, query, key):
     Returns:
       Tensor of shape `[batch_size, Tq, Tv]`.
     """
-    scores = tf.matmul(query, key, transpose_b=True)
-    if self.scale is not None:
-      scores *= self.scale
+    if self.score_mode == 'dot':
+      scores = tf.matmul(query, key, transpose_b=True)
+      if self.scale is not None:
+        scores *= self.scale
+    elif self.score_mode == 'concat':
+      # Reshape tensors to enable broadcasting.
+      # Reshape into [batch_size, Tq, 1, dim].
+      q_reshaped = tf.expand_dims(query, axis=-2)
+      # Reshape into [batch_size, 1, Tv, dim].
+      k_reshaped = tf.expand_dims(key, axis=-3)
+      if self.scale is not None:
+        scores = self.concat_score_weight * tf.reduce_sum(
+         tf.tanh(self.scale * (q_reshaped + k_reshaped)), axis=-1)
+      else:
+        scores = self.concat_score_weight * tf.reduce_sum(
+         tf.tanh(q_reshaped + k_reshaped), axis=-1)
+
     return scores
 
   def get_config(self):
-    config = {'use_scale': self.use_scale}
+    config = {'use_scale': self.use_scale,
+              'score_mode': self.score_mode}
     base_config = super(Attention, self).get_config()
     return dict(list(base_config.items()) + list(config.items()))
 


Test patch:
diff --git a/keras/layers/dense_attention_test.py b/keras/layers/dense_attention_test.py
--- a/keras/layers/dense_attention_test.py
+++ b/keras/layers/dense_attention_test.py
@@ -203,6 +203,31 @@ def test_calculate_scores_multi_dim(self):
     expected = np.array([[[7.64, 12.24, 16.84], [14.24, 22.84, 31.44]]],
                         dtype=np.float32)
     self.assertAllClose(expected, actual)
+ 
+  def test_calculate_scores_multi_dim_concat(self):
+    # Query tensor of shape [1, 2, 4]
+    q = np.array([[[1., 1.1, 1.2, 1.3], [2., 2.1, 2.2, 2.3]]], dtype=np.float32)
+    # Key tensor of shape [1, 3, 4]
+    k = np.array(
+        [[[1.5, 1.6, 1.7, 1.8], [2.5, 2.6, 2.7, 2.8], [3.5, 3.6, 3.7, 3.8]]],
+        dtype=np.float32)
+    attention_layer = dense_attention.Attention(score_mode='concat')
+    attention_layer.concat_score_weight = 1
+    attention_layer.build(input_shape=([1, 2, 4], [1, 3, 4]))
+    actual = keras.backend.get_value(
+            attention_layer._calculate_scores(query=q, key=k))
+
+    # pylint:disable=line-too-long
+    # expected000 = tanh(1.+1.5) + tanh(1.1+1.6) + tanh(1.2+1.7) + tanh(1.3+1.8) = 3.96753427840
+    # expected001 = tanh(1.+2.5) + tanh(1.1+2.6) + tanh(1.2+2.7) + tanh(1.3+2.8) = 3.99558784825
+    # expected002 = tanh(1.+3.5) + tanh(1.1+3.6) + tanh(1.2+3.7) + tanh(1.3+3.8) = 3.99940254147
+    # expected010 = tanh(2.+1.5) + tanh(2.1+1.6) + tanh(2.2+1.7) + tanh(2.3+1.8) = 3.99558784825
+    # expected011 = tanh(2.+2.5) + tanh(2.1+2.6) + tanh(2.2+2.7) + tanh(2.3+2.8) = 3.99940254147
+    # expected012 = tanh(2.+3.5) + tanh(2.1+3.6) + tanh(2.2+3.7) + tanh(2.3+3.8) = 3.99991913657
+    expected = np.array([[[3.96753427840, 3.99558784825, 3.99940254147],
+                          [3.99558784825, 3.99940254147, 3.99991913657]]],
+                        dtype=np.float32)
+    self.assertAllClose(expected, actual)
 
   def test_calculate_scores_one_dim_batch_size_two(self):
     # Query tensor of shape [2, 1, 1]
@@ -234,6 +259,24 @@ def test_calculate_scores_one_dim_with_scale(self):
     # expected000 = -2*1.1*1.6 = -3.52
     expected = np.array([[[-3.52]]], dtype=np.float32)
     self.assertAllClose(expected, actual)
+   
+  def test_calculate_scores_one_dim_with_scale_concat(self):
+    """Tests that scores are multiplied by scale."""
+    # Query tensor of shape [1, 1, 1]
+    q = np.array([[[1.1]]], dtype=np.float32)
+    # Key tensor of shape [1, 1, 1]
+    k = np.array([[[1.6]]], dtype=np.float32)
+    attention_layer = dense_attention.Attention(use_scale=True, score_mode='concat')
+    attention_layer.concat_score_weight = 1
+    attention_layer.build(input_shape=([1, 1, 1], [1, 1, 1]))
+    attention_layer.scale = 2.
+    actual = keras.backend.get_value(
+            attention_layer._calculate_scores(query=q, key=k))
+
+    # Expected tensor of shape [1, 1, 1].
+    # expected000 = tanh(2*(1.1+1.6)) = 0.9999592018254402
+    expected = np.array([[[0.999959202]]], dtype=np.float32)
+    self.assertAllClose(expected, actual)
 
   def test_shape(self):
     # Query tensor of shape [1, 2, 4]
@@ -249,6 +292,22 @@ def test_shape(self):
 
     expected_shape = [1, 2, 4]
     self.assertAllEqual(expected_shape, tf.shape(actual))
+  
+  def test_shape_concat(self):
+    # Query tensor of shape [1, 2, 4]
+    q = np.array([[[1., 1.1, 1.2, 1.3], [2., 2.1, 2.2, 2.3]]], dtype=np.float32)
+    # Value tensor of shape [1, 3, 4]
+    v = np.array(
+        [[[1.5, 1.6, 1.7, 1.8], [2.5, 2.6, 2.7, 2.8], [3.5, 3.6, 3.7, 3.8]]],
+        dtype=np.float32)
+    # Value mask tensor of shape [1, 3]
+    v_mask = np.array([[True, True, False]], dtype=np.bool_)
+    attention_layer = dense_attention.Attention(score_mode='concat')
+    attention_layer.concat_score_weight = 1
+    actual = attention_layer([q, v], mask=[None, v_mask])
+
+    expected_shape = [1, 2, 4]
+    self.assertAllEqual(expected_shape, tf.shape(actual))
 
   def test_shape_with_key(self):
     # Query tensor of shape [1, 2, 4]
@@ -268,6 +327,26 @@ def test_shape_with_key(self):
 
     expected_shape = [1, 2, 4]
     self.assertAllEqual(expected_shape, tf.shape(actual))
+   
+  def test_shape_with_key_concat(self):
+    # Query tensor of shape [1, 2, 4]
+    q = np.array([[[1., 1.1, 1.2, 1.3], [2., 2.1, 2.2, 2.3]]], dtype=np.float32)
+    # Value tensor of shape [1, 3, 4]
+    v = np.array(
+        [[[1.5, 1.6, 1.7, 1.8], [2.5, 2.6, 2.7, 2.8], [3.5, 3.6, 3.7, 3.8]]],
+        dtype=np.float32)
+    # Key tensor of shape [1, 3, 4]
+    k = np.array(
+        [[[1.5, 1.6, 1.7, 1.8], [2.5, 2.6, 2.7, 2.8], [3.5, 3.6, 3.7, 3.8]]],
+        dtype=np.float32)
+    # Value mask tensor of shape [1, 3]
+    v_mask = np.array([[True, True, False]], dtype=np.bool_)
+    attention_layer = dense_attention.Attention(score_mode='concat')
+    attention_layer.concat_score_weight = 1
+    actual = attention_layer([q, v, k], mask=[None, v_mask])
+
+    expected_shape = [1, 2, 4]
+    self.assertAllEqual(expected_shape, tf.shape(actual))
 
   def test_multi_dim(self):
     # Query tensor of shape [1, 1, 1]