Repository: keras-team/keras

Issue:
in_top_k() gives different results for Theano and TensorFlow backends
Hello,

The `in_top_k()` function in `theano_backend.py` gives incorrect results when there are identical values in the input tensor `predictions`.

For example, when `predictions[i] = [0.2, 0.2, 0.4, 0.5]` and we want to know whether `predictions[i, 0]` (i.e., 0.2) is in the top-3 list.

Here's a simple script that reproduces the issue:
https://gist.github.com/myutwo150/2400260865122cc509b0025452d1db0f

When running with `KERAS_BACKEND='tensorflow'`, the output is:
```
Using TensorFlow backend.
row number: 0   1   2
in top-1: [ 0.  0.  0.]
in top-2: [ 0.  0.  1.]
in top-3: [ 1.  0.  1.]
in top-4: [ 1.  1.  1.]
in top-5: [ 1.  1.  1.]
in top-6: [ 1.  1.  1.]
in top-7: [ 1.  1.  1.]
in top-8: [ 1.  1.  1.]
in top-9: [ 1.  1.  1.]
```

While with `KERAS_BACKEND='theano'`, the output is:
```
Using Theano backend.
row number: 0   1   2
in top-1: [ 0.  0.  0.]
in top-2: [ 0.  0.  0.]
in top-3: [ 0.  0.  0.]
in top-4: [ 0.  0.  0.]
in top-5: [ 0.  0.  0.]
in top-6: [ 0.  0.  1.]
in top-7: [ 1.  0.  1.]
in top-8: [ 1.  1.  1.]
in top-9: [ 1.  1.  1.]
```

I think it's because in `theano_backend.py`, the "top-k" judgement is done by comparing the indices, instead of values.


By changing the `in_top_k()` function from:
```python
predictions_top_k = T.argsort(predictions)[:, -k:]
result, _ = theano.map(lambda prediction, target: any(equal(prediction, target)), sequences=[predictions_top_k, targets])
return result
```
to:
```python
predictions_k = T.sort(predictions)[:, -k]
targets_values = predictions[T.arange(targets.shape[0]), targets]
return T.ge(targets_values, predictions_k)
```
the problem can be fixed.

I can submit a PR if the `tensorflow_backend.py` version is indeed the desired behavior of this function.
Thanks. 


Discussion:

Resolving patch:
diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -2729,13 +2729,14 @@ def in_top_k(predictions, targets, k):
     """Returns whether the `targets` are in the top `k` `predictions`.
 
     # Arguments
-        predictions: A tensor of shape `batch_size` x classes and type `float32`.
-        targets: A tensor of shape batch_size and type `int32` or `int64`.
+        predictions: A tensor of shape `(batch_size, classes)` and type `float32`.
+        targets: A 1D tensor of length `batch_size` and type `int32` or `int64`.
         k: An `int`, number of top elements to consider.
 
     # Returns
-        A tensor of shape `batch_size` and type `bool`. `output_i` is `True` if
-        `targets_i` is within top-k values of `predictions_i`
+        A 1D tensor of length `batch_size` and type `bool`.
+        `output[i]` is `True` if `predictions[i, targets[i]]` is within top-`k`
+        values of `predictions[i]`.
     """
     return tf.nn.in_top_k(predictions, targets, k)
 
diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py
--- a/keras/backend/theano_backend.py
+++ b/keras/backend/theano_backend.py
@@ -1494,20 +1494,35 @@ def l2_normalize(x, axis):
 
 
 def in_top_k(predictions, targets, k):
-    """Returns whether the `targets` are in the top `k` `predictions`
+    """Returns whether the `targets` are in the top `k` `predictions`.
 
     # Arguments
-        predictions: A tensor of shape batch_size x classess and type float32.
-        targets: A tensor of shape batch_size and type int32 or int64.
-        k: An int, number of top elements to consider.
+        predictions: A tensor of shape `(batch_size, classes)` and type `float32`.
+        targets: A 1D tensor of length `batch_size` and type `int32` or `int64`.
+        k: An `int`, number of top elements to consider.
 
     # Returns
-        A tensor of shape batch_size and type int. output_i is 1 if
-        targets_i is within top-k values of predictions_i
+        A 1D tensor of length `batch_size` and type `bool`.
+        `output[i]` is `True` if `predictions[i, targets[i]]` is within top-`k`
+        values of `predictions[i]`.
     """
-    predictions_top_k = T.argsort(predictions)[:, -k:]
-    result, _ = theano.map(lambda prediction, target: any(equal(prediction, target)), sequences=[predictions_top_k, targets])
-    return result
+    # handle k < 1 and k >= predictions.shape[1] cases to match TF behavior
+    if k < 1:
+        # dtype='bool' is only available since Theano 0.9.0
+        try:
+            return T.zeros_like(targets, dtype='bool')
+        except TypeError:
+            return T.zeros_like(targets, dtype='int8')
+
+    if k >= int_shape(predictions)[1]:
+        try:
+            return T.ones_like(targets, dtype='bool')
+        except TypeError:
+            return T.ones_like(targets, dtype='int8')
+
+    predictions_k = T.sort(predictions)[:, -k]
+    targets_values = predictions[T.arange(targets.shape[0]), targets]
+    return T.ge(targets_values, predictions_k)
 
 
 # CONVOLUTIONS


Test patch:
diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py
--- a/tests/keras/backend/backend_test.py
+++ b/tests/keras/backend/backend_test.py
@@ -618,6 +618,46 @@ def test_nn_operations(self):
         check_single_tensor_operation('l2_normalize', (4, 3), axis=-1)
         check_single_tensor_operation('l2_normalize', (4, 3), axis=1)
 
+    def test_in_top_k(self):
+        batch_size = 20
+        num_classes = 10
+
+        # Random prediction test case
+        predictions = np.random.random((batch_size, num_classes)).astype('float32')
+        targets = np.random.randint(num_classes, size=batch_size, dtype='int32')
+
+        predictions_th = KTH.variable(predictions, dtype='float32')
+        targets_th = KTH.variable(targets, dtype='int32')
+        predictions_tf = KTF.variable(predictions, dtype='float32')
+        targets_tf = KTF.variable(targets, dtype='int32')
+
+        for k in range(1, num_classes + 1):
+            res_th = KTH.eval(KTH.in_top_k(predictions_th, targets_th, k))
+            res_tf = KTF.eval(KTF.in_top_k(predictions_tf, targets_tf, k))
+
+            assert res_th.shape == res_tf.shape
+            assert_allclose(res_th, res_tf, atol=1e-05)
+
+        # Identical prediction test case:
+        # randomly set half of the predictions to an identical value
+        num_identical = num_classes // 2
+        for i in range(batch_size):
+            idx_identical = np.random.choice(num_classes, size=num_identical, replace=False)
+            predictions[i, idx_identical] = predictions[i, 0]
+        targets = np.zeros(batch_size, dtype='int32')
+
+        predictions_th = KTH.variable(predictions, dtype='float32')
+        targets_th = KTH.variable(targets, dtype='int32')
+        predictions_tf = KTF.variable(predictions, dtype='float32')
+        targets_tf = KTF.variable(targets, dtype='int32')
+
+        for k in range(1, num_classes + 1):
+            res_th = KTH.eval(KTH.in_top_k(predictions_th, targets_th, k))
+            res_tf = KTF.eval(KTF.in_top_k(predictions_tf, targets_tf, k))
+
+            assert res_th.shape == res_tf.shape
+            assert_allclose(res_th, res_tf, atol=1e-05)
+
     def test_conv2d(self):
         # TF kernel shape: (rows, cols, input_depth, depth)