Repository: pandas-dev/pandas

Issue:
pd.merge_asof() matches out of tolerance when timestamps are duplicated
This is a continuation of #13695.

Starting with the original DataFrames from that issue:

```
df1 = pd.DataFrame({'time':pd.to_datetime(['2016-07-15 13:30:00.030']), 'username':['bob']})
df2 = pd.DataFrame({'time':pd.to_datetime(['2016-07-15 13:30:00.000', '2016-07-15 13:30:00.030']), 'version':[1, 2]})
```

I now get the null:

```
In [82]: pd.merge_asof(df1, df2, on='time', allow_exact_matches=False, tolerance=pd.Timedelta('10ms'))
Out[82]:
                     time username  version
0 2016-07-15 13:30:00.030      bob      NaN
```

However, if I change the first DataFrame to have duplicate timestamps:

```
df1 = pd.DataFrame({'time':pd.to_datetime(['2016-07-15 13:30:00.030', '2016-07-15 13:30:00.030']), 'username':['bob', 'charlie']})
```

then the bug reappears:

```
In [85]: pd.merge_asof(df1, df2, on='time', allow_exact_matches=False, tolerance=pd.Timedelta('10ms'))
Out[85]:
                     time username  version
0 2016-07-15 13:30:00.030      bob        1
1 2016-07-15 13:30:00.030  charlie        1
```

This is in pandas version 0.18.0+418.gc46dcfa.



Discussion:
I can take a stab at this. I should dive into the asof internals anyway.

great @chrisaycock yeah the fix is trivial, BUT it breaks the full on test, not entirely sure why, so its not exactly right


Resolving patch:
diff --git a/asv_bench/benchmarks/join_merge.py b/asv_bench/benchmarks/join_merge.py
--- a/asv_bench/benchmarks/join_merge.py
+++ b/asv_bench/benchmarks/join_merge.py
@@ -293,6 +293,43 @@ def time_join_dataframe_integer_key(self):
         merge(self.df, self.df2, on='key1')
 
 
+class merge_asof_noby(object):
+
+    def setup(self):
+        np.random.seed(0)
+        one_count = 200000
+        two_count = 1000000
+        self.df1 = pd.DataFrame({'time': np.random.randint(0, one_count/20, one_count),
+                                 'value1': np.random.randn(one_count)})
+        self.df2 = pd.DataFrame({'time': np.random.randint(0, two_count/20, two_count),
+                                 'value2': np.random.randn(two_count)})
+        self.df1 = self.df1.sort_values('time')
+        self.df2 = self.df2.sort_values('time')
+
+    def time_merge_asof_noby(self):
+        merge_asof(self.df1, self.df2, on='time')
+
+
+class merge_asof_by(object):
+
+    def setup(self):
+        import string
+        np.random.seed(0)
+        one_count = 200000
+        two_count = 1000000
+        self.df1 = pd.DataFrame({'time': np.random.randint(0, one_count/20, one_count),
+                                 'key': np.random.choice(list(string.uppercase), one_count),
+                                 'value1': np.random.randn(one_count)})
+        self.df2 = pd.DataFrame({'time': np.random.randint(0, two_count/20, two_count),
+                                 'key': np.random.choice(list(string.uppercase), two_count),
+                                 'value2': np.random.randn(two_count)})
+        self.df1 = self.df1.sort_values('time')
+        self.df2 = self.df2.sort_values('time')
+
+    def time_merge_asof_by(self):
+        merge_asof(self.df1, self.df2, on='time', by='key')
+
+
 class join_non_unique_equal(object):
     goal_time = 0.2
 
diff --git a/pandas/tools/merge.py b/pandas/tools/merge.py
--- a/pandas/tools/merge.py
+++ b/pandas/tools/merge.py
@@ -258,8 +258,7 @@ def merge_asof(left, right, on=None,
                by=None,
                suffixes=('_x', '_y'),
                tolerance=None,
-               allow_exact_matches=True,
-               check_duplicates=True):
+               allow_exact_matches=True):
     """Perform an asof merge. This is similar to a left-join except that we
     match on nearest key rather than equal keys.
 
@@ -304,14 +303,6 @@ def merge_asof(left, right, on=None,
         - If False, don't match the same 'on' value
           (i.e., stricly less-than)
 
-    check_duplicates : boolean, default True
-
-        - If True, check and remove duplicates for the right
-          DataFrame, on the [by, on] combination, keeping the last value.
-        - If False, no check for duplicates. If you *know* that
-          you don't have duplicates, then turning off the check for duplicates
-          can be more performant.
-
     Returns
     -------
     merged : DataFrame
@@ -436,7 +427,7 @@ def _merger(x, y):
     if by is not None:
         result, groupby = _groupby_and_merge(by, on, left, right,
                                              lambda x, y: _merger(x, y),
-                                             check_duplicates=check_duplicates)
+                                             check_duplicates=False)
 
         # we want to preserve the original order
         # we had grouped, so need to reverse this
@@ -446,20 +437,12 @@ def _merger(x, y):
         sorter = _ensure_platform_int(
             np.concatenate([groupby.indices[g] for g, _ in groupby]))
         if len(result) != len(sorter):
-            if check_duplicates:
-                raise AssertionError("invalid reverse grouping")
             return result
 
         rev = np.empty(len(sorter), dtype=np.int_)
         rev.put(sorter, np.arange(len(sorter)))
         return result.take(rev).reset_index(drop=True)
 
-    if check_duplicates:
-        if on is None:
-            on = []
-        elif not isinstance(on, (list, tuple)):
-            on = [on]
-
         if right.duplicated(on).any():
             right = right.drop_duplicates(on, keep='last')
 
@@ -1067,8 +1050,8 @@ def _get_join_indexers(self):
                 lt = lt.view('i8')
                 t = t.value
                 rt = rt.view('i8')
-            kwargs['left_distance'] = lt
-            kwargs['right_distance'] = rt
+            kwargs['left_values'] = lt
+            kwargs['right_values'] = rt
             kwargs['tolerance'] = t
 
         return _get_join_indexers(self.left_join_keys,


Test patch:
diff --git a/pandas/tools/tests/test_merge_asof.py b/pandas/tools/tests/test_merge_asof.py
--- a/pandas/tools/tests/test_merge_asof.py
+++ b/pandas/tools/tests/test_merge_asof.py
@@ -184,43 +184,8 @@ def test_with_duplicates(self):
         expected = self.read_data('asof.csv')
         assert_frame_equal(result, expected)
 
-        result = merge_asof(self.trades, q,
-                            on='time',
-                            by='ticker',
-                            check_duplicates=False)
-        expected = self.read_data('asof.csv')
-        expected = pd.concat([expected, expected]).sort_values(
-            ['time', 'ticker']).reset_index(drop=True)
-
-        # the results are not ordered in a meaningful way
-        # nor are the exact matches duplicated, so comparisons
-        # are pretty tricky here, however the uniques are the same
-
-        def aligner(x, ticker):
-            return (x[x.ticker == ticker]
-                    .sort_values(['time', 'ticker', 'quantity', 'price',
-                                  'marketCenter', 'bid', 'ask'])
-                    .drop_duplicates(keep='last')
-                    .reset_index(drop=True)
-                    )
-
-        for ticker in expected.ticker.unique():
-            r = aligner(result, ticker)
-            e = aligner(expected, ticker)
-            assert_frame_equal(r, e)
-
     def test_with_duplicates_no_on(self):
 
-        df1 = pd.DataFrame({'key': [1, 1, 3],
-                            'left_val': [1, 2, 3]})
-        df2 = pd.DataFrame({'key': [1, 3, 3],
-                            'right_val': [1, 2, 3]})
-        result = merge_asof(df1, df2, on='key', check_duplicates=False)
-        expected = pd.DataFrame({'key': [1, 1, 3, 3],
-                                 'left_val': [1, 2, 3, 3],
-                                 'right_val': [1, 1, 2, 3]})
-        assert_frame_equal(result, expected)
-
         df1 = pd.DataFrame({'key': [1, 1, 3],
                             'left_val': [1, 2, 3]})
         df2 = pd.DataFrame({'key': [1, 2, 2],
@@ -379,6 +344,26 @@ def test_allow_exact_matches_and_tolerance2(self):
             'version': [np.nan]})
         assert_frame_equal(result, expected)
 
+    def test_allow_exact_matches_and_tolerance3(self):
+        # GH 13709
+        df1 = pd.DataFrame({
+            'time': pd.to_datetime(['2016-07-15 13:30:00.030',
+                                   '2016-07-15 13:30:00.030']),
+            'username': ['bob', 'charlie']})
+        df2 = pd.DataFrame({
+            'time': pd.to_datetime(['2016-07-15 13:30:00.000',
+                                    '2016-07-15 13:30:00.030']),
+            'version': [1, 2]})
+
+        result = pd.merge_asof(df1, df2, on='time', allow_exact_matches=False,
+                               tolerance=pd.Timedelta('10ms'))
+        expected = pd.DataFrame({
+            'time': pd.to_datetime(['2016-07-15 13:30:00.030',
+                                    '2016-07-15 13:30:00.030']),
+            'username': ['bob', 'charlie'],
+            'version': [np.nan, np.nan]})
+        assert_frame_equal(result, expected)
+
 
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],