Repository: adamchainz/django-perf-rec

Issue:
Don't replace ORDER BY elements with "..."
Ordering a query on columns or not, and the specifics of what column and in what order can be extremely relevant to understand why 2 seemingly identical queries run in either milliseconds or minutes. Given that the number of elements in the `ORDER BY` clause is generally quite limited, the interest of replacing the columns by `...` in the `ORDER BY` clause can be limited too.

I read the source of django-perf-rec and the source of sqlparse it uses, and my understanding is that this replacement is actually a side effect of sqlparse not identifying directly when a list of columns is in an `ORDER BY` or not, but I believe we could make a (probably a bit contrieved) improvement to `sql_recursively_simplify` to avoid that. Would it be accepted as a PR ? Would it need a special setting ?

I think the idea would be that whatever (non whitespace) token follows a `Keyword.ORDER` and a `Keyword.BY` would be left as-is.


Discussion:

Resolving patch:
diff --git a/django_perf_rec/sql.py b/django_perf_rec/sql.py
--- a/django_perf_rec/sql.py
+++ b/django_perf_rec/sql.py
@@ -58,10 +58,21 @@ def sql_recursively_simplify(node):
     if node.tokens[0].value.startswith('"_django_curs_'):
         node.tokens[0].value = '"_django_curs_#"'
 
-    for i, token in enumerate(node.tokens):
+    # Memorize the 2 last non-whitespace tokens
+    two_before = None
+    one_before = None
+
+    for token in node.tokens:
         ttype = getattr(token, 'ttype', None)
 
-        if isinstance(token, IdentifierList):
+        # Detect if the token is an ORDER BY, GROUP BY or HAVING clause.
+        # If so, we don't want to replace its value with "..."
+        keywords_with_by = match_keyword(two_before, ["ORDER", "GROUP"])
+        is_by = match_keyword(one_before, "BY")
+        is_having = match_keyword(one_before, "HAVING")
+        disable_replacement = (keywords_with_by and is_by) or is_having
+
+        if isinstance(token, IdentifierList) and not disable_replacement:
             token.tokens = [Token(tokens.Punctuation, '...')]
         elif hasattr(token, 'tokens'):
             sql_recursively_simplify(token)
@@ -74,6 +85,21 @@ def sql_recursively_simplify(node):
         elif getattr(token, 'value', None) == 'NULL':
             token.value = '#'
 
+        if not token.is_whitespace:
+            two_before, one_before = one_before, token
+
+
+def match_keyword(token, keywords):
+    """
+    Checks if the given token represents one of the given keywords
+    """
+    if not token:
+        return False
+    if not token.is_keyword:
+        return False
+
+    return token.value.upper() in keywords
+
 
 def _is_group(token):
     """


Test patch:
diff --git a/tests/test_sql.py b/tests/test_sql.py
--- a/tests/test_sql.py
+++ b/tests/test_sql.py
@@ -35,21 +35,21 @@ def test_select_join():
 def test_select_order_by():
     assert (
         sql_fingerprint('SELECT f1, f2 FROM a ORDER BY f3, f4') ==
-        'SELECT ... FROM a ORDER BY ...'
+        'SELECT ... FROM a ORDER BY f3, f4'
     )
 
 
 def test_select_group_by():
     assert (
-        sql_fingerprint('SELECT f1, f2 FROM a GROUP BY f1') ==
-        'SELECT ... FROM a GROUP BY f1'
+        sql_fingerprint('SELECT f1, f2 FROM a GROUP BY f1, f2') ==
+        'SELECT ... FROM a GROUP BY f1, f2'
     )
 
 
 def test_select_group_by_having():
     assert (
-        sql_fingerprint('SELECT f1, f2 FROM a GROUP BY f1 HAVING f1 > 21') ==
-        'SELECT ... FROM a GROUP BY f1 HAVING f1 > #'
+        sql_fingerprint('SELECT f1, f2 FROM a GROUP BY f1 HAVING f1 > 21, f2 < 42') ==
+        'SELECT ... FROM a GROUP BY f1 HAVING f1 > #, f2 < #'
     )