Repository: adamchainz/django-mysql

Issue:
Support reverse ordering for Smart Iteration
Let's say our queryset looks like is so:
`qs = MyModel.objects.all()`

`list(qs.iter_smart())` returns non-empty list
`list(qs.reverse().iter_smart())` returns empty list `[]`

The bug is caused by a `max_pk` and `min_pk` assuming `min_pk <= max_pk` which isn't the case for `reverse`.
https://github.com/adamchainz/django-mysql/blob/66788e709c7b5196a9bc3f97f17993f039be9e36/django_mysql/models/query.py#L173



Discussion:
`reverse()` isn't necessarily the right thing to be calling, as it just means "invert" the ordering - you might mean `order_by('-pk')`. However it is true that SmartIterator is assuming forwards PK order - e.g. line 223 `min_qs = base_qs.order_by('pk').values_list('pk', flat=True)`.

Ideally it would work with, and only with, ordering set to `order_by('pk')` or `order_by('-pk')`.


Resolving patch:
diff --git a/django_mysql/models/query.py b/django_mysql/models/query.py
--- a/django_mysql/models/query.py
+++ b/django_mysql/models/query.py
@@ -1,6 +1,7 @@
 # -*- coding:utf-8 -*-
 from __future__ import print_function, unicode_literals
 
+import operator
 import sys
 from copy import copy
 from subprocess import PIPE, Popen
@@ -252,23 +253,37 @@ def __init__(self, queryset, atomically=True, status_thresholds=None,
         self.total = total
 
     def __iter__(self):
-        min_pk, max_pk = self.get_min_and_max()
-        current_pk = min_pk
+        first_pk, last_pk = self.get_first_and_last()
+        if first_pk <= last_pk:
+            comp = operator.le  # <=
+            direction = 1
+        else:
+            comp = operator.ge  # >=
+            direction = -1
+        current_pk = first_pk
         db_alias = self.queryset.db
         status = GlobalStatus(db_alias)
 
         self.init_progress()
 
-        while current_pk <= max_pk:
+        while comp(current_pk, last_pk):
             status.wait_until_load_low(self.status_thresholds)
 
             start_pk = current_pk
-            current_pk = current_pk + self.chunk_size
+            current_pk = current_pk + self.chunk_size * direction
             # Don't process rows that didn't exist at start of iteration
-            end_pk = min(current_pk, max_pk + 1)
+            if direction == 1:
+                end_pk = min(current_pk, last_pk + 1)
+            else:
+                end_pk = max(current_pk, last_pk - 1)
 
             with StopWatch() as timer, self.maybe_atomic(using=db_alias):
-                chunk = self.queryset.filter(pk__gte=start_pk, pk__lt=end_pk)
+                if direction == 1:
+                    chunk = self.queryset.filter(pk__gte=start_pk,
+                                                 pk__lt=end_pk)
+                else:
+                    chunk = self.queryset.filter(pk__lte=start_pk,
+                                                 pk__gt=end_pk)
                 # Attach the start_pk, end_pk onto the chunk queryset so they
                 # can be read by SmartRangeIterator or other client code
                 chunk._smart_iterator_pks = (start_pk, end_pk)
@@ -309,8 +324,14 @@ def sanitize_queryset(self, queryset):
         models.ForeignKey  # Should always point to an integer
     )
 
-    def get_min_and_max(self):
+    def get_first_and_last(self):
         if isinstance(self.pk_range, tuple) and len(self.pk_range) == 2:
+            should_be_reversed = (
+                self.pk_range[1] < self.pk_range[0] and
+                self.queryset.query.standard_ordering
+            )
+            if should_be_reversed:
+                self.queryset = self.queryset.reverse()
             return self.pk_range
         elif self.pk_range == 'all':
             base_qs = self.queryset.model.objects.using(self.queryset.db).all()
@@ -320,6 +341,9 @@ def get_min_and_max(self):
             raise ValueError("Unrecognized value for pk_range: {}"
                              .format(self.pk_range))
 
+        if not base_qs.query.standard_ordering:  # It's reverse()d
+            base_qs = base_qs.reverse()
+
         min_qs = base_qs.order_by('pk').values_list('pk', flat=True)
         max_qs = base_qs.order_by('-pk').values_list('pk', flat=True)
         try:
@@ -336,7 +360,10 @@ def get_min_and_max(self):
                 # between finding min_pk and the above [0]
                 max_pk = min_pk
 
-        return (min_pk, max_pk)
+        if self.queryset.query.standard_ordering:
+            return (min_pk, max_pk)
+        else:
+            return (max_pk, min_pk)
 
     def adjust_chunk_size(self, chunk, chunk_time):
         # If the queryset is not being fetched as-is, e.g. its .delete() is


Test patch:
diff --git a/tests/testapp/test_models.py b/tests/testapp/test_models.py
--- a/tests/testapp/test_models.py
+++ b/tests/testapp/test_models.py
@@ -423,6 +423,12 @@ def test_objects_non_atomic(self):
                                      .values_list('id', flat=True))
         assert seen == all_ids
 
+    def test_objects_reverse(self):
+        seen = [author.id for author in Author.objects.reverse().iter_smart()]
+        all_ids = list(Author.objects.order_by('-id')
+                                     .values_list('id', flat=True))
+        assert seen == all_ids
+
     def test_objects_pk_range_all(self):
         seen = [author.id for author in
                 Author.objects.iter_smart(pk_range='all')]
@@ -431,10 +437,6 @@ def test_objects_pk_range_all(self):
         assert seen == all_ids
 
     def test_objects_pk_range_tuple(self):
-        seen = [author.id for author in
-                Author.objects.iter_smart(pk_range=(0, 0))]
-        assert seen == []
-
         min_id = Author.objects.earliest('id').id
         max_id = Author.objects.order_by('id')[5].id
 
@@ -445,6 +447,21 @@ def test_objects_pk_range_tuple(self):
                                      .values_list('id', flat=True))
         assert seen == cut_ids
 
+    def test_objects_pk_range_tuple_0_0(self):
+        seen = [author.id for author in
+                Author.objects.iter_smart(pk_range=(0, 0))]
+        assert seen == []
+
+    def test_objects_pk_range_reversed(self):
+        min_id = Author.objects.earliest('id').id
+        max_id = Author.objects.latest('id').id
+
+        seen = [author.id for author in
+                Author.objects.iter_smart(pk_range=(max_id, min_id))]
+        all_ids = list(Author.objects.order_by('-id')
+                                     .values_list('id', flat=True))
+        assert seen == all_ids
+
     def test_objects_pk_range_bad(self):
         with pytest.raises(ValueError) as excinfo:
             list(Author.objects.iter_smart(pk_range="My Bad Value"))