Repository: adamchainz/django-mysql

Issue:
Smart Iteration `report_progress` could output a time estimate
As suggested by @tolomea . The `WeightedAverageRate` should give a good guide, as long as it doesn't end up like:

![xkcd](http://imgs.xkcd.com/comics/estimation.png)



Discussion:
I'll pick this up


Resolving patch:
diff --git a/django_mysql/models/query.py b/django_mysql/models/query.py
--- a/django_mysql/models/query.py
+++ b/django_mysql/models/query.py
@@ -18,8 +18,8 @@
 from django_mysql.rewrite_query import REWRITE_MARKER
 from django_mysql.status import GlobalStatus
 from django_mysql.utils import (
-    StopWatch, WeightedAverageRate, have_program, noop_context,
-    settings_to_cmd_args
+    StopWatch, WeightedAverageRate, format_duration, have_program,
+    noop_context, settings_to_cmd_args
 )
 
 
@@ -422,7 +422,7 @@ def init_progress(self, direction):
         if not self.report_progress:
             return
 
-        self.have_reported = False
+        self.old_report = ""
         self.objects_done = 0
         self.chunks_done = 0
         if self.total is None:  # User didn't pass in a total
@@ -455,30 +455,41 @@ def update_progress(self, direction, chunk=None, end_pk=None):
         except (ZeroDivisionError, ValueError):
             percent_complete = 0.0
 
-        if not self.have_reported:
-            self.have_reported = True
-        else:
-            # Reset line on successive outputs
-            sys.stdout.write("\r")
-
-        sys.stdout.write(
-            "{} processed {}/{} objects ({:.2f}%) in {} chunks".format(
-                self.model_name + self.__class__.__name__,
-                self.objects_done,
-                self.total,
-                percent_complete,
-                self.chunks_done,
-            )
+        report = "{} {} processed {}/{} objects ({:.2f}%) in {} chunks".format(
+            self.model_name,
+            self.__class__.__name__,
+            self.objects_done,
+            self.total,
+            percent_complete,
+            self.chunks_done,
         )
+
         if end_pk is not None:
-            sys.stdout.write(
-                "; {dir} pk so far {end_pk}".format(
-                    dir="highest" if direction == 1 else "lowest",
-                    end_pk=end_pk,
-                )
+            report += "; {dir} pk so far {end_pk}".format(
+                dir="highest" if direction == 1 else "lowest",
+                end_pk=end_pk,
             )
+
+            if self.objects_done != '???' and self.rate.avg_rate:
+                n_remaining = self.total - self.objects_done
+                s_remaining = max(0, int(n_remaining // self.rate.avg_rate))
+                report += ', {} remaining'.format(
+                    format_duration(s_remaining)
+                )
+
+        # Add spaces to avoid problem with reverse iteration, see #177.
+        spacing = " " * max(0, len(self.old_report) - len(report))
+
+        if self.old_report:
+            # Reset line on successive outputs
+            sys.stdout.write("\r")
+
+        sys.stdout.write(report)
+        sys.stdout.write(spacing)
         sys.stdout.flush()
 
+        self.old_report = report
+
     def end_progress(self):
         if not self.report_progress:
             return
diff --git a/django_mysql/utils.py b/django_mysql/utils.py
--- a/django_mysql/utils.py
+++ b/django_mysql/utils.py
@@ -49,14 +49,16 @@ def update(self, n, t):
             self.avg_n = n
             self.avg_t = t
 
+        new_n = int(self.avg_rate * self.target_t)
+        return new_n
+
+    @property
+    def avg_rate(self):
         try:
-            avg_rate = self.avg_n / self.avg_t
+            return self.avg_n / self.avg_t
         except ZeroDivisionError:
             # Assume a small amount of time, not 0
-            avg_rate = self.avg_n / 0.001
-
-        new_n = int(avg_rate * self.target_t)
-        return new_n
+            return self.avg_n / 0.001
 
 
 class StopWatch(object):
@@ -77,6 +79,20 @@ def noop_context(*args, **kwargs):
     yield
 
 
+def format_duration(total_seconds):
+    hours = total_seconds // 3600
+    minutes = (total_seconds % 3600) // 60
+    seconds = total_seconds % 60
+
+    out = []
+    if hours > 0:
+        out.extend([str(hours), 'h'])
+    if hours or minutes:
+        out.extend([str(minutes), 'm'])
+    out.extend([str(seconds), 's'])
+    return ''.join(out)
+
+
 def settings_to_cmd_args(settings_dict):
     """
     Copied from django 1.8 MySQL backend DatabaseClient - where the runshell


Test patch:
diff --git a/tests/testapp/test_models.py b/tests/testapp/test_models.py
--- a/tests/testapp/test_models.py
+++ b/tests/testapp/test_models.py
@@ -401,7 +401,7 @@ class SmartIteratorTests(TestCase):
 
     def setUp(self):
         super(SmartIteratorTests, self).setUp()
-        Author.objects.bulk_create([Author() for i in range(10)])
+        Author.objects.bulk_create([Author(id=i + 1) for i in range(10)])
 
     def test_bad_querysets(self):
         with pytest.raises(ValueError) as excinfo:
@@ -623,17 +623,21 @@ def test_reporting(self):
         reports = lines[0].split('\r')
         for report in reports:
             assert re.match(
-                r"^AuthorSmartChunkedIterator processed \d+/10 objects "
-                r"\(\d+\.\d+%\) in \d+ chunks(; highest pk so far \d+)?$",
+                r"^Author SmartChunkedIterator processed \d+/10 objects "
+                r"\(\d+\.\d+%\) in \d+ chunks"
+                r"(; highest pk so far \d+(, [\dhms]+ remaining)?)?$",
                 report
             )
 
         assert lines[1] == 'Finished!'
 
     def test_reporting_reverse(self):
+        Author.objects.create(id=10000)
+
         with captured_stdout() as output:
             qs = Author.objects.reverse()
-            for authors in qs.iter_smart_chunks(report_progress=True):
+            for authors in qs.iter_smart_chunks(report_progress=True,
+                                                chunk_min=100):
                 list(authors)  # fetch them
 
         lines = output.getvalue().split('\n')
@@ -642,8 +646,9 @@ def test_reporting_reverse(self):
         assert len(reports) > 0
         for report in reports:
             assert re.match(
-                r"^AuthorSmartChunkedIterator processed \d+/10 objects "
-                r"\(\d+\.\d+%\) in \d+ chunks(; lowest pk so far \d+)?$",
+                r"^Author SmartChunkedIterator processed \d+/11 objects "
+                r"\(\d+\.\d+%\) in \d+ chunks"
+                r"(; lowest pk so far \d+(, [\dhms]+ remaining)?)?( )*$",
                 report
             )
 
@@ -660,8 +665,9 @@ def test_reporting_with_total(self):
         reports = lines[0].split('\r')
         for report in reports:
             assert re.match(
-                r"^AuthorSmartChunkedIterator processed \d+/4 objects "
-                r"\(\d+\.\d+%\) in \d+ chunks(; highest pk so far \d+)?$",
+                r"^Author SmartChunkedIterator processed \d+/4 objects "
+                r"\(\d+\.\d+%\) in \d+ chunks"
+                r"(; highest pk so far \d+(, [\dhms]+ remaining)?)?$",
                 report
             )
 
@@ -682,7 +688,7 @@ def test_reporting_on_uncounted_qs(self):
             assert re.match(
                 # We should have ??? since the deletion means the objects
                 # aren't fetched into python
-                r"AuthorSmartChunkedIterator processed (0|\?\?\?)/1 objects "
+                r"Author SmartChunkedIterator processed (0|\?\?\?)/1 objects "
                 r"\(\d+\.\d+%\) in \d+ chunks(; highest pk so far \d+)?",
                 report
             )
diff --git a/tests/testapp/test_utils.py b/tests/testapp/test_utils.py
--- a/tests/testapp/test_utils.py
+++ b/tests/testapp/test_utils.py
@@ -9,8 +9,8 @@
 from django.utils import six
 
 from django_mysql.utils import (
-    PTFingerprintThread, WeightedAverageRate, have_program, index_name,
-    pt_fingerprint
+    PTFingerprintThread, WeightedAverageRate, format_duration, have_program,
+    index_name, pt_fingerprint
 )
 from testapp.models import Author, AuthorMultiIndex
 
@@ -54,6 +54,25 @@ def test_zero_division(self):
         assert rate.update(1, 0.0) == 500
 
 
+class FormatDurationTests(SimpleTestCase):
+
+    def test_seconds(self):
+        assert format_duration(0) == '0s'
+        assert format_duration(1) == '1s'
+        assert format_duration(30) == '30s'
+        assert format_duration(59) == '59s'
+
+    def test_minutes(self):
+        assert format_duration(60) == '1m0s'
+        assert format_duration(61) == '1m1s'
+        assert format_duration(120) == '2m0s'
+        assert format_duration(3599) == '59m59s'
+
+    def test_hours(self):
+        assert format_duration(3600) == '1h0m0s'
+        assert format_duration(3601) == '1h0m1s'
+
+
 @skipUnless(have_program('pt-fingerprint'),
             "pt-fingerprint must be installed")
 class PTFingerprintTests(SimpleTestCase):