Repository: petl-developers/petl

Issue:
Add progress function that uses a logging handler
It would be nice to be able to direct the progress funciton (or a similar function) to a logging handler instead of a i/o stream.

Comments:

Patch:
diff --git a/docs/changes.rst b/docs/changes.rst
index efa84aa..1a5245d 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -13,6 +13,10 @@ Version 1.3.0
   `"inline"` (use any exceptions as the output value). The default value is `False` which 
   maintains compatibility with previous releases. By :user:`bmaggard`, :issue:`460`, 
   :issue:`406`, :issue:`365`.
+  
+* A new function :func:`petl.util.timing.log_progress` has been added, which behaves
+  in a similar way to :func:`petl.util.timing.progress` but writes to a Python logger.
+  By :user:`dusktreader`, :issue:`408`, :issue:`407`.
 
 * Pass through keyword arguments to :func:`xlrd.open_workbook`. By :user:`gjunqueira`,
   :issue:`470`, :issue:`473`.
diff --git a/petl/util/__init__.py b/petl/util/__init__.py
index 2f66b48..5eb62fd 100644
--- a/petl/util/__init__.py
+++ b/petl/util/__init__.py
@@ -21,7 +21,7 @@ from petl.util.counting import parsecounter, parsecounts, typecounter, \
 from petl.util.materialise import listoflists, listoftuples, tupleoflists, \
     tupleoftuples, columns, facetcolumns
 
-from petl.util.timing import progress, clock
+from petl.util.timing import progress, log_progress, clock
 
 from petl.util.statistics import limits, stats
 
diff --git a/petl/util/timing.py b/petl/util/timing.py
index c3b564c..5d3a20c 100644
--- a/petl/util/timing.py
+++ b/petl/util/timing.py
@@ -1,6 +1,8 @@
 from __future__ import absolute_import, print_function, division
 
 
+import abc
+import logging
 import sys
 import time
 
@@ -9,9 +11,12 @@ from petl.util.base import Table
 from petl.util.statistics import onlinestats
 
 
-def progress(table, batchsize=1000, prefix="", out=sys.stderr):
+def progress(table, batchsize=1000, prefix="", out=None):
     """
-    Report progress on rows passing through. E.g.::
+    Report progress on rows passing through to a file or file-like object
+    (defaults to sys.stderr)
+
+    E.g.::
 
         >>> import petl as etl
         >>> table = etl.dummytable(100000)
@@ -35,16 +40,52 @@ def progress(table, batchsize=1000, prefix="", out=sys.stderr):
     return ProgressView(table, batchsize, prefix, out)
 
 
+def log_progress(table, batchsize=1000, prefix="", logger=None, level=logging.INFO):
+    """
+    Report progress on rows passing through to a python logger. If logger is
+    none, a new logger will be created that, by default, streams to stdout
+
+    E.g.::
+
+        >>> import petl as etl
+        >>> table = etl.dummytable(100000)
+        >>> table.log_progress(10000).tocsv('example.csv')
+        10000 rows in 0.13s (78363 row/s); batch in 0.13s (78363 row/s)
+        20000 rows in 0.22s (91679 row/s); batch in 0.09s (110448 row/s)
+        30000 rows in 0.31s (96573 row/s); batch in 0.09s (108114 row/s)
+        40000 rows in 0.40s (99535 row/s); batch in 0.09s (109625 row/s)
+        50000 rows in 0.49s (101396 row/s); batch in 0.09s (109591 row/s)
+        60000 rows in 0.59s (102245 row/s); batch in 0.09s (106709 row/s)
+        70000 rows in 0.68s (103221 row/s); batch in 0.09s (109498 row/s)
+        80000 rows in 0.77s (103810 row/s); batch in 0.09s (108126 row/s)
+        90000 rows in 0.90s (99465 row/s); batch in 0.13s (74516 row/s)
+        100000 rows in 1.02s (98409 row/s); batch in 0.11s (89821 row/s)
+        100000 rows in 1.02s (98402 row/s); batches in 0.10 +/- 0.02s [0.09-0.13] (100481 +/- 13340 rows/s [74516-110448])
+
+    See also :func:`petl.util.timing.clock`.
+
+    """
+
+    return LoggingProgressView(table, batchsize, prefix, logger, level=level)
+
+
 Table.progress = progress
+Table.log_progress = log_progress
 
 
-class ProgressView(Table):
+class ProgressViewBase(Table):
+    """
+    Abstract base class for reporting on proecessing status
+    """
 
-    def __init__(self, inner, batchsize, prefix, out):
+    def __init__(self, inner, batchsize, prefix):
         self.inner = inner
         self.batchsize = batchsize
         self.prefix = prefix
-        self.out = out
+
+    @abc.abstractmethod
+    def print_message(self, message):
+        pass
 
     def __iter__(self):
         start = time.time()
@@ -76,9 +117,7 @@ class ProgressView(Table):
                 message = self.prefix + \
                     '%s rows in %.2fs (%s row/s); ' \
                     'batch in %.2fs (%s row/s)' % v
-                print(message, file=self.out)
-                if hasattr(self.out, 'flush'):
-                    self.out.flush()
+                self.print_message(message)
                 batchstart = batchend
                 batchtimemean, batchtimevar = \
                     onlinestats(batchtime, batchn, mean=batchtimemean,
@@ -120,9 +159,43 @@ class ProgressView(Table):
             v = (n, elapsedtime, rate)
             message = self.prefix + '%s rows in %.2fs (%s row/s)' % v
 
-        print(message, file=self.out)
-        if hasattr(self.out, 'flush'):
-            self.out.flush()
+        self.print_message(message)
+
+
+class ProgressView(ProgressViewBase):
+    """
+    Reports progress to a file_object like sys.stdout or a file handler
+    """
+
+    def __init__(self, inner, batchsize, prefix, out):
+        if out is None:
+            self.file_object = sys.stderr
+        else:
+            self.file_object = out
+        super(ProgressView, self).__init__(inner, batchsize, prefix)
+
+    def print_message(self, message):
+        print(message, file=self.file_object)
+        if hasattr(self.file_object, 'flush'):
+            self.file_object.flush()
+
+
+class LoggingProgressView(ProgressViewBase):
+    """
+    Reports progress to a logger, log handler, or log adapter
+    """
+
+    def __init__(self, inner, batchsize, prefix, logger, level=logging.INFO):
+        if logger is None:
+            self.logger = logging.getLogger(__name__)
+            self.logger.setLevel(level)
+        else:
+            self.logger = logger
+        self.level = level
+        super(LoggingProgressView, self).__init__(inner, batchsize, prefix)
+
+    def print_message(self, message):
+        self.logger.log(self.level, message)
 
 
 def clock(table):


Test patch:
diff --git a/petl/test/util/test_timing.py b/petl/test/util/test_timing.py
index 33e262c..418cc88 100644
--- a/petl/test/util/test_timing.py
+++ b/petl/test/util/test_timing.py
@@ -2,7 +2,7 @@ from __future__ import absolute_import, print_function, division
 
 
 from petl.util.counting import nrows
-from petl.util.timing import progress
+from petl.util.timing import progress, log_progress
 
 
 def test_progress():
@@ -13,3 +13,10 @@ def test_progress():
              ('b', 3))
     nrows(progress(table))
 
+def test_log_progress():
+    # make sure log_progress doesn't raise exception
+    table = (('foo', 'bar', 'baz'),
+             ('a', 1, True),
+             ('b', 2, True),
+             ('b', 3))
+    nrows(log_progress(table))