Repository: petl-developers/petl

Issue:
convert() and fieldmap() swallow errors silently by default
This has led to a lot of difficult-to-debug scenarios in my experience. 
As you're developing an ETL pipeline, you're often iterating on your conversion functions quickly, which often leads to exceptions being raised. Currently petl eats those errors silently, making it far more difficult to navigate this process.
In the regular operation of an ETL pipeline, it's also undesirable for new, unexpected errors to pass silently.

The current workaround is to set `failonerror=True` in every call to `fieldmap()` and the like. I suggest you make `failonerror=True` be the default. This might be something that would require bumping to version 2.0 since it will likely cause a lot of exceptions to be raised that were previously ignored (for better or for worse!).


Discussion:
coveralls: 
[![Coverage Status](https://coveralls.io/builds/19127569/badge)](https://coveralls.io/builds/19127569)

Coverage decreased (-5.3%) to 84.766% when pulling **c67b5f2fa1743e489bc1dc82541f7c35057f13f4 on bmaggard:pr406** into **9372b2ab3c4f7e7cd826ad8eb1cbe28e7047d7da on petl-developers:master**.


Patch:
diff --git a/docs/changes.rst b/docs/changes.rst
index 7c79fb7..efa84aa 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -4,6 +4,16 @@ Changes
 Version 1.3.0
 -------------
 
+* A new configuration option `failonerror` has been added to the :mod:`petl.config` 
+  module. This option affects various transformation functions including 
+  :func:`petl.transform.conversions.convert`, :func:`petl.transform.maps.fieldmap`, 
+  :func:`petl.transform.maps.rowmap` and :func:`petl.transform.maps.rowmapmany`. 
+  The option can have values `True` (raise any exceptions encountered during conversion), 
+  `False` (silently use a given `errorvalue` if any exceptions arise during conversion) or 
+  `"inline"` (use any exceptions as the output value). The default value is `False` which 
+  maintains compatibility with previous releases. By :user:`bmaggard`, :issue:`460`, 
+  :issue:`406`, :issue:`365`.
+
 * Pass through keyword arguments to :func:`xlrd.open_workbook`. By :user:`gjunqueira`,
   :issue:`470`, :issue:`473`.
 
diff --git a/docs/config.rst b/docs/config.rst
new file mode 100644
index 0000000..663d3b8
--- /dev/null
+++ b/docs/config.rst
@@ -0,0 +1,6 @@
+Configuration
+===============================
+
+.. automodule:: petl.config
+    :members:
+
diff --git a/docs/index.rst b/docs/index.rst
index 81d503c..5c192f6 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -36,6 +36,7 @@ see the :ref:`genindex`.
    io
    transform
    util
+   config
    changes
    contributing
    acknowledgments
diff --git a/petl/config.py b/petl/config.py
index eb1a6c1..4815a15 100644
--- a/petl/config.py
+++ b/petl/config.py
@@ -16,3 +16,15 @@ display_limit = 5
 display_index_header = False
 display_vrepr = text_type
 sort_buffersize = 100000
+failonerror=False # False, True, 'inline'
+"""
+Controls what happens when unhandled exceptions are raised in a
+transformation:
+
+    - If `False`, exceptions are suppressed.  If present, the value
+      provided in the `errorvalue` argument is returned.
+
+    - If `True`, the first unhandled exception is raised.
+
+    - If `'inline'`, unhandled exceptions are returned.
+"""
diff --git a/petl/transform/conversions.py b/petl/transform/conversions.py
index b0c7a1d..00a3152 100644
--- a/petl/transform/conversions.py
+++ b/petl/transform/conversions.py
@@ -4,6 +4,7 @@ from __future__ import absolute_import, print_function, division
 from petl.compat import next, integer_types, string_types, text_type
 
 
+import petl.config as config
 from petl.errors import ArgumentError, FieldSelectionError
 from petl.util.base import Table, expr, header, Record
 from petl.util.parsers import numparser
@@ -165,6 +166,9 @@ def convert(table, *args, **kwargs):
     arguments to the conversion function (so, i.e., the conversion function
     should accept two arguments).
 
+    Also accepts `failonerror` and `errorvalue` keyword arguments,
+    documented under :func:`petl.config.failonerror`
+
     """
 
     converters = None
@@ -286,7 +290,7 @@ Table.convertnumbers = convertnumbers
 
 class FieldConvertView(Table):
 
-    def __init__(self, source, converters=None, failonerror=False,
+    def __init__(self, source, converters=None, failonerror=None,
                  errorvalue=None, where=None, pass_row=False):
         self.source = source
         if converters is None:
@@ -297,7 +301,8 @@ class FieldConvertView(Table):
             self.converters = dict([(i, v) for i, v in enumerate(converters)])
         else:
             raise ArgumentError('unexpected converters: %r' % converters)
-        self.failonerror = failonerror
+        self.failonerror = (config.failonerror if failonerror is None
+                                else failonerror)
         self.errorvalue = errorvalue
         self.where = where
         self.pass_row = pass_row
@@ -366,7 +371,9 @@ def iterfieldconvert(source, converters, failonerror, errorvalue, where,
             try:
                 return converter_functions[i](v, *args)
             except Exception as e:
-                if failonerror:
+                if failonerror == 'inline':
+                    return e
+                elif failonerror:
                     raise e
                 else:
                     return errorvalue
diff --git a/petl/transform/maps.py b/petl/transform/maps.py
index 3e76509..ff64afe 100644
--- a/petl/transform/maps.py
+++ b/petl/transform/maps.py
@@ -6,12 +6,13 @@ from collections import OrderedDict
 from petl.compat import next, string_types, text_type
 
 
+import petl.config as config
 from petl.errors import ArgumentError
 from petl.util.base import Table, expr, rowgroupby, Record
 from petl.transform.sorts import sort
 
 
-def fieldmap(table, mappings=None, failonerror=False, errorvalue=None):
+def fieldmap(table, mappings=None, failonerror=None, errorvalue=None):
     """
     Transform a table, mapping fields arbitrarily between input and output.
     E.g.::
@@ -53,6 +54,8 @@ def fieldmap(table, mappings=None, failonerror=False, errorvalue=None):
     Note also that the mapping value can be an expression string, which will be
     converted to a lambda function via :func:`petl.util.base.expr`.
 
+    The `failonerror` and `errorvalue` keyword arguments are documented
+    under :func:`petl.config.failonerror`
     """
 
     return FieldMapView(table, mappings=mappings, failonerror=failonerror,
@@ -64,14 +67,15 @@ Table.fieldmap = fieldmap
 
 class FieldMapView(Table):
 
-    def __init__(self, source, mappings=None, failonerror=False,
+    def __init__(self, source, mappings=None, failonerror=None,
                  errorvalue=None):
         self.source = source
         if mappings is None:
             self.mappings = OrderedDict()
         else:
             self.mappings = mappings
-        self.failonerror = failonerror
+        self.failonerror = (config.failonerror if failonerror is None
+                                else failonerror)
         self.errorvalue = errorvalue
 
     def __setitem__(self, key, value):
@@ -119,7 +123,9 @@ def iterfieldmap(source, mappings, failonerror, errorvalue):
             try:
                 val = mapfuns[outfld](row)
             except Exception as e:
-                if failonerror:
+                if failonerror == 'inline':
+                    val = e
+                elif failonerror:
                     raise e
                 else:
                     val = errorvalue
@@ -143,7 +149,7 @@ def composedict(d, srcfld):
     return g
 
 
-def rowmap(table, rowmapper, header, failonerror=False):
+def rowmap(table, rowmapper, header, failonerror=None):
     """
     Transform rows via an arbitrary function. E.g.::
 
@@ -182,6 +188,8 @@ def rowmap(table, rowmapper, header, failonerror=False):
     The `rowmapper` function should accept a single row and return a single
     row (list or tuple).
 
+    The `failonerror` keyword argument is documented under
+    :func:`petl.config.failonerror`
     """
 
     return RowMapView(table, rowmapper, header, failonerror=failonerror)
@@ -192,11 +200,12 @@ Table.rowmap = rowmap
 
 class RowMapView(Table):
 
-    def __init__(self, source, rowmapper, header, failonerror=False):
+    def __init__(self, source, rowmapper, header, failonerror=None):
         self.source = source
         self.rowmapper = rowmapper
         self.header = header
-        self.failonerror = failonerror
+        self.failonerror = (config.failonerror if failonerror is None
+                                else failonerror)
 
     def __iter__(self):
         return iterrowmap(self.source, self.rowmapper, self.header,
@@ -214,11 +223,13 @@ def iterrowmap(source, rowmapper, header, failonerror):
             outrow = rowmapper(row)
             yield tuple(outrow)
         except Exception as e:
-            if failonerror:
+            if failonerror == 'inline':
+                yield tuple([e])
+            elif failonerror:
                 raise e
 
 
-def rowmapmany(table, rowgenerator, header, failonerror=False):
+def rowmapmany(table, rowgenerator, header, failonerror=None):
     """
     Map each input row to any number of output rows via an arbitrary
     function. E.g.::
@@ -268,6 +279,9 @@ def rowmapmany(table, rowgenerator, header, failonerror=False):
     The `rowgenerator` function should accept a single row and yield zero or
     more rows (lists or tuples).
 
+    The `failonerror` keyword argument is documented under
+    :func:`petl.config.failonerror`
+
     See also the :func:`petl.transform.reshape.melt` function.
 
     """
@@ -280,11 +294,12 @@ Table.rowmapmany = rowmapmany
 
 class RowMapManyView(Table):
 
-    def __init__(self, source, rowgenerator, header, failonerror=False):
+    def __init__(self, source, rowgenerator, header, failonerror=None):
         self.source = source
         self.rowgenerator = rowgenerator
         self.header = header
-        self.failonerror = failonerror
+        self.failonerror = (config.failonerror if failonerror is None
+                                else failonerror)
 
     def __iter__(self):
         return iterrowmapmany(self.source, self.rowgenerator, self.header,
@@ -302,7 +317,9 @@ def iterrowmapmany(source, rowgenerator, header, failonerror):
             for outrow in rowgenerator(row):
                 yield tuple(outrow)
         except Exception as e:
-            if failonerror:
+            if failonerror == 'inline':
+                yield tuple([e])
+            elif failonerror:
                 raise e
             else:
                 pass


Test patch:
diff --git a/petl/test/failonerror.py b/petl/test/failonerror.py
new file mode 100644
index 0000000..dd85715
--- /dev/null
+++ b/petl/test/failonerror.py
@@ -0,0 +1,118 @@
+from petl.test.helpers import ieq, eq_
+import petl.config as config
+
+from nose.tools import nottest
+
+
+@nottest
+def test_failonerror(input_fn, expected_output):
+    """In the input rows, the first row should process through the
+    transformation cleanly.  The second row should generate an
+    exception.  There are no requirements for any other rows."""
+    #=========================================================
+    # Test function parameters with default config settings
+    #=========================================================
+    # test the default config setting: failonerror == False
+    eq_(config.failonerror, False)
+
+    # By default, a bad conversion does not raise an exception, and
+    # values for the failed conversion are returned as None
+    table2 = input_fn()
+    ieq(expected_output, table2)
+    ieq(expected_output, table2)
+
+    # When called with failonerror is False or None, a bad conversion
+    # does not raise an exception, and values for the failed conversion
+    # are returned as None
+    table3 = input_fn(failonerror=False)
+    ieq(expected_output, table3)
+    ieq(expected_output, table3)
+    table3 = input_fn(failonerror=None)
+    ieq(expected_output, table3)
+    ieq(expected_output, table3)
+
+    # When called with failonerror=True, a bad conversion raises an
+    # exception
+    try:
+        table4 = input_fn(failonerror=True)
+        table4.nrows()
+    except Exception:
+        pass
+    else:
+        raise Exception('expected exception not raised')
+
+    # When called with failonerror='inline', a bad conversion
+    # does not raise an exception, and an Exception for the failed
+    # conversion is returned in the result.
+    expect5 = expected_output[0], expected_output[1]
+    table5 = input_fn(failonerror='inline')
+    ieq(expect5, table5.head(1))
+    ieq(expect5, table5.head(1))
+    excp = table5[2][0]
+    assert isinstance(excp, Exception)
+
+    #=========================================================
+    # Test config settings
+    #=========================================================
+    # Save config setting
+    saved_config_failonerror = config.failonerror
+
+    # When config.failonerror == True, a bad conversion raises an
+    # exception
+    config.failonerror = True
+    try:
+        table6 = input_fn()
+        table6.nrows()
+    except Exception:
+        pass
+    else:
+        raise Exception('expected exception not raised')
+
+    # When config.failonerror == 'inline', a bad conversion
+    # does not raise an exception, and an Exception for the failed
+    # conversion is returned in the result.
+    expect7 = expected_output[0], expected_output[1]
+    config.failonerror = 'inline'
+    table7 = input_fn()
+    ieq(expect7, table7.head(1))
+    ieq(expect7, table7.head(1))
+    excp = table7[2][0]
+    assert isinstance(excp, Exception)
+
+    # When config.failonerror is an invalid value, but still truthy, it
+    # behaves the same as if == True
+    config.failonerror = 'invalid'
+    try:
+        table8 = input_fn()
+        table8.nrows()
+    except Exception:
+        pass
+    else:
+        raise Exception('expected exception not raised')
+
+    # When config.failonerror is None, it behaves the same as if
+    # config.failonerror is False
+    config.failonerror = None
+    table9 = input_fn()
+    ieq(expected_output, table9)
+    ieq(expected_output, table9)
+
+    # A False keyword parameter overrides config.failonerror == True
+    config.failonerror = True
+    table10 = input_fn(failonerror=False)
+    ieq(expected_output, table10)
+    ieq(expected_output, table10)
+
+    # A None keyword parameter uses config.failonerror == True
+    config.failonerror = True
+    try:
+        table11 = input_fn(failonerror=None)
+        table11.nrows()
+    except Exception:
+        pass
+    else:
+        raise Exception('expected exception not raised')
+
+    # restore config setting
+    config.failonerror = saved_config_failonerror
+
diff --git a/petl/test/transform/test_conversions.py b/petl/test/transform/test_conversions.py
index 9aff0a8..928f31b 100644
--- a/petl/test/transform/test_conversions.py
+++ b/petl/test/transform/test_conversions.py
@@ -1,10 +1,13 @@
 from __future__ import absolute_import, print_function, division
 
 
+from petl.test.failonerror import test_failonerror
 from petl.test.helpers import ieq
 from petl.transform.conversions import convert, convertall, convertnumbers, \
     replace, update, format, interpolate
 
+from functools import partial
+
 
 def test_convert():
 
@@ -289,6 +292,16 @@ def test_convert_where():
     ieq(expect, actual)
 
 
+def test_convert_failonerror():
+    input_  = (('foo',), ('A',), (1,))
+    cvt_    = {'foo': 'lower'}
+    expect_ = (('foo',), ('a',), (None,))
+
+    test_failonerror(
+            input_fn=partial(convert, input_, cvt_),
+            expected_output=expect_)
+
+
 def test_replace_where():
 
     tbl1 = (('foo', 'bar'),
@@ -363,3 +376,4 @@ def test_interpolate():
     actual = interpolate(table, 'bar', '%02d')
     ieq(expect, actual)
     ieq(expect, actual)
+
diff --git a/petl/test/transform/test_maps.py b/petl/test/transform/test_maps.py
index 478f10d..6e26efc 100644
--- a/petl/test/transform/test_maps.py
+++ b/petl/test/transform/test_maps.py
@@ -1,10 +1,13 @@
 from __future__ import absolute_import, print_function, division
 
-
 from collections import OrderedDict
+
+from petl.test.failonerror import test_failonerror
 from petl.test.helpers import ieq
 from petl.transform.maps import fieldmap, rowmap, rowmapmany
 
+from functools import partial
+
 
 def test_fieldmap():
     table = (('id', 'sex', 'age', 'height', 'weight'),
@@ -88,6 +91,16 @@ def test_fieldmap_empty():
     ieq(expect, actual)
 
 
+def test_fieldmap_failonerror():
+    input_  = (('foo',), ('A',), (1,))
+    mapper_ = {'bar': ('foo', lambda v: v.lower())}
+    expect_ = (('bar',), ('a',), (None,))
+
+    test_failonerror(
+            input_fn=partial(fieldmap, input_, mapper_),
+            expected_output=expect_)
+
+
 def test_rowmap():
     table = (('id', 'sex', 'age', 'height', 'weight'),
              (1, 'male', 16, 1.45, 62.0),
@@ -147,6 +160,17 @@ def test_rowmap_empty():
     ieq(expect, actual)
 
 
+def test_rowmap_failonerror():
+    input_  = (('foo',), ('A',), (1,), ('B',))
+    mapper  = lambda r: [r[0].lower()]
+    # exceptions in rowmappers do not generate an output row
+    expect_ = (('foo',), ('a',), ('b',))
+
+    test_failonerror(
+            input_fn=partial(rowmap, input_, mapper, header=('foo',)),
+            expected_output=expect_)
+
+
 def test_recordmap():
     table = (('id', 'sex', 'age', 'height', 'weight'),
              (1, 'male', 16, 1.45, 62.0),
@@ -222,6 +246,16 @@ def test_rowmapmany():
     ieq(expect, actual)  # can iteratate twice?
 
 
+def test_rowmapmany_failonerror():
+    input_  = (('foo',), ('A',), (1,), ('B',))
+    mapper  = lambda r: [r[0].lower()]
+    expect_ = (('foo',), ('a',), ('b',),)
+
+    test_failonerror(
+            input_fn=partial(rowmapmany, input_, mapper, header=('foo',)),
+            expected_output=expect_)
+
+
 def test_recordmapmany():
     table = (('id', 'sex', 'age', 'height', 'weight'),
              (1, 'male', 16, 1.45, 62.0),
@@ -252,3 +286,4 @@ def test_recordmapmany():
               (4, 'age_months', 21 * 12))
     ieq(expect, actual)
     ieq(expect, actual)  # can iteratate twice?
+