Repository: dask/dask

Issue:
DataFrame: Splitting a list to multiple columns is throwing error 
I was trying to split a string and saved value in a temp column and use that column to populate additional columns.  I am able to split string and convert to a list. But while fetching elements from list and assigning to new column is throwing error. 

I have tried to recreate the error with below code:

```
df = pd.DataFrame({ 'a' : [1, 2, 3] ,
      'b' : ['a|b|c|d|e|f|g', 'a|b|c|d|e|f|g', 'a|b|c|d|e|f|g']
            })
ddf = dd.from_pandas(df,1)
ddf['c'] = ddf.b.str.split('|')
ddf = ddf.assign(col1= ddf.c.map(lambda x: x[0]))
ddf = ddf.assign(col2= ddf.c.map(lambda x: x[1]))
ddf = ddf.assign(col3= ddf.c.map(lambda x: x[2]))
ddf = ddf.assign(col4= ddf.c.map(lambda x: x[3]))
```
Getting error while executing last line >> ddf = ddf.assign(col4= ddf.c.map(lambda x: x[3]))
   IndexError('string index out of range',) 




Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -1970,7 +1970,7 @@ def map(self, arg, na_action=None, meta=no_default):
                enumerate(self.__dask_keys__())}
         dsk.update(self.dask)
         if meta is no_default:
-            meta = _emulate(M.map, self, arg, na_action=na_action)
+            meta = _emulate(M.map, self, arg, na_action=na_action, udf=True)
         else:
             meta = make_meta(meta)
 
@@ -2121,7 +2121,7 @@ def apply(self, func, convert_dtype=True, meta=no_default, args=(), **kwds):
 
             meta = _emulate(M.apply, self._meta_nonempty, func,
                             convert_dtype=convert_dtype,
-                            args=args, **kwds)
+                            args=args, udf=True, **kwds)
 
         return map_partitions(M.apply, self, func,
                               convert_dtype, args, meta=meta, **kwds)
@@ -2794,7 +2794,7 @@ def apply(self, func, axis=0, args=(), meta=no_default, **kwds):
             warnings.warn(msg)
 
             meta = _emulate(M.apply, self._meta_nonempty, func,
-                            axis=axis, args=args, **kwds)
+                            axis=axis, args=args, udf=True, **kwds)
 
         return map_partitions(M.apply, self, func, axis,
                               False, False, None, args, meta=meta, **kwds)
@@ -3291,8 +3291,8 @@ def apply_concat_apply(args, chunk=None, aggregate=None, combine=None,
             dsk[(b, j)] = (aggregate, conc)
 
     if meta is no_default:
-        meta_chunk = _emulate(chunk, *args, **chunk_kwargs)
-        meta = _emulate(aggregate, _concat([meta_chunk]),
+        meta_chunk = _emulate(chunk, *args, udf=True, **chunk_kwargs)
+        meta = _emulate(aggregate, _concat([meta_chunk]), udf=True,
                         **aggregate_kwargs)
     meta = make_meta(meta)
 
@@ -3332,7 +3332,7 @@ def _emulate(func, *args, **kwargs):
     Apply a function using args / kwargs. If arguments contain dd.DataFrame /
     dd.Series, using internal cache (``_meta``) for calculation
     """
-    with raise_on_meta_error(funcname(func)):
+    with raise_on_meta_error(funcname(func), udf=kwargs.pop('udf', False)):
         return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))
 
 
@@ -3369,7 +3369,7 @@ def map_partitions(func, *args, **kwargs):
     args = _maybe_align_partitions(args)
 
     if meta is no_default:
-        meta = _emulate(func, *args, **kwargs)
+        meta = _emulate(func, *args, udf=True, **kwargs)
 
     if all(isinstance(arg, Scalar) for arg in args):
         dask = {(name, 0):
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -124,7 +124,7 @@ def insert_meta_param_description(*args, **kwargs):
 
 
 @contextmanager
-def raise_on_meta_error(funcname=None):
+def raise_on_meta_error(funcname=None, udf=False):
     """Reraise errors in this block to show metadata inference failure.
 
     Parameters
@@ -138,15 +138,19 @@ def raise_on_meta_error(funcname=None):
     except Exception as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         tb = ''.join(traceback.format_tb(exc_traceback))
-        msg = ("Metadata inference failed{0}.\n\n"
-               "Original error is below:\n"
-               "------------------------\n"
-               "{1}\n\n"
-               "Traceback:\n"
-               "---------\n"
-               "{2}"
-               ).format(" in `{0}`".format(funcname) if funcname else "",
-                        repr(e), tb)
+        msg = "Metadata inference failed{0}.\n\n"
+        if udf:
+            msg += ("You have supplied a custom function and Dask is unable to \n"
+                    "determine the type of output that that function returns. \n\n"
+                    "To resolve this please provide a meta= keyword.\n"
+                    "The docstring of the Dask function you ran should have more information.\n\n")
+        msg += ("Original error is below:\n"
+                "------------------------\n"
+                "{1}\n\n"
+                "Traceback:\n"
+                "---------\n"
+                "{2}")
+        msg = msg.format(" in `{0}`".format(funcname) if funcname else "", repr(e), tb)
         raise ValueError(msg)
 
 


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -3042,3 +3042,24 @@ def test_mixed_dask_array_multi_dimensional():
     assert_eq(ddf + dx + 1, df + x + 1)
     assert_eq(ddf + dx.rechunk((None, 1)) + 1, df + x + 1)
     assert_eq(ddf[['y', 'x']] + dx + 1, df[['y', 'x']] + x + 1)
+
+
+def test_meta_raises():
+    # Raise when we use a user defined fucntion
+    s = pd.Series(['abcd', 'abcd'])
+    ds = dd.from_pandas(s, npartitions=2)
+    try:
+        ds.map(lambda x: x[3])
+    except ValueError as e:
+        assert "meta=" in str(e)
+
+    # But not otherwise
+    df = pd.DataFrame({'a': ['x', 'y', 'y'],
+                       'b': ['x', 'y', 'z'],
+                       'c': [1, 2, 3]})
+    ddf = dd.from_pandas(df, npartitions=1)
+
+    with pytest.raises(Exception) as info:
+        ddf.a + ddf.c
+
+    assert "meta=" not in str(info.value)