Repository: NVIDIA-Merlin/NVTabular

Issue:
[BUG] Movielens 02 ETL error with CPU
When running Notebook: https://github.com/NVIDIA/NVTabular/blob/main/examples/getting-started-movielens/02-ETL-with-NVTabular.ipynb there is an error in Categorify related to: https://github.com/NVIDIA/NVTabular/blob/main/nvtabular/ops/categorify.py#L829

```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<timed eval> in <module>

/NVTA/nvtabular/workflow.py in fit(self, dataset)
    150                 results = [r.result() for r in self.client.compute(stats)]
    151             else:
--> 152                 results = dask.compute(stats, scheduler="synchronous")[0]
    153 
    154             for computed_stats, op in zip(results, ops):

/usr/local/lib/python3.8/dist-packages/dask/base.py in compute(*args, **kwargs)
    565         postcomputes.append(x.__dask_postcompute__())
    566 
--> 567     results = schedule(dsk, keys, **kwargs)
    568     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    569 

/usr/local/lib/python3.8/dist-packages/dask/local.py in get_sync(dsk, keys, **kwargs)
    558     """
    559     kwargs.pop("num_workers", None)  # if num_workers present, remove it
--> 560     return get_async(
    561         synchronous_executor.submit,
    562         synchronous_executor._max_workers,

/usr/local/lib/python3.8/dist-packages/dask/local.py in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)
    501             while state["waiting"] or state["ready"] or state["running"]:
    502                 fire_tasks(chunksize)
--> 503                 for key, res_info, failed in queue_get(queue).result():
    504                     if failed:
    505                         exc, tb = loads(res_info)

/usr/lib/python3.8/concurrent/futures/_base.py in result(self, timeout)
    430                 raise CancelledError()
    431             elif self._state == FINISHED:
--> 432                 return self.__get_result()
    433 
    434             self._condition.wait(timeout)

/usr/lib/python3.8/concurrent/futures/_base.py in __get_result(self)
    386     def __get_result(self):
    387         if self._exception:
--> 388             raise self._exception
    389         else:
    390             return self._result

/usr/local/lib/python3.8/dist-packages/dask/local.py in submit(self, fn, *args, **kwargs)
    543         fut = Future()
    544         try:
--> 545             fut.set_result(fn(*args, **kwargs))
    546         except BaseException as e:
    547             fut.set_exception(e)

/usr/local/lib/python3.8/dist-packages/dask/local.py in batch_execute_tasks(it)
    235     Batch computing of multiple tasks with `execute_task`
    236     """
--> 237     return [execute_task(*a) for a in it]
    238 
    239 

/usr/local/lib/python3.8/dist-packages/dask/local.py in <listcomp>(.0)
    235     Batch computing of multiple tasks with `execute_task`
    236     """
--> 237     return [execute_task(*a) for a in it]
    238 
    239 

/usr/local/lib/python3.8/dist-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    226         failed = False
    227     except BaseException as e:
--> 228         result = pack_exception(e, dumps)
    229         failed = True
    230     return key, result, failed

/usr/local/lib/python3.8/dist-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    221     try:
    222         task, data = loads(task_info)
--> 223         result = _execute_task(task, data)
    224         id = get_id()
    225         result = dumps((result, id))

/usr/local/lib/python3.8/dist-packages/dask/core.py in _execute_task(arg, cache, dsk)
    119         # temporaries by their reference count and can execute certain
    120         # operations in-place.
--> 121         return func(*(_execute_task(a, cache) for a in args))
    122     elif not ishashable(arg):
    123         return arg

/usr/lib/python3.8/contextlib.py in inner(*args, **kwds)
     73         def inner(*args, **kwds):
     74             with self._recreate_cm():
---> 75                 return func(*args, **kwds)
     76         return inner
     77 

/NVTA/nvtabular/ops/categorify.py in _write_uniques(dfs, base_path, col_group, options)
    827                 nulls_missing = True
    828                 new_cols[col] = _concat(
--> 829                     [df._constructor_sliced([None], dtype=df[col].dtype), df[col]],
    830                     ignore_index=True,
    831                 )

/usr/local/lib/python3.8/dist-packages/pandas/core/series.py in __init__(self, data, index, dtype, name, copy, fastpath)
    325                     data = data.copy()
    326             else:
--> 327                 data = sanitize_array(data, index, dtype, copy, raise_cast_failure=True)
    328 
    329                 data = SingleBlockManager.from_array(data, index)

/usr/local/lib/python3.8/dist-packages/pandas/core/construction.py in sanitize_array(data, index, dtype, copy, raise_cast_failure)
    445 
    446         if dtype is not None:
--> 447             subarr = _try_cast(data, dtype, copy, raise_cast_failure)
    448         else:
    449             subarr = maybe_convert_platform(data)

/usr/local/lib/python3.8/dist-packages/pandas/core/construction.py in _try_cast(arr, dtype, copy, raise_cast_failure)
    553         if is_integer_dtype(dtype):
    554             # this will raise if we have e.g. floats
--> 555             maybe_cast_to_integer_array(arr, dtype)
    556             subarr = arr
    557         else:

/usr/local/lib/python3.8/dist-packages/pandas/core/dtypes/cast.py in maybe_cast_to_integer_array(arr, dtype, copy)
   1672     try:
   1673         if not hasattr(arr, "astype"):
-> 1674             casted = np.array(arr, dtype=dtype, copy=copy)
   1675         else:
   1676             casted = arr.astype(dtype, copy=copy)

TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'
```


Discussion:
any ideas @benfred @rjzamora ?
@jperez999 were you looking into anything similar?
Can you verify the version of dask you're running? We need to be using 2021.4.1 here - we've seen this issue with newer versions of dask
Solved with our latests container using dask and dsitributed 2021.04.1
Still getting errors with latest containers
It seems this problem is related to nullabe-integer types in Pandas.  Since cudf allows null values in its `"int32"`/`"int64"` columns, there are many places in NVTabular where we require/expect the data to be casted to these types.  More-recent version of Pandas seem to throw errors when you try to create an `"int32"`/`"int64"`  `Series`/column when null values are present.

Resolving patch:
diff --git a/nvtabular/dispatch.py b/nvtabular/dispatch.py
--- a/nvtabular/dispatch.py
+++ b/nvtabular/dispatch.py
@@ -68,6 +68,13 @@ def inner2(*args, **kwargs):
     DataFrameType = Union[pd.DataFrame]
     SeriesType = Union[pd.Series]
 
+# Define mapping between non-nullable,
+# and nullable types in Pandas
+_PD_NULLABLE_MAP = {
+    "int32": "Int32",
+    "int64": "Int64",
+}
+
 
 class ExtData(enum.Enum):
     """Simple Enum to track external-data types"""
@@ -94,6 +101,17 @@ def _is_dataframe_object(x):
     return isinstance(x, (cudf.DataFrame, pd.DataFrame))
 
 
+def _nullable_series(data, like_df, dtype):
+    # Return a Series containing the elements in `data`,
+    # with a nullable version of `dtype`, using `like_df`
+    # to infer the Series constructor
+    if isinstance(like_df, pd.DataFrame):
+        # Note that we cannot use "int32"/"int64" to
+        # represent nullable data in pandas
+        return like_df._constructor_sliced(data, dtype=_PD_NULLABLE_MAP.get(str(dtype), dtype))
+    return like_df._constructor_sliced(data, dtype=dtype)
+
+
 def _is_series_object(x):
     # Simple check if object is a cudf or pandas
     # Series object
diff --git a/nvtabular/ops/categorify.py b/nvtabular/ops/categorify.py
--- a/nvtabular/ops/categorify.py
+++ b/nvtabular/ops/categorify.py
@@ -34,7 +34,7 @@
 from pyarrow import parquet as pq
 
 from nvtabular import dispatch
-from nvtabular.dispatch import DataFrameType, annotate
+from nvtabular.dispatch import DataFrameType, _nullable_series, annotate
 from nvtabular.ops.internal import ConcatColumns, Identity, SubsetColumns
 from nvtabular.worker import fetch_table_data, get_worker_cache
 
@@ -892,7 +892,7 @@ def _write_uniques(dfs, base_path, col_selector: ColumnSelector, options: FitOpt
 
                 nulls_missing = True
                 new_cols[col] = _concat(
-                    [df._constructor_sliced([None], dtype=df[col].dtype), df[col]],
+                    [_nullable_series([None], df, df[col].dtype), df[col]],
                     ignore_index=True,
                 )
             else:
@@ -1079,7 +1079,7 @@ def _encode(
         value = type(df)()
         for c in selection_r:
             typ = df[selection_l[0]].dtype if len(selection_l) == 1 else df[c].dtype
-            value[c] = df._constructor_sliced([None], dtype=typ)
+            value[c] = _nullable_series([None], df, typ)
         value.index.name = "labels"
         value.reset_index(drop=False, inplace=True)
 
@@ -1089,8 +1089,8 @@ def _encode(
             codes["order"] = dispatch._arange(len(codes), like_df=df)
         else:
             codes = type(df)({"order": dispatch._arange(len(df), like_df=df)}, index=df.index)
-            for c in selection_l:
-                codes[c] = df[c].copy()
+            for cl, cr in zip(selection_l, selection_r):
+                codes[cl] = df[cl].copy().astype(value[cr].dtype)
         if buckets and storage_name in buckets:
             na_sentinel = _hash_bucket(df, buckets, selection_l, encode_type=encode_type)
         # apply frequency hashing


Test patch: