Repository: NVIDIA-Merlin/NVTabular

Issue:
[BUG] Problem on writing to_parquet after transforming 
```python
#######################################
trasforming Code:
class Processjson(Operator):
    def transform(self, columns, gdf):        
        col = gdf['event_properties']
        gdf['item_id'] = col.str.extract('\'product_id\'\s*:\s*\'([^\']+)\'')
        gdf['event_time'] = (gdf['event_time'] - pd.Timestamp("1970-01-01")) // pd.Timedelta('1s')
        gdf['device_brand'] = gdf['device_brand'].fillna('Apple')  
        return gdf

    def output_column_names(self, columns):
        return [i for i in columns if (i != 'event_properties')] + ['item_id']

    def dependencies(self):
        return None

filtered = COLUMNS >> nvt.ops.Filter(lambda df: df['event_type'].isin(['Value'])) 
filtered = filtered >> nvt.ops.JoinExternal(df_ext=fid_map,on='user_id',columns_ext=['user_id','memberID'])
filtered = filtered>>Processjson() >> nvt.ops.Dropna()
workflow = nvt.Workflow(filtered)

dataset_file = glob.glob('raw/*')
subdataset_file = dataset_file[6:8]
dataset = nvt.Dataset(subdataset_file, part_size="500MB")
workflow.transform(dataset).to_parquet(f'processed/test')

############################################
```

    
I follow exam and edited to served my data.
When I set multiple files as input for Dataset class, there is some specific files that I loaded and this exception was thrown.
It might be because there is no handling, where some file have data and some have no data after the filtering. 
This doesn't happen if I loaded a single file and process separately.

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-60-2a116bd489a4> in <module>
      2 # for i in dataset_file:
      3 dataset = nvt.Dataset(subdataset_file, part_size="500MB")
----> 4 workflow.transform(dataset).to_parquet(f'processed/test')
      5 

/usr/local/lib/python3.8/dist-packages/nvtabular/io/dataset.py in to_parquet(self, output_path, shuffle, preserve_files, output_files, out_files_per_proc, num_threads, dtypes, cats, conts, labels, suffix, partition_on)
    763 
    764         # Output dask_cudf DataFrame to dataset
--> 765         _ddf_to_dataset(
    766             ddf,
    767             fs,

/usr/local/lib/python3.8/dist-packages/nvtabular/io/dask.py in _ddf_to_dataset(ddf, fs, output_path, shuffle, file_partition_map, out_files_per_proc, cat_names, cont_names, label_names, output_format, client, num_threads, cpu, suffix, partition_on)
    364         out = client.compute(out).result()
    365     else:
--> 366         out = dask.compute(out, scheduler="synchronous")[0]
    367 
    368     if cached_writers:

/usr/local/lib/python3.8/dist-packages/dask/base.py in compute(*args, **kwargs)
    564         postcomputes.append(x.__dask_postcompute__())
    565 
--> 566     results = schedule(dsk, keys, **kwargs)
    567     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    568 

/usr/local/lib/python3.8/dist-packages/dask/local.py in get_sync(dsk, keys, **kwargs)
    558     """
    559     kwargs.pop("num_workers", None)  # if num_workers present, remove it
--> 560     return get_async(
    561         synchronous_executor.submit,
    562         synchronous_executor._max_workers,

/usr/local/lib/python3.8/dist-packages/dask/local.py in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)
    501             while state["waiting"] or state["ready"] or state["running"]:
    502                 fire_tasks(chunksize)
--> 503                 for key, res_info, failed in queue_get(queue).result():
    504                     if failed:
    505                         exc, tb = loads(res_info)

/usr/lib/python3.8/concurrent/futures/_base.py in result(self, timeout)
    430                 raise CancelledError()
    431             elif self._state == FINISHED:
--> 432                 return self.__get_result()
    433 
    434             self._condition.wait(timeout)

/usr/lib/python3.8/concurrent/futures/_base.py in __get_result(self)
    386     def __get_result(self):
    387         if self._exception:
--> 388             raise self._exception
    389         else:
    390             return self._result

/usr/local/lib/python3.8/dist-packages/dask/local.py in submit(self, fn, *args, **kwargs)
    543         fut = Future()
    544         try:
--> 545             fut.set_result(fn(*args, **kwargs))
    546         except BaseException as e:
    547             fut.set_exception(e)

/usr/local/lib/python3.8/dist-packages/dask/local.py in batch_execute_tasks(it)
    235     Batch computing of multiple tasks with `execute_task`
    236     """
--> 237     return [execute_task(*a) for a in it]
    238 
    239 

/usr/local/lib/python3.8/dist-packages/dask/local.py in <listcomp>(.0)
    235     Batch computing of multiple tasks with `execute_task`
    236     """
--> 237     return [execute_task(*a) for a in it]
    238 
    239 

/usr/local/lib/python3.8/dist-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    226         failed = False
    227     except BaseException as e:
--> 228         result = pack_exception(e, dumps)
    229         failed = True
    230     return key, result, failed

/usr/local/lib/python3.8/dist-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    221     try:
    222         task, data = loads(task_info)
--> 223         result = _execute_task(task, data)
    224         id = get_id()
    225         result = dumps((result, id))

/usr/local/lib/python3.8/dist-packages/dask/core.py in _execute_task(arg, cache, dsk)
    119         # temporaries by their reference count and can execute certain
    120         # operations in-place.
--> 121         return func(*(_execute_task(a, cache) for a in args))
    122     elif not ishashable(arg):
    123         return arg

/usr/lib/python3.8/contextlib.py in inner(*args, **kwds)
     73         def inner(*args, **kwds):
     74             with self._recreate_cm():
---> 75                 return func(*args, **kwds)
     76         return inner
     77 

/usr/local/lib/python3.8/dist-packages/nvtabular/io/dask.py in _write_output_partition(df, processed_path, shuffle, out_files_per_proc, fs, cat_names, cont_names, label_names, output_format, num_threads, cpu, suffix)
     92 
     93         # Add data
---> 94         writer.add_data(df)
     95 
     96     return df_size

/usr/lib/python3.8/contextlib.py in inner(*args, **kwds)
     73         def inner(*args, **kwds):
     74             with self._recreate_cm():
---> 75                 return func(*args, **kwds)
     76         return inner
     77 

/usr/local/lib/python3.8/dist-packages/nvtabular/io/writer.py in add_data(self, df)
    137             # Only writing to a single file. No need to
    138             # scatter or slice the data before writing
--> 139             self._add_single_file(df)
    140         else:
    141             # Use different mechanism to decompose and write each df

/usr/local/lib/python3.8/dist-packages/nvtabular/io/writer.py in _add_single_file(self, df)
    224             self.queue.put((0, df))
    225         else:
--> 226             self._write_table(0, df)
    227 
    228     def package_general_metadata(self):

/usr/local/lib/python3.8/dist-packages/nvtabular/io/parquet.py in _write_table(self, idx, data)
    788     def _write_table(self, idx, data):
    789         writer = self._get_or_create_writer(idx)
--> 790         writer.write_table(data)
    791 
    792     @classmethod

cudf/_lib/parquet.pyx in cudf._lib.parquet.ParquetWriter.write_table()

RuntimeError: cuDF failure at: /workspace/build-env/cpp/src/io/parquet/writer_impl.cu:462: Leaf column's corresponding metadata cannot have children



Discussion:
The bug is related, when NVTabular iterates over the partitions of the dataset and the DropNA operator produces an empy dataframe (all rows are filtered due to missing values).

Quick fix could be resolved by increasing the partition size or merging the data, that there will be no partition, which results in an empty dataframe.
We need to handle that edge case when writing the file to disk 
@thanatdam (and maybe @bschifferer ) - If possible, could you share a self-contained reproducer (inlcuding some dummy input data to produce the error)?  I am having trouble reproducing locally.

Also, can you share the version of NVTabular that you are using? The traceback makes it look like the **old** `method="worker"` algorithm is being used for the write.
I wrote this using NVT 0.5.3, but this also happen with 0.6.
As for the sample data and code, I already provided that to Ettikan and @bschifferer via email, since it contains corporate data. 
@rjzamora  @thanatdam thanks for sharing. I provided an own toy example, which reproduces the same error
@bschifferer shared a simple toy reproducer with me offline, and it seems that the error is caused by the dtypes of an empty partition being converted to `float64` in Dask's `map_partitions` call. The problem is not that we are trying to write an empty partition, but that the empty partition has an inconsistent dtype, and this makes the parquet writer unhappy.

As far as I can tell, we should be able to use `ddf.map_partitions(..., enforce_metadata=False)` in cases where we already know `ddf._meta`  is wrong.  With `enforce_metadata=False`, this error should be avoided.

Resolving patch:
diff --git a/nvtabular/workflow/workflow.py b/nvtabular/workflow/workflow.py
--- a/nvtabular/workflow/workflow.py
+++ b/nvtabular/workflow/workflow.py
@@ -433,6 +433,7 @@ def _transform_ddf(ddf, workflow_nodes, meta=None, additional_columns=None):
         workflow_nodes,
         additional_columns=additional_columns,
         meta=meta,
+        enforce_metadata=False,
     )
 
 


Test patch:
diff --git a/tests/unit/test_dask_nvt.py b/tests/unit/test_dask_nvt.py
--- a/tests/unit/test_dask_nvt.py
+++ b/tests/unit/test_dask_nvt.py
@@ -20,8 +20,10 @@
 
 import cudf
 import dask_cudf
+import pandas as pd
 import pytest
 from dask.dataframe import assert_eq
+from dask.dataframe import from_pandas as dd_from_pandas
 from dask.dataframe import read_parquet as dd_read_parquet
 
 from nvtabular import ColumnSelector, Dataset, Workflow, ops
@@ -276,3 +278,18 @@ def test_dask_preproc_cpu(client, tmpdir, datasets, engine, shuffle, cpu):
         df_disk.sort_values(["id", "x"])[["name-string", "label"]],
         check_index=False,
     )
+
+
+@pytest.mark.parametrize("cpu", [None, True])
+def test_filtered_partition(tmpdir, cpu):
+    # Toy DataFrame example
+    df = pd.DataFrame({"col": range(100)})
+    ddf = dd_from_pandas(df, npartitions=5)
+    dataset = Dataset(ddf, cpu=cpu)
+
+    # Workflow
+    filtered = ["col"] >> ops.Filter(lambda df: df["col"] < 75)
+    workflow = Workflow(filtered)
+
+    # Write result to disk
+    workflow.transform(dataset).to_parquet(str(tmpdir))