# Buggy code
def add_data(self, gdf):
    # Populate columns idxs
    if not self.col_idx:
        for i, x in enumerate(gdf.columns.values):
            self.col_idx[str(x)] = i

    # list columns in cudf don't currently support chunked writing in parquet.
    # hack around this by just writing a single file with this partition
    # this restriction can be removed once cudf supports chunked writing
    # in parquet
    if any(is_list_dtype(gdf[col].dtype) for col in gdf.columns):
        self._write_table(gdf, 0, True)
        return

    # Generate `ind` array to map each row to an output file.
    # This approach is certainly more optimized for shuffling
    # than it is for non-shuffling, but using a single code
    # path is probably worth the (possible) minor overhead.
    nrows = gdf.shape[0]
    typ = np.min_scalar_type(nrows * 2)
    if self.shuffle:
        ind = cp.random.choice(cp.arange(self.num_out_files, dtype=typ), nrows)
    else:
        ind = cp.arange(nrows, dtype=typ)
        cp.floor_divide(ind, math.ceil(nrows / self.num_out_files), out=ind)
    for x, group in enumerate(
        gdf.scatter_by_map(ind, map_size=self.num_out_files, keep_index=False)
    ):
        self.num_samples[x] += len(group)
        if self.num_threads > 1:
            self.queue.put((x, group))
        else:
            self._write_table(x, group)

    # wait for all writes to finish before exiting
    # (so that we aren't using memory)
    if self.num_threads > 1:
        self.queue.join()


# Traceback (AttributeError)
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-19-f93c44c3b381> in <module>
11 proc.add_preprocess(JoinExternal(df_grouped, on= ['doc_id'], on_ext= ['doc_id'], kind_ext=kind_ext, columns_ext=columns_ext, cache='device', how='left'))
12 train_dataset = nvt.Dataset(df2)
---> 13 proc.apply(train_dataset, apply_offline=True, record_stats=True, output_path='./output/', shuffle=True, out_files_per_proc=1)

~/ronaya/NVTabular/nvtabular/workflow.py in apply(self, dataset, apply_offline, record_stats, shuffle, output_path, output_format, out_files_per_proc, num_io_threads)
738                 output_format=output_format,
739                 out_files_per_proc=out_files_per_proc,
--> 740                 num_io_threads=num_io_threads,
741             )
742         else:

~/ronaya/NVTabular/nvtabular/workflow.py in build_and_process_graph(self, dataset, end_phase, output_path, record_stats, shuffle, output_format, out_files_per_proc, apply_ops, num_io_threads)
845                 shuffle=shuffle,
846                 out_files_per_proc=out_files_per_proc,
--> 847                 num_threads=num_io_threads,
848             )
849

~/ronaya/NVTabular/nvtabular/workflow.py in ddf_to_dataset(self, output_path, shuffle, out_files_per_proc, output_format, num_threads)
931                 output_format,
932                 self.client,
--> 933                 num_threads,
934             )
935             return

~/ronaya/NVTabular/nvtabular/io/dask.py in _ddf_to_dataset(ddf, fs, output_path, shuffle, out_files_per_proc, cat_names, cont_names, label_names, output_format, client, num_threads)
110         out = client.compute(out).result()
111     else:
--> 112         out = dask.compute(out, scheduler="synchronous")[0]
113
114     # Follow-up Shuffling and _metadata creation

~/miniconda3/envs/1019/lib/python3.7/site-packages/dask/base.py in compute(*args, **kwargs)
450         postcomputes.append(x.__dask_postcompute__())
451
--> 452     results = schedule(dsk, keys, **kwargs)
453     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
454

~/miniconda3/envs/1019/lib/python3.7/site-packages/dask/local.py in get_sync(dsk, keys, **kwargs)
525     """
526     kwargs.pop("num_workers", None)  # if num_workers present, remove it
--> 527     return get_async(apply_sync, 1, dsk, keys, **kwargs)
528
529

~/miniconda3/envs/1019/lib/python3.7/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
492
493                 while state["ready"] and len(state["running"]) < num_workers:
--> 494                     fire_task()
495
496             succeeded = True

~/miniconda3/envs/1019/lib/python3.7/site-packages/dask/local.py in fire_task()
464                         pack_exception,
465                     ),
--> 466                     callback=queue.put,
467                 )
468

~/miniconda3/envs/1019/lib/python3.7/site-packages/dask/local.py in apply_sync(func, args, kwds, callback)
514 def apply_sync(func, args=(), kwds={}, callback=None):
515     """ A naive synchronous version of apply_async """
--> 516     res = func(*args, **kwds)
517     if callback is not None:
518         callback(res)

~/miniconda3/envs/1019/lib/python3.7/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
225         failed = False
226     except BaseException as e:
--> 227         result = pack_exception(e, dumps)
228         failed = True
229     return key, result, failed

~/miniconda3/envs/1019/lib/python3.7/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
220     try:
221         task, data = loads(task_info)
--> 222         result = _execute_task(task, data)
223         id = get_id()
224         result = dumps((result, id))

~/miniconda3/envs/1019/lib/python3.7/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
119         # temporaries by their reference count and can execute certain
120         # operations in-place.
--> 121         return func(*(_execute_task(a, cache) for a in args))
122     elif not ishashable(arg):
123         return arg

~/miniconda3/envs/1019/lib/python3.7/contextlib.py in inner(*args, **kwds)
72         def inner(*args, **kwds):
73             with self._recreate_cm():
---> 74                 return func(*args, **kwds)
75         return inner
76

~/ronaya/NVTabular/nvtabular/io/dask.py in _write_output_partition(gdf, processed_path, shuffle, out_files_per_proc, fs, cat_names, cont_names, label_names, output_format, num_threads)
61
62         # Add data
---> 63         writer.add_data(gdf)
64
65     return gdf_size

~/miniconda3/envs/1019/lib/python3.7/contextlib.py in inner(*args, **kwds)
72         def inner(*args, **kwds):
73             with self._recreate_cm():
---> 74                 return func(*args, **kwds)
75         return inner
76

~/ronaya/NVTabular/nvtabular/io/writer.py in add_data(self, gdf)
125         # in parquet
126         if any(is_list_dtype(gdf[col].dtype) for col in gdf.columns):
--> 127             self._write_table(gdf, 0, True)
128             return
129

~/ronaya/NVTabular/nvtabular/io/parquet.py in _write_table(self, idx, data, has_list_column)
210             # write out a new file, rather than stream multiple chunks to a single file
211             filename = self._get_filename(len(self.data_paths))
--> 212             data.to_parquet(filename)
213             self.data_paths.append(filename)
214         else:

AttributeError: 'int' object has no attribute 'to_parquet'

# Fixed code
def add_data(self, gdf):
    # Populate columns idxs
    if not self.col_idx:
        for i, x in enumerate(gdf.columns.values):
            self.col_idx[str(x)] = i

    # list columns in cudf don't currently support chunked writing in parquet.
    # hack around this by just writing a single file with this partition
    # this restriction can be removed once cudf supports chunked writing
    # in parquet
    if any(is_list_dtype(gdf[col].dtype) for col in gdf.columns):
        self._write_table(0, gdf, True)
        return

    # Generate `ind` array to map each row to an output file.
    # This approach is certainly more optimized for shuffling
    # than it is for non-shuffling, but using a single code
    # path is probably worth the (possible) minor overhead.
    nrows = gdf.shape[0]
    typ = np.min_scalar_type(nrows * 2)
    if self.shuffle:
        ind = cp.random.choice(cp.arange(self.num_out_files, dtype=typ), nrows)
    else:
        ind = cp.arange(nrows, dtype=typ)
        cp.floor_divide(ind, math.ceil(nrows / self.num_out_files), out=ind)
    for x, group in enumerate(
        gdf.scatter_by_map(ind, map_size=self.num_out_files, keep_index=False)
    ):
        self.num_samples[x] += len(group)
        if self.num_threads > 1:
            self.queue.put((x, group))
        else:
            self._write_table(x, group)

    # wait for all writes to finish before exiting
    # (so that we aren't using memory)
    if self.num_threads > 1:
        self.queue.join()