# Buggy code
def get_ddf(self):
    if self.ddf is None:
        raise ValueError("No dask_cudf frame available.")
    elif isinstance(self.ddf, Dataset):
        columns = self.columns_ctx["all"]["base"]
        return self.ddf.to_ddf(columns=columns, shuffle=self._shuffle_parts)
    return self.ddf


# Traceback (ValueError)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-13-b133e2b51cbf> in <module>
2 valid_dataset = nvt.Dataset(OUTPUT_BUCKET_FOLDER+'valid_gdf.parquet', part_mem_fraction=0.12)
3
----> 4 workflow.apply(train_dataset, record_stats=True, output_path=output_train_dir, shuffle=True, out_files_per_proc=5)
5 workflow.apply(valid_dataset, record_stats=False, output_path=output_valid_dir, shuffle=False, out_files_per_proc=5)

/rapids/notebooks/benf/NVTabular/nvtabular/workflow.py in apply(self, dataset, apply_offline, record_stats, shuffle, output_path, output_format, out_files_per_proc, num_io_threads, dtypes)
782                 out_files_per_proc=out_files_per_proc,
783                 num_io_threads=num_io_threads,
--> 784                 dtypes=dtypes,
785             )
786         else:

/rapids/notebooks/benf/NVTabular/nvtabular/workflow.py in build_and_process_graph(self, dataset, end_phase, output_path, record_stats, shuffle, output_format, out_files_per_proc, apply_ops, num_io_threads, dtypes)
885             self._base_phase = 0  # Set _base_phase
886             for idx, _ in enumerate(self.phases[:end]):
--> 887                 self.exec_phase(idx, record_stats=record_stats, update_ddf=(idx == (end - 1)))
888             self._base_phase = 0  # Re-Set _base_phase
889

/rapids/notebooks/benf/NVTabular/nvtabular/workflow.py in exec_phase(self, phase_index, record_stats, update_ddf)
631
632         # Perform transforms as single dask task (per ddf partition)
--> 633         _ddf = self.get_ddf()
634         if transforms:
635             _ddf = self._aggregated_dask_transform(_ddf, transforms)

/rapids/notebooks/benf/NVTabular/nvtabular/workflow.py in get_ddf(self)
587         elif isinstance(self.ddf, Dataset):
588             columns = self.columns_ctx["all"]["base"]
--> 589             return self.ddf.to_ddf(columns=columns, shuffle=self._shuffle_parts)
590         return self.ddf
591

/rapids/notebooks/benf/NVTabular/nvtabular/io/dataset.py in to_ddf(self, columns, shuffle, seed)
263         """
264         # Use DatasetEngine to create ddf
--> 265         ddf = self.engine.to_ddf(columns=columns)
266
267         # Shuffle the partitions of ddf (optional)

/rapids/notebooks/benf/NVTabular/nvtabular/io/parquet.py in to_ddf(self, columns)
102             gather_statistics=False,
103             split_row_groups=self.row_groups_per_part,
--> 104             storage_options=self.storage_options,
105         )
106

/opt/conda/envs/rapids/lib/python3.7/site-packages/dask_cudf/io/parquet.py in read_parquet(path, columns, split_row_groups, row_groups_per_part, **kwargs)
192         split_row_groups=split_row_groups,
193         engine=CudfEngine,
--> 194         **kwargs,
195     )
196

/opt/conda/envs/rapids/lib/python3.7/site-packages/dask/dataframe/io/parquet/core.py in read_parquet(path, columns, filters, categories, index, storage_options, engine, gather_statistics, split_row_groups, chunksize, **kwargs)
248     # Modify `meta` dataframe accordingly
249     meta, index, columns = set_index_columns(
--> 250         meta, index, columns, index_in_columns, auto_index_allowed
251     )
252     if meta.index.name == NONE_LABEL:

/opt/conda/envs/rapids/lib/python3.7/site-packages/dask/dataframe/io/parquet/core.py in set_index_columns(meta, index, columns, index_in_columns, auto_index_allowed)
771             "The following columns were not found in the dataset %s\n"
772             "The following columns were found %s"
--> 773             % (set(columns) - set(meta.columns), meta.columns)
774         )
775

ValueError: The following columns were not found in the dataset {'document_id_promo_count', 'publish_time_days_since_published', 'campaign_id_clicked_sum_ctr', 'ad_id_count', 'ad_id_clicked_sum_ctr', 'source_id_clicked_sum_ctr', 'publish_time_promo_days_since_published', 'advertiser_id_clicked_sum_ctr', 'document_id_promo_clicked_sum_ctr', 'publisher_id_clicked_sum_ctr', 'geo_location_country', 'geo_location_state'}
The following columns were found Index(['display_id', 'ad_id', 'clicked', 'uuid', 'document_id', 'timestamp',
'platform', 'geo_location', 'document_id_promo', 'campaign_id',
'advertiser_id', 'source_id', 'publisher_id', 'publish_time',
'source_id_promo', 'publisher_id_promo', 'publish_time_promo',
'day_event'],
dtype='object')

# Fixed code
def get_ddf(self):
    if self.ddf is None:
        raise ValueError("No dask_cudf frame available.")
    elif isinstance(self.ddf, Dataset):
        # Right now we can't distinguish between input columns and generated columns
        # in the dataset, we don't limit the columm set right now in the to_ddf call
        # (https://github.com/NVIDIA/NVTabular/issues/409 )
        return self.ddf.to_ddf(shuffle=self._shuffle_parts)
    return self.ddf