# Buggy code
def get_feature_meta(column, preprocessing_parameters, backend):
    idx2str, str2idx, str2freq, max_length, _, _, _ = create_vocabulary(
        column,
        preprocessing_parameters["tokenizer"],
        lowercase=preprocessing_parameters["lowercase"],
        num_most_frequent=preprocessing_parameters["most_common"],
        vocab_file=preprocessing_parameters["vocab_file"],
        unknown_symbol=preprocessing_parameters["unknown_symbol"],
        padding_symbol=preprocessing_parameters["padding_symbol"],
        processor=backend.df_engine,
    )
    max_length = min(preprocessing_parameters["sequence_length_limit"], max_length)
    return {
        "idx2str": idx2str,
        "str2idx": str2idx,
        "str2freq": str2freq,
        "vocab_size": len(idx2str),
        "max_sequence_length": max_length,
    }


# Traceback (AttributeError)
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-13-c2e6c8f1ae7c> in <module>()
10     skip_save_progress=True,
11     skip_save_model=False,
---> 12     skip_save_processed_input=True
13 )
/usr/local/lib/python3.6/dist-packages/ludwig/hyperopt/run.py in hyperopt(config, dataset, training_set, validation_set, test_set, training_set_metadata, data_format, experiment_name, model_name, skip_save_training_description, skip_save_training_statistics, skip_save_model, skip_save_progress, skip_save_log, skip_save_processed_input, skip_save_unprocessed_output, skip_save_predictions, skip_save_eval_stats, skip_save_hyperopt_statistics, output_directory, gpus, gpu_memory_limit, allow_parallel_threads, use_horovod, random_seed, debug, **kwargs)
289         random_seed=random_seed,
290         debug=debug,
--> 291         **kwargs
292     )
293
/usr/local/lib/python3.6/dist-packages/ludwig/hyperopt/execution.py in execute(self, config, dataset, training_set, validation_set, test_set, training_set_metadata, data_format, experiment_name, model_name, skip_save_training_description, skip_save_training_statistics, skip_save_model, skip_save_progress, skip_save_log, skip_save_processed_input, skip_save_unprocessed_output, skip_save_predictions, skip_save_eval_stats, output_directory, gpus, gpu_memory_limit, allow_parallel_threads, use_horovod, random_seed, debug, **kwargs)
149                     skip_collect_overall_stats=False,
150                     random_seed=random_seed,
--> 151                     debug=debug,
152                 )
153                 metric_score = self.get_metric_score(eval_stats)
/usr/local/lib/python3.6/dist-packages/ludwig/api.py in experiment(self, dataset, training_set, validation_set, test_set, training_set_metadata, data_format, experiment_name, model_name, model_load_path, model_resume_path, eval_split, skip_save_training_description, skip_save_training_statistics, skip_save_model, skip_save_progress, skip_save_log, skip_save_processed_input, skip_save_unprocessed_output, skip_save_predictions, skip_save_eval_stats, skip_collect_predictions, skip_collect_overall_stats, output_directory, random_seed, debug, **kwargs)
1054             output_directory=output_directory,
1055             random_seed=random_seed,
-> 1056             debug=debug,
1057         )
1058
/usr/local/lib/python3.6/dist-packages/ludwig/api.py in train(self, dataset, training_set, validation_set, test_set, training_set_metadata, data_format, experiment_name, model_name, model_resume_path, skip_save_training_description, skip_save_training_statistics, skip_save_model, skip_save_progress, skip_save_log, skip_save_processed_input, output_directory, random_seed, debug, **kwargs)
415                     random_seed=random_seed,
416                     devbug=debug,
--> 417                     **kwargs,
418                 )
419                 (training_set,
/usr/local/lib/python3.6/dist-packages/ludwig/api.py in preprocess(self, dataset, training_set, validation_set, test_set, training_set_metadata, data_format, skip_save_processed_input, random_seed, debug, **kwargs)
1261             preprocessing_params=self.config[PREPROCESSING],
1262             backend=self.backend,
-> 1263             random_seed=random_seed
1264         )
1265
/usr/local/lib/python3.6/dist-packages/ludwig/data/preprocessing.py in preprocess_for_training(config, dataset, training_set, validation_set, test_set, training_set_metadata, data_format, skip_save_processed_input, preprocessing_params, backend, random_seed)
1395         preprocessing_params=preprocessing_params,
1396         backend=backend,
-> 1397         random_seed=random_seed
1398     )
1399     training_set, test_set, validation_set, training_set_metadata = processed
/usr/local/lib/python3.6/dist-packages/ludwig/data/preprocessing.py in preprocess_for_training(features, dataset, training_set, validation_set, test_set, training_set_metadata, skip_save_processed_input, preprocessing_params, backend, random_seed)
183             preprocessing_params=preprocessing_params,
184             backend=backend,
--> 185             random_seed=random_seed
186         )
187
/usr/local/lib/python3.6/dist-packages/ludwig/data/preprocessing.py in _preprocess_df_for_training(features, dataset, training_set, validation_set, test_set, training_set_metadata, preprocessing_params, backend, random_seed)
1645         metadata=training_set_metadata,
1646         random_seed=random_seed,
-> 1647         backend=backend
1648     )
1649
/usr/local/lib/python3.6/dist-packages/ludwig/data/preprocessing.py in build_dataset(dataset_df, features, global_preprocessing_parameters, metadata, backend, random_seed)
1013             features,
1014             global_preprocessing_parameters,
-> 1015             backend
1016         )
1017
/usr/local/lib/python3.6/dist-packages/ludwig/data/preprocessing.py in build_metadata(dataset_df, features, global_preprocessing_parameters, backend)
1122                 column,
1123                 preprocessing_parameters,
-> 1124                 backend
1125             )
1126
/usr/local/lib/python3.6/dist-packages/ludwig/features/category_feature.py in get_feature_meta(column, preprocessing_parameters, backend)
65             lowercase=preprocessing_parameters['lowercase'],
66             add_padding=False,
---> 67             processor=backend.df_engine
68         )
69         return {
/usr/local/lib/python3.6/dist-packages/ludwig/utils/strings_utils.py in create_vocabulary(data, tokenizer_type, add_unknown, add_padding, lowercase, num_most_frequent, vocab_file, unknown_symbol, padding_symbol, pretrained_model_name_or_path, processor)
139         vocab = load_vocabulary(vocab_file)
140
--> 141     processed_lines = data.map(lambda line: tokenizer(line.lower() if lowercase else line))
142     processed_counts = processed_lines.explode().value_counts(sort=False)
143     processed_counts = processor.compute(processed_counts)
/usr/local/lib/python3.6/dist-packages/pandas/core/series.py in map(self, arg, na_action)
3980         dtype: object
3981         """
-> 3982         new_values = super()._map_values(arg, na_action=na_action)
3983         return self._constructor(new_values, index=self.index).__finalize__(
3984             self, method="map"
/usr/local/lib/python3.6/dist-packages/pandas/core/base.py in _map_values(self, mapper, na_action)
1158
1159         # mapper is a function
-> 1160         new_values = map_f(values, mapper)
1161
1162         return new_values
pandas/_libs/lib.pyx in pandas._libs.lib.map_infer()
/usr/local/lib/python3.6/dist-packages/ludwig/utils/strings_utils.py in <lambda>(line)
139         vocab = load_vocabulary(vocab_file)
140
--> 141     processed_lines = data.map(lambda line: tokenizer(line.lower() if lowercase else line))
142     processed_counts = processed_lines.explode().value_counts(sort=False)
143     processed_counts = processor.compute(processed_counts)
/usr/local/lib/python3.6/dist-packages/ludwig/utils/strings_utils.py in __call__(self, text)
305 class StrippedStringToListTokenizer(BaseTokenizer):
306     def __call__(self, text):
--> 307         return [text.strip()]
308
309
AttributeError: 'int' object has no attribute 'strip'

# Fixed code
def get_feature_meta(column, preprocessing_parameters, backend):
    column = column.astype(str)
    idx2str, str2idx, str2freq, max_length, _, _, _ = create_vocabulary(
        column,
        preprocessing_parameters["tokenizer"],
        lowercase=preprocessing_parameters["lowercase"],
        num_most_frequent=preprocessing_parameters["most_common"],
        vocab_file=preprocessing_parameters["vocab_file"],
        unknown_symbol=preprocessing_parameters["unknown_symbol"],
        padding_symbol=preprocessing_parameters["padding_symbol"],
        processor=backend.df_engine,
    )
    max_length = min(preprocessing_parameters["sequence_length_limit"], max_length)
    return {
        "idx2str": idx2str,
        "str2idx": str2idx,
        "str2freq": str2freq,
        "vocab_size": len(idx2str),
        "max_sequence_length": max_length,
    }