# Buggy code
def _tf_glue_convert_examples_to_features(
    examples: tf.data.Dataset,
    tokenizer: PreTrainedTokenizer,
    task=str,
    max_length: Optional[int] = None,
) -> tf.data.Dataset:
    """
    Returns:
        A ``tf.data.Dataset`` containing the task-specific features.

    """
    processor = glue_processors[task]()
    examples = [
        processor.tfds_map(processor.get_example_from_tensor_dict(example))
        for example in examples
    ]
    features = glue_convert_examples_to_features(
        examples, tokenizer, max_length=max_length, task=task
    )

    def gen():
        for ex in features:
            yield (
                {
                    "input_ids": ex.input_ids,
                    "attention_mask": ex.attention_mask,
                    "token_type_ids": ex.token_type_ids,
                },
                ex.label,
            )

    return tf.data.Dataset.from_generator(
        gen,
        (
            {
                "input_ids": tf.int32,
                "attention_mask": tf.int32,
                "token_type_ids": tf.int32,
            },
            tf.int64,
        ),
        (
            {
                "input_ids": tf.TensorShape([None]),
                "attention_mask": tf.TensorShape([None]),
                "token_type_ids": tf.TensorShape([None]),
            },
            tf.TensorShape([]),
        ),
    )


# Traceback (tensorflow.python.framework.errors_impl.InvalidArgumentError)
Traceback (most recent call last):
File "run_glue.py", line 229, in <module>
main()
File "run_glue.py", line 199, in main
compute_metrics=compute_metrics,
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/transformers/trainer_tf.py", line 48, in __init__
self._setup_training()
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/transformers/trainer_tf.py", line 58, in _setup_training
self._prepare_dataset()
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/transformers/trainer_tf.py", line 95, in _prepare_dataset
self.num_train_examples = self.train_dataset.reduce(tf.constant(0), lambda x, _: x + 1).numpy()
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 1934, in reduce
output_types=structure.get_flat_tensor_types(state_structure)))
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/ops/gen_dataset_ops.py", line 4661, in reduce_dataset
_ops.raise_from_not_ok_status(e, name)
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py", line 6606, in raise_from_not_ok_status
six.raise_from(core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: TypeError: `generator` yielded an element that could not be converted to the expected type. The expected type was int32, but the yielded element was None.
Traceback (most recent call last):

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 805, in generator_py_func
ret, dtype=dtype.as_numpy_dtype))

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/ops/script_ops.py", line 196, in _convert
result = np.asarray(value, dtype=dtype, order="C")

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray
return array(a, dtype, copy=False, order=order)

TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'


During handling of the above exception, another exception occurred:


Traceback (most recent call last):

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/ops/script_ops.py", line 236, in __call__
ret = func(*args)

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 810, in generator_py_func
"element was %s." % (dtype.name, ret)), sys.exc_info()[2])

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/six.py", line 702, in reraise
raise value.with_traceback(tb)

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/data/ops/dataset_ops.py", line 805, in generator_py_func
ret, dtype=dtype.as_numpy_dtype))

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorflow_core/python/ops/script_ops.py", line 196, in _convert
result = np.asarray(value, dtype=dtype, order="C")

File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray
return array(a, dtype, copy=False, order=order)

TypeError: `generator` yielded an element that could not be converted to the expected type. The expected type was int32, but the yielded element was None.


[[{{node PyFunc}}]] [Op:ReduceDataset]

# Fixed code
def _tf_glue_convert_examples_to_features(
    examples: tf.data.Dataset,
    tokenizer: PreTrainedTokenizer,
    task=str,
    max_length: Optional[int] = None,
) -> tf.data.Dataset:
    """
    Returns:
        A ``tf.data.Dataset`` containing the task-specific features.

    """
    processor = glue_processors[task]()
    examples = [
        processor.tfds_map(processor.get_example_from_tensor_dict(example))
        for example in examples
    ]
    features = glue_convert_examples_to_features(
        examples, tokenizer, max_length=max_length, task=task
    )

    def gen():
        for ex in features:
            d = {k: v for k, v in asdict(ex).items() if v is not None}
            label = d.pop("label")
            yield (d, label)

    input_names = ["input_ids"] + tokenizer.model_input_names

    return tf.data.Dataset.from_generator(
        gen,
        ({k: tf.int32 for k in input_names}, tf.int64),
        ({k: tf.TensorShape([None]) for k in input_names}, tf.TensorShape([])),
    )