# Buggy code
def _intra_word_tokenize(
    self, string_tokens: List[str]
) -> Tuple[List[Token], List[Optional[Tuple[int, int]]]]:
    tokens: List[Token] = []
    offsets: List[Optional[Tuple[int, int]]] = []
    for token_string in string_tokens:
        wordpieces = self.tokenizer.encode_plus(
            token_string,
            add_special_tokens=False,
            return_tensors=None,
            return_offsets_mapping=False,
            return_attention_mask=False,
            return_token_type_ids=False,
        )
        wp_ids = wordpieces["input_ids"]

        if len(wp_ids) > 0:
            offsets.append((len(tokens), len(tokens) + len(wp_ids) - 1))
            tokens.extend(
                Token(text=wp_text, text_id=wp_id)
                for wp_id, wp_text in zip(
                    wp_ids, self.tokenizer.convert_ids_to_tokens(wp_ids)
                )
            )
        else:
            offsets.append(None)
    return tokens, offsets


# Traceback (ValueError)
Traceback (most recent call last):
File "bug_example.py", line 4, in <module>
tokenizer_kwargs={"use_fast": False}).intra_word_tokenize(["My", "text", "will"])
File "X/venv/lib/python3.6/site-packages/allennlp-1.2.0rc1-py3.6.egg/allennlp/data/tokenizers/pretrained_transformer_tokenizer.py", line 387, in intra_word_tokenize
tokens, offsets = self._intra_word_tokenize(string_tokens)
File "X/venv/lib/python3.6/site-packages/allennlp-1.2.0rc1-py3.6.egg/allennlp/data/tokenizers/pretrained_transformer_tokenizer.py", line 354, in _intra_word_tokenize
return_token_type_ids=False,
File "X/venv/lib/python3.6/site-packages/transformers-3.4.0-py3.6.egg/transformers/tokenization_utils_base.py", line 2229, in encode_plus
**kwargs,
File "X/venv/lib/python3.6/site-packages/transformers-3.4.0-py3.6.egg/transformers/tokenization_utils.py", line 490, in _encode_plus
verbose=verbose,
File "X/venv/lib/python3.6/site-packages/transformers-3.4.0-py3.6.egg/transformers/tokenization_utils_base.py", line 2617, in prepare_for_model
"Asking to return token_type_ids while setting add_special_tokens to False "
ValueError: Asking to return token_type_ids while setting add_special_tokens to False results in an undefined behavior. Please set add_special_tokens to True or set return_token_type_ids to None.

# Fixed code
def _intra_word_tokenize(
    self, string_tokens: List[str]
) -> Tuple[List[Token], List[Optional[Tuple[int, int]]]]:
    tokens: List[Token] = []
    offsets: List[Optional[Tuple[int, int]]] = []
    for token_string in string_tokens:
        wordpieces = self.tokenizer.encode_plus(
            token_string,
            add_special_tokens=False,
            return_tensors=None,
            return_offsets_mapping=False,
            return_attention_mask=False,
        )
        wp_ids = wordpieces["input_ids"]

        if len(wp_ids) > 0:
            offsets.append((len(tokens), len(tokens) + len(wp_ids) - 1))
            tokens.extend(
                Token(text=wp_text, text_id=wp_id)
                for wp_id, wp_text in zip(
                    wp_ids, self.tokenizer.convert_ids_to_tokens(wp_ids)
                )
            )
        else:
            offsets.append(None)
    return tokens, offsets