# Buggy code
def tokenize(self, tokenizer):
    passage = (
        self.passage.lower()
        if resolve_is_lower_case(tokenizer=tokenizer)
        else self.passage
    )
    passage_tokens = tokenizer.tokenize(passage)
    token_aligner = TokenAligner(source=passage, target=passage_tokens)
    answer_token_span = token_aligner.project_char_to_token_span(
        self.answer_char_span[0], self.answer_char_span[1], inclusive=True
    )

    return TokenizedExample(
        guid=self.guid,
        passage=passage_tokens,
        question=tokenizer.tokenize(self.question),
        answer_str=self.answer,
        passage_str=passage,
        answer_token_span=answer_token_span,
        token_idx_to_char_idx_map=token_aligner.source_char_idx_to_target_token_idx.T,
    )


# Traceback (ValueError)
Traceback (most recent call last):
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/proj/main/tokenize_and_cache.py", line 214, in <module>
main(args=RunConfiguration.run_cli_json_prepend())
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/proj/main/tokenize_and_cache.py", line 168, in main
args=args,
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/proj/main/tokenize_and_cache.py", line 53, in chunk_and_save
args=args,
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/proj/main/tokenize_and_cache.py", line 129, in iter_chunk_and_save
recorder_callback=max_valid_length_recorder,
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/shared/caching.py", line 91, in iter_chunk_and_save
for datum in data:
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/proj/main/preprocessing.py", line 145, in iter_chunk_convert_examples_to_dataset
verbose=verbose,
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/proj/main/preprocessing.py", line 216, in iter_chunk_tokenize_and_featurize
yield example.tokenize(tokenizer).featurize(tokenizer, feat_spec)
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/tasks/lib/templates/span_prediction.py", line 38, in tokenize
self.answer_char_span[0], self.answer_char_span[1], inclusive=True
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/utils/retokenize.py", line 276, in project_char_to_token_span
mat=self.source_char_idx_to_target_token_idx, start=start, end=end, inclusive=inclusive
File "/scratch/pmh330/nyu-mll-jiant/jiant/jiant/utils/retokenize.py", line 192, in _project_span
raise ValueError(f"Project {(start, end)} into empty span in target sequence")
ValueError: Project (0, 2) into empty span in target sequence

# Fixed code
def tokenize(self, tokenizer):
    passage = (
        self.passage.lower()
        if model_resolution.resolve_is_lower_case(tokenizer=tokenizer)
        else self.passage
    )
    passage_tokens = tokenizer.tokenize(passage)
    token_aligner = TokenAligner(source=passage, target=passage_tokens)
    answer_token_span = token_aligner.project_char_to_token_span(
        self.answer_char_span[0], self.answer_char_span[1], inclusive=True
    )

    return TokenizedExample(
        guid=self.guid,
        passage=passage_tokens,
        question=tokenizer.tokenize(self.question),
        answer_str=self.answer,
        passage_str=passage,
        answer_token_span=answer_token_span,
        token_idx_to_char_idx_map=token_aligner.source_char_idx_to_target_token_idx.T,
    )