maskrcnn_benchmark/modeling/language_backbone/build.py

from .simple_tokenizer import SimpleTokenizer


def build_tokenizer(tokenizer_name):
    tokenizer = None
    if tokenizer_name == "clip":
        tokenizer = SimpleTokenizer()
    elif "hf_" in tokenizer_name:
        from .hfpt_tokenizer import HFPTTokenizer

        tokenizer = HFPTTokenizer(pt_name=tokenizer_name[3:])
    elif "hfc_" in tokenizer_name:
        from .hfpt_tokenizer import HFPTTokenizer

        tokenizer = HFPTTokenizer(pt_name=tokenizer_name[4:])
    else:
        raise ValueError("Unknown tokenizer")

    return tokenizer