tokenizer.py

import spacy
from spacy.tokenizer import Tokenizer
from spacy.lang.en import English
nlp = English()


tokenizer = nlp.tokenizer