examples/sent_tokenize/spacy_sent_tokenize.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
import argparse
import os
import warnings
import time

import spacy

from project_settings import project_path


def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--text",
        type=str,
        default="Mr. Honey Tian. How are you."
    )
    # https://huggingface.co/stanfordnlp
    parser.add_argument(
        "--language",
        type=str,
        default="english"
    )
    args = parser.parse_args()
    return args


# https://spacy.io/models
language_to_models = {
    "english": "en_core_web_sm",
    # "english": "en_core_web_md",
    # "english": "en_core_web_lg",
    # "english": "en_core_web_trf",

}


def main():
    args = get_args()

    model_name = language_to_models[args.language]

    spacy_nlp = spacy.load(model_name)

    begin_time = time.time()
    doc = spacy_nlp(args.text)
    sentences = [sentence.text for sentence in doc.sents]

    cost = time.time() - begin_time
    print(f"time cost: {cost}")

    print(sentences)
    return


if __name__ == "__main__":
    main()