Spaces:
Building
Building
| import spacy | |
| import pickle | |
| from nltk.corpus import wordnet | |
| def load_spacy_values(model = "en_core_web_md", filepath_docs_spacy = 'dict_spacy_object.pkl'): | |
| ''' | |
| Loads a spaCy language model and a dictionary of spaCy Doc objects from a pickle file. | |
| Parameters | |
| ---------- | |
| model : str | |
| The name or local path of the spaCy model to be loaded for processing text. | |
| For example, "en_core_web_sm" or a custom model path. | |
| filepath_docs_spacy : str | |
| The path to the pickle file containing a dictionary where the keys are tokens | |
| (strings) and the values are the corresponding serialized spaCy Doc objects. | |
| Returns | |
| ------- | |
| nlp : spacy.language.Language | |
| The loaded spaCy language model. | |
| dict_docs_spacy : dict | |
| A dictionary where the keys are tokens (strings) and the values are spaCy Doc | |
| objects reconstructed from the serialized bytes stored in the pickle file. | |
| ''' | |
| # ---- Load the spaCy NLP model | |
| # | |
| nlp = spacy.load(model) | |
| # ---- Load pickle file and reconstruct the dictionary with tokens as keys and spaCy Doc objects as values | |
| # | |
| with open(filepath_docs_spacy, 'rb') as file: | |
| dict_docs_spacy_bytes = pickle.load(file) | |
| dict_docs_spacy = {key: spacy.tokens.Doc(nlp.vocab).from_bytes(doc_bytes) for key, doc_bytes in dict_docs_spacy_bytes.items()} | |
| return nlp, dict_docs_spacy | |
| def find_antonyms(word): | |
| ''' | |
| Generate a set of all the antonyms of a given word | |
| Parameters | |
| ---------- | |
| word : str | |
| The word that we want to find the antonyms | |
| Returns | |
| ------- | |
| antonyms : set of str | |
| A set of all the antonym detected using nltk and WordNet | |
| ''' | |
| antonyms = set() | |
| # ---- Load all the set of synonyms of the word recorded from wordnet | |
| # | |
| syn_set = wordnet.synsets(word) | |
| # ---- Loop over each set of synonyms | |
| # | |
| for syn in syn_set: | |
| # ---- Loop over each synonym | |
| # | |
| for lemma in syn.lemmas(): | |
| # ---- Add antonyms of the synonyms to the antonyms set | |
| # | |
| if lemma.antonyms(): | |
| antonyms.add(lemma.antonyms()[0].name()) | |
| return antonyms | |
| def find_synonyms(word, model, dict_embedding, list_2000_tokens): | |
| ''' | |
| Finds the most similar token to a given word. | |
| Parameters | |
| ---------- | |
| word : str | |
| The word that we want to find the most similar word | |
| model : spacy.language.Language | |
| spaCy language model to use for the detection of the synonym | |
| dict_embedding: dict | |
| A dictionary where the keys are tokens (str) and the values are spaCy Doc objects | |
| list_2000_tokens : list of str | |
| A list of 2000 tokens against which the gloss will be checked. | |
| Returns | |
| ------- | |
| most_similar_token : str | |
| The most similar token to the given word | |
| ''' | |
| # ---- Skip synonym detection if the word is already in the list_2000_token | |
| # | |
| if word in list_2000_tokens: | |
| return word | |
| else: | |
| # ---- Remove antonyms of the given word of the list_2000_tokens (a word and an antonym might be similar in embedding representation) | |
| # | |
| antonyms = find_antonyms(word) | |
| list_2000_tokens_less_antonyms = [token for token in list_2000_tokens if token not in antonyms] | |
| # ---- Generate a list of tuple (token, similarities values between the embedding of the given word and the embedding of each token of the list_2000_tokens) | |
| # | |
| word_embedding = model(word) | |
| similarities=[] | |
| for token in list_2000_tokens_less_antonyms: | |
| similarities.append((token, dict_embedding.get(token).similarity(word_embedding))) | |
| # ---- Extract the most similar token of the list | |
| # | |
| most_similar_token = sorted(similarities, key=lambda item: -item[1])[0][0] | |
| return most_similar_token |