AnalogyArcade / word_embedding.py
Mila
still broken?
862ca42
raw
history blame
22.4 kB
from datasets import load_dataset
import shutil
import json
from collections import defaultdict
import multiprocessing
import gensim
from sklearn.metrics import classification_report
from gensim import corpora
from gensim.test.utils import common_texts
from gensim.models import Word2Vec
from gensim.models import KeyedVectors
from gensim.models import fasttext
from gensim.test.utils import datapath
from wefe.datasets import load_bingliu
from wefe.metrics import RNSB
from wefe.query import Query
from wefe.word_embedding_model import WordEmbeddingModel
from wefe.utils import plot_queries_results, run_queries
import pandas as pd
import gensim.downloader as api
import glob
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from wefe.metrics import WEAT
from wefe.datasets import load_weat
from wefe.utils import run_queries
from wefe.utils import plot_queries_results
import random
from scipy.special import expit
import math
import sys
import os
import argparse
import nltk
import scipy.sparse
import numpy as np
import string
import io
from sklearn.model_selection import train_test_split
'''STEPS FOR CODE:
1. Train word embeddings on Simple English Wikipedia;
2. Compare these to other pre-trained embeddings;
3. Quantify biases that exist in these word embeddings;
4. Use your word embeddings as features in a simple text classifier;
'''
def load_vectors(fname):
fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
n, d = map(int, fin.readline().split())
data = {}
# print("Hello", n, d)
for line in fin:
tokens = line.rstrip().split(' ')
data[tokens[0]] = map(float, tokens[1:])
# print(data)
print(data)
return data
def train_embeddings():
'''TRAIN WORD EMBEDDINGS
This will be making use of the dataset from wikipedia and the first step'''
dataset = load_dataset("wikipedia", "20220301.simple")
cores = multiprocessing.cpu_count()
# check the first example of the training portion of the dataset :
# print(dataset['train'][0])
dataset_size = len(dataset)
### BUILD VOCAB ###
# print(type(dataset["train"][0]))
vocab = set()
vocab_size = 0
count = 0
## Generate vocab and split sentances and words?
data = []
for index, page in enumerate(dataset["train"]):
document = page["text"]
document = document.replace("\n", ". ")
# print(document)
for sent in document.split("."):
# print("Sentance:", sent)
new_sent = []
clean_sent =[s for s in sent if s.isalnum() or s.isspace()]
clean_sent = "".join(clean_sent)
for word in clean_sent.split(" "):
if len(word) > 0:
new_word = word.lower()
# print("Word:", new_word)
if new_word[0] not in string.punctuation:
new_sent.append(new_word)
if len(new_sent) > 0:
data.append(new_sent)
# print("New Sent:", new_sent)
for index, page in enumerate(dataset["train"]):
# print(page["text"])
# for text in page:
# print(text)
text = page["text"]
clean_text = [s for s in text if s.isalnum() or s.isspace()]
clean_text = "".join(clean_text)
clean_text = clean_text.replace("\n", " ")
# text = text.replace('; ', ' ').replace(", ", " ").replace("\n", " ").replace(":", " ").replace(". ", " ").replace("! ", " ").replace("? ", " ").replace()
for word in clean_text.split(" "):
# print(word)
if word != "\n" and word != " " and word not in vocab:
vocab.add(word)
vocab_size += 1
# if index == 10:
# break
# print(f"word #{index}/{count} is {word}")
count += 1
# print(f"There are {vocab_size} vocab words")
embeddings_model = Word2Vec(
data,
epochs= 10,
window=10,
vector_size= 50)
embeddings_model.save("word2vec.model")
skip_model = Word2Vec(
data,
epochs= 10,
window=10,
vector_size= 50,
sg=1)
skip_model.save("skip2vec.model")
embeddings_model = Word2Vec.load("word2vec.model")
skip_model = Word2Vec.load("skip2vec.model")
# embeddings_model.train(dataset, total_examples=dataset_size, epochs=15)
# print(embeddings_model['train'])
# print(embeddings_model.wv["france"])
return embeddings_model, skip_model
def get_data():
dataset = load_dataset("wikipedia", "20220301.simple")
cores = multiprocessing.cpu_count()
# check the first example of the training portion of the dataset :
# print(dataset['train'][0])
dataset_size = len(dataset)
### BUILD VOCAB ###
# print(type(dataset["train"][0]))
vocab = set()
vocab_size = 0
count = 0
## Generate vocab and split sentances and words?
data = []
num_sents = 0
for index, page in enumerate(dataset["train"]):
document = page["text"]
document = document.replace("\n", ". ")
# print(document)
for sent in document.split("."):
num_sents += 1
# print("Sentance:", sent)
new_sent = []
clean_sent =[s for s in sent if s.isalnum() or s.isspace()]
clean_sent = "".join(clean_sent)
for word in clean_sent.split(" "):
if len(word) > 0:
new_word = word.lower()
# print("Word:", new_word)
if new_word[0] not in string.punctuation:
new_sent.append(new_word)
if len(new_sent) > 0:
data.append(new_sent)
# print("New Sent:", new_sent)
return data, num_sents
def compare_embeddings(cbow, skip, urban, fasttext):
'''COMPARE EMBEDDINGS'''
print("Most Similar to dog")
print("cbow", cbow.wv.most_similar(positive=['dog'], negative=[], topn=2))
print("skip", skip.wv.most_similar(positive=['dog'], negative=[], topn=2))
print("urban", urban.most_similar(positive=['dog'], negative=[], topn=2))
print("fasttext", fasttext.most_similar(positive=['dog'], negative=[], topn=2))
print("\nMost Similar to Pizza - Pepperoni + Pretzel")
print("cbow", cbow.wv.most_similar(positive=['pizza', 'pretzel'], negative=['pepperoni'], topn=2))
print("skip", skip.wv.most_similar(positive=['pizza', 'pretzel'], negative=['pepperoni'], topn=2))
print("urban", urban.most_similar(positive=['pizza', 'pretzel'], negative=['pepperoni'], topn=2))
print("fasttext", fasttext.most_similar(positive=['pizza', 'pretzel'], negative=['pepperoni'], topn=2))
print("\nMost Similar to witch - woman + man")
print("cbow", cbow.wv.most_similar(positive=['witch', 'man'], negative=['woman'], topn=2))
print("skip", skip.wv.most_similar(positive=['witch', 'man'], negative=['woman'], topn=2))
print("urban", urban.most_similar(positive=['witch', 'man'], negative=['woman'], topn=2))
print("fasttext", fasttext.most_similar(positive=['witch', 'man'], negative=['woman'], topn=2))
print("\nMost Similar to mayor - town + country")
print("cbow", cbow.wv.most_similar(positive=['mayor', 'country'], negative=['town'], topn=2))
print("skip", skip.wv.most_similar(positive=['mayor', 'country'], negative=['town'], topn=2))
print("urban", urban.most_similar(positive=['mayor', 'country'], negative=['town'], topn=2))
print("fasttext", fasttext.most_similar(positive=['mayor', 'country'], negative=['town'], topn=2))
print("\nMost Similar to death")
print("cbow", cbow.wv.most_similar(positive=['death'], negative=[], topn=2))
print("skip", skip.wv.most_similar(positive=['death'], negative=[], topn=2))
print("urban", urban.most_similar(positive=['death'], negative=[], topn=2))
print("fasttext", fasttext.most_similar(positive=['death'], negative=[], topn=2))
def quantify_bias(cbow, skip, urban, fasttext):
'''QUANTIFY BIASES'''
'''Using WEFE, RNSB'''
RNSB_words = [
['christianity'],
['catholicism'],
['islam'],
['judaism'],
['hinduism'],
['buddhism'],
['mormonism'],
['scientology'],
['taoism']]
weat_wordset = load_weat()
models = [WordEmbeddingModel(cbow.wv, "CBOW"),
WordEmbeddingModel(skip.wv, "skip-gram"),
WordEmbeddingModel(urban, "urban dictionary"),
WordEmbeddingModel(fasttext, "fasttext")]
# Define the 10 Queries:
# print(weat_wordset["science"])
religions = ['christianity',
'catholicism',
'islam',
'judaism',
'hinduism',
'buddhism',
'mormonism',
'scientology',
'taoism',
'atheism']
queries = [
# Flowers vs Insects wrt Pleasant (5) and Unpleasant (5)
Query([religions, weat_wordset['arts']],
[weat_wordset['career'], weat_wordset['family']],
['Religion', 'Art'], ['Career', 'Family']),
Query([religions, weat_wordset['weapons']],
[weat_wordset['male_terms'], weat_wordset['female_terms']],
['Religion', 'Weapons'], ['Male terms', 'Female terms']),
]
wefe_results = run_queries(WEAT,
queries,
models,
metric_params ={
'preprocessors': [
{},
{'lowercase': True }
]
},
warn_not_found_words = True
).T.round(2)
print(wefe_results)
plot_queries_results(wefe_results).show()
def text_classifier(cbow):
'''SIMPLE TEXT CLASSIFIER'''
'''For each document, average together all embeddings for the
individual words in that document to get a new, d-dimensional representation
of that document (this is essentially a “continuous bag-of-words”). Note that
your input feature size is only d now, instead of the size of your entire vocabulary.
Compare the results of training a model using these “CBOW” input features to
your original (discrete) BOW model.'''
pos_train_files = glob.glob('aclImdb/train/pos/*')
neg_train_files = glob.glob('aclImdb/train/neg/*')
# print(pos_train_files[:5])
num_files_per_class = 1000
# bow_train_files = cbow
all_train_files = pos_train_files[:num_files_per_class] + neg_train_files[:num_files_per_class]
# vectorizer = TfidfVectorizer(input="filename", stop_words="english")
# vectors = vectorizer.fit_transform(all_train_files)
d = len(cbow.wv["man"])
vectors = np.empty([len(all_train_files), d])
count = 0
vocab = set()
for doc in all_train_files:
temp_array = avg_embeddings(doc, cbow, vocab)
if len(temp_array) > 0:
vectors[count] = temp_array
count += 1
else:
vectors = np.delete(vectors, count)
# vectors = np.array(avg_embeddings(doc, cbow) for doc in all_train_files)
# print(vectors)
# print(vocab)
# len(vectorizer.vocabulary_)
vectors[0].sum()
# print("Vector at 0", vectors[0])
X = vectors
y = [1] * num_files_per_class + [0] * num_files_per_class
len(y)
x_0 = X[0]
w = np.zeros(X.shape[1])
# x_0_dense = x_0.todense()
x_0.dot(w)
w,b = sgd_for_lr_with_ce(X,y)
# w
# sorted_vocab = sorted([(k,v) for k,v in vectorizer.vocabulary_.items()],key=lambda x:x[1])
sorted_vocab = sorted(vocab)
# sorted_vocab = [a for (a,b) in sorted_vocab]
sorted_words_weights = sorted([x for x in zip(sorted_vocab, w)], key=lambda x:x[1])
sorted_words_weights[-50:]
preds = predict_y_lr(w,b,X)
preds
w,b = sgd_for_lr_with_ce(X, y, num_passes=10)
y_pred = predict_y_lr(w,b,X)
print(classification_report(y, y_pred))
# compute for dev set
# pos_dev_files = glob.glob('aclImdb/test/pos/*')
# neg_dev_files = glob.glob('aclImdb/test/neg/*')
# num_dev_files_per_class = 100
# all_dev_files = pos_dev_files[:num_dev_files_per_class] + neg_dev_files[:num_dev_files_per_class]
# # use the same vectorizer from before! otherwise features won't line up
# # don't fit it again, just use it to transform!
# X_dev = vectorizer.transform(all_dev_files)
# y_dev = [1]* num_dev_files_per_class + [0]* num_dev_files_per_class
# # don't need new w and b, these are from out existing model
# y_dev_pred = predict_y_lr(w,b,X_dev)
# print(classification_report(y_dev, y_dev_pred))
def avg_embeddings(doc, model, vocab: set):
words = []
# remove out-of-vocabulary words
with open(doc, "r") as file:
for line in file:
for word in line.split():
words.append(word)
vocab.add(word)
words = [word for word in words if word in model.wv.index_to_key]
if len(words) >= 1:
return np.mean(model.wv[words], axis=0)
else:
return []
def sent_vec(sent, cbow):
vector_size = cbow.wv.vector_size
wv_res = np.zeros(vector_size)
# print(wv_res)
ctr = 1
for w in sent:
if w in cbow.wv:
ctr += 1
wv_res += cbow.wv[w]
wv_res = wv_res/ctr
return wv_res
def spacy_tokenizer(sentence):
# Creating our token object, which is used to create documents with linguistic annotations.
# doc = nlp(sentence)
# print(doc)
# print(type(doc))
# Lemmatizing each token and converting each token into lowercase
# mytokens = [ word.lemma_.lower().strip() for word in doc ]
# print(mytokens)
# Removing stop words
# mytokens = [ word for word in mytokens if word not in stop_words and word not in punctuations ]
# return preprocessed list of tokens
return 0
def cbow_classifier(cbow, data, num_sentances):
vocab_len = len(cbow.wv.index_to_key)
embeddings = []
embedding_dict = {}
vocab = set(cbow.wv.index_to_key)
# print("Data len", len(data))
# print("Data at 0", data[0])
X_temp = np.empty([len(data), 1])
X_train_vect = np.array([np.array([cbow.wv[i] for i in ls if i in vocab])
for ls in data])
X_test_vect = np.array([np.array([cbow.wv[i] for i in ls if i in vocab])
for ls in data])
# words = [word for word in words if word in cbow.wv.index_to_key]
for word in vocab:
# embedding[word] = cbow.wv[word]
embeddings.append(np.mean(cbow.wv[word], axis=0))
embedding_dict[word] = np.mean(cbow.wv[word], axis=0)
X = embeddings
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,stratify=y)
# print(embeddings)
# print(vocab_len)
# X_train_vect_avg = []
# for v in X_train_vect:
# if v.size:
# X_train_vect_avg.append(v.mean(axis=0))
# else:
# X_train_vect_avg.append(np.zeros(100, dtype=float))
# X_test_vect_avg = []
# for v in X_test_vect:
# if v.size:
# X_test_vect_avg.append(v.mean(axis=0))
# else:
# X_test_vect_avg.append(np.zeros(100, dtype=float))
# # for i, v in enumerate(X_train_vect_avg):
# # print(len(data.iloc[i]), len(v))
# x_0 = X_train_vect_avg[0]
# num_files_per_class = 100
# y = [1] * num_files_per_class + [0] * num_files_per_class
# w = np.zeros(X_train_vect_avg.shape[1])
# x_0_dense = x_0.todense()
# x_0.dot(w)
# w,b = sgd_for_lr_with_ce(X_train_vect_avg, y)
# w
# sorted_vocab = sorted([(k,v) for k,v in enumerate(embedding_dict)],key=lambda x:x[1])
# sorted_vocab = [a for (a,b) in sorted_vocab]
# sorted_words_weights = sorted([x for x in zip(sorted_vocab, w)], key=lambda x:x[1])
# sorted_words_weights[-50:]
# preds = predict_y_lr(w,b,X_train_vect_avg)
# preds
# w,b = sgd_for_lr_with_ce(X_train_vect_avg, y, num_passes=10)
# y_pred = predict_y_lr(w,b,X_train_vect_avg)
# print(classification_report(y, y_pred))
# # compute for dev set
# pos_dev_files = glob.glob('aclImdb/test/pos/*')
# neg_dev_files = glob.glob('aclImdb/test/neg/*')
# num_dev_files_per_class = 100
# all_dev_files = pos_dev_files[:num_dev_files_per_class] + neg_dev_files[:num_dev_files_per_class]
# # use the same vectorizer from before! otherwise features won't line up
# # don't fit it again, just use it to transform!
# # X_dev = vectorizer.transform(all_dev_files)
# # y_dev = [1]* num_dev_files_per_class + [0]* num_dev_files_per_class
# # # don't need new w and b, these are from out existing model
# # y_dev_pred = predict_y_lr(w,b,X_dev)
# # print(classification_report(y_dev, y_dev_pred))
def sgd_for_lr_with_ce(X, y, num_passes=5, learning_rate = 0.1):
num_data_points = X.shape[0]
# Initialize theta -> 0
num_features = X.shape[1]
w = np.zeros(num_features)
b = 0.0
# repeat until done
# how to define "done"? let's just make it num passes for now
# we can also do norm of gradient and when it is < epsilon (something tiny)
# we stop
for current_pass in range(num_passes):
# iterate through entire dataset in random order
order = list(range(num_data_points))
random.shuffle(order)
for i in order:
# compute y-hat for this value of i given y_i and x_i
x_i = X[i]
y_i = y[i]
# need to compute based on w and b
# sigmoid(w dot x + b)
z = x_i.dot(w) + b
y_hat_i = expit(z)
# for each w (and b), modify by -lr * (y_hat_i - y_i) * x_i
w = w - learning_rate * (y_hat_i - y_i) * x_i
b = b - learning_rate * (y_hat_i - y_i)
# return theta
return w,b
def predict_y_lr(w,b,X,threshold=0.5):
# use our matrix operation version of the logistic regression model
# X dot w + b
# need to make w a column vector so the dimensions line up correctly
y_hat = X.dot( w.reshape((-1,1)) ) + b
# then just check if it's > threshold
preds = np.where(y_hat > threshold,1,0)
return preds
def main():
parser = argparse.ArgumentParser(
prog='word_embedding',
description='This program will train a word embedding model using simple wikipedia.',
epilog='To skip training the model and to used the saved model "word2vec.model", use the command --skip or -s.'
)
parser.add_argument('-s', '--skip', action='store_true')
parser.add_argument('-e', '--extra', action='store_true')
parser.add_argument('-b', '--bias', action='store_true')
parser.add_argument('-c', '--compare', action='store_true')
parser.add_argument('-t', '--text', action='store_true')
args = parser.parse_args()
skip_model = None
cbow_model = None
ud_model = None
wiki_model = None
if args.compare:
if args.skip:
# print("Skipping")
cbow_model = Word2Vec.load("word2vec.model")
skip_model = Word2Vec.load("skip2vec.model")
ud_model = KeyedVectors.load("urban2vec.model")
wiki_model = KeyedVectors.load("wiki2vec.model")
elif args.extra:
# print("Extra mode")
cbow_model = Word2Vec.load("word2vec.model")
skip_model = Word2Vec.load("skip2vec.model")
wiki_model = KeyedVectors.load_word2vec_format("wiki-news-300d-1M-subwords.vec", binary=False)
ud_model = KeyedVectors.load_word2vec_format("ud_basic.vec", binary=False)
wiki_model.save("wiki2vec.model")
ud_model.save("urban2vec.model")
else:
cbow_model, skip_model = train_embeddings()
wiki_model = KeyedVectors.load_word2vec_format("wiki-news-300d-1M-subwords.vec", binary=False)
ud_model = KeyedVectors.load_word2vec_format("ud_basic.vec", binary=False)
wiki_model.save("wiki2vec.model")
ud_model.save("urban2vec.model")
compare_embeddings(cbow_model, skip_model, ud_model, wiki_model)
if args.bias:
if args.skip:
# print("Skipping")
cbow_model = Word2Vec.load("word2vec.model")
skip_model = Word2Vec.load("skip2vec.model")
ud_model = KeyedVectors.load("urban2vec.model")
wiki_model = KeyedVectors.load("wiki2vec.model")
elif args.extra:
# print("Extra mode")
cbow_model = Word2Vec.load("word2vec.model")
skip_model = Word2Vec.load("skip2vec.model")
wiki_model = KeyedVectors.load_word2vec_format("wiki-news-300d-1M-subwords.vec", binary=False)
ud_model = KeyedVectors.load_word2vec_format("ud_basic.vec", binary=False)
wiki_model.save("wiki2vec.model")
ud_model.save("urban2vec.model")
else:
cbow_model, skip_model = train_embeddings()
wiki_model = KeyedVectors.load_word2vec_format("wiki-news-300d-1M-subwords.vec", binary=False)
ud_model = KeyedVectors.load_word2vec_format("ud_basic.vec", binary=False)
wiki_model.save("wiki2vec.model")
ud_model.save("urban2vec.model")
quantify_bias(cbow_model, skip_model, ud_model, wiki_model)
if args.text:
if args.skip:
# print("Skipping")
cbow_model = Word2Vec.load("word2vec.model")
else:
cbow_model, skip_model = train_embeddings()
text_classifier(cbow_model)
# data, sents = get_data()
# cbow_classifier(cbow_model, data, sents)
# print("No errors?")
if __name__ == "__main__":
main()