import numpy as np from sklearn.cluster import AgglomerativeClustering, KMeans from web.datasets.similarity import fetch_MEN, fetch_WS353, fetch_SimLex999, fetch_MTurk, fetch_RG65, fetch_RW, fetch_TR9856 from web.datasets.categorization import fetch_AP, fetch_battig, fetch_BLESS, fetch_ESSLI_1a, fetch_ESSLI_2b, \ fetch_ESSLI_2c from web.analogy import * from six import iteritems from web.embedding import Embedding from web.evaluate import calculate_purity, evaluate_categorization, evaluate_on_semeval_2012_2, evaluate_analogy, \ evaluate_on_WordRep, evaluate_similarity def evaluate_similarity_pearson(w, X, y): """ Calculate Pearson correlation between cosine similarity of the model and human rated similarity of word pairs Parameters ---------- w : Embedding or dict Embedding or dict instance. X: array, shape: (n_samples, 2) Word pairs y: vector, shape: (n_samples,) Human ratings Returns ------- cor: float Pearson correlation """ if isinstance(w, dict): w = Embedding.from_dict(w) missing_words = 0 words = w.vocabulary.word_id for query in X: for query_word in query: if query_word not in words: missing_words += 1 if missing_words > 0: print("Missing {} words. Will replace them with mean vector".format(missing_words)) new_x = [] new_y = [] for i in range(len(X)): if X[i, 0] in words and X[i, 1] in words: new_x.append(X[i]) new_y.append(y[i]) X = np.array(new_x) y = np.array(new_y) mean_vector = np.mean(w.vectors, axis=0, keepdims=True) A = np.vstack(list(w.get(word, mean_vector) for word in X[:, 0])) B = np.vstack(list(w.get(word, mean_vector) for word in X[:, 1])) scores = np.array([v1.dot(v2.T)/(np.linalg.norm(v1)*np.linalg.norm(v2)) for v1, v2 in zip(A, B)]) return scipy.stats.pearsonr(scores, y.squeeze()) def evaluate_similarity(w, X, y): """ Calculate Spearman correlation between cosine similarity of the model and human rated similarity of word pairs Parameters ---------- w : Embedding or dict Embedding or dict instance. X: array, shape: (n_samples, 2) Word pairs y: vector, shape: (n_samples,) Human ratings Returns ------- cor: float Spearman correlation """ if isinstance(w, dict): w = Embedding.from_dict(w) missing_words = 0 words = w.vocabulary.word_id for query in X: for query_word in query: if query_word not in words: missing_words += 1 # if missing_words > 0: # print("Missing {} words. Will replace them with mean vector".format(missing_words)) new_x = [] new_y = [] exist_cnt = 0 for i in range(len(X)): if X[i, 0] in words and X[i, 1] in words: new_x.append(X[i]) new_y.append(y[i]) exist_cnt += 1 print('exist {} in {}'.format(exist_cnt, len(X))) X = np.array(new_x) y = np.array(new_y) mean_vector = np.mean(w.vectors, axis=0, keepdims=True) A = np.vstack(w.get(word, mean_vector) for word in X[:, 0]) B = np.vstack(w.get(word, mean_vector) for word in X[:, 1]) # scores = np.array([v1.dot(v2.T)/(np.linalg.norm(v1)*np.linalg.norm(v2)) for v1, v2 in zip(A, B)]) scores = np.array([v1.dot(v2.T) for v1, v2 in zip(A, B)]) return scipy.stats.spearmanr(scores, y).correlation def evaluate_simi(wv, w2i, vocab): wv_dict = dict() for w in vocab: wv_dict[w] = wv[w2i[w], :] if isinstance(wv_dict, dict): w = Embedding.from_dict(wv_dict) # Calculate results on similarity print("Calculating similarity benchmarks") similarity_tasks = { "WS353": fetch_WS353(), "RG65": fetch_RG65(), # "WS353R": fetch_WS353(which="relatedness"), # "WS353S": fetch_WS353(which="similarity"), "SimLex999": fetch_SimLex999(), "MTurk": fetch_MTurk(), "RW": fetch_RW(), "MEN": fetch_MEN(), } # similarity_results = {} for name, data in iteritems(similarity_tasks): print("Sample data from {}, num of samples: {} : pair \"{}\" and \"{}\" is assigned score {}".format( name, len(data.X), data.X[0][0], data.X[0][1], data.y[0])) score = evaluate_similarity(w, data.X, data.y) print("Spearman correlation of scores on {} {}".format(name, score)) # score, p_value = evaluate_similarity_pearson(w, data.X, data.y) # print("Pearson correlation of scores on {} {}, p value: {}".format(name, score, p_value)) def evaluate_categorization(w, X, y, method="kmeans", seed=None): """ Evaluate embeddings on categorization task. Parameters ---------- w: Embedding or dict Embedding to test. X: vector, shape: (n_samples, ) Vector of words. y: vector, shape: (n_samples, ) Vector of cluster assignments. method: string, default: "all" What method to use. Possible values are "agglomerative", "kmeans", "all. If "agglomerative" is passed, method will fit AgglomerativeClustering (with very crude hyperparameter tuning to avoid overfitting). If "kmeans" is passed, method will fit KMeans. In both cases number of clusters is preset to the correct value. seed: int, default: None Seed passed to KMeans. Returns ------- purity: float Purity of the best obtained clustering. Notes ----- KMedoids method was excluded as empirically didn't improve over KMeans (for categorization tasks available in the package). """ if isinstance(w, dict): w = Embedding.from_dict(w) assert method in ["all", "kmeans", "agglomerative"], "Uncrecognized method" mean_vector = np.mean(w.vectors, axis=0, keepdims=True) new_x = [] new_y = [] exist_cnt = 0 for idx, word in enumerate(X.flatten()): if word in w : new_x.append(X[idx]) new_y.append(y[idx]) exist_cnt += 1 print('exist {} in {}'.format(exist_cnt, len(X))) X = np.array(new_x) y = np.array(new_y) words = np.vstack([w.get(word, mean_vector) for word in X.flatten()]) ids = np.random.RandomState(seed).choice(range(len(X)), len(X), replace=False) # Evaluate clustering on several hyperparameters of AgglomerativeClustering and # KMeans best_purity = 0 if method == "all" or method == "agglomerative": best_purity = calculate_purity(y[ids], AgglomerativeClustering(n_clusters=len(set(y)), affinity="euclidean", linkage="ward").fit_predict(words[ids])) logger.debug("Purity={:.3f} using affinity={} linkage={}".format(best_purity, 'euclidean', 'ward')) for affinity in ["cosine", "euclidean"]: for linkage in ["average", "complete"]: purity = calculate_purity(y[ids], AgglomerativeClustering(n_clusters=len(set(y)), affinity=affinity, linkage=linkage).fit_predict(words[ids])) logger.debug("Purity={:.3f} using affinity={} linkage={}".format(purity, affinity, linkage)) best_purity = max(best_purity, purity) if method == "all" or method == "kmeans": purity = calculate_purity(y[ids], KMeans(random_state=seed, n_init=10, n_clusters=len(set(y))). fit_predict(words[ids])) logger.debug("Purity={:.3f} using KMeans".format(purity)) best_purity = max(purity, best_purity) return best_purity def evaluate_cate(wv, w2i, vocab, method="all", seed=None): """ method: string, default: "all" What method to use. Possible values are "agglomerative", "kmeans", "all. If "agglomerative" is passed, method will fit AgglomerativeClustering (with very crude hyperparameter tuning to avoid overfitting). If "kmeans" is passed, method will fit KMeans. In both cases number of clusters is preset to the correct value. seed: int, default: None Seed passed to KMeans. """ wv_dict = dict() for w in vocab: wv_dict[w] = wv[w2i[w], :] if isinstance(wv_dict, dict): w = Embedding.from_dict(wv_dict) # Calculate results on categorization print("Calculating categorization benchmarks") categorization_tasks = { "AP": fetch_AP(), "ESSLI_2c": fetch_ESSLI_2c(), "ESSLI_2b": fetch_ESSLI_2b(), "ESSLI_1a": fetch_ESSLI_1a(), "Battig": fetch_battig(), "BLESS": fetch_BLESS(), } categorization_results = {} # Calculate results using helper function for name, data in iteritems(categorization_tasks): print("Sample data from {}, num of samples: {} : \"{}\" is assigned class {}".format( name, len(data.X), data.X[0], data.y[0])) categorization_results[name] = evaluate_categorization(w, data.X, data.y, method=method, seed=None) print("Cluster purity on {} {}".format(name, categorization_results[name])) def evaluate_analogy_google(W, vocab): """Evaluate the trained w vectors on a variety of tasks""" filenames = [ 'capital-common-countries.txt', 'capital-world.txt', 'currency.txt', 'city-in-state.txt', 'family.txt', 'gram1-adjective-to-adverb.txt', 'gram2-opposite.txt', 'gram3-comparative.txt', 'gram4-superlative.txt', 'gram5-present-participle.txt', 'gram6-nationality-adjective.txt', 'gram7-past-tense.txt', 'gram8-plural.txt', 'gram9-plural-verbs.txt', ] prefix = '/zf15/tw8cb/summer_2019/code/GloVe/eval/question-data/' # to avoid memory overflow, could be increased/decreased # depending on system and vocab size split_size = 100 correct_sem = 0; # count correct semantic questions correct_syn = 0; # count correct syntactic questions correct_tot = 0 # count correct questions count_sem = 0; # count all semantic questions count_syn = 0; # count all syntactic questions count_tot = 0 # count all questions full_count = 0 # count all questions, including those with unknown words for i in range(len(filenames)): with open('%s/%s' % (prefix, filenames[i]), 'r') as f: full_data = [line.rstrip().split(' ') for line in f] full_count += len(full_data) data = [x for x in full_data if all(word in vocab for word in x)] indices = np.array([[vocab[word] for word in row] for row in data]) ind1, ind2, ind3, ind4 = indices.T predictions = np.zeros((len(indices),)) num_iter = int(np.ceil(len(indices) / float(split_size))) for j in range(num_iter): subset = np.arange(j*split_size, min((j + 1)*split_size, len(ind1))) pred_vec = (W[ind2[subset], :] - W[ind1[subset], :] + W[ind3[subset], :]) #cosine similarity if input W has been normalized dist = np.dot(W, pred_vec.T) for k in range(len(subset)): dist[ind1[subset[k]], k] = -np.Inf dist[ind2[subset[k]], k] = -np.Inf dist[ind3[subset[k]], k] = -np.Inf # predicted word index predictions[subset] = np.argmax(dist, 0).flatten() val = (ind4 == predictions) # correct predictions count_tot = count_tot + len(ind1) correct_tot = correct_tot + sum(val) if i < 5: count_sem = count_sem + len(ind1) correct_sem = correct_sem + sum(val) else: count_syn = count_syn + len(ind1) correct_syn = correct_syn + sum(val) print("%s:" % filenames[i]) print('ACCURACY TOP1: %.2f%% (%d/%d)' % (np.mean(val) * 100, np.sum(val), len(val))) print('Questions seen/total: %.2f%% (%d/%d)' % (100 * count_tot / float(full_count), count_tot, full_count)) print('Semantic accuracy: %.2f%% (%i/%i)' % (100 * correct_sem / float(count_sem), correct_sem, count_sem)) print('Syntactic accuracy: %.2f%% (%i/%i)' % (100 * correct_syn / float(count_syn), correct_syn, count_syn)) print('Total accuracy: %.2f%% (%i/%i)' % (100 * correct_tot / float(count_tot), correct_tot, count_tot)) def evaluate_analogy_msr(W, vocab, file_name='EN-MSR.txt'): """Evaluate the trained word vectors on a variety of tasks""" prefix = '/zf15/tw8cb/summer_2019/code/GloVe/eval/question-data/' # to avoid memory overflow, could be increased/decreased # depending on system and vocab size split_size = 100 correct_sem = 0; # count correct semantic questions correct_syn = 0; # count correct syntactic questions correct_tot = 0 # count correct questions count_sem = 0; # count all semantic questions count_syn = 0; # count all syntactic questions count_tot = 0 # count all questions full_count = 0 # count all questions, including those with unknown words with open('%s/%s' % (prefix, file_name), 'r') as f: full_data = [] for line in f: tokens = line.rstrip().split(' ') full_data.append([tokens[0], tokens[1], tokens[2], tokens[4]]) full_count += len(full_data) data = [x for x in full_data if all(word in vocab for word in x)] indices = np.array([[vocab[word] for word in row] for row in data]) ind1, ind2, ind3, ind4 = indices.T predictions = np.zeros((len(indices),)) num_iter = int(np.ceil(len(indices) / float(split_size))) for j in range(num_iter): subset = np.arange(j*split_size, min((j + 1)*split_size, len(ind1))) pred_vec = (W[ind2[subset], :] - W[ind1[subset], :] + W[ind3[subset], :]) #cosine similarity if input W has been normalized dist = np.dot(W, pred_vec.T) for k in range(len(subset)): dist[ind1[subset[k]], k] = -np.Inf dist[ind2[subset[k]], k] = -np.Inf dist[ind3[subset[k]], k] = -np.Inf # predicted word index predictions[subset] = np.argmax(dist, 0).flatten() val = (ind4 == predictions) # correct predictions count_tot = count_tot + len(ind1) correct_tot = correct_tot + sum(val) # print("%s:" % filenames[i]) print(len(val)) print('ACCURACY TOP1-MSR: %.2f%% (%d/%d)' % (np.mean(val) * 100, np.sum(val), len(val))) def evaluate_analogy_semeval2012(w_dict): score = evaluate_on_semeval_2012_2(w_dict)['all'] print("Analogy prediction accuracy on {} {}".format("SemEval2012", score)) def evaluate_ana(wv, w2i, vocab): W_norm = np.zeros(wv.shape) d = (np.sum(wv ** 2, 1) ** (0.5)) W_norm = (wv.T / d).T evaluate_analogy_msr(W_norm, w2i) evaluate_analogy_google(W_norm, w2i) wv_dict = dict() for w in vocab: wv_dict[w] = W_norm[w2i[w], :] if isinstance(wv_dict, dict): w = Embedding.from_dict(wv_dict) evaluate_analogy_semeval2012(w) # analogy_tasks = { # "Google": fetch_google_analogy(), # "MSR": fetch_msr_analogy() # } # analogy_results = {} # for name, data in iteritems(analogy_tasks): # analogy_results[name] = evaluate_analogy(w, data.X, data.y) # print("Analogy prediction accuracy on {} {}".format(name, analogy_results[name]))