Spaces:
Runtime error
Runtime error
| import numpy as np | |
| from sklearn.cluster import AgglomerativeClustering, KMeans | |
| from web.datasets.similarity import fetch_MEN, fetch_WS353, fetch_SimLex999, fetch_MTurk, fetch_RG65, fetch_RW, fetch_TR9856 | |
| from web.datasets.categorization import fetch_AP, fetch_battig, fetch_BLESS, fetch_ESSLI_1a, fetch_ESSLI_2b, \ | |
| fetch_ESSLI_2c | |
| from web.analogy import * | |
| from six import iteritems | |
| from web.embedding import Embedding | |
| from web.evaluate import calculate_purity, evaluate_categorization, evaluate_on_semeval_2012_2, evaluate_analogy, \ | |
| evaluate_on_WordRep, evaluate_similarity | |
| def evaluate_similarity_pearson(w, X, y): | |
| """ | |
| Calculate Pearson correlation between cosine similarity of the model | |
| and human rated similarity of word pairs | |
| Parameters | |
| ---------- | |
| w : Embedding or dict | |
| Embedding or dict instance. | |
| X: array, shape: (n_samples, 2) | |
| Word pairs | |
| y: vector, shape: (n_samples,) | |
| Human ratings | |
| Returns | |
| ------- | |
| cor: float | |
| Pearson correlation | |
| """ | |
| if isinstance(w, dict): | |
| w = Embedding.from_dict(w) | |
| missing_words = 0 | |
| words = w.vocabulary.word_id | |
| for query in X: | |
| for query_word in query: | |
| if query_word not in words: | |
| missing_words += 1 | |
| if missing_words > 0: | |
| print("Missing {} words. Will replace them with mean vector".format(missing_words)) | |
| new_x = [] | |
| new_y = [] | |
| for i in range(len(X)): | |
| if X[i, 0] in words and X[i, 1] in words: | |
| new_x.append(X[i]) | |
| new_y.append(y[i]) | |
| X = np.array(new_x) | |
| y = np.array(new_y) | |
| mean_vector = np.mean(w.vectors, axis=0, keepdims=True) | |
| A = np.vstack(list(w.get(word, mean_vector) for word in X[:, 0])) | |
| B = np.vstack(list(w.get(word, mean_vector) for word in X[:, 1])) | |
| scores = np.array([v1.dot(v2.T)/(np.linalg.norm(v1)*np.linalg.norm(v2)) for v1, v2 in zip(A, B)]) | |
| return scipy.stats.pearsonr(scores, y.squeeze()) | |
| def evaluate_similarity(w, X, y): | |
| """ | |
| Calculate Spearman correlation between cosine similarity of the model | |
| and human rated similarity of word pairs | |
| Parameters | |
| ---------- | |
| w : Embedding or dict | |
| Embedding or dict instance. | |
| X: array, shape: (n_samples, 2) | |
| Word pairs | |
| y: vector, shape: (n_samples,) | |
| Human ratings | |
| Returns | |
| ------- | |
| cor: float | |
| Spearman correlation | |
| """ | |
| if isinstance(w, dict): | |
| w = Embedding.from_dict(w) | |
| missing_words = 0 | |
| words = w.vocabulary.word_id | |
| for query in X: | |
| for query_word in query: | |
| if query_word not in words: | |
| missing_words += 1 | |
| # if missing_words > 0: | |
| # print("Missing {} words. Will replace them with mean vector".format(missing_words)) | |
| new_x = [] | |
| new_y = [] | |
| exist_cnt = 0 | |
| for i in range(len(X)): | |
| if X[i, 0] in words and X[i, 1] in words: | |
| new_x.append(X[i]) | |
| new_y.append(y[i]) | |
| exist_cnt += 1 | |
| print('exist {} in {}'.format(exist_cnt, len(X))) | |
| X = np.array(new_x) | |
| y = np.array(new_y) | |
| mean_vector = np.mean(w.vectors, axis=0, keepdims=True) | |
| A = np.vstack(w.get(word, mean_vector) for word in X[:, 0]) | |
| B = np.vstack(w.get(word, mean_vector) for word in X[:, 1]) | |
| # scores = np.array([v1.dot(v2.T)/(np.linalg.norm(v1)*np.linalg.norm(v2)) for v1, v2 in zip(A, B)]) | |
| scores = np.array([v1.dot(v2.T) for v1, v2 in zip(A, B)]) | |
| return scipy.stats.spearmanr(scores, y).correlation | |
| def evaluate_simi(wv, w2i, vocab): | |
| wv_dict = dict() | |
| for w in vocab: | |
| wv_dict[w] = wv[w2i[w], :] | |
| if isinstance(wv_dict, dict): | |
| w = Embedding.from_dict(wv_dict) | |
| # Calculate results on similarity | |
| print("Calculating similarity benchmarks") | |
| similarity_tasks = { | |
| "WS353": fetch_WS353(), | |
| "RG65": fetch_RG65(), | |
| # "WS353R": fetch_WS353(which="relatedness"), | |
| # "WS353S": fetch_WS353(which="similarity"), | |
| "SimLex999": fetch_SimLex999(), | |
| "MTurk": fetch_MTurk(), | |
| "RW": fetch_RW(), | |
| "MEN": fetch_MEN(), | |
| } | |
| # similarity_results = {} | |
| for name, data in iteritems(similarity_tasks): | |
| print("Sample data from {}, num of samples: {} : pair \"{}\" and \"{}\" is assigned score {}".format( | |
| name, len(data.X), data.X[0][0], data.X[0][1], data.y[0])) | |
| score = evaluate_similarity(w, data.X, data.y) | |
| print("Spearman correlation of scores on {} {}".format(name, score)) | |
| # score, p_value = evaluate_similarity_pearson(w, data.X, data.y) | |
| # print("Pearson correlation of scores on {} {}, p value: {}".format(name, score, p_value)) | |
| def evaluate_categorization(w, X, y, method="kmeans", seed=None): | |
| """ | |
| Evaluate embeddings on categorization task. | |
| Parameters | |
| ---------- | |
| w: Embedding or dict | |
| Embedding to test. | |
| X: vector, shape: (n_samples, ) | |
| Vector of words. | |
| y: vector, shape: (n_samples, ) | |
| Vector of cluster assignments. | |
| method: string, default: "all" | |
| What method to use. Possible values are "agglomerative", "kmeans", "all. | |
| If "agglomerative" is passed, method will fit AgglomerativeClustering (with very crude | |
| hyperparameter tuning to avoid overfitting). | |
| If "kmeans" is passed, method will fit KMeans. | |
| In both cases number of clusters is preset to the correct value. | |
| seed: int, default: None | |
| Seed passed to KMeans. | |
| Returns | |
| ------- | |
| purity: float | |
| Purity of the best obtained clustering. | |
| Notes | |
| ----- | |
| KMedoids method was excluded as empirically didn't improve over KMeans (for categorization | |
| tasks available in the package). | |
| """ | |
| if isinstance(w, dict): | |
| w = Embedding.from_dict(w) | |
| assert method in ["all", "kmeans", "agglomerative"], "Uncrecognized method" | |
| mean_vector = np.mean(w.vectors, axis=0, keepdims=True) | |
| new_x = [] | |
| new_y = [] | |
| exist_cnt = 0 | |
| for idx, word in enumerate(X.flatten()): | |
| if word in w : | |
| new_x.append(X[idx]) | |
| new_y.append(y[idx]) | |
| exist_cnt += 1 | |
| print('exist {} in {}'.format(exist_cnt, len(X))) | |
| X = np.array(new_x) | |
| y = np.array(new_y) | |
| words = np.vstack([w.get(word, mean_vector) for word in X.flatten()]) | |
| ids = np.random.RandomState(seed).choice(range(len(X)), len(X), replace=False) | |
| # Evaluate clustering on several hyperparameters of AgglomerativeClustering and | |
| # KMeans | |
| best_purity = 0 | |
| if method == "all" or method == "agglomerative": | |
| best_purity = calculate_purity(y[ids], AgglomerativeClustering(n_clusters=len(set(y)), | |
| affinity="euclidean", | |
| linkage="ward").fit_predict(words[ids])) | |
| logger.debug("Purity={:.3f} using affinity={} linkage={}".format(best_purity, 'euclidean', 'ward')) | |
| for affinity in ["cosine", "euclidean"]: | |
| for linkage in ["average", "complete"]: | |
| purity = calculate_purity(y[ids], AgglomerativeClustering(n_clusters=len(set(y)), | |
| affinity=affinity, | |
| linkage=linkage).fit_predict(words[ids])) | |
| logger.debug("Purity={:.3f} using affinity={} linkage={}".format(purity, affinity, linkage)) | |
| best_purity = max(best_purity, purity) | |
| if method == "all" or method == "kmeans": | |
| purity = calculate_purity(y[ids], KMeans(random_state=seed, n_init=10, n_clusters=len(set(y))). | |
| fit_predict(words[ids])) | |
| logger.debug("Purity={:.3f} using KMeans".format(purity)) | |
| best_purity = max(purity, best_purity) | |
| return best_purity | |
| def evaluate_cate(wv, w2i, vocab, method="all", seed=None): | |
| """ | |
| method: string, default: "all" | |
| What method to use. Possible values are "agglomerative", "kmeans", "all. | |
| If "agglomerative" is passed, method will fit AgglomerativeClustering (with very crude | |
| hyperparameter tuning to avoid overfitting). | |
| If "kmeans" is passed, method will fit KMeans. | |
| In both cases number of clusters is preset to the correct value. | |
| seed: int, default: None | |
| Seed passed to KMeans. | |
| """ | |
| wv_dict = dict() | |
| for w in vocab: | |
| wv_dict[w] = wv[w2i[w], :] | |
| if isinstance(wv_dict, dict): | |
| w = Embedding.from_dict(wv_dict) | |
| # Calculate results on categorization | |
| print("Calculating categorization benchmarks") | |
| categorization_tasks = { | |
| "AP": fetch_AP(), | |
| "ESSLI_2c": fetch_ESSLI_2c(), | |
| "ESSLI_2b": fetch_ESSLI_2b(), | |
| "ESSLI_1a": fetch_ESSLI_1a(), | |
| "Battig": fetch_battig(), | |
| "BLESS": fetch_BLESS(), | |
| } | |
| categorization_results = {} | |
| # Calculate results using helper function | |
| for name, data in iteritems(categorization_tasks): | |
| print("Sample data from {}, num of samples: {} : \"{}\" is assigned class {}".format( | |
| name, len(data.X), data.X[0], data.y[0])) | |
| categorization_results[name] = evaluate_categorization(w, data.X, data.y, method=method, seed=None) | |
| print("Cluster purity on {} {}".format(name, categorization_results[name])) | |
| def evaluate_analogy_google(W, vocab): | |
| """Evaluate the trained w vectors on a variety of tasks""" | |
| filenames = [ | |
| 'capital-common-countries.txt', 'capital-world.txt', 'currency.txt', | |
| 'city-in-state.txt', 'family.txt', 'gram1-adjective-to-adverb.txt', | |
| 'gram2-opposite.txt', 'gram3-comparative.txt', 'gram4-superlative.txt', | |
| 'gram5-present-participle.txt', 'gram6-nationality-adjective.txt', | |
| 'gram7-past-tense.txt', 'gram8-plural.txt', 'gram9-plural-verbs.txt', | |
| ] | |
| prefix = '/zf15/tw8cb/summer_2019/code/GloVe/eval/question-data/' | |
| # to avoid memory overflow, could be increased/decreased | |
| # depending on system and vocab size | |
| split_size = 100 | |
| correct_sem = 0; # count correct semantic questions | |
| correct_syn = 0; # count correct syntactic questions | |
| correct_tot = 0 # count correct questions | |
| count_sem = 0; # count all semantic questions | |
| count_syn = 0; # count all syntactic questions | |
| count_tot = 0 # count all questions | |
| full_count = 0 # count all questions, including those with unknown words | |
| for i in range(len(filenames)): | |
| with open('%s/%s' % (prefix, filenames[i]), 'r') as f: | |
| full_data = [line.rstrip().split(' ') for line in f] | |
| full_count += len(full_data) | |
| data = [x for x in full_data if all(word in vocab for word in x)] | |
| indices = np.array([[vocab[word] for word in row] for row in data]) | |
| ind1, ind2, ind3, ind4 = indices.T | |
| predictions = np.zeros((len(indices),)) | |
| num_iter = int(np.ceil(len(indices) / float(split_size))) | |
| for j in range(num_iter): | |
| subset = np.arange(j*split_size, min((j + 1)*split_size, len(ind1))) | |
| pred_vec = (W[ind2[subset], :] - W[ind1[subset], :] | |
| + W[ind3[subset], :]) | |
| #cosine similarity if input W has been normalized | |
| dist = np.dot(W, pred_vec.T) | |
| for k in range(len(subset)): | |
| dist[ind1[subset[k]], k] = -np.Inf | |
| dist[ind2[subset[k]], k] = -np.Inf | |
| dist[ind3[subset[k]], k] = -np.Inf | |
| # predicted word index | |
| predictions[subset] = np.argmax(dist, 0).flatten() | |
| val = (ind4 == predictions) # correct predictions | |
| count_tot = count_tot + len(ind1) | |
| correct_tot = correct_tot + sum(val) | |
| if i < 5: | |
| count_sem = count_sem + len(ind1) | |
| correct_sem = correct_sem + sum(val) | |
| else: | |
| count_syn = count_syn + len(ind1) | |
| correct_syn = correct_syn + sum(val) | |
| print("%s:" % filenames[i]) | |
| print('ACCURACY TOP1: %.2f%% (%d/%d)' % | |
| (np.mean(val) * 100, np.sum(val), len(val))) | |
| print('Questions seen/total: %.2f%% (%d/%d)' % | |
| (100 * count_tot / float(full_count), count_tot, full_count)) | |
| print('Semantic accuracy: %.2f%% (%i/%i)' % | |
| (100 * correct_sem / float(count_sem), correct_sem, count_sem)) | |
| print('Syntactic accuracy: %.2f%% (%i/%i)' % | |
| (100 * correct_syn / float(count_syn), correct_syn, count_syn)) | |
| print('Total accuracy: %.2f%% (%i/%i)' % (100 * correct_tot / float(count_tot), correct_tot, count_tot)) | |
| def evaluate_analogy_msr(W, vocab, file_name='EN-MSR.txt'): | |
| """Evaluate the trained word vectors on a variety of tasks""" | |
| prefix = '/zf15/tw8cb/summer_2019/code/GloVe/eval/question-data/' | |
| # to avoid memory overflow, could be increased/decreased | |
| # depending on system and vocab size | |
| split_size = 100 | |
| correct_sem = 0; # count correct semantic questions | |
| correct_syn = 0; # count correct syntactic questions | |
| correct_tot = 0 # count correct questions | |
| count_sem = 0; # count all semantic questions | |
| count_syn = 0; # count all syntactic questions | |
| count_tot = 0 # count all questions | |
| full_count = 0 # count all questions, including those with unknown words | |
| with open('%s/%s' % (prefix, file_name), 'r') as f: | |
| full_data = [] | |
| for line in f: | |
| tokens = line.rstrip().split(' ') | |
| full_data.append([tokens[0], tokens[1], tokens[2], tokens[4]]) | |
| full_count += len(full_data) | |
| data = [x for x in full_data if all(word in vocab for word in x)] | |
| indices = np.array([[vocab[word] for word in row] for row in data]) | |
| ind1, ind2, ind3, ind4 = indices.T | |
| predictions = np.zeros((len(indices),)) | |
| num_iter = int(np.ceil(len(indices) / float(split_size))) | |
| for j in range(num_iter): | |
| subset = np.arange(j*split_size, min((j + 1)*split_size, len(ind1))) | |
| pred_vec = (W[ind2[subset], :] - W[ind1[subset], :] | |
| + W[ind3[subset], :]) | |
| #cosine similarity if input W has been normalized | |
| dist = np.dot(W, pred_vec.T) | |
| for k in range(len(subset)): | |
| dist[ind1[subset[k]], k] = -np.Inf | |
| dist[ind2[subset[k]], k] = -np.Inf | |
| dist[ind3[subset[k]], k] = -np.Inf | |
| # predicted word index | |
| predictions[subset] = np.argmax(dist, 0).flatten() | |
| val = (ind4 == predictions) # correct predictions | |
| count_tot = count_tot + len(ind1) | |
| correct_tot = correct_tot + sum(val) | |
| # print("%s:" % filenames[i]) | |
| print(len(val)) | |
| print('ACCURACY TOP1-MSR: %.2f%% (%d/%d)' % | |
| (np.mean(val) * 100, np.sum(val), len(val))) | |
| def evaluate_analogy_semeval2012(w_dict): | |
| score = evaluate_on_semeval_2012_2(w_dict)['all'] | |
| print("Analogy prediction accuracy on {} {}".format("SemEval2012", score)) | |
| def evaluate_ana(wv, w2i, vocab): | |
| W_norm = np.zeros(wv.shape) | |
| d = (np.sum(wv ** 2, 1) ** (0.5)) | |
| W_norm = (wv.T / d).T | |
| evaluate_analogy_msr(W_norm, w2i) | |
| evaluate_analogy_google(W_norm, w2i) | |
| wv_dict = dict() | |
| for w in vocab: | |
| wv_dict[w] = W_norm[w2i[w], :] | |
| if isinstance(wv_dict, dict): | |
| w = Embedding.from_dict(wv_dict) | |
| evaluate_analogy_semeval2012(w) | |
| # analogy_tasks = { | |
| # "Google": fetch_google_analogy(), | |
| # "MSR": fetch_msr_analogy() | |
| # } | |
| # analogy_results = {} | |
| # for name, data in iteritems(analogy_tasks): | |
| # analogy_results[name] = evaluate_analogy(w, data.X, data.y) | |
| # print("Analogy prediction accuracy on {} {}".format(name, analogy_results[name])) |