# Download and preprocessing data - do not change
import requests
import numpy as np
from tqdm.notebook import tqdm

url = "http://download.tensorflow.org/data/questions-words.txt"
response = requests.get(url)
response.raise_for_status() # Raise an exception for invalid HTTP status codes
analogy = response.text.split('\n')

# filtering out analogies that do not exist in our vocabulary
task = []
for line in tqdm(analogy, total=len(analogy)):
  row = line.split(' ')
  if len(row) != 4 or line.startswith(':'):
    continue
  if sum(_ in vocab for _ in row) == 4:
    task.append(row)
len(task), task[-1]

import scipy.special
import numpy as np

D = 50 # embedding_dimensions
w_vectors = np.random.rand(len(vocab), D) # target
c_vectors = np.random.rand(len(vocab), D) # context and noise

start_time = time.time()
LR = 1e-2 # learning_rate
K = 15 # num_negatives
span_length = 5 # number of words before and after target in context
num_iter = 500_000 # number of iterations or update steps

losses = []
for iter in tqdm(range(num_iter)):
    # Randomly select a target word index
    i = np.random.randint(span_length, len(data) - span_length)
    target_word = data[i]
    target_idx = revmap[target_word]
    v_target = w_vectors[target_idx]

    # Get context word indices within the window
    context_indices = []
    for offset in range(-span_length, span_length + 1):
        if offset == 0:
            continue  # Skip target word itself
        context_word = data[i + offset]
        context_idx = revmap[context_word]
        context_indices.append(context_idx)

    # Iterate over context words
    for context_idx in context_indices:
        u_context = c_vectors[context_idx]

        # Generate K negative samples
        negative_indices = []
        for _ in range(K):
            negative_idx = np.random.randint(0, len(vocab))
            negative_indices.append(negative_idx)
        u_negatives = [c_vectors[neg_idx] for neg_idx in negative_indices]

        # Compute positive and negative scores
        pos_score = scipy.special.expit(np.dot(u_context, v_target))
        neg_scores = []
        for u_negative in u_negatives:
            neg_score = scipy.special.expit(-np.dot(u_negative, v_target))
            neg_scores.append(neg_score)

        # Calculate loss
        neg_scores_array = np.array(neg_scores)
        loss = -np.log(pos_score) - np.sum(np.log(neg_scores_array))
        losses.append(loss)

        # Compute gradients
        grad_v_target = (1 - pos_score) * u_context.copy()
        for k in range(K):
            grad_v_target -= (1 - neg_scores[k]) * u_negatives[k]

        grad_u_context = (1 - pos_score) * v_target.copy()

        grad_u_negatives = []
        for k in range(K):
            grad_u_negative = (1 - neg_scores[k]) * (-v_target)
            grad_u_negatives.append(grad_u_negative)

        # Update embeddings
        w_vectors[target_idx] += LR * grad_v_target
        c_vectors[context_idx] += LR * grad_u_context
        for idx, neg_idx in enumerate(negative_indices):
            c_vectors[neg_idx] += LR * grad_u_negatives[idx]

    # Logging the average loss every 10K iterations
    if iter % 10_000 == 0:
        if losses:
            print(f"Iteration {iter}, Average Loss: {sum(losses) / len(losses)}")
            losses = []  # Resetting losses
        else:
            print(f"Iteration {iter}, No losses recorded.")

print('training time:', time.time()-start_time)

w_vectors_norms = np.linalg.norm(w_vectors, axis=1)
c_vectors_norms = np.linalg.norm(c_vectors, axis=1)

get_nearest_neighbours('First', vectors=w_vectors, top_k=5, vocab=vocab,
                       norms=w_vectors_norms)

get_nearest_neighbours('First', vectors=c_vectors, top_k=5, vocab=vocab,
                       norms=c_vectors_norms)

# evaluation - do not change
start_time = time.time()
hit_at_k = 0.
for a,b,c,d in tqdm(task): # Note down this Inference Time
  d_ = get_nearest_neighbours(
        word_or_vector=w_vectors[revmap[a]]+w_vectors[revmap[b]]-w_vectors[revmap[c]],
        top_k=100,
        vectors=w_vectors,
        norms=w_vectors_norms,
      )
  # Note that we could have used c_vectors as well instead of w_vectors
  if d in d_:
    hit_at_k += 1
hit_at_k/=len(task)
print('inference time:', time.time() - start_time)
print(hit_at_k)

import sys  # For getting the size of objects in bytes
import numpy as np

# For Co-occurrence Embeddings
# Assuming 'counts' and 'ppmi' matrices are already computed
counts_size_bytes = counts.nbytes
ppmi_size_bytes = ppmi.nbytes

# Total size in bytes
cooccur_total_size_bytes = counts_size_bytes + ppmi_size_bytes

# Convert to Megabytes
cooccur_total_size_mb = cooccur_total_size_bytes / (1024 ** 2)
print(f"Co-occurrence Embeddings Total Size: {cooccur_total_size_mb:.2f} MB")

# For Word2Vec Embeddings
# Assuming 'w_vectors' and 'c_vectors' are already computed
w_vectors_size_bytes = w_vectors.nbytes
c_vectors_size_bytes = c_vectors.nbytes

# Total size in bytes
word2vec_total_size_bytes = w_vectors_size_bytes + c_vectors_size_bytes

# Convert to Megabytes
word2vec_total_size_mb = word2vec_total_size_bytes / (1024 ** 2)
print(f"Word2Vec Embeddings Total Size: {word2vec_total_size_mb:.2f} MB")

# Training Times
# Assuming you have recorded the training times previously as 'cooccur_training_time' and 'word2vec_training_time'
# For example:
# cooccur_training_time = ...  # The time printed after training co-occurrence embeddings
# word2vec_training_time = ...  # The time printed after training Word2Vec embeddings

# As an example, let's say the times were recorded as:
# (Please replace these with your actual recorded times)
cooccur_training_time = 150.0  # in seconds
word2vec_training_time = 1800.0  # in seconds

# Convert times to minutes for easier comparison
cooccur_training_time_minutes = cooccur_training_time / 60
word2vec_training_time_minutes = word2vec_training_time / 60

print(f"Co-occurrence Embeddings Training Time: {cooccur_training_time_minutes:.2f} minutes")
print(f"Word2Vec Embeddings Training Time: {word2vec_training_time_minutes:.2f} minutes")

# Inference Speed (Nearest Neighbors)
# Let's measure the time it takes to find nearest neighbors for a sample word
import time

sample_word = 'eye'  # You can choose any word from your vocabulary

# For Co-occurrence Embeddings
start_time = time.time()
nearest_neighbors_ppmi = get_nearest_neighbours(sample_word, vectors=ppmi, top_k=5, vocab=vocab, norms=ppmi_norms)
ppmi_inference_time = time.time() - start_time
print(f"Co-occurrence Embeddings Inference Time: {ppmi_inference_time*1000:.2f} ms")

# For Word2Vec Embeddings
# First, we need to compute the combined embeddings (average of input and output embeddings)
word2vec_vectors = (w_vectors + c_vectors) / 2
word2vec_norms = np.linalg.norm(word2vec_vectors, axis=1)

start_time = time.time()
nearest_neighbors_word2vec = get_nearest_neighbours(sample_word, vectors=word2vec_vectors, top_k=5, vocab=vocab, norms=word2vec_norms)
word2vec_inference_time = time.time() - start_time
print(f"Word2Vec Embeddings Inference Time: {word2vec_inference_time*1000:.2f} ms")

# Summary
print("\nSummary:")
print(f"Co-occurrence Embeddings:")
print(f"- Total Size: {cooccur_total_size_mb:.2f} MB")
print(f"- Training Time: {cooccur_training_time_minutes:.2f} minutes")
print(f"- Inference Time: {ppmi_inference_time*1000:.2f} ms")

print(f"\nWord2Vec Embeddings:")
print(f"- Total Size: {word2vec_total_size_mb:.2f} MB")
print(f"- Training Time: {word2vec_training_time_minutes:.2f} minutes")
print(f"- Inference Time: {word2vec_inference_time*1000:.2f} ms")

\# COMPLETE THIS TABLE.

Round each value up to two decimal points and use seconds for speed, MBs for memory, and % for Analogy Hit@100 scores.

Method|Training Speed|Inference Speed|Memory|Analogy
---|---|---|---|---
PPMI|2.50 minutes|50.52 ms|5008.88 MB|
SGNS|30.00 minutes|1.35 ms|13.82 MB|

Write the code in the last code block for computing analogy hit