Upload 4 files

data/dataset.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6c10dbf7a899fbf0553bf6cab5fd11abf35cf224e4e6e4f7843fdd19144c550
+size 19266108

data/embeddings_main.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b33d9e4726eff511c3f0f74dd9d1f22f863828aa0c03ff060c2983be3dce0115
+size 45892736

data/faiss_index_main.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5fbaa50af8354c8a54372b1c763337f98792c351fa2e3aa266f448ec8266da2
+size 45892653

data/func.py

@@ -0,0 +1,53 @@
+import pandas as pd
+import numpy as np
+import torch
+from transformers import BertModel, BertTokenizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+
+tokenizer = BertTokenizer.from_pretrained("DeepPavlov/rubert-base-cased-sentence")
+model = BertModel.from_pretrained("DeepPavlov/rubert-base-cased-sentence")
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+
+def filter_by_ganre(df: pd.DataFrame, ganre_list: list):
+    filtered_df = df[df['ganres'].apply(lambda x: any(g in ganre_list for g in(x)))]
+    filt_ind = filtered_df.index.to_list()
+    return filt_ind
+
+# def mean_pooling(model_output, attention_mask):
+#     token_embeddings = model_output['last_hidden_state']
+#     input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+#     sum_embeddings = torch.sum(token_embeddings * input_mask_expanded, 1)
+#     sum_mask = torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+#     return sum_embeddings / sum_mask
+
+# def recommendation(filt_ind: list, embeddings: np.array, user_text: str, n=10):
+#     token_user_text = tokenizer(user_text, return_tensors='pt', padding='max_length', truncation=True, max_length=512)
+#     user_embeddings = torch.Tensor().to(device)
+#     model.to(device)
+#     model.eval()
+#     with torch.no_grad():
+#         batch = {k: v.to(device) for k, v in token_user_text.items()}
+#         outputs = model(**batch)
+#         user_embeddings = torch.cat([user_embeddings, mean_pooling(outputs, batch['attention_mask'])])
+#         user_embeddings = user_embeddings.cpu().numpy()
+#     cosine_similarities = cosine_similarity(embeddings[filt_ind], user_embeddings.reshape(1, -1))
+#     df_res = pd.DataFrame(cosine_similarities.ravel(), columns=['cos_sim']).sort_values('cos_sim', ascending=False)
+#     dict_topn = df_res.iloc[:n, :].cos_sim.to_dict()
+#     return dict_topn
+
+
+def embed_user(filt_ind: list, embeddings:np.array, user_text: str, n=10):
+    tokens = tokenizer(user_text, return_tensors="pt", padding=True, truncation=True).to(device)
+    model.to(device)
+    model.eval()
+    with torch.no_grad():
+        outputs = model(**tokens)
+        user_embedding = outputs.last_hidden_state.mean(dim=1).cpu().numpy().reshape(1, -1)
+    return user_embedding
+
+    # cosine_similarities = cosine_similarity(embeddings[filt_ind], user_embedding.reshape(1, -1))
+    # df_res = pd.DataFrame(cosine_similarities.ravel(), columns=['cos_sim']).sort_values('cos_sim', ascending=False)
+    # dict_topn = df_res.iloc[:n, :].cos_sim.to_dict()
+    # return dict_topn