Upload 7 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Themes_And_Concepts.csv filter=lfs diff=lfs merge=lfs -text

ModelSimilarity.py

@@ -0,0 +1,166 @@
+import pandas as pd
+import torch
+import numpy as np
+from ast import literal_eval
+from transformers import AutoModel, AutoTokenizer
+from torch.utils.data import TensorDataset
+
+class MovieSimilarityModel:
+    def __init__(self, csv_file_path, model_name='sentence-transformers/all-mpnet-base-v2', max_length=32):
+        self.dataset = pd.read_csv(csv_file_path)
+        self.columns_to_eval = ['genres', 'Keywords', 'Negative_Reviews', 'Positive_Reviews', 'positive_relationship', 'negative_relationship', 'Themes', 'Concepts']
+        self._preprocess_data()
+        
+        
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.model = AutoModel.from_pretrained(model_name)  # Push model to GPU
+        self.max_length = max_length
+        
+        self.sentences = (self.dataset['Themes'] + self.dataset['Concepts']).to_list()
+        
+        self.genres = self.dataset['genres'].to_list()
+        
+        self.data = self.SentenceDataset(self.sentences, self.genres, self.tokenizer, max_length=self.max_length)
+
+    def _preprocess_data(self):
+        for col in self.columns_to_eval:
+            self.dataset[col] = self.dataset[col].apply(self._safe_literal_eval)
+
+    @staticmethod
+    def _safe_literal_eval(value):
+        try:
+            return literal_eval(value)
+        except (ValueError, SyntaxError):
+            return value
+
+    class SentenceDataset(TensorDataset):
+        def __init__(self, sentences, genres, tokenizer, max_length=32):
+            self.sentences = sentences
+            self.genres = genres
+            self.tokenizer = tokenizer
+            self.max_length = max_length
+            self.encoded_sentences =  [
+                [tokenizer.encode_plus(word_list, padding='max_length', return_attention_mask=True, truncation=True, max_length=max_length, is_split_into_words=False)
+                 for word_list in word_lists]
+                for word_lists in sentences
+            ]
+            self.genres_categories = [
+                tokenizer.encode_plus(word_lists, padding='max_length', return_attention_mask=True, return_tensors='pt', truncation=True, max_length=32, is_split_into_words=True)
+                for word_lists in genres
+            ]
+
+        def __len__(self):
+            return len(self.sentences)
+
+        def __getitem__(self, idx):
+            sentences = self.encoded_sentences[idx]
+            genres = self.genres_categories[idx]
+            input_ids = torch.tensor([d['input_ids'] for d in sentences])
+            attention_masks = torch.tensor([d['attention_mask'] for d in sentences])
+            return [{'input_ids': input_ids, 'attention_mask': attention_masks}, genres]
+
+    @staticmethod
+    def mean_pooling(model_output, attention_mask):
+        token_embeddings = model_output[0]  # First element contains all token embeddings
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+        return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+    def compute_sentence_embeddings(self):
+        sentence_embedded_data = []
+        genres_embedded_data = []
+        for i in range(len(self.data)):
+            with torch.no_grad():
+                embeddings_themes_concepts = self.model(**self.data[i][0])
+                embeddings_genres = self.model(**self.data[i][1])
+                sentence_embeddings = self.mean_pooling(embeddings_themes_concepts, self.data[i][0]['attention_mask'])
+                genres_embeddings = self.mean_pooling(embeddings_genres, self.data[i][1]['attention_mask'])
+                sentence_embedded_data.append(sentence_embeddings)  # Keep as tensors
+                genres_embedded_data.append(genres_embeddings)  # Keep as tensors
+        return sentence_embedded_data, genres_embedded_data
+
+    @staticmethod
+    def similarity_matrix(a, b, eps=1e-9):
+        a_n, b_n = a.norm(dim=1)[:, None], b.norm(dim=1)[:, None]
+        a_norm = a / torch.clamp(a_n, min=eps)
+        b_norm = b / torch.clamp(b_n, min=eps)
+        a_norm, b_norm, combinations = MovieSimilarityModel.tensor_pad(a_norm, b_norm)
+        sim_mt = torch.mm(a_norm, b_norm.transpose(0, 1))
+        return sim_mt, combinations
+
+    @staticmethod
+    def tensor_pad(a, b):
+        x_a, y_a = a.shape
+        x_b, y_b = b.shape
+        combinations = x_a * x_b
+        
+        if x_a < x_b:
+            padding = (0, 0, 0, x_b - x_a)
+            a_padded = torch.nn.functional.pad(a, padding)
+            return a_padded, b, combinations
+        elif x_b < x_a:
+            padding = (0, 0, 0, x_a - x_b)
+            b_padded = torch.nn.functional.pad(b, padding)
+            return b_padded, a, combinations
+        else:
+            return a, b, combinations
+
+    def compute_similarity_matrix(self):
+        Y = []
+        for sentence_embed_x, genres_embed_x in zip(self.sentence_embedded_data, self.genres_embedded_data):
+            X = []
+            for sentence_embed_y, genres_embed_y in zip(self.sentence_embedded_data, self.genres_embedded_data):
+                sim_mt, combinations = self.similarity_matrix(sentence_embed_x, sentence_embed_y)
+                thc_similarity = (sim_mt.sum() / combinations)
+                genres_similarity = torch.nn.functional.cosine_similarity(genres_embed_x, genres_embed_y)
+                final_similarity = (0.70 * thc_similarity) + (0.30 * genres_similarity)
+                X.append(final_similarity.item())
+            Y.append(X)
+        return Y
+
+    def encode_input_movie(self, input_themes, input_genres):
+        input_data = self.SentenceDataset(input_themes, input_genres, self.tokenizer, max_length=self.max_length)
+        
+        with torch.no_grad():
+            embeddings_themes_concepts = self.model(**input_data[0][0])
+            embeddings_genres = self.model(**input_data[0][1])
+            input_sentence_embeddings = self.mean_pooling(embeddings_themes_concepts, input_data[0][0]['attention_mask'])
+            input_genres_embeddings = self.mean_pooling(embeddings_genres, input_data[0][1]['attention_mask'])
+            
+        return input_sentence_embeddings, input_genres_embeddings
+
+    def compare_input_vs_dataset(self, input_sentence_embeddings, input_genres_embeddings):
+        similarity_scores = []
+        for dataset_sentence_embeddings, dataset_genres_embeddings in zip(self.sentence_embedded_data, self.genres_embedded_data):
+            sim_mt, combinations = self.similarity_matrix(input_sentence_embeddings, dataset_sentence_embeddings)
+            thc_similarity = (sim_mt.sum() / combinations)
+            genres_similarity = torch.nn.functional.cosine_similarity(input_genres_embeddings, dataset_genres_embeddings)
+            final_similarity = (0.70 * thc_similarity) + (0.30 * genres_similarity)
+            similarity_scores.append(final_similarity.item())
+
+        # Sort by similarity scores and get top 5
+        top_5_indices = np.argsort(similarity_scores)[-5:][::-1]
+        top_5_scores = np.array(similarity_scores)[top_5_indices]
+        top_5_movies = self.dataset['Movie'].iloc[top_5_indices].values
+        
+        # Return the top 5 movies with their similarity scores
+        return pd.DataFrame(list(zip(top_5_movies, top_5_scores)), columns=['Movie', 'Similarity Score'])
+
+    def load_tensor_list(self):
+        # Load precomputed embeddings and map them to CUDA
+        self.sentence_embedded_data = torch.load('./tac_embeddings.pt')
+        self.genres_embedded_data = torch.load('./genres_embeddings.pt')
+
+        
+    def save_embeddings_to_dataframe(self):
+        # Compute embeddings
+        sentence_embedded_data, genres_embedded_data = self.compute_sentence_embeddings()
+        
+        # Convert to dictionary for easier saving
+        embeddings_dict = {
+            'Movie': self.dataset['Movie'],
+            'Sentence_Embeddings': sentence_embedded_data,
+            'Genres_Embeddings': genres_embedded_data
+        }
+        
+        return embeddings_dict
+

README.md

@@ -1,13 +1,13 @@
----
-title: Similarity Model
-emoji: 🦀
-colorFrom: yellow
-colorTo: green
-sdk: gradio
-sdk_version: 4.41.0
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Similarity Model
+emoji: 🦀
+colorFrom: yellow
+colorTo: green
+sdk: gradio
+sdk_version: 4.41.0
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Themes_And_Concepts.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:538e953c7030591110f38921f4a2252a5e61400a6daad84f1896159b2b9b1300
+size 78347252

app.py

@@ -0,0 +1,31 @@
+import gradio as gr
+import torch
+import pandas as pd
+from ModelSimilarity import MovieSimilarityModel
+
+# Assume the MovieSimilarityModel class is already defined above
+csv_file_path = './Themes_And_Concepts.csv'
+movie_similarity_model = MovieSimilarityModel(csv_file_path)
+
+# Load precomputed embeddings if available
+movie_similarity_model.load_tensor_list()
+
+# Define a function to wrap the comparison
+def get_top_5_similar_movies(input_themes, input_genres):
+    # Directly use the input arrays
+    input_sentence_embeddings, input_genres_embeddings = movie_similarity_model.encode_input_movie(input_themes, input_genres)
+    
+    # Get top 5 similar movies
+    top_5_similar_movies = movie_similarity_model.compare_input_vs_dataset(input_sentence_embeddings, input_genres_embeddings)
+    
+    return top_5_similar_movies
+
+# Define Gradio interface
+input_themes = gr.List(headers=["Themes"], label="Input Movie Themes (array of strings)")
+input_genres = gr.List(headers=["Genres"], label="Input Movie Genres (array of strings)")
+output = gr.Dataframe(type="pandas", label="Top 5 Similar Movies")
+
+interface = gr.Interface(fn=get_top_5_similar_movies, inputs=[input_themes, input_genres], outputs=output)
+
+# Launch the Gradio interface
+interface.launch()

genres_embeddings.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c70d517b5cd91db2a14677e2c2162d14ee13a5f1cdbf0c2e996f411b9752078b
+size 2881850

requirements.txt

@@ -0,0 +1,4 @@
+torch == 2.4.0
+transformers == 4.44.0
+numpy == 2.0.1
+pandas == 2.2.2

tac_embeddings.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09d628a47bbc59f74def388861a49e27c9401bb67d9b66bfd1daaee7d9d006ad
+size 22871834