import pandas as pd from gensim.models import Word2Vec import numpy as np from sklearn.metrics.pairwise import cosine_similarity from nltk.tokenize import word_tokenize from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences def recommend_papers(positive_csv_path, unlabelled_csv_path, n): # Load positive labelled data and unlabelled labelled data positive_labelled_info = pd.read_csv(positive_csv_path) unlabelled_labelled = pd.read_csv(unlabelled_csv_path) # Combine title and abstract for positive labelled data and unlabelled labelled data positive_labelled_info['text'] = positive_labelled_info['title'] + ' ' + positive_labelled_info['abstract'] unlabelled_labelled['text'] = unlabelled_labelled['title'] + ' ' + unlabelled_labelled['abstract'] # Preprocess text and tokenize positive_labelled_info['tokenized_text'] = positive_labelled_info['text'].apply(word_tokenize) unlabelled_labelled['tokenized_text'] = unlabelled_labelled['text'].apply(word_tokenize) # Train Word2Vec model on the tokenized text of positive labelled data word2vec_model = Word2Vec(sentences=positive_labelled_info['tokenized_text'], vector_size=100, window=5, min_count=1, workers=4) # Initialize tokenizer tokenizer = Tokenizer() tokenizer.fit_on_texts(positive_labelled_info['text']) # Convert text to sequences of integer indices positive_sequences = tokenizer.texts_to_sequences(positive_labelled_info['text']) unlabelled_sequences = tokenizer.texts_to_sequences(unlabelled_labelled['text']) # Pad or truncate sequences to a fixed length max_length = 50 # Adjust as needed padded_positive_sequences = pad_sequences(positive_sequences, maxlen=max_length, padding='post', truncating='post') padded_unlabelled_sequences = pad_sequences(unlabelled_sequences, maxlen=max_length, padding='post', truncating='post') # Compute centroid of positive examples' word embeddings centroid = np.mean([word2vec_model.wv[token] for token in tokenizer.index_word.values() if token in word2vec_model.wv], axis=0) # Compute similarity scores for each document in the unlabelled labelled data similarity_scores = [] for sequence in padded_unlabelled_sequences: embeddings = [word2vec_model.wv[token] for token in sequence if token in word2vec_model.wv] if len(embeddings) > 0: similarity_score = cosine_similarity([centroid], [np.mean(embeddings, axis=0)])[0][0] else: similarity_score = 0 # Default similarity score for empty embeddings similarity_scores.append(similarity_score) # Add similarity scores to unlabelled labelled data unlabelled_labelled['similarity_score'] = similarity_scores # Sort unlabelled labelled data by similarity score in descending order recommended_documents = unlabelled_labelled.sort_values(by='similarity_score', ascending=False) # Return top n recommended documents return recommended_documents[['id', 'title', 'similarity_score']].head(n)