Upload 10 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+pages/MiniLM/MiniLM_index.index filter=lfs diff=lfs merge=lfs -text

main.py

@@ -0,0 +1,71 @@
+import streamlit as st
+from pages.rubert.app_bert import ruBert_page
+from pages.MiniLM.MiniLm_app import MiniLm_page
+
+# st.markdown("""
+#     <style>
+#         /* Основной фон страницы */
+#         body {
+#             background-color: #50C878;  /* Изумрудный цвет */
+#         }
+
+#         /* Фон основной панели, чтобы убрать несоответствующие цвета */
+#         .stApp {
+#             background-color: #50C878;
+#         }
+
+#         /* Фон и текст боковой панели */
+#         .stSidebar > div:first-child {
+#             background-color: #50C878; /* Изумрудный цвет для боковой панели */
+#             color: #FFFFFF; /* Белый цвет текста для контраста */
+#         }
+#         .stSidebar .sidebar-content {
+#             color: #FFFFFF;
+#         }
+
+#         /* Стиль заголовка боковой панели */
+#         .stSidebar .sidebar-content h1, .stSidebar .sidebar-content h2, .stSidebar .sidebar-content h3 {
+#             color: #FFFFFF;
+#         }
+
+#         /* Стиль кнопок */
+#         .stButton > button {
+#             background-color: #006400; /* Темно-изумрудный цвет кнопок */
+#             color: #FFFFFF; /* Белый текст на кнопках */
+#         }
+
+#         /* Стили текстового поля */
+#         .stTextInput input {
+#             background-color: #FFFFFF; /* Белый фон текстового поля */
+#             color: #006400; /* Темно-изумрудный текст */
+#         }
+
+#         /* Стили активной кнопки радио и чекбокса */
+#         .stRadio > label > div:first-of-type > div, .stCheckbox > label > div:first-of-type > div {
+#             background-color: #006400; /* Темно-изумрудный фон для активных кнопок радио и чекбоксов */
+#             border-color: #006400;
+#         }
+
+#         /* Стили слайдера */
+#         .stSlider > div > div > div > div {
+#             background-color: #006400; /* Темно-изумрудный цвет слайдера */
+#         }
+#     </style>
+# """, unsafe_allow_html=True)
+
+def app_description_page():
+    st.title("Welcome to My App!")
+
+def main():
+    st.sidebar.title("Book app")
+    page = st.sidebar.radio("Select page:", ["About Project", "📚 Book search", "🔍 Book search (faiss)"])
+    if page == "About Project":
+        app_description_page()
+    if page == "📚 Book search":
+        ruBert_page()
+    if page == "🔍 Book search (faiss)":
+        MiniLm_page()
+
+
+if __name__ == "__main__":
+    main()

pages/MiniLM/MiniLM_embeddings.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8b48f9594e9a78afbdd85f8acc801df16e30f90641e5e9d2c6b1b4dc66c65bf
+size 7578787

pages/MiniLM/MiniLM_index.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a85188579f9f15ea1809ecfd0478c026c449b019775ffee3b8202d85ce787de8
+size 7578669

pages/MiniLM/MiniLm_app.py

@@ -0,0 +1,75 @@
+import streamlit as st
+import pandas as pd
+from sentence_transformers import SentenceTransformer
+import faiss
+import pickle
+import time
+
+# Загрузка данных
+@st.cache_data
+def load_data():
+    data = pd.read_csv('Data/books_data2.csv')
+    with open('pages/MiniLM/MiniLM_embeddings.pkl', 'rb') as f:
+        book_embeddings = pickle.load(f)
+    index = faiss.read_index('pages/MiniLM/MiniLM_index.index')
+
+    embedder = SentenceTransformer("all-MiniLM-L6-v2")
+    return data, index, embedder
+
+# Функция поиска
+def search_books(embedder, index, data, query, n_results):
+    query_embedding = embedder.encode([query])
+    D, I = index.search(query_embedding, n_results)
+    
+    result_books = []
+    for i in range(n_results):
+        book_index = I[0][i]
+        book_info = data.iloc[book_index]
+        book_link = book_info['page_url']
+        book_image = book_info['image_url']
+        book_genre = book_info['genre']
+        book_title = book_info['title']
+        book_description = book_info['annotation']
+        book_author = book_info['author']
+        similarity_score = 1 / (1 + D[0][i])
+        
+        result_books.append({
+            'Link': book_link,
+            'Image': book_image,
+            'Genre': book_genre,
+            'Title': book_title,
+            'Description': book_description,
+            'Author': book_author,
+            'Similarity': similarity_score
+        })
+    
+    return result_books
+
+# Streamlit
+def MiniLm_page():
+    st.title('Поиск книг')
+    data, index, embedder = load_data()
+    query = st.text_input('Введите цитату или автора:')
+    n_results = st.slider("Количество результатов", min_value=1, max_value=20, value=5)
+
+    if st.button("Искать"):
+        if query:
+            start_time = time.time() 
+            result_books = search_books(embedder, index, data, query, n_results)
+            end_time = time.time()  
+            search_time = end_time - start_time 
+            st.write("Результаты поиска:")
+            
+            for book in result_books:
+                st.write('---')
+                st.image(book['Image'], width=250)
+                st.write(f"**Название:** {book['Title']}")
+                st.write(f"**Автор:** {book['Author']}")
+                st.write(f"**Жанр:** {book['Genre']}")
+                if len(book['Description']) > 50:
+                    book['Description'] = ' '.join(book['Description'].split()[:50]) + '...'
+                st.write(f"**Описание:** {book['Description']}")
+                st.write(f"**Сходство:** {book['Similarity']:.2f}")
+                st.write(f'**Время поиска:** {search_time:.4f} секунд')
+                st.write(f"[Читать подробнее]({book['Link']})")
+                st.text("") 

pages/rubert/app_bert.py

@@ -0,0 +1,70 @@
+import streamlit as st
+import numpy as np
+import torch
+from transformers import AutoTokenizer, AutoModel
+from scipy.spatial.distance import cosine
+import pandas as pd
+import time 
+
+@st.cache_data
+def load_data():
+    book_embeddings = np.load('pages/rubert/embeddings.npz')
+    all_embeddings = book_embeddings['embeddings']
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    tokenizer = AutoTokenizer.from_pretrained("cointegrated/rubert-tiny2")
+    model = AutoModel.from_pretrained("cointegrated/rubert-tiny2").to(device)
+    data = pd.read_csv('Data/books_data2.csv')
+    return all_embeddings, device, tokenizer, model, data
+
+# Функция для получения эмбеддинга
+
+def embed_text(text, tokenizer, model, device):
+    encoded_input = tokenizer(text, padding=True, truncation=True, max_length=128, return_tensors='pt')
+    with torch.no_grad():
+        model_output = model(**encoded_input.to(device))
+        embeddings = model_output.last_hidden_state[:,0,:].cpu().numpy()
+        return np.mean(embeddings, axis=0)
+    
+# Функция поиска
+def search_books(user_query, all_embeddings, device, tokenizer, model, data, n_results):
+    query_embedding = embed_text(user_query, tokenizer, model, device)
+    
+    similarities = [1 - cosine(query_embedding, book_embedding) for book_embedding in all_embeddings]
+    top_results_indices = np.argsort(similarities)[::-1][:n_results]
+    top_similarities = np.sort(similarities)[::-1][:n_results]
+    
+    return top_results_indices, top_similarities
+
+# Streamlit
+def ruBert_page():
+    st.title("Поиск книг")
+    all_embeddings, device, tokenizer, model, data = load_data()
+
+    user_query = st.text_input("Введите цитату или автора:")
+    n_results = st.slider("Количество результатов", min_value=1, max_value=20, value=5)
+
+    if st.button("Искать"):
+        start_time = time.time() 
+        top_books_indices, top_similarities = search_books(user_query, all_embeddings, device, tokenizer, model, data, n_results)
+        end_time = time.time()  
+        search_time = end_time - start_time  
+        st.write("Результаты поиска:")
+        for i, idx in enumerate(top_books_indices):
+            st.write('---')
+            similarity = top_similarities[i]
+            st.image(data.loc[idx, 'image_url'], width=250)
+            st.write(f"**Название:** {data.loc[idx, 'title']}")
+            st.write(f"**Автор:** {data.loc[idx, 'author']}")
+            st.write(f"**Жанр:** {data.loc[idx, 'genre']}")
+            annotation = data.loc[idx, 'annotation']
+            if len(annotation) > 50:
+                annotation = ' '.join(annotation.split()[:50]) + '...'
+            st.write(f"**Описание:** {annotation}")
+            st.write(f"**Косинусное сходство:** {similarity:.3f}")
+            st.write(f'**Время поиска:** {search_time:.4f} секунд')
+            st.markdown(f"[Читать подробнее]({data.loc[idx, 'page_url']})")
+    
+
+        

pages/rubert/embeddings.npz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:567061998a07f76e3290d80b1cebfd6945baf186a0d1c920bd1071f40e0d1e64
+size 3544594

requirements.txt

@@ -0,0 +1,11 @@
+huggingface-hub
+numpy
+pandas
+streamlit
+tokenizers
+torch
+transformers
+faiss-cpu
+sentence_transformers
+scipy
+