pages

main.py

@@ -1 +1,26 @@
-print()
+import streamlit as st
+
+
+st.markdown("""
+    <style>
+        section[data-testid="stSidebar"][aria-expanded="true"]{
+            display: none;
+        }
+    </style>
+    """, unsafe_allow_html=True)
+
+
+
+st.title('📝 &  ⚡💨🍃🪫💡')
+
+
+
+st.write('choose your option')
+
+
+
+st.page_link("pages/task1.py", label="Kinopoisk", icon='🎥')
+st.page_link("pages/toxicapp.py", label="Personality", icon='☠️')
+st.page_link("pages/task3.py", label="GPT", icon='📲')
+
+st.header(f'''made by: Alexey Kamaev & Marina Kochetova''')

media/page1.py

@@ -1 +0,0 @@
-print()

media/page2.py

@@ -1 +0,0 @@
-print()

pages/page1.py

@@ -1 +0,0 @@
-print()

pages/page2.py

@@ -1 +0,0 @@
-print()

pages/task1.py

@@ -0,0 +1,109 @@
+import json 
+import numpy as np
+import joblib
+
+from transformers import AutoTokenizer, AutoModel
+import torch
+import pickle
+
+import matplotlib.pyplot as plt
+
+with open('srcs/vocab_to_int.json', encoding='utf-8') as f:
+    vocab_to_int = json.load(f)
+
+with open('srcs/int_to_vocab.json', encoding='utf-8') as f:
+    int_to_vocab = json.load(f)
+
+VOCAB_SIZE = len(vocab_to_int) + 1
+EMBEDDING_DIM = 64 # embedding_dim
+SEQ_LEN = 350
+HIDDEN_SIZE = 64
+
+with open('srcs/embedding_matrix.npy', 'rb') as f:
+    embedding_matrix = np.load(f)
+
+
+
+from srcs.srcs import LSTMConcatAttentionB
+from srcs.srcs import Text_ex, clean
+
+
+log_reg_vec = joblib.load('srcs/log_reg_vec.sav')
+
+log_reg_bert = joblib.load('srcs/log_reg_bert.sav')
+
+texter = Text_ex(clean, vocab_to_int, SEQ_LEN)
+lstm = LSTMConcatAttentionB()
+
+lstm.load_state_dict(torch.load('srcs/lstm.pt'))
+
+
+vectorizer = pickle.load(open("srcs/vectorizer.pickle", "rb"))
+tokenizer = AutoTokenizer.from_pretrained('srcs/tokenzier', local_files_only=True)
+bert = torch.load('srcs/bert.pt')
+
+
+
+from srcs.srcs import PredMaker
+predM = PredMaker(model1=log_reg_vec, model2=lstm, rubert=bert, model3=log_reg_bert, vectorizer=vectorizer, texter=texter, clean_func=clean, tokenizer=tokenizer, itc=int_to_vocab)
+
+
+
+
+import streamlit as st
+st.markdown("""
+    <style>
+        section[data-testid="stSidebar"][aria-expanded="true"]{
+            display: none;
+        }
+    </style>
+    """, unsafe_allow_html=True)
+
+
+st.title('Кинопоиск')
+st.page_link("main.py", label="Home", icon='🏠')
+
+import streamlit as st
+
+txt = st.text_area(
+    "Введите сюда отзыв на фильм:",
+    "",
+    )
+
+# st.write(f'Введено {len(txt)} символов.')
+
+if txt == '' or len(txt) < 12:
+    if len(txt) >= 1:
+        st.write('Введи что-нибудь нормальное')
+else:
+    text = txt
+    res1, res2, res3, t, att, *times = predM(text)
+    t_ = t[0].numpy()[0]
+    k = len(t[1].split()) + 1
+
+    labels = [int_to_vocab[str(x)] for x in t_ if int_to_vocab.get(str(x))]
+
+    if list(set(labels[-k:])) == ["<pad>"]:
+        st.write('Давай по новой миша, всё @**##')
+        st.write(set(labels[-k:]))
+    else:
+        st.toast('!', icon='🎉')
+        di = {0:'Плохо',1:'Нейтрально',2:'Хорошо'}
+        d = {0: st.error, 1: st.warning, 2: st.success}
+
+        d[res1](f'Предсказание 1-й модели: {di[res1]}')
+        st.write(f'время = {round(times[0],3)}c, f1-score = 0.64')
+
+        d[res2](f'Предсказание 2-й модели: {di[res2]}')
+        st.write(f'время = {round(times[0],3)}c, f1-score = 0.70')
+
+        d[res3](f'Предсказание 3-й модели: {di[res3]}')
+        st.write(f'время = {round(times[0],3)}c, f1-score = 0.66')
+
+
+        plt.figure(figsize=(8, 8))
+        plt.barh(np.arange(len(t_))[-k:], att[-k:])
+        plt.yticks(ticks = np.arange(len(t_))[-k:], labels = labels[-k:])
+        plt.title(f'f1-score = 0.7\npred = {di[res2]}\ntime = {round(times[1],3)}c');
+        st.set_option('deprecation.showPyplotGlobalUse', False)
+        st.pyplot()

pages/task3.py

@@ -0,0 +1,89 @@
+import streamlit as st
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+import time
+
+def generate_text(model, tokenizer, prompt, max_length, num_generations, temperature):
+    generated_texts = []
+
+    for _ in range(num_generations):
+        input_ids = tokenizer.encode(prompt, return_tensors="pt")
+        output = model.generate(
+            input_ids,
+            max_length=max_length,
+            temperature=temperature,
+            num_return_sequences=1
+        )
+        generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
+        generated_texts.append(generated_text)
+
+    return generated_texts
+
+button_style = """
+    <style>
+    .center-align {
+        display: flex;
+        justify-content: center;
+    
+    </style>
+"""
+
+DEVICE = 'cpu' 
+
+# Загрузка пользовательской модели и токенизатора (замените на свои пути и модель)
+# model_path = "sberbank-ai/rugpt3small_based_on_gpt2"
+# tokenizer_path = "sberbank-ai/rugpt3small_based_on_gpt2"
+
+# model = GPT2LMHeadModel.from_pretrained(model_path).to(DEVICE)
+# tokenizer = GPT2Tokenizer.from_pretrained(tokenizer_path)
+
+st.markdown("""
+    <style>
+        section[data-testid="stSidebar"][aria-expanded="true"]{
+            display: none;
+        }
+    </style>
+    """, unsafe_allow_html=True)
+
+st.write("## Text generator")
+st.page_link("main.py", label="Home", icon='🏠')
+st.markdown(
+        """
+        This streamlit-app can generate text using your prompt 
+    """
+)
+# Ввод пользовательского prompt
+prompt = st.text_area("Enter your prompt:")
+
+# Параметры генерации
+max_length = st.slider("Max length of generated text:", min_value=10, max_value=500, value=100, step=10)
+num_generations = st.slider("Number of generations:", min_value=1, max_value=10, value=3, step=1)
+temperature = st.slider("Temperature:", min_value=0.1, max_value=2.0, value=1.0, step=0.1)
+try:
+    if st.button("Generate text"):
+        start_time = time.time()
+        generated_texts = generate_text(model, tokenizer, prompt, max_length, num_generations, temperature)
+        end_time = time.time()
+
+        st.subheader("Сгенерированный текст:")
+        for i, text in enumerate(generated_texts, start=1):
+            st.write(f"Генерация {i}:\n{text}")
+
+        generation_time = end_time - start_time
+        st.write(f"\nВремя генерации: {generation_time:.2f} секунд")
+
+    st.markdown(button_style, unsafe_allow_html=True)  # Применяем стиль к кнопке
+    st.markdown(
+        """
+        <style>
+            div[data-baseweb="textarea"] {
+                border: 2px solid #3498db;  /* Цвет границы */
+                border-radius: 5px;  /* Закругленные углы */
+                background-color: #ecf0f1;  /* Цвет фона */
+                padding: 10px;  /* Поля вокруг текстового поля */
+            }
+        </style>
+        """,
+        unsafe_allow_html=True,
+    )
+except:
+    st.write('Модель в разработке ( ﾉ ﾟｰﾟ)ﾉ( ﾉ ﾟｰﾟ)ﾉ( ﾉ ﾟｰﾟ)ﾉ( ﾉ ﾟｰﾟ)ﾉ')  

pages/toxicapp.py

@@ -0,0 +1,156 @@
+import streamlit as st
+import torch
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import pandas as pd
+import matplotlib.pyplot as plt
+from PIL import Image
+import os
+import numpy as np
+
+model_path = 'srcs/model_modify.pth'
+
+#  токенизатор
+tokenizer = AutoTokenizer.from_pretrained('cointegrated/rubert-tiny-toxicity')
+
+model = AutoModelForSequenceClassification.from_pretrained('cointegrated/rubert-tiny-toxicity', num_labels=1, ignore_mismatched_sizes=True)
+# весов модифицированной модели
+model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')), strict=False)
+image = Image.open("media/oritoxic.jpg")
+
+# df = pd.read_csv("media/Toxic_labeled.csv")
+loss_values = [0.4063596375772262, 0.402279906166038, 0.3998144585561736, 0.39567733055365567,
+               0.3921396666608141, 0.38956182373070186, 0.3866641920902114, 0.3879134839351564,
+               0.38288725781591604, 0.38198364493999004]
+
+#Боковая панель
+selected_option = st.sidebar.selectbox("Выберите из списка", ["Определение токсичность текста", "Информация о датасете", "Информация о модели"])
+
+#st.title("Главная страница")
+
+
+if selected_option == "Определение токсичность текста":
+
+
+    st.markdown("<h1 style='text-align: center;'>Приложение для определения токсичности текста</h1>",
+                unsafe_allow_html=True)
+    st.image(image, use_column_width=True)
+    user_input = st.text_area("")
+
+
+    # Функция предсказания токсичности
+
+    def predict_toxicity(text):
+        inputs = tokenizer(text, return_tensors="pt")
+        outputs = model(**inputs)
+        logits = outputs.logits
+        probability = torch.sigmoid(logits).item()
+        prediction = "токсичный" if probability >= 0.5 else "не токсичный"
+        return prediction, probability
+    # Тык на кнопу
+    if st.button("Оценить токсичность"):
+        if user_input:
+            prediction, toxicity_probability = predict_toxicity(user_input)
+            st.write(f'Вероятность токсичности: {toxicity_probability:.4f}')
+
+    # Прогресс бар
+    if 'toxicity_probability' in locals():
+        progress_percentage = int(toxicity_probability * 100)
+        progress_bar_color = f'linear-gradient(to right, rgba(0, 0, 255, 0.5) {progress_percentage}%, rgba(255, 0, 0, 0.5) {progress_percentage}%)'
+        st.markdown(f'<div style="background: {progress_bar_color}; height: 20px; border-radius: 5px;"></div>',
+                    unsafe_allow_html=True)
+
+elif selected_option == "Информация о датасете":
+    st.header("Информация о датасете:")
+    st.dataframe(df.head())
+    st.write(f"Объем выборки: 14412")
+    st.subheader("Баланс классов в датасете:")
+    st.write(f"Количество записей в классе 0.0: {len(df[df['toxic'] == 0.0])}")
+    st.write(f"Количество записей в классе 1.0: {len(df[df['toxic'] == 1.0])}")
+    fig, ax = plt.subplots()
+    df['toxic'].value_counts().plot(kind='bar', ax=ax, color=['skyblue', 'orange'])
+    ax.set_xticklabels(['Не токсичный', 'Токсичный'], rotation=0)
+    ax.set_xlabel('Класс')
+    ax.set_ylabel('Количество записей')
+    ax.set_title('Распределение по классам')
+    st.pyplot(fig)
+
+elif selected_option == "Информация о модели":
+    st.subheader("Информация о модели:")
+    st.write(f"Модель: Rubert tiny toxicity")
+    st.subheader("Информация о процессе обучения")
+
+# график лосса
+#st.subheader("График потерь в процессе обучения")
+#st.line_chart([0.5181976270121774, 0.4342067330899996, 0.41386983832460666])  # Замените данными из ваших эпох
+    for epoch, loss in enumerate(loss_values, start=1):
+        st.write(f"<b>Epoch {epoch}/{len(loss_values)}, Loss:</b> {loss}<br>", unsafe_allow_html=True)
+    st.markdown(
+        """
+        <b>Количество эпох:</b> 10
+        <b>Размер батча:</b> 8
+        <b>Оптимизатор:</b> Adam
+        <b>Функция потерь:</b> BCEWithLogitsLoss
+        <b>learning rate:</b> 0.00001
+        """,
+        unsafe_allow_html=True
+    )
+
+    st.subheader("Метрики модели:")
+    st.write(f"Accuracy: {0.8366:.4f}")
+    st.write(f"Precision: {0.8034:.4f}")
+    st.write(f"Recall: {0.6777:.4f}")
+    st.write(f"F1 Score: {0.7352:.4f}")
+
+
+    st.subheader("Код")
+
+
+    bert_model_code = """
+    
+    model = BertModel(
+        embeddings=BertEmbeddings(
+            word_embeddings=Embedding(29564, 312, padding_idx=0),
+            position_embeddings=Embedding(512, 312),
+            token_type_embeddings=Embedding(2, 312),
+            LayerNorm=LayerNorm((312,), eps=1e-12, elementwise_affine=True),
+            dropout=Dropout(p=0.1, inplace=False),
+        ),
+        encoder=BertEncoder(
+            layer=ModuleList(
+                BertLayer(
+                    attention=BertAttention(
+                        self=BertSelfAttention(
+                            query=Linear(in_features=312, out_features=312, bias=True),
+                            key=Linear(in_features=312, out_features=312, bias=True),
+                            value=Linear(in_features=312, out_features=312, bias=True),
+                            dropout=Dropout(p=0.1, inplace=False),
+                        ),
+                        output=BertSelfOutput(
+                            dense=Linear(in_features=312, out_features=312, bias=True),
+                            LayerNorm=LayerNorm((312,), eps=1e-12, elementwise_affine=True),
+                            dropout=Dropout(p=0.1, inplace=False),
+                        ),
+                    ),
+                    intermediate=BertIntermediate(
+                        dense=Linear(in_features=312, out_features=600, bias=True),
+                        intermediate_act_fn=GELUActivation(),
+                    ),
+                    output=BertOutput(
+                        dense=Linear(in_features=600, out_features=312, bias=True),
+                        LayerNorm=LayerNorm((312,), eps=1e-12, elementwise_affine=True),
+                        dropout=Dropout(p=0.1, inplace=False),
+                    ),
+                )
+            )
+        ),
+        pooler=BertPooler(
+            dense=Linear(in_features=312, out_features=312, bias=True),
+            activation=Tanh(),
+        ),
+        dropout=Dropout(p=0.1, inplace=False),
+        classifier=Linear(in_features=312, out_features=1, bias=True),
+    )
+    """
+
+    # Отображение кода в Streamlit
+    st.code(bert_model_code, language="python")

requirements.txt

@@ -0,0 +1,109 @@
+altair==5.2.0
+asttokens==2.4.1
+attrs==23.2.0
+blinker==1.7.0
+cachetools==5.3.3
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+comm==0.2.1
+contourpy==1.2.0
+cycler==0.12.1
+debugpy==1.8.1
+decorator==5.1.1
+executing==2.0.1
+filelock==3.9.0
+fonttools==4.49.0
+fsspec==2024.2.0
+gensim==4.3.2
+gitdb==4.0.11
+GitPython==3.1.42
+huggingface-hub==0.21.4
+idna==3.6
+importlib-metadata==7.0.1
+ipykernel==6.29.3
+ipython==8.22.1
+jedi==0.19.1
+Jinja2==3.1.2
+joblib==1.3.2
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.0
+jupyter_core==5.7.1
+kiwisolver==1.4.5
+lightning-utilities==0.10.1
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+matplotlib==3.8.3
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+mpmath==1.3.0
+nest-asyncio==1.6.0
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.3
+opencv-python==4.9.0.80
+packaging==23.2
+pandas==2.2.1
+parso==0.8.3
+pillow==10.2.0
+platformdirs==4.2.0
+prompt-toolkit==3.0.43
+protobuf==4.25.3
+psutil==5.9.8
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==15.0.0
+pydeck==0.8.1b0
+Pygments==2.17.2
+pymystem3==0.2.0
+pyparsing==3.1.1
+pyproject-toml==0.0.10
+python-dateutil==2.8.2
+pytz==2024.1
+pywin32==306
+PyYAML==6.0.1
+pyzmq==25.1.2
+referencing==0.33.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.1
+rpds-py==0.18.0
+safetensors==0.4.2
+scikit-learn==1.4.1.post1
+scipy==1.12.0
+seaborn==0.13.2
+setuptools==69.1.1
+six==1.16.0
+smart-open==7.0.1
+smmap==5.0.1
+stack-data==0.6.3
+stqdm==0.0.5
+streamlit==1.31.1
+sympy==1.12
+tenacity==8.2.3
+thop==0.1.1.post2209072238
+threadpoolctl==3.3.0
+tokenizers==0.15.2
+toml==0.10.2
+toolz==0.12.1
+torch==2.2.1+cu121
+torchaudio==2.2.1+cu121
+torchmetrics==1.3.1
+torchvision==0.17.1+cu121
+tornado==6.4
+tqdm==4.66.2
+traitlets==5.14.1
+transformers==4.38.2
+typing_extensions==4.8.0
+tzdata==2024.1
+tzlocal==5.2
+ultralytics==8.1.20
+urllib3==2.2.1
+validators==0.22.0
+watchdog==4.0.0
+wcwidth==0.2.13
+wheel==0.42.0
+wrapt==1.16.0
+zipp==3.17.0

srcs/model_modify.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b97eff933dde60a624143ec9416613b9ffc74b8c4877a00e05ebecefeb1e485
+size 47160284