Upload 2 files

pages/1_🗒️_Tahmin.py

@@ -1,24 +1,24 @@
 import json
+
+import numpy as np
 import requests
 import streamlit as st
-from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
+from simpletransformers.classification import MultiLabelClassificationModel
 import clean
+from simpletransformers.config.model_args import ClassificationArgs
 
 st.set_page_config(page_title='Teknofest 2023 Türkçe Doğal Dil İşleme Yarışması ', page_icon=':bar_chart:', layout='wide')
 st.title('🗒️ Aşağılayıcı Söylem Tespit Uygulaması')
 
-model=AutoModelForSequenceClassification.from_pretrained("BRAIN-TR/insult-bert-uncased")
-tokenizer=AutoTokenizer.from_pretrained("BRAIN-TR/insult-bert-uncased")
+target_names = ['is_offensive', 'insult', 'other', 'profanity', 'racist', 'sexist']
 
-nlp = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer)
+model_args = ClassificationArgs(
+        use_multiprocessing=True,
+        use_multiprocessing_for_evaluation=True,
+        no_cache= False
+    )
 
-label_dict = {
-    'LABEL_0': 'insult',
-    'LABEL_1': 'other',
-    'LABEL_2': 'profanity',
-    'LABEL_3': 'racist',
-    'LABEL_4': 'sexist'
-}
+bert_model = MultiLabelClassificationModel('bert','BRAIN-TR/acikhack_multilabel_insult_uncased', use_cuda=False, args=model_args)
 
 with open('style.css')as f:
     st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html = True)
@@ -28,8 +28,14 @@ st.subheader('Söylem Tespiti')
 def text_input_screen():
     def predict(text):
         cleaned_text = clean.CLEANING(text, True, False, True).clean()
-        out = nlp(cleaned_text)
-        return out
+        output_arr = bert_model.predict([cleaned_text])
+
+        def decode_one_hot(input, target_names):
+            return input[0][0], target_names[np.argmax(input[0][1:]) + 1], np.max(output_arr[1][0][1:])
+
+        io, category, score = decode_one_hot(output_arr[0], target_names)
+
+        return io, category, score
 
     def callback():
         st.session_state.predict_input_text
@@ -56,20 +62,18 @@ def text_input_screen():
 
                 feedback_file.write(f"{st.session_state.predict_input_text}|{text_data}\n{file_data}")
 
-        def load_feedback(value):
+        def load_feedback(p_value, p_score):
             feedback_data = []
             feedback_data.append("Bildirim Gönder")
-            p_value = label_dict[predict_value[0]['label']]
-            p_score = predict_value[0]['score']
 
             for item in ["profanity", "racist", "sexist", "insult", "other"]:
                 feedback_data.append(f"{item} ")
             return feedback_data
 
-        predict_value = predict(text_input)
+        is_offansive, categories, score = predict(text_input)
 
-        p_value = label_dict[predict_value[0]['label']]
-        p_score = predict_value[0]['score']
+        p_value = categories
+        p_score = score
 
         class_desc = {"profanity": "İçerikte küfür ifadesi bulunmatadır. [profanity]",
                       "racist": "İçerik ırkı ifadeler içermektedir. [racist]",
@@ -83,7 +87,7 @@ def text_input_screen():
         else:
             st.info(f"Bu metin için bir tahminde buladım. Size daha iyi hizmet vermek için bana metnin sınıfını söyler misiniz?")
 
-        feedback = st.selectbox("Geri Bildirim Gönder", load_feedback(predict_value), key="colour",label_visibility="collapsed", on_change=save_feedback)
+        feedback = st.selectbox("Geri Bildirim Gönder", load_feedback(categories, score), key="colour",label_visibility="collapsed", on_change=save_feedback)
 
         if not feedback in ["Bildirim Gönder"]:
             st.success(f"Geri bildiriminiz alındı, teşekkür ederiz.")

pages/2_⏱️_Canlı İzle.py

@@ -2,30 +2,30 @@ import os.path
 import json
 import time
 
+import numpy as np
 import requests
 import streamlit as st
 import pandas as pd
 import tweepy
 from matplotlib import pyplot as plt
-from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline
+from simpletransformers.classification import MultiLabelClassificationModel
 import clean
+from simpletransformers.config.model_args import ClassificationArgs
 
 
 st.set_page_config(page_title='Fees - Cross Chain Monitoring', page_icon=':bar_chart:', layout='wide')
 st.title('⏱️ Aşağılayıcı Tweet Tespiti')
 
-model=AutoModelForSequenceClassification.from_pretrained("BRAIN-TR/insult-bert-uncased")
-tokenizer=AutoTokenizer.from_pretrained("BRAIN-TR/insult-bert-uncased")
+target_names = ['is_offensive', 'insult', 'other', 'profanity', 'racist', 'sexist']
 
-nlp = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer)
+model_args = ClassificationArgs(
+        use_multiprocessing=True,
+        use_multiprocessing_for_evaluation=True,
+        no_cache= False
+    )
+
+bert_model = MultiLabelClassificationModel('bert','BRAIN-TR/acikhack_multilabel_insult_uncased', use_cuda=False, args=model_args)
 
-label_dict = {
-    'LABEL_0': 'insult',
-    'LABEL_1': 'other',
-    'LABEL_2': 'profanity',
-    'LABEL_3': 'racist',
-    'LABEL_4': 'sexist'
-}
 
 with open('style.css')as f:
     st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html = True)
@@ -45,8 +45,13 @@ def tweets_df(results):
 
     def predict(text):
         cleaned_text = clean.CLEANING(text, True, False,True).clean()
-        out = nlp(cleaned_text)
-        predict_label = label_dict[out[0]['label']]
+        output_arr = bert_model.predict([cleaned_text])
+        def decode_one_hot(input, target_names):
+            return input[0][0], target_names[np.argmax(input[0][1:]) + 1], np.max(output_arr[1][0][1:])
+
+        io, category, score = decode_one_hot(output_arr[0], target_names)
+
+        predict_label = category
         return predict_label
     def has_it_tweet(df, column,  tweet_id):
         source_data = df[(df[column] == tweet_id)]