Upload 13 files

app.py

@@ -1,21 +1,55 @@
 import streamlit as st
 import diary as d
 import sentiment_parser as sp
+import network_builder as nb
+from pyvis.network import Network
+import streamlit.components.v1 as components
 # import altair as alt
 
 st.title('Автоматический аннотатор')
 
 st.markdown("Скопируйте текст дневика в это поле или выберите для теста один из подготовленных отрывков.")
 
+
 with open('test.txt', 'r') as f:
     TEST = f.read()
 # diary = st.text_area('Текст дневника')
 if st.button('Быстрая обработка на тестовом тексте '):
     df = d.analyze(TEST)
-    st.dataframe(df)
-    for_chart = sp.data_for_sentiment_chart(df).set_index('n_date')
-    st.markdown('### График сентимента по записям дневника (тест)')
-    st.line_chart(data=for_chart)
+    # st.dataframe(df)
+    # for_chart = sp.data_for_sentiment_chart(df).set_index('n_date')
+    # st.markdown('### График сентимента по записям дневника (тест)')
+    # st.line_chart(data=for_chart)
+    # st.experimental_memo.clear()
+    graph = nb.build_graph(df)
+
+    textnet = Network( height='400px',
+                       width='100%',
+                       bgcolor='white',
+                       font_color='black'
+                      )
+    
+    textnet.from_nx(graph)
+
+    textnet.repulsion(
+                        node_distance=420,
+                        central_gravity=0.33,
+                        spring_length=110,
+                        spring_strength=0.10,
+                        damping=0.95
+                       )
+    
+    try:
+        path = '/tmp'
+        textnet.save_graph(f'{path}/pyvis_graph.html')
+        HtmlFile = open(f'{path}/pyvis_graph.html', 'r', encoding='utf-8')
+
+    except:
+        path = '/html_files'
+        textnet.save_graph(f'{path}/pyvis_graph.html')
+        HtmlFile = open(f'{path}/pyvis_graph.html', 'r', encoding='utf-8')
+
+    components.html(HtmlFile.read(), height=435)
 
 # if st.button('Обработать'):
 #     df = d.analyze(diary)

category_parser.py

@@ -126,7 +126,8 @@ def get_facts(tokens, category):
                 for word in find_need_word_by_lemma(sent_tokens, w):
                     fact = construct_fact(sent_tokens, word, category)
                     if fact: 
-                        facts.append(fact)
+                        # facts.append(fact)
+                       facts.append([w, fact]) 
     return facts
 
 

date_parser.py

@@ -9,7 +9,7 @@ from yargy.predicates import (
     dictionary, normalized,
 )
 import pandas as pd
-
+import streamlit as st
 
 DateRange = fact(
     'DateRange',
@@ -309,7 +309,7 @@ DATE_RANGE = or_(
     DateRange
 )
 
-
+@st.experimental_memo
 def date_extractor_for_diary(text):
     res = {
         'date_start' : [],

diary.py

@@ -2,11 +2,12 @@ import date_parser as dp
 import preproc
 import category_parser as cp
 import sentiment_parser as sp
-
+import streamlit as st
 
 # def get_gender(tokens):
 #     r = [token.feats['Gender'] for sent in tokenizing(text) for token in sent if (token.feats.get('Gender') and token.feats.get('Voice')) ]
 
+# @st.experimental_memo
 def analyze(text):
     # Разделение текста на датированные куски
     diary = dp.date_extractor_for_diary(text)
@@ -17,12 +18,12 @@ def analyze(text):
     # Токенизация текста дневника по предложениям
     diary['tokens'] = diary['text'].apply(lambda text: preproc.tokenizing(text))
     
-    # Выделение фактов из текста
+    # # Выделение фактов из текста
     diary['loc_facts'] = diary['tokens'].apply(lambda tokens: cp.get_facts(tokens, 'locations'))
-    diary['loc_words'] = diary['tokens'].apply(lambda tokens: cp.get_mentioned_words(tokens, 'locations'))
+    # diary['loc_words'] = diary['tokens'].apply(lambda tokens: cp.get_mentioned_words(tokens, 'locations'))
 
     # Определение сентимента по записям
-    diary['sent'] = diary['tokens'].apply(lambda tokens: sp.get_overall_sentiment(tokens))
-    diary['sent_index'] = diary['sent'].apply(lambda sent: sp.get_sentiment_index(sent))
+    # diary['sent'] = diary['tokens'].apply(lambda tokens: sp.get_overall_sentiment(tokens))
+    # diary['sent_index'] = diary['sent'].apply(lambda sent: sp.get_sentiment_index(sent))
 
     return diary

network_builder.py

@@ -0,0 +1,21 @@
+import networkx as nx
+
+
+def build_graph(df):
+    G = nx.Graph()
+
+    # Связывание дат 
+    for previous, current in zip(df['date_start'], df['date_start'][1:]):
+        G.add_node(previous, group="Date", color = "blue")
+        G.add_node(current, group="Date", color = "blue")
+        G.add_edge(previous, current)
+
+    # Добавление связей дат и фактов, фактов и слов
+    for index, row in df[['date_start', 'loc_facts']].iterrows():
+        for fact in row['loc_facts']:
+            G.add_node(fact[0], group="Category_word", color = "green")
+            G.add_node(fact[1], group="Fact", color = "red")
+            G.add_edge(row['date_start'], fact[1])
+            G.add_edge(fact[0], fact[1])
+
+    return G

requirements.txt

@@ -1,4 +1,8 @@
-natasha==1.5.0
+natasha
 streamlit==1.17.0
-pandas
-altair<5
+networkx==2.8.6
+numpy==1.23.3
+pandas==1.4.4
+altair<5
+pyvis==0.2.1
+pymorphy2

sentiment_parser.py

@@ -3,6 +3,7 @@ import os
 import preproc, date_parser
 from collections import Counter
 import pandas as pd
+import streamlit as st
 
 SCRIPT_DIR = os.path.dirname(__file__)
 
@@ -10,13 +11,17 @@ def sentiment_verbs():
     with open(f'{SCRIPT_DIR}/sentiment/verbs.json', 'r', encoding='utf-8') as file:
         return json.load(file)
 
+
 def sentiment_nouns():
     with open(f'{SCRIPT_DIR}/sentiment/emo_clean.json', 'r', encoding='utf-8') as file:
         return json.load(file)
 
+VERBS = sentiment_verbs()
+NOUNS = sentiment_nouns()
+
+
 def get_sentiment_from_verbs(lemmas):
     res = []
-    VERBS = sentiment_verbs()
     matching = set(lemmas) & set(VERBS.keys())
     lemmas_dict = Counter(lemmas)
     if matching:
@@ -26,10 +31,9 @@ def get_sentiment_from_verbs(lemmas):
                 res.extend(s)
         return Counter(res)
     else: return Counter()
-       
+   
 def get_sentiment_from_nouns(lemmas):
     res = []
-    NOUNS = sentiment_nouns()
     matching = set(lemmas) & set(NOUNS.keys())
     lemmas_dict = Counter(lemmas)
     if matching:
@@ -46,9 +50,11 @@ def get_overall_sentiment(tokens):
     nouns = get_sentiment_from_nouns(lemmas)
     return verbs + nouns
 
+
 def get_sentiment_index(sentiments):
   return sentiments['positive'] - sentiments['negative']
 
+
 def get_most_sentiment(sentiment_index):
   sentiments = []
   for index in sentiment_index:
@@ -61,6 +67,7 @@ def get_most_sentiment(sentiment_index):
   sentiments = Counter(sentiments)
   return sentiments.most_common(1)[0][0]
 
+@st.experimental_memo
 def data_for_sentiment_chart(df):
     df = df.copy()
     df['n_date'] = df.apply(lambda row: