Upload 6 files

README.txt

@@ -0,0 +1,13 @@
+---
+title: Preprocessing
+emoji: 🔥
+colorFrom: gray
+colorTo: purple
+sdk: gradio
+sdk_version: 3.32.0
+app_file: app.py
+pinned: false
+duplicated_from: veneta/preprocessing
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,26 +1,30 @@
 import pandas as pd
 import gradio as gr
 
-from helper_funcs import functions, INPUT_FILE_TYPE, OUTPUT_FILE_TYPE
+from helper_funcs import functions, INPUT_FILE_TYPE, OUTPUT_FILE_TYPE, get_classla_stats_df
 
 
 def run_function(selected_function, file_obj, input_column, output_column, output_type):
+
     if 'json' in file_obj.name.lower():
         df = pd.read_json(file_obj.name)
     if any([x in file_obj.name.lower() for x in ['csv', 'txt']]):
-        df = pd.read_csv(file_obj.name)
+        df = pd.read_csv(file_obj.name, encoding='utf-8')
 
     output_file = 'result' + output_type
 
     if input_column not in list(df.columns):
         raise gr.Error("Input column name: such column does not exist in dataframe!")
 
-    return functions[selected_function](df, input_column, output_column, output_file)
+    funcs = functions[selected_function](df, input_column, output_column, output_file)
+    return funcs
 
 
 app = gr.Blocks()
 
+
 with app:
+    process_status = gr.State(False)
     gr.Markdown(
         """
         # Instructions
@@ -78,10 +82,19 @@ with app:
                 file_types=OUTPUT_FILE_TYPE
             )
 
-    gr.Button("Process").click(
+    stats_plot = gr.BarPlot(
+        value = pd.DataFrame(columns=['value', 'count']),
+        x = 'value',
+        y = 'count'
+    )
+
+    process_button = gr.Button("Process")
+    process_button.click(
         run_function,
         inputs=[selected_function, file_obj, input_column, output_column, output_type],
-        outputs=[output_dataframe, output_csv]
+        outputs=[output_dataframe, output_csv],
     )
+    strats_button = gr.Button("Get Stats")
+    strats_button.click(get_classla_stats_df, inputs=None, outputs=stats_plot)
 
 app.launch()

gitattributes.txt

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

helper_funcs.py

@@ -1,5 +1,6 @@
 import ast
 import warnings
+from collections import Counter
 
 import classla
 import pandas as pd
@@ -12,6 +13,8 @@ warnings.filterwarnings('ignore')
 INPUT_FILE_TYPE = ['.csv', '.json', '.txt']
 OUTPUT_FILE_TYPE = ['.csv', '.xlsx']
 
+STATS_OUTPUT = 'classla_stats'
+OUTPUT_FILE_NAME = 'result.csv'
 
 def to_output(df, output_file):
     if 'xlsx' in output_file:
@@ -137,6 +140,45 @@ def get_classla_all(df, input_column, output_column, output_file):
     df[output_column] = [clarin_classla_result[index] for index in range(df.shape[0])]
     return to_output(df, output_file)
 
+def classla_stats(df, input_column, output_column, output_file):
+
+    def count_ner(ner_list: []):
+        counter = Counter()
+        for el in ner_list:
+            counter += Counter(el)
+        return str(dict(counter))
+
+
+    global STATS_OUTPUT
+    STATS_OUTPUT = output_column
+    global OUTPUT_FILE_NAME
+    OUTPUT_FILE_NAME = output_file
+
+
+    df[input_column] = df[input_column].apply(lambda x: ast.literal_eval(x))
+
+    if input_column != output_column:
+        df[output_column] = df[input_column]
+
+    clarin_classla_result = [count_ner(df.iloc[index][input_column]) for index in range(df.shape[0])]
+    df[output_column] = [clarin_classla_result[index] for index in range(df.shape[0])]
+    return to_output(df, output_file)
+
+
+def get_classla_stats_df():
+    print(OUTPUT_FILE_NAME)
+    df = pd.read_csv(OUTPUT_FILE_NAME, encoding='utf-8')
+    df[STATS_OUTPUT] = df[STATS_OUTPUT].apply(lambda x: ast.literal_eval(x))
+
+    counter = Counter()
+    for _, line in df.iterrows():
+        counter += Counter(line[STATS_OUTPUT])
+
+    r = pd.DataFrame(dict(counter), index=range(len(dict(counter))))
+    r.drop_duplicates(inplace=True)
+    r = r.melt(var_name='value', value_name='count')
+    return r
+
 
 def run_all(df, input_column, output_column, output_file):
     def load_file(output_file):
@@ -160,6 +202,9 @@ def run_all(df, input_column, output_column, output_file):
     _, _ = get_classla_all(df, 'extracted_sentences', 'classla_all', output_file)
     df = load_file(output_file)
     _, _ = get_classla_ner(df, 'extracted_sentences', 'classla_ner', output_file)
+    df = load_file(output_file)
+    _, _ = classla_stats(df, 'classla_ner', 'classla_stats', output_file)
+    df = load_file(output_file)
     return df.head(10), output_file
 
 
@@ -171,5 +216,8 @@ functions = {
     'separate sentences': get_sentences,
     'Classla NER': get_classla_ner,
     'Classla full result': get_classla_all,
-    'run all': run_all
+    'classla stats': classla_stats,
+    'run all': run_all,
 }
+
+