refresh log files not working

app_text_classification.py

@@ -1,7 +1,8 @@
 import gradio as gr
+import uuid
 from io_utils import read_scanners, write_scanners, read_inference_type, write_inference_type
 from wordings import INTRODUCTION_MD, CONFIRM_MAPPING_DETAILS_MD
-from text_classification_ui_helpers import try_submit, check_dataset_and_get_config, check_dataset_and_get_split, check_model_and_show_prediction, write_column_mapping_to_config
+from text_classification_ui_helpers import try_submit, check_dataset_and_get_config, check_dataset_and_get_split, check_model_and_show_prediction, write_column_mapping_to_config, get_logs_file
 
 MAX_LABELS = 20
 MAX_FEATURES = 20
@@ -11,93 +12,125 @@ EXAMPLE_DATA_ID = 'tweet_eval'
 CONFIG_PATH='./config.yaml'
 
 def get_demo():
-    with gr.Row():
-        gr.Markdown(INTRODUCTION_MD)
-    with gr.Row():
-        model_id_input = gr.Textbox(
-            label="Hugging Face model id",
-            placeholder=EXAMPLE_MODEL_ID + " (press enter to confirm)",
-        )
+    with gr.Blocks() as demo:
+        with gr.Row():
+            gr.Markdown(INTRODUCTION_MD)
+        with gr.Row():
+            model_id_input = gr.Textbox(
+                label="Hugging Face model id",
+                placeholder=EXAMPLE_MODEL_ID + " (press enter to confirm)",
+            )
 
-        dataset_id_input = gr.Textbox(
-            label="Hugging Face Dataset id",
-            placeholder=EXAMPLE_DATA_ID + " (press enter to confirm)",
-        )
-    
-    with gr.Row():
-        dataset_config_input = gr.Dropdown(label='Dataset Config', visible=False)
-        dataset_split_input = gr.Dropdown(label='Dataset Split', visible=False)
-    
-    with gr.Row():
-        example_input = gr.Markdown('Example Input', visible=False)
-    with gr.Row():
-        example_prediction = gr.Label(label='Model Prediction Sample', visible=False)
-    
-    with gr.Row():
-        with gr.Accordion(label='Label and Feature Mapping', visible=False, open=False) as column_mapping_accordion:
-            with gr.Row():
-                gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
-            column_mappings = []
-            with gr.Row():
-                with gr.Column():
-                    for _ in range(MAX_LABELS):
-                        column_mappings.append(gr.Dropdown(visible=False))
-                with gr.Column():    
-                    for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
-                        column_mappings.append(gr.Dropdown(visible=False))
-    
-    with gr.Accordion(label='Model Wrap Advance Config (optional)', open=False):
-        run_local = gr.Checkbox(value=True, label="Run in this Space")
-        use_inference = read_inference_type('./config.yaml') == 'hf_inference_api'
-        run_inference = gr.Checkbox(value=use_inference, label="Run with Inference API")
-    
-    with gr.Accordion(label='Scanner Advance Config (optional)', open=False):
-        selected = read_scanners('./config.yaml')
-        scan_config = selected + ['data_leakage']
-        scanners = gr.CheckboxGroup(choices=scan_config, value=selected, label='Scan Settings', visible=True)
+            dataset_id_input = gr.Textbox(
+                label="Hugging Face Dataset id",
+                placeholder=EXAMPLE_DATA_ID + " (press enter to confirm)",
+            )
+        
+        with gr.Row():
+            dataset_config_input = gr.Dropdown(label='Dataset Config', visible=False)
+            dataset_split_input = gr.Dropdown(label='Dataset Split', visible=False)
+        
+        with gr.Row():
+            example_input = gr.Markdown('Example Input', visible=False)
+        with gr.Row():
+            example_prediction = gr.Label(label='Model Prediction Sample', visible=False)
+        
+        with gr.Row():
+            with gr.Accordion(label='Label and Feature Mapping', visible=False, open=False) as column_mapping_accordion:
+                with gr.Row():
+                    gr.Markdown(CONFIRM_MAPPING_DETAILS_MD)
+                column_mappings = []
+                with gr.Row():
+                    with gr.Column():
+                        for _ in range(MAX_LABELS):
+                            column_mappings.append(gr.Dropdown(visible=False))
+                    with gr.Column():    
+                        for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
+                            column_mappings.append(gr.Dropdown(visible=False))
+        
+        with gr.Accordion(label='Model Wrap Advance Config (optional)', open=False):
+            run_local = gr.Checkbox(value=True, label="Run in this Space")
+            use_inference = read_inference_type('./config.yaml') == 'hf_inference_api'
+            run_inference = gr.Checkbox(value=use_inference, label="Run with Inference API")
+        
+        with gr.Accordion(label='Scanner Advance Config (optional)', open=False):
+            selected = read_scanners('./config.yaml')
+            # currently we remove data_leakage from the default scanners
+            # Reason: data_leakage barely raises any issues and takes too many requests
+            # when using inference API, causing rate limit error
+            scan_config = selected + ['data_leakage']
+            scanners = gr.CheckboxGroup(choices=scan_config, value=selected, label='Scan Settings', visible=True)
 
-    with gr.Row():
-        run_btn = gr.Button(
-            "Get Evaluation Result",
-            variant="primary",
-            interactive=True,
-            size="lg",
-        )
-    
-    with gr.Row():
-        logs = gr.Textbox(label="Giskard Bot Evaluation Log:", visible=False)
+        with gr.Row():
+            run_btn = gr.Button(
+                "Get Evaluation Result",
+                variant="primary",
+                interactive=True,
+                size="lg",
+            )
         
-    gr.on(triggers=[label.change for label in column_mappings], 
-          fn=write_column_mapping_to_config,
-           inputs=[dataset_id_input, dataset_config_input, dataset_split_input, *column_mappings])
+        with gr.Row():
+            uid = uuid.uuid4()
+            uid_label = gr.Textbox(label="Evaluation ID:", value=uid, visible=False)
+            logs = gr.Textbox(label="Giskard Bot Evaluation Log:", visible=False)
+            demo.load(get_logs_file, uid_label, logs, every=0.5)
+            
+        gr.on(triggers=[label.change for label in column_mappings], 
+            fn=write_column_mapping_to_config,
+            inputs=[dataset_id_input, dataset_config_input, dataset_split_input, *column_mappings])
 
-    gr.on(triggers=[model_id_input.change, dataset_config_input.change, dataset_split_input.change],
-        fn=check_model_and_show_prediction,
-        inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
-        outputs=[example_input, example_prediction, column_mapping_accordion, *column_mappings])
+        gr.on(triggers=[model_id_input.change, dataset_config_input.change, dataset_split_input.change],
+            fn=check_model_and_show_prediction,
+            inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input], 
+            outputs=[example_input, example_prediction, column_mapping_accordion, *column_mappings])
 
-    dataset_id_input.blur(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
+        dataset_id_input.blur(check_dataset_and_get_config, dataset_id_input, dataset_config_input)
 
-    dataset_config_input.change(
-        check_dataset_and_get_split, 
-        inputs=[dataset_id_input, dataset_config_input], 
-        outputs=[dataset_split_input])
+        dataset_config_input.change(
+            check_dataset_and_get_split, 
+            inputs=[dataset_id_input, dataset_config_input], 
+            outputs=[dataset_split_input])
 
-    scanners.change(
-        write_scanners,
-        inputs=scanners
-    )
+        scanners.change(
+            write_scanners,
+            inputs=scanners
+        )
 
-    run_inference.change(
-        write_inference_type,
-        inputs=[run_inference]
-    )
+        run_inference.change(
+            write_inference_type,
+            inputs=[run_inference]
+        )
 
-    gr.on(
-        triggers=[
-            run_btn.click,
-            ],
-        fn=try_submit,
-        inputs=[model_id_input, dataset_id_input, dataset_config_input, dataset_split_input, run_local],
-        outputs=[run_btn, logs])
-        
+        gr.on(
+            triggers=[
+                run_btn.click,
+                ],
+            fn=try_submit,
+            inputs=[
+                model_id_input, 
+                dataset_id_input, 
+                dataset_config_input, 
+                dataset_split_input, 
+                run_local, 
+                uid_label],
+            outputs=[run_btn, logs])
+        
+        def enable_run_btn():
+            return gr.update(interactive=True)
+        gr.on(
+            triggers=[
+                    model_id_input.change, 
+                    dataset_config_input.change, 
+                    dataset_split_input.change, 
+                    run_inference.change, 
+                    run_local.change, 
+                    scanners.change],
+            fn=enable_run_btn,
+            inputs=None,
+            outputs=[run_btn])
+        
+        gr.on(
+            triggers=[label.change for label in column_mappings],
+            fn=enable_run_btn,
+            inputs=None,
+            outputs=[run_btn])

text_classification_ui_helpers.py

@@ -3,7 +3,6 @@ from wordings import CONFIRM_MAPPING_DETAILS_FAIL_RAW
 import json
 import os
 import logging
-import uuid
 import threading
 from io_utils import read_column_mapping, write_column_mapping, save_job_to_pipe, write_log_to_user_file
 import datasets
@@ -125,12 +124,14 @@ def check_model_and_show_prediction(model_id, dataset_id, dataset_config, datase
     )
 
 def get_logs_file(uid):
-    file = open(f"./tmp/{uid}_log")
+    print("read log file")
+    file = open(f"./tmp/{uid}_log", "r")
     contents = file.readlines()
+    print(contents)
     file.close()
     return '\n'.join(contents)
 
-def try_submit(m_id, d_id, config, split, local):
+def try_submit(m_id, d_id, config, split, local, uid):
     all_mappings = read_column_mapping(CONFIG_PATH)
 
     if all_mappings is None:
@@ -168,14 +169,14 @@ def try_submit(m_id, d_id, config, split, local):
 
         eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
         logging.info(f"Start local evaluation on {eval_str}")
-        uid = uuid.uuid4()
+        # uid = uuid.uuid4()
         save_job_to_pipe(uid, command, threading.Lock())
         write_log_to_user_file(uid, f"Start local evaluation on {eval_str}. Please wait for your job to start...\n")
         gr.Info(f"Start local evaluation on {eval_str}")
 
         return (
             gr.update(interactive=False),
-            gr.update(value=get_logs_file(uid), visible=True, interactive=False, every=5))
+            gr.update(lines=5, visible=True, interactive=False))
 
     else:
         gr.Info("TODO: Submit task to an endpoint")

wordings.py

@@ -8,7 +8,7 @@ CONFIRM_MAPPING_DETAILS_MD = '''
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            Please confirm the pre-processing details below. If you are not sure, please double check your model and dataset.
+                            Please confirm the pre-processing details below. Align the column names of your model in the <b>dropdown</b> menu to your dataset's. If you are not sure, please double check your model and dataset.
                             '''
 CONFIRM_MAPPING_DETAILS_FAIL_MD = '''
                             <h1 style="text-align: center;">