More restrictions for HF spaces to stabilize against GPU OOM and hide unusable options

app.py

@@ -5,7 +5,7 @@ import os
 import traceback
 import typing
 
-from utils import set_seed, flatten_list, clear_torch_cache
+from utils import set_seed, flatten_list, clear_torch_cache, system_info_print
 
 SEED = 1236
 set_seed(SEED)
@@ -210,6 +210,16 @@ def main(
                             traceback.print_exc()
                             score = 0.0
                             clear_torch_cache()
+                        except RuntimeError as e:
+                            if 'Expected all tensors to be on the same device' in str(
+                                    e) or 'expected scalar type Half but found Float' in str(e):
+                                print("GPU error: question: %s answer: %s exception: %s" % (prompt, res, str(e)),
+                                      flush=True)
+                                traceback.print_exc()
+                                score = 0.0
+                                clear_torch_cache()
+                            else:
+                                raise
                         print("SCORE %s: %s" % (exi, score), flush=True)
                         score_dump.append(ex + [prompt, res, score])
                         # dump every score in case abort
@@ -515,7 +525,7 @@ def go_gradio(**kwargs):
     if os.environ.get("HUGGINGFACE_SPACES"):
         description += """<p><b> DISCLAIMERS: </b><ul><i><li>The data used to train this model include The Pile and other sources. These may contain objectionable content, so the model may reproduce that material. Use application and responses at own risk.</i></li>"""
         if kwargs['load_8bit']:
-            description += """<i><li> Model is loaded in 8-bit and 768 token context length to fit on HF GPUs, so model may perform worse than 16-bit with 2048 token limit.</i></li>"""
+            description += """<i><li> Model is loaded in 8-bit and HF spaces version has other limitations in order to fit on HF GPUs, so UX can be worse than native app.</i></li>"""
         description += """<i><li>Model loading and unloading disabled on HF SPACES to avoid GPU OOM for multi-user environment.</i></li></ul></p>"""
 
     if kwargs['verbose']:
@@ -668,7 +678,8 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
                             stream_output = gr.components.Checkbox(label="Stream output",
                                                                    value=kwargs['stream_output'])
                             prompt_type = gr.Dropdown(prompt_types_strings,
-                                                      value=kwargs['prompt_type'], label="Prompt Type")
+                                                      value=kwargs['prompt_type'], label="Prompt Type",
+                                                      visible=not os.environ.get("HUGGINGFACE_SPACES"))
                             temperature = gr.Slider(minimum=0, maximum=3,
                                                     value=kwargs['temperature'],
                                                     label="Temperature",
@@ -681,35 +692,43 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
                                 value=kwargs['top_k'], label="Top k",
                                 info='Num. tokens to sample from'
                             )
-                            num_beams = gr.Slider(minimum=1, maximum=8, step=1,
-                                                  value=kwargs['num_beams'], label="Beams",
-                                                  info="Number of searches for optimal overall probability.  Uses more GPU memory/compute")
+                            max_beams = 8 if not os.environ.get("HUGGINGFACE_SPACES") else 2
+                            num_beams = gr.Slider(minimum=1, maximum=max_beams, step=1,
+                                                  value=min(max_beams, kwargs['num_beams']), label="Beams",
+                                                  info="Number of searches for optimal overall probability.  "
+                                                       "Uses more GPU memory/compute")
+                            max_max_new_tokens = 2048 if not os.environ.get("HUGGINGFACE_SPACES") else kwargs['max_new_tokens']
                             max_new_tokens = gr.Slider(
-                                minimum=1, maximum=2048, step=1,
-                                value=kwargs['max_new_tokens'], label="Max output length"
+                                minimum=1, maximum=max_max_new_tokens, step=1,
+                                value=min(max_max_new_tokens, kwargs['max_new_tokens']), label="Max output length",
                             )
                             min_new_tokens = gr.Slider(
-                                minimum=0, maximum=2048, step=1,
-                                value=kwargs['min_new_tokens'], label="Min output length"
+                                minimum=0, maximum=max_max_new_tokens, step=1,
+                                value=min(max_max_new_tokens, kwargs['min_new_tokens']), label="Min output length",
                             )
                             early_stopping = gr.Checkbox(label="EarlyStopping", info="Stop early in beam search",
                                                          value=kwargs['early_stopping'])
-                            max_time = gr.Slider(minimum=0, maximum=60 * 5, step=1,
-                                                 value=kwargs['max_time'], label="Max. time",
+                            max_max_time = 60 * 5 if not os.environ.get("HUGGINGFACE_SPACES") else 60
+                            max_time = gr.Slider(minimum=0, maximum=max_max_time, step=1,
+                                                 value=min(max_max_time, kwargs['max_time']), label="Max. time",
                                                  info="Max. time to search optimal output.")
                             repetition_penalty = gr.Slider(minimum=0.01, maximum=3.0,
                                                            value=kwargs['repetition_penalty'],
                                                            label="Repetition Penalty")
                             num_return_sequences = gr.Slider(minimum=1, maximum=10, step=1,
                                                              value=kwargs['num_return_sequences'],
-                                                             label="Number Returns", info="Must be <= num_beams")
+                                                             label="Number Returns", info="Must be <= num_beams",
+                                                             visible=not os.environ.get("HUGGINGFACE_SPACES"))
                             do_sample = gr.Checkbox(label="Sample", info="Sample, for diverse output(s)",
                                                     value=kwargs['do_sample'])
                             if kwargs['chat']:
                                 iinput = gr.Textbox(lines=4, label="Input",
-                                                    placeholder=kwargs['placeholder_input'])
+                                                    placeholder=kwargs['placeholder_input'],
+                                                    visible=not os.environ.get("HUGGINGFACE_SPACES"))
+                            # nominally empty for chat mode
                             context = gr.Textbox(lines=1, label="Context",
-                                                 info="Ignored in chat mode.")  # nominally empty for chat mode
+                                                 info="Ignored in chat mode.",
+                                                 visible=not os.environ.get("HUGGINGFACE_SPACES"))
 
                 with gr.TabItem("Models"):
                     with gr.Row():
@@ -731,6 +750,12 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
                                 with gr.Column(scale=1):
                                     add_model_button = gr.Button("Add new model name")
                                     add_lora_button = gr.Button("Add new LORA name", visible=kwargs['show_lora'])
+                with gr.TabItem("System"):
+                    with gr.Row():
+                        with gr.Column():
+                            system_text = gr.Textbox(label='System Info')
+                            system_btn = gr.Button(value='Get System Info')
+
 
         inputs_list = get_inputs_list(locals(), kwargs['model_lower'])
         from functools import partial
@@ -801,6 +826,14 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
                     traceback.print_exc()
                     clear_torch_cache()
                     return 'Response Score: GPU OOM'
+                except RuntimeError as e:
+                    if 'Expected all tensors to be on the same device' in str(e) or 'expected scalar type Half but found Float' in str(e):
+                        print("GPU Error: question: %s answer: %s exception: %s" % (question, answer, str(e)), flush=True)
+                        traceback.print_exc()
+                        clear_torch_cache()
+                        return 'Response Score: GPU Error'
+                    else:
+                        raise
                 os.environ['TOKENIZERS_PARALLELISM'] = 'true'
                 return 'Response Score: {:.1%}'.format(score)
             else:
@@ -987,7 +1020,10 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
         prompt_update_args = dict(fn=dropdown_prompt_type_list, inputs=prompt_type, outputs=prompt_type)
         chatbot_update_args = dict(fn=chatbot_list, inputs=[text_output, model_used], outputs=text_output)
         if not os.environ.get("HUGGINGFACE_SPACES"):
-            load_model_event = load_model_button.click(**load_model_args).then(**prompt_update_args).then(**chatbot_update_args).then(clear_torch_cache)
+            load_model_event = load_model_button.click(**load_model_args) \
+                                                 .then(**prompt_update_args) \
+                                                 .then(**chatbot_update_args) \
+                                                 .then(clear_torch_cache)
 
         def dropdown_model_list(list0, x):
             new_state = [list0[0] + [x]]
@@ -1015,6 +1051,12 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
         callback.setup(inputs_list + [text_output], "flagged_data_points")
         flag_btn.click(lambda *args: callback.flag(args), inputs_list + [text_output], None, preprocess=False,
                        api_name='flag')
+
+        def get_system_info():
+            return gr.Textbox.update(value=system_info_print())
+
+        system_event = system_btn.click(get_system_info, outputs=system_text, api_name='system_info')
+
         if kwargs['chat']:
 
             # don't pass text_output, don't want to clear output, just stop it
@@ -1275,13 +1317,25 @@ def evaluate(
                 try:
                     model.generate(**kwargs)
                 except torch.cuda.OutOfMemoryError as e:
-                    print("GPU OOM: prompt: %s inputs_decoded: %s exception: %s" % (prompt, inputs_decoded, str(e)), flush=True)
+                    print("GPU OOM: prompt: %s inputs_decoded: %s exception: %s" % (prompt, inputs_decoded, str(e)),
+                          flush=True)
                     if kwargs['input_ids'] is not None:
                         kwargs['input_ids'].cpu()
                     kwargs['input_ids'] = None
                     traceback.print_exc()
                     clear_torch_cache()
                     return
+                except RuntimeError as e:
+                    if 'Expected all tensors to be on the same device' in str(
+                            e) or 'expected scalar type Half but found Float' in str(e):
+                        print(
+                            "GPU Error: prompt: %s inputs_decoded: %s exception: %s" % (prompt, inputs_decoded, str(e)),
+                            flush=True)
+                        traceback.print_exc()
+                        clear_torch_cache()
+                        return
+                    else:
+                        raise
 
             for output in CallbackToGenerator(generate, callback=None, **gen_kwargs):
                 decoded_output = decoder(output)

requirements.txt

@@ -23,6 +23,10 @@ git+https://github.com/huggingface/peft.git@098962fa6515f2e4fe83a757f5995d3ffbb1
 transformers==4.28.1
 tokenizers==0.13.3
 
+# optional for generate
+pynvml==11.5.0
+psutil==5.9.4
+
 # optional for finetune
 tensorboard==2.12.1
 neptune==1.1.1

utils.py

@@ -1,7 +1,9 @@
 import os
 import gc
 import random
+import time
 import numpy as np
+import pandas as pd
 import torch
 
 
@@ -37,3 +39,51 @@ def clear_torch_cache():
         torch.cuda.empty_cache()
         torch.cuda.ipc_collect()
         gc.collect()
+
+
+def system_info():
+    import psutil
+
+    system = {}
+    # https://stackoverflow.com/questions/48951136/plot-multiple-graphs-in-one-plot-using-tensorboard
+    # https://arshren.medium.com/monitoring-your-devices-in-python-5191d672f749
+    temps = psutil.sensors_temperatures(fahrenheit=False)
+    if 'coretemp' in temps:
+        coretemp = temps['coretemp']
+        temp_dict = {k.label: k.current for k in coretemp}
+        for k, v in temp_dict.items():
+            system['CPU_C/%s' % k] = v
+
+    # https://github.com/gpuopenanalytics/pynvml/blob/master/help_query_gpu.txt
+    from pynvml.smi import nvidia_smi
+    nvsmi = nvidia_smi.getInstance()
+
+    gpu_power_dict = {'W_gpu%d' % i: x['power_readings']['power_draw'] for i, x in
+                      enumerate(nvsmi.DeviceQuery('power.draw')['gpu'])}
+    for k, v in gpu_power_dict.items():
+        system['GPU_W/%s' % k] = v
+
+    gpu_temp_dict = {'C_gpu%d' % i: x['temperature']['gpu_temp'] for i, x in
+                     enumerate(nvsmi.DeviceQuery('temperature.gpu')['gpu'])}
+    for k, v in gpu_temp_dict.items():
+        system['GPU_C/%s' % k] = v
+
+    gpu_memory_free_dict = {'MiB_gpu%d' % i: x['fb_memory_usage']['free'] for i, x in
+                            enumerate(nvsmi.DeviceQuery('memory.free')['gpu'])}
+    gpu_memory_total_dict = {'MiB_gpu%d' % i: x['fb_memory_usage']['total'] for i, x in
+                             enumerate(nvsmi.DeviceQuery('memory.total')['gpu'])}
+    gpu_memory_frac_dict = {k: gpu_memory_free_dict[k] / gpu_memory_total_dict[k] for k in gpu_memory_total_dict}
+    for k, v in gpu_memory_frac_dict.items():
+        system[f'GPU_M/%s' % k] = v
+
+    return system
+
+
+def system_info_print():
+    try:
+        df = pd.DataFrame.from_dict(system_info(), orient='index')
+        # avoid slamming GPUs
+        time.sleep(1)
+        return df.to_markdown()
+    except Exception as e:
+        return "Error: %s" % str(e)