Migrate to ZeroGPU attempt

app.py

@@ -5,6 +5,76 @@ import re
 import os
 import shutil
 from PIL import Image
+import spaces
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from threading import Thread
+import time
+
+# Initialize the zero tensor on CUDA
+zero = torch.Tensor([0]).cuda()
+print(zero.device)  # This will print 'cpu' outside the @spaces.GPU decorated function
+
+# Load the model and tokenizer
+llmguide_model = AutoModelForCausalLM.from_pretrained(
+    "Qwen/Qwen2-0.5B-Instruct",
+    torch_dtype="auto",
+    device_map="auto"
+)
+llmguide_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
+
+@spaces.GPU
+def llmguide_generate_response(prompt, stream=False):
+    print(zero.device)  # This will print 'cuda:0' inside the @spaces.GPU decorated function
+    
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt}
+    ]
+    text = llmguide_tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    model_inputs = llmguide_tokenizer([text], return_tensors="pt").to(zero.device)
+
+    start_time = time.time()
+    total_tokens = 0
+
+    if stream:
+        streamer = TextIteratorStreamer(llmguide_tokenizer, skip_special_tokens=True)
+        generation_kwargs = dict(
+            model_inputs,
+            streamer=streamer,
+            max_new_tokens=512,
+            temperature=0.7,
+        )
+        thread = Thread(target=llmguide_model.generate, kwargs=generation_kwargs)
+        thread.start()
+
+        generated_text = ""
+        for new_text in streamer:
+            generated_text += new_text
+            total_tokens += 1
+            current_time = time.time()
+            tokens_per_second = total_tokens / (current_time - start_time)
+            yield generated_text, f"{tokens_per_second:.2f}"
+    else:
+        generated_ids = llmguide_model.generate(
+            model_inputs.input_ids,
+            max_new_tokens=512
+        )
+        generated_ids = [
+            output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+        ]
+        response = llmguide_tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+        total_tokens = len(generated_ids[0])
+        end_time = time.time()
+        tokens_per_second = total_tokens / (end_time - start_time)
+        yield response, f"{tokens_per_second:.2f}"
+
+
+#--------------------------------------------------------------------------------------------------------------------------------
 
 # Default configuration template
 default_config = {
@@ -648,6 +718,25 @@ with gr.Blocks() as demo:
     with gr.Accordion("Qwen 0.5B as Space Guide Tests", open=False):
         gr.HTML("Placeholder for FAQ type - front end as prompt engineering for the first message to force direction of conversion")
         gr.HTML("Placeholder for weak RAG Type - Explanations through an opensource embeddings engine")
+        gr.Markdown("# Qwen-0.5B-Instruct Language Model")
+        gr.Markdown("This demo uses the Qwen-0.5B-Instruct model to generate responses based on your input.")
+        
+        with gr.Row():
+            with gr.Column():
+                llmguide_prompt = gr.Textbox(lines=2, placeholder="Enter your prompt here...")
+                llmguide_stream_checkbox = gr.Checkbox(label="Enable streaming")
+                llmguide_submit_button = gr.Button("Generate")
+            
+            with gr.Column():
+                llmguide_output = gr.Textbox(lines=10, label="Generated Response")
+                llmguide_tokens_per_second = gr.Textbox(label="Tokens per Second")
+        
+        llmguide_submit_button.click(
+            llmguide_generate_response,
+            inputs=[llmguide_prompt, llmguide_stream_checkbox],
+            outputs=[llmguide_output, llmguide_tokens_per_second],
+        )
+
     with gr.Accordion("Temporary Asset Management Assist - click to open", open=False):
         gr.HTML("Make Files and Text ideas for the field and paste <br>When Space is restarted it will clear - zip export and import will be added later")
         with gr.Accordion("Upload Files for config"):
@@ -1468,12 +1557,20 @@ Would you like me to elaborate on any of these ideas or show how to implement th
 
         with gr.Tab("Images"):
             with gr.Accordion("Image Gen or Animation HF Spaces/Sites (Click Here to Open) - Have to download and upload at the the top", open=False):
-                with gr.Tabs("General"):
-                    with gr.Row():
-                        linktoimagegen = gr.Dropdown(choices=["https://scribble-sdxl-flash.space", "https://prodia-sdxl-stable-diffusion-xl.hf.space", "https://prodia-fast-stable-diffusion.hf.space", "https://lllyasviel-ic-light.hf.space", "https://gparmar-img2img-turbo-sketch.hf.space", "https://artificialguybr-artificialguybr-demo-lora.hf.space", "https://kadirnar-open-sora.hf.space", "https://bytedance-animatediff-lightning.hf.space", "https://bytedance-hyper-sdxl-1step-t2i.hf.space", "https://ehristoforu-dalle-3-xl-lora-v2.hf.space", "https://multimodalart-cosxl.hf.space", "https://radames-real-time-text-to-image-sdxl-lightning.hf.space", "https://cagliostrolab-animagine-xl-3-1.hf.space", "https://wangfuyun-animatelcm-svd.hf.space" "https://modelscope-transferanything.hf.space", "https://visionmaze-magic-me.hf.space", "https://wangfuyun-animatelcm.hf.space", "https://artgan-diffusion-api.hf.space", "https://multimodalart-stable-cascade.hf.space", "https://ap123-sdxl-lightning.hf.space", "https://google-sdxl.hf.space", "https://guoyww-animatediff.hf.space", "https://segmind-segmind-stable-diffusion.hf.space", "https://simianluo-latent-consistency-model.hf.space", "https://artificialguybr-studio-ghibli-lora-sdxl.hf.space", "https://artificialguybr-pixel-art-generator.hf.space", "https://fffiloni-sdxl-control-loras.hf.space"], label="Choose/Cancel type any .hf.space link here (can also type a link)'", allow_custom_value=True)
-                        imagegenspacebtn = gr.Button("Use the chosen URL to load interface with a image generation model")
+                # with gr.Tabs("General"):
+                with gr.Row():
+                    linktoimagegen = gr.Dropdown(choices=["General", "https://prodia-sdxl-stable-diffusion-xl.hf.space", "https://prodia-fast-stable-diffusion.hf.space", "https://bytedance-hyper-sdxl-1step-t2i.hf.space",  "https://multimodalart-cosxl.hf.space", "https://cagliostrolab-animagine-xl-3-1.hf.space",
+                                                            "Speed", "https://radames-real-time-text-to-image-sdxl-lightning.hf.space", "https://ap123-sdxl-lightning.hf.space",
+                                                            "LORA Support", "https://artificialguybr-artificialguybr-demo-lora.hf.space", "https://ehristoforu-dalle-3-xl-lora-v2.hf.space",
+                                                            "Image to Image", "https://lllyasviel-ic-light.hf.space", "https://gparmar-img2img-turbo-sketch.hf.space",
+                                                            "Control of Pose", "https://instantx-instantid.hf.space",
+                                                            "Control of Shapes", "https://linoyts-scribble-sdxl-flash.hf.space",    "https://modelscope-transferanything.hf.space", "https://visionmaze-magic-me.hf.space", "https://wangfuyun-animatelcm.hf.space",   "https://guoyww-animatediff.hf.space", "https://segmind-segmind-stable-diffusion.hf.space", "https://simianluo-latent-consistency-model.hf.space", "https://artificialguybr-studio-ghibli-lora-sdxl.hf.space", "https://artificialguybr-pixel-art-generator.hf.space", "https://fffiloni-sdxl-control-loras.hf.space"], label="Choose/Cancel type any .hf.space link here (can also type a link)'", allow_custom_value=True)
+                    imagegenspacebtn = gr.Button("Use the chosen URL to load interface with a image generation model")
+            
                 imagegenspace = gr.HTML("Chat Space Chosen will load here")
                 imagegenspacebtn.click(display_website, inputs=linktoimagegen, outputs=imagegenspace)
+            
+            linkstobecollectednoembed = "https://artgan-diffusion-api.hf.space", "https://multimodalart-stable-cascade.hf.space", "https://google-sdxl.hf.space",
             gr.HTML("Concept Art, UI elements, Static/3D Characters, Environments and Objects")
             gr.HTML("Image Caption =  https://huggingface.co/spaces/microsoft/Promptist, https://huggingface.co/spaces/gokaygokay/SD3-Long-Captioner, https://huggingface.co/spaces/gokaygokay/Florence-2,  ")
             gr.HTML("Images Generation Portraits = https://huggingface.co/spaces/okaris/omni-zero")
@@ -1490,6 +1587,8 @@ Would you like me to elaborate on any of these ideas or show how to implement th
             gr.HTML("Placeholder for models small enough to run on cpu here in this space that can assist")
 
         with gr.Tab("Video"):
+
+            linkstobecollectednoembed2 = "https://kadirnar-open-sora.hf.space", "https://wangfuyun-animatelcm-svd.hf.space", "https://bytedance-animatediff-lightning.hf.space",
             gr.HTML("Cutscenes, Tutorials, Trailers")
             gr.HTML("Portrait Video eg. Solo Taking NPC - https://huggingface.co/spaces/fffiloni/tts-hallo-talking-portrait (Image + Audio and combination)  https://huggingface.co/spaces/KwaiVGI/LivePortrait (Non verbal communication eg. in a library, when running from a pursuer)")
             gr.HTML("Placeholder for huggingface spaces that can assist - https://huggingface.co/spaces/KingNish/Instant-Video, https://huggingface.co/spaces/multimodalart/stable-video-diffusion, https://huggingface.co/spaces/multimodalart/stable-video-diffusion")
@@ -1671,4 +1770,4 @@ Would you like me to elaborate on any of these ideas or show how to implement th
     </div>
     """)
 
-demo.launch()
+demo.queue().launch()

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+accelerate