Update web_demo.py

web_demo.py

@@ -8,11 +8,9 @@ import requests
 from argparse import ArgumentParser
 
 import torchaudio
-from transformers import WhisperFeatureExtractor, AutoTokenizer, AutoModel
+from transformers import WhisperFeatureExtractor, AutoTokenizer
 from speech_tokenizer.modeling_whisper import WhisperVQEncoder
-#import os
 
-#os.environ["no_proxy"]="localhost,127.0.0.1,::1"
 
 sys.path.insert(0, "./cosyvoice")
 sys.path.insert(0, "./third_party/Matcha-TTS")
@@ -22,19 +20,28 @@ from speech_tokenizer.utils import extract_speech_token
 import gradio as gr
 import torch
 
+
 audio_token_pattern = re.compile(r"<\|audio_(\d+)\|>")
 
 from flow_inference import AudioDecoder
 
+use_local_interface = True  
+if use_local_interface :
+    from model_server import ModelWorker 
+
 if __name__ == "__main__":
     parser = ArgumentParser()
     parser.add_argument("--host", type=str, default="0.0.0.0")
     parser.add_argument("--port", type=int, default="8888")
     parser.add_argument("--flow-path", type=str, default="./glm-4-voice-decoder")
     parser.add_argument("--model-path", type=str, default="THUDM/glm-4-voice-9b")
-    parser.add_argument("--tokenizer-path", type=str, default="THUDM/glm-4-voice-tokenizer")
+    parser.add_argument("--tokenizer-path", type= str, default="THUDM/glm-4-voice-tokenizer")
     args = parser.parse_args()
-
+    # --tokenizer-path  /home/hanrf/llm/voice/model/ZhipuAI/glm-4-voice-tokenizer --model-path /home/hanrf/llm/voice/model/ZhipuAI/glm-4-voice-9b --flow-path /home/hanrf/llm/voice/model/ZhipuAI/glm-4-voice-decoder 
+    # args.tokenizer_path = '/home/hanrf/llm/voice/model/ZhipuAI/glm-4-voice-tokenizer'
+    # args.model_path = '/home/hanrf/llm/voice/model/ZhipuAI/glm-4-voice-9b'
+    # args.flow_path = '/home/hanrf/llm/voice/model/ZhipuAI/glm-4-voice-decoder'
+    
     flow_config = os.path.join(args.flow_path, "config.yaml")
     flow_checkpoint = os.path.join(args.flow_path, 'flow.pt')
     hift_checkpoint = os.path.join(args.flow_path, 'hift.pt')
@@ -42,7 +49,7 @@ if __name__ == "__main__":
     device = "cuda"
     audio_decoder: AudioDecoder = None
     whisper_model, feature_extractor = None, None
-
+    worker = None 
 
     def initialize_fn():
         global audio_decoder, feature_extractor, whisper_model, glm_model, glm_tokenizer
@@ -61,12 +68,18 @@ if __name__ == "__main__":
         whisper_model = WhisperVQEncoder.from_pretrained(args.tokenizer_path).eval().to(device)
         feature_extractor = WhisperFeatureExtractor.from_pretrained(args.tokenizer_path)
 
+        global use_local_interface, worker  
+        if  use_local_interface :
+            model_path0 = 'THUDM/glm-4-voice-9b '
+            # dtype = 'bfloat16'
+            device0 = 'cuda:0'
+            worker = ModelWorker(model_path0,device0) 
 
     def clear_fn():
         return [], [], '', '', '', None, None
 
 
-def inference_fn(
+    def inference_fn(
             temperature: float,
             top_p: float,
             max_new_token: int,
@@ -105,17 +118,26 @@ def inference_fn(
             inputs += f"<|system|>\n{system_prompt}"
         inputs += f"<|user|>\n{user_input}<|assistant|>streaming_transcription\n"
 
+        global use_local_interface , worker  
         with torch.no_grad():
-            response = requests.post(
-                "http://localhost:10000/generate_stream",
-                data=json.dumps({
-                    "prompt": inputs,
-                    "temperature": temperature,
-                    "top_p": top_p,
-                    "max_new_tokens": max_new_token,
-                }),
-                stream=True
-            )
+            if  use_local_interface :
+                params = {  "prompt": inputs,
+                            "temperature": temperature,
+                            "top_p": top_p,
+                            "max_new_tokens": max_new_token, } 
+                response = worker.generate_stream( params ) 
+                
+            else :
+                response = requests.post(
+                    "http://localhost:10000/generate_stream",
+                    data=json.dumps({
+                        "prompt": inputs,
+                        "temperature": temperature,
+                        "top_p": top_p,
+                        "max_new_tokens": max_new_token,
+                    }),
+                    stream=True
+                )
             text_tokens, audio_tokens = [], []
             audio_offset = glm_tokenizer.convert_tokens_to_ids('<|audio_0|>')
             end_token_id = glm_tokenizer.convert_tokens_to_ids('<|user|>')
@@ -128,7 +150,8 @@ def inference_fn(
             prev_mel = None
             is_finalize = False
             block_size = 10
-            for chunk in response.iter_lines():
+            # for chunk in response.iter_lines():
+            for chunk in response :
                 token_id = json.loads(chunk)["token_id"]
                 if token_id == end_token_id:
                     is_finalize = True
@@ -165,15 +188,15 @@ def inference_fn(
         yield history, inputs, complete_text, '', None, (22050, tts_speech.numpy())
 
 
-def update_input_interface(input_mode):
+    def update_input_interface(input_mode):
         if input_mode == "audio":
             return [gr.update(visible=True), gr.update(visible=False)]
         else:
             return [gr.update(visible=False), gr.update(visible=True)]
 
 
-# Create the Gradio interface
-with gr.Blocks(title="GLM-4-Voice Demo", fill_height=True) as demo:
+    # Create the Gradio interface
+    with gr.Blocks(title="GLM-4-Voice Demo", fill_height=True) as demo:
         with gr.Row():
             temperature = gr.Number(
                 label="Temperature",
@@ -200,7 +223,9 @@ with gr.Blocks(title="GLM-4-Voice Demo", fill_height=True) as demo:
         with gr.Row():
             with gr.Column():
                 input_mode = gr.Radio(["audio", "text"], label="Input Mode", value="audio")
-                audio = gr.Audio(label="Input audio", type='filepath', show_download_button=True, visible=True)
+                # audio = gr.Audio(label="Input audio", type='filepath', show_download_button=True, visible=True)
+                audio = gr.Audio(sources=["upload","microphone"], label="Input audio", type='filepath', show_download_button=True, visible=True)
+                # audio = gr.Audio(source="microphone", label="Input audio", type='filepath', show_download_button=True, visible=True)
                 text_input = gr.Textbox(label="Input text", placeholder="Enter your text here...", lines=2, visible=False)
 
             with gr.Column():
@@ -252,10 +277,16 @@ with gr.Blocks(title="GLM-4-Voice Demo", fill_height=True) as demo:
         reset_btn.click(clear_fn, outputs=[chatbot, history_state, input_tokens, completion_tokens, detailed_error, output_audio, complete_audio])
         input_mode.input(clear_fn, outputs=[chatbot, history_state, input_tokens, completion_tokens, detailed_error, output_audio, complete_audio]).then(update_input_interface, inputs=[input_mode], outputs=[audio, text_input])
 
-initialize_fn()
-# Launch the interface
-demo.launch(
-    server_port=args.port,
-    server_name=args.host,
-    share=True
-)
+    initialize_fn()
+    # Launch the interface
+    demo.launch(
+        server_port=args.port,
+        server_name=args.host,
+        ssl_verify=False,
+        share=True
+    )
+
+'''
+server.launch(share=True)
+https://1a9b77cb89ac33f546.gradio.live
+'''