ffreemt commited on
Commit
93ccd13
1 Parent(s): 2006331

Update vertical stream

Browse files
Files changed (1) hide show
  1. app.py +14 -8
app.py CHANGED
@@ -1,17 +1,16 @@
1
  """Refer to https://github.com/abacaj/mpt-30B-inference."""
2
- # pylint: disable=invalid-name, missing-function-docstring, missing-class-docstring, redefined-outer-name, broad-except, line-too-long
3
  import os
4
  import time
5
  from dataclasses import asdict, dataclass
6
  from types import SimpleNamespace
7
 
8
- from about_time import about_time
9
  import gradio as gr
 
10
  from ctransformers import AutoConfig, AutoModelForCausalLM
11
-
12
- from mcli import predict
13
  from huggingface_hub import hf_hub_download
14
  from loguru import logger
 
15
 
16
  URL = os.getenv("URL", "")
17
  MOSAICML_API_KEY = os.getenv("MOSAICML_API_KEY", "")
@@ -26,6 +25,7 @@ ns = SimpleNamespace(response="")
26
  def predict0(prompt, bot):
27
  # logger.debug(f"{prompt=}, {bot=}, {timeout=}")
28
  logger.debug(f"{prompt=}, {bot=}")
 
29
  ns.response = ""
30
  with about_time() as atime:
31
  try:
@@ -43,6 +43,7 @@ def predict0(prompt, bot):
43
  buff.update(value=response)
44
  print("")
45
  logger.debug(f"{response=}")
 
46
  except Exception as exc:
47
  logger.error(exc)
48
  response = f"{exc=}"
@@ -52,10 +53,14 @@ def predict0(prompt, bot):
52
  f"(time elapsed: {atime.duration_human}, "
53
  f"{atime.duration/(len(prompt) + len(response)):.1f}s/char)"
54
  )
 
 
55
  bot.append([prompt, f"{response} {_}"])
56
 
57
  return prompt, bot
58
 
 
 
59
 
60
  def predict_api(prompt):
61
  logger.debug(f"{prompt=}")
@@ -230,11 +235,12 @@ def call_inf_server(prompt):
230
  generator = generate(
231
  llm, generation_config, system_prompt, user_prompt.strip()
232
  )
233
- print(assistant_prefix, end=" ", flush=True)
 
234
  for word in generator:
235
  print(word, end="", flush=True)
 
236
  print("")
237
- response = word
238
  except Exception as exc:
239
  logger.error(exc)
240
  response = f"{exc=}"
@@ -475,8 +481,8 @@ with gr.Blocks(
475
  api_name="predict",
476
  )
477
  submit.click(
478
- # fn=conversation.user_turn,
479
- fn=predict0,
480
  inputs=[msg, chatbot],
481
  outputs=[msg, chatbot],
482
  queue=True,
 
1
  """Refer to https://github.com/abacaj/mpt-30B-inference."""
2
+ # pylint: disable=invalid-name, missing-function-docstring, missing-class-docstring, redefined-outer-name, broad-except, line-too-long, too-many-instance-attributes
3
  import os
4
  import time
5
  from dataclasses import asdict, dataclass
6
  from types import SimpleNamespace
7
 
 
8
  import gradio as gr
9
+ from about_time import about_time
10
  from ctransformers import AutoConfig, AutoModelForCausalLM
 
 
11
  from huggingface_hub import hf_hub_download
12
  from loguru import logger
13
+ from mcli import predict
14
 
15
  URL = os.getenv("URL", "")
16
  MOSAICML_API_KEY = os.getenv("MOSAICML_API_KEY", "")
 
25
  def predict0(prompt, bot):
26
  # logger.debug(f"{prompt=}, {bot=}, {timeout=}")
27
  logger.debug(f"{prompt=}, {bot=}")
28
+
29
  ns.response = ""
30
  with about_time() as atime:
31
  try:
 
43
  buff.update(value=response)
44
  print("")
45
  logger.debug(f"{response=}")
46
+ bot[-1] = [prompt, response]
47
  except Exception as exc:
48
  logger.error(exc)
49
  response = f"{exc=}"
 
53
  f"(time elapsed: {atime.duration_human}, "
54
  f"{atime.duration/(len(prompt) + len(response)):.1f}s/char)"
55
  )
56
+
57
+ # bot[-1] = [prompt, f"{response} {_}"]
58
  bot.append([prompt, f"{response} {_}"])
59
 
60
  return prompt, bot
61
 
62
+ # for stream refer to https://gradio.app/creating-a-chatbot/#a-simple-chatbot-demo
63
+
64
 
65
  def predict_api(prompt):
66
  logger.debug(f"{prompt=}")
 
235
  generator = generate(
236
  llm, generation_config, system_prompt, user_prompt.strip()
237
  )
238
+ # print(assistant_prefix, end=" ", flush=True)
239
+ print(assistant_prefix, flush=True) # vertical
240
  for word in generator:
241
  print(word, end="", flush=True)
242
+ response = word
243
  print("")
 
244
  except Exception as exc:
245
  logger.error(exc)
246
  response = f"{exc=}"
 
481
  api_name="predict",
482
  )
483
  submit.click(
484
+ # fn=predict0,
485
+ fn=lambda x, y: ("",) + predict0(x, y)[1:], # clear msg
486
  inputs=[msg, chatbot],
487
  outputs=[msg, chatbot],
488
  queue=True,