ffreemt commited on
Commit
d8b0712
1 Parent(s): 9f406cd

Update bot[-1] for testing streaming

Browse files
Files changed (2) hide show
  1. app.py +30 -21
  2. requirements.txt +2 -1
app.py CHANGED
@@ -1,10 +1,11 @@
1
- """Refer to https://github.com/abacaj/mpt-30B-inference/blob/main/download_model.py."""
2
- # pylint: disable=invalid-name, missing-function-docstring, missing-class-docstring, redefined-outer-name, broad-except
3
  import os
4
  import time
5
  from dataclasses import asdict, dataclass
6
  from types import SimpleNamespace
7
 
 
8
  import gradio as gr
9
  from ctransformers import AutoConfig, AutoModelForCausalLM
10
 
@@ -26,25 +27,33 @@ def predict0(prompt, bot):
26
  # logger.debug(f"{prompt=}, {bot=}, {timeout=}")
27
  logger.debug(f"{prompt=}, {bot=}")
28
  ns.response = ""
29
- try:
30
- user_prompt = prompt
31
- generator = generate(llm, generation_config, system_prompt, user_prompt.strip())
32
- print(assistant_prefix, end=" ", flush=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
33
 
34
- response = ""
35
- buff.update(value="diggin...")
36
- for word in generator:
37
- print(word, end="", flush=True)
38
- response += word
39
- ns.response = response
40
- buff.update(value=response)
41
- print("")
42
- logger.debug(f"{response=}")
43
- except Exception as exc:
44
- logger.error(exc)
45
- response = f"{exc=}"
46
  # bot = {"inputs": [response]}
47
- bot = [(prompt, response)]
 
 
 
 
48
 
49
  return prompt, bot
50
 
@@ -53,8 +62,8 @@ def predict_api(prompt):
53
  logger.debug(f"{prompt=}")
54
  ns.response = ""
55
  try:
56
- user_prompt = prompt
57
- generator = generate(llm, generation_config, system_prompt, user_prompt.strip())
58
  print(assistant_prefix, end=" ", flush=True)
59
 
60
  response = ""
 
1
+ """Refer to https://github.com/abacaj/mpt-30B-inference."""
2
+ # pylint: disable=invalid-name, missing-function-docstring, missing-class-docstring, redefined-outer-name, broad-except, line-too-long
3
  import os
4
  import time
5
  from dataclasses import asdict, dataclass
6
  from types import SimpleNamespace
7
 
8
+ from about_time import about_time
9
  import gradio as gr
10
  from ctransformers import AutoConfig, AutoModelForCausalLM
11
 
 
27
  # logger.debug(f"{prompt=}, {bot=}, {timeout=}")
28
  logger.debug(f"{prompt=}, {bot=}")
29
  ns.response = ""
30
+ with about_time() as atime:
31
+ try:
32
+ # user_prompt = prompt
33
+ generator = generate(llm, generation_config, system_prompt, prompt.strip())
34
+ print(assistant_prefix, end=" ", flush=True)
35
+
36
+ response = ""
37
+ buff.update(value="diggin...")
38
+ bot[-1] = [(prompt, "diggin...")]
39
+ for word in generator:
40
+ print(word, end="", flush=True)
41
+ response += word
42
+ ns.response = response
43
+ buff.update(value=response)
44
+ bot[-1] = [(prompt, response)]
45
+ print("")
46
+ logger.debug(f"{response=}")
47
+ except Exception as exc:
48
+ logger.error(exc)
49
+ response = f"{exc=}"
50
 
 
 
 
 
 
 
 
 
 
 
 
 
51
  # bot = {"inputs": [response]}
52
+ _ = (
53
+ f"(time elapsed: {atime.duration_human}, "
54
+ f"{atime.duration/(len(prompt) + len(response))} s/char)"
55
+ )
56
+ bot = [(prompt, f"{response} {_}")]
57
 
58
  return prompt, bot
59
 
 
62
  logger.debug(f"{prompt=}")
63
  ns.response = ""
64
  try:
65
+ # user_prompt = prompt
66
+ generator = generate(llm, generation_config, system_prompt, prompt.strip())
67
  print(assistant_prefix, end=" ", flush=True)
68
 
69
  response = ""
requirements.txt CHANGED
@@ -2,4 +2,5 @@ mosaicml-cli
2
  ctransformers==0.2.10
3
  transformers==4.30.2
4
  huggingface_hub
5
- loguru
 
 
2
  ctransformers==0.2.10
3
  transformers==4.30.2
4
  huggingface_hub
5
+ loguru
6
+ about-time