Luke Stanley commited on
Commit
a0f49a0
1 Parent(s): ddb0d91

Make llm_stream_sans_network actually stream to stdout

Browse files
Files changed (1) hide show
  1. utils.py +10 -3
utils.py CHANGED
@@ -95,14 +95,21 @@ def llm_stream_sans_network(
95
  json_schema = json.dumps(schema)
96
  grammar = LlamaGrammar.from_json_schema(json_schema)
97
 
98
- output_text = in_memory_llm(
99
  prompt,
100
  max_tokens=1000,
101
  temperature=0.7,
102
  grammar=grammar,
103
- )["choices"][0]["text"]
 
 
 
 
 
 
 
104
 
105
- print(output_text)
106
 
107
  if return_pydantic_object:
108
  model_object = pydantic_model_class.model_validate_json(output_text)
 
95
  json_schema = json.dumps(schema)
96
  grammar = LlamaGrammar.from_json_schema(json_schema)
97
 
98
+ stream = in_memory_llm(
99
  prompt,
100
  max_tokens=1000,
101
  temperature=0.7,
102
  grammar=grammar,
103
+ stream=True
104
+ )
105
+
106
+ output_text = ""
107
+ for chunk in stream:
108
+ result = chunk["choices"][0]
109
+ print(result["text"], end='', flush=True)
110
+ output_text = output_text + result["text"]
111
 
112
+ print('\n')
113
 
114
  if return_pydantic_object:
115
  model_object = pydantic_model_class.model_validate_json(output_text)