Added streamed generation

app.py

@@ -1,3 +1,4 @@
+# gpt Gradio App by Craig Brennan
 import torch
 import tiktoken
 import gradio as gr
@@ -13,6 +14,9 @@ n_vocab = 50257
 max_seq_len = 740
 dropout = 0.0
 
+# Number of tokens per update interval
+update_interval = 16
+
 @spaces.GPU
 def load_model():
     from model import GPTModel
@@ -30,9 +34,17 @@ def load_model():
 @spaces.GPU
 def generate(prompt,out_tokens,temperature,top_p_value):
     model.to(device)
-    inputs = tokenizer.encode(prompt)
-    outputs = model.generate(inputs, int(out_tokens), temperature, top_p_value)
-    return tokenizer.decode(outputs) 
+    outputs = tokenizer.encode(prompt)
+    tokens_remaining = int(out_tokens)
+    out_text = prompt
+    yield out_text
+
+    while tokens_remaining:
+        new_inputs_len = update_interval if tokens_remaining >= update_interval else tokens_remaining % update_interval
+        outputs = model.generate(outputs, len(outputs)+new_inputs_len, temperature, top_p_value)
+        tokens_remaining -= new_inputs_len
+        out_text += tokenizer.decode(outputs[-new_inputs_len:])
+        yield out_text
 
 # Create the model
 model, tokenizer, device = load_model()
@@ -41,10 +53,10 @@ app = gr.Interface(
     generate,
     [
         gr.Textbox(label='Prompt', lines=3), 
-        gr.Number(label='Output Tokens', value=54),
+        gr.Number(label='Output Tokens', value=150),
         gr.Slider(0.1, 1.0, step=0.05, value=0.95, label='Top-p Value'),
         gr.Slider(0.1, 2.0, step=0.05, value=0.95, label='Temperature')
     ],
-    gr.Textbox(label='Output', lines=15)
+    gr.Textbox(label='Output', lines=15, max_lines=15)
 )
 app.queue().launch(ssr_mode=False, share=True)