[#547] fix stream generation bug (#561)

* fix stream generation bug

* Update src/routes/conversation/[id]/+server.ts

Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>

* fix types

* lint

* modified empty token return

* fix lint

* move empty token control flow

* add type check

---------

Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>

src/routes/conversation/[id]/+page.svelte

@@ -140,11 +140,14 @@
 			const reader = response?.body?.pipeThrough(encoder).getReader();
 			let finalAnswer = "";
 
+			// set str queue
+			// ex) if the last response is => {"type": "stream", "token":
+			// It should be => {"type": "stream", "token": "Hello"} = prev_input_chunk + "Hello"}
+			let prev_input_chunk = [""];
+
 			// this is a bit ugly
 			// we read the stream until we get the final answer
 			while (finalAnswer === "") {
-				await new Promise((r) => setTimeout(r, 25));
-
 				// check for abort
 				if (isAborted) {
 					reader?.cancel();
@@ -163,6 +166,8 @@
 						return;
 					}
 
+					value = prev_input_chunk.pop() + value;
+
 					// if it's not done we parse the value, which contains all messages
 					const inputs = value.split("\n");
 					inputs.forEach(async (el: string) => {
@@ -210,6 +215,10 @@
 							}
 						} catch (parseError) {
 							// in case of parsing error we wait for the next message
+
+							if (el === inputs[inputs.length - 1]) {
+								prev_input_chunk.push(el);
+							}
 							return;
 						}
 					});

src/routes/conversation/[id]/+server.ts

@@ -194,7 +194,16 @@ export async function POST({ request, locals, params, getClientAddress }) {
 				if (newUpdate.type !== "stream") {
 					updates.push(newUpdate);
 				}
+
+				if (newUpdate.type === "stream" && newUpdate.token === "") {
+					return;
+				}
 				controller.enqueue(JSON.stringify(newUpdate) + "\n");
+
+				if (newUpdate.type === "finalAnswer") {
+					// 4096 of spaces to make sure the browser doesn't blocking buffer that holding the response
+					controller.enqueue(" ".repeat(4096));
+				}
 			}
 
 			update({ type: "status", status: "started" });