Matthew Current nsarrazin HF staff commited on
Commit
9c5a826
1 Parent(s): 3abaf81

Bug Fix: Json Decoder aggessively pulls json (#867)

Browse files

* wait for newline char b4 decoding json

* remove noisy console log

* linters

---------

Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>

src/lib/server/endpoints/llamacpp/endpointLlamacpp.ts CHANGED
@@ -55,10 +55,13 @@ export function endpointLlamacpp(
55
  let stop = false;
56
  let generatedText = "";
57
  let tokenId = 0;
 
 
58
  while (!stop) {
59
- // read the stream and log the outputs to console
60
  const out = (await reader?.read()) ?? { done: false, value: undefined };
61
- // we read, if it's done we cancel
 
62
  if (out.done) {
63
  reader?.cancel();
64
  return;
@@ -68,31 +71,49 @@ export function endpointLlamacpp(
68
  return;
69
  }
70
 
71
- if (out.value.startsWith("data: ")) {
72
- let data = null;
73
- try {
74
- data = JSON.parse(out.value.slice(6));
75
- } catch (e) {
76
- return;
77
- }
78
- if (data.content || data.stop) {
79
- generatedText += data.content;
80
- const output: TextGenerationStreamOutput = {
81
- token: {
82
- id: tokenId++,
83
- text: data.content ?? "",
84
- logprob: 0,
85
- special: false,
86
- },
87
- generated_text: data.stop ? generatedText : null,
88
- details: null,
89
- };
90
- if (data.stop) {
91
- stop = true;
92
- reader?.cancel();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
93
  }
94
- yield output;
95
- // take the data.content value and yield it
96
  }
97
  }
98
  }
 
55
  let stop = false;
56
  let generatedText = "";
57
  let tokenId = 0;
58
+ let accumulatedData = ""; // Buffer to accumulate data chunks
59
+
60
  while (!stop) {
61
+ // Read the stream and log the outputs to console
62
  const out = (await reader?.read()) ?? { done: false, value: undefined };
63
+
64
+ // If it's done, we cancel
65
  if (out.done) {
66
  reader?.cancel();
67
  return;
 
71
  return;
72
  }
73
 
74
+ // Accumulate the data chunk
75
+ accumulatedData += out.value;
76
+
77
+ // Process each complete JSON object in the accumulated data
78
+ while (accumulatedData.includes("\n")) {
79
+ // Assuming each JSON object ends with a newline
80
+ const endIndex = accumulatedData.indexOf("\n");
81
+ let jsonString = accumulatedData.substring(0, endIndex).trim();
82
+
83
+ // Remove the processed part from the buffer
84
+ accumulatedData = accumulatedData.substring(endIndex + 1);
85
+
86
+ if (jsonString.startsWith("data: ")) {
87
+ jsonString = jsonString.slice(6);
88
+ let data = null;
89
+
90
+ try {
91
+ data = JSON.parse(jsonString);
92
+ } catch (e) {
93
+ console.error("Failed to parse JSON", e);
94
+ console.error("Problematic JSON string:", jsonString);
95
+ continue; // Skip this iteration and try the next chunk
96
+ }
97
+
98
+ // Handle the parsed data
99
+ if (data.content || data.stop) {
100
+ generatedText += data.content;
101
+ const output: TextGenerationStreamOutput = {
102
+ token: {
103
+ id: tokenId++,
104
+ text: data.content ?? "",
105
+ logprob: 0,
106
+ special: false,
107
+ },
108
+ generated_text: data.stop ? generatedText : null,
109
+ details: null,
110
+ };
111
+ if (data.stop) {
112
+ stop = true;
113
+ reader?.cancel();
114
+ }
115
+ yield output;
116
  }
 
 
117
  }
118
  }
119
  }