File size: 870 Bytes
6e6dab9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b1cb16f
 
 
 
 
 
6e6dab9
 
 
d1f0f2d
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
const decoder = new TextDecoder("utf-8");

export function stringify_stream_bytes(bytes) {
    return decoder.decode(bytes);
}

export function jsonize_stream_data(data) {
    var json_chunks = [];
    data = data
        .replace(/^data:\s*/gm, "")
        .replace(/\[DONE\]/gm, "")
        .split("\n")
        .filter(function (line) {
            return line.trim().length > 0;
        })
        .map(function (line) {
            try {
                // ToFix: Single line broken into multiple chunks
                json_chunks.push(JSON.parse(line.trim()));
            } catch {
                console.log(`Failed to parse: ${line}`);
            }
        });
    return json_chunks;
}

export function transform_footnote(text) {
    return text
        .replace(/\[\^(\d+)\^\]\[\d+\]/g, "[$1]")
        .replace(/\[(\d+)\]:\s*(.*)\s*""/g, "[$1] $2 \n");
}