Spaces:
Running
Running
Update script1.js
Browse files- script1.js +527 -1
script1.js
CHANGED
@@ -1 +1,527 @@
|
|
1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
const startStopButton = document.getElementById('startStopButton');
|
2 |
+
const voiceSelectionDropdown = document.getElementById('voiceSelect');
|
3 |
+
const modelSelectionDropdown = document.getElementById('modelSelect');
|
4 |
+
const noiseSuppressionCheckbox = document.getElementById('noiseSuppression');
|
5 |
+
const responseTimeDisplay = document.getElementById('responseTime');
|
6 |
+
const userActivityIndicator = document.getElementById('userIndicator');
|
7 |
+
const aiActivityIndicator = document.getElementById('aiIndicator');
|
8 |
+
const transcriptDiv = document.getElementById('transcript');
|
9 |
+
|
10 |
+
let speechRecognizer;
|
11 |
+
let activeQuery = null;
|
12 |
+
let queryStartTime = 0;
|
13 |
+
let completeTranscript = '';
|
14 |
+
let isRequestInProgress = false;
|
15 |
+
let isUserSpeaking = false;
|
16 |
+
let isSpeechRecognitionActive = false;
|
17 |
+
let requestAbortController = null;
|
18 |
+
let partialTranscript = '';
|
19 |
+
let lastUserSpeechTimestamp = null;
|
20 |
+
let prefetchTextQuery = "";
|
21 |
+
let firstResponseTextTimestamp = null;
|
22 |
+
|
23 |
+
// Configuration
|
24 |
+
const USER_SPEECH_INTERRUPT_DELAY = 500;
|
25 |
+
const TEXT_TO_SPEECH_API_ENDPOINT = "https://api.streamelements.com/kappa/v2/speech";
|
26 |
+
const CHUNK_SIZE = 300;
|
27 |
+
|
28 |
+
// Audio Management
|
29 |
+
let currentAudio = null;
|
30 |
+
let audioPlaybackQueue = [];
|
31 |
+
let prefetchQueue = [];
|
32 |
+
|
33 |
+
// Enhanced Prefetching and Caching
|
34 |
+
const prefetchCache = new Map();
|
35 |
+
const pendingPrefetchRequests = new Map();
|
36 |
+
const MAX_PREFETCH_REQUESTS = 10;
|
37 |
+
const prefetchCacheExpiration = 60000; // 1 minute
|
38 |
+
|
39 |
+
// Global Conversation History
|
40 |
+
let conversationHistory = [];
|
41 |
+
|
42 |
+
// Audio Caching
|
43 |
+
const audioCache = new Map();
|
44 |
+
const audioCacheExpiration = 3600000; // 1 hour
|
45 |
+
|
46 |
+
// Normalize query text
|
47 |
+
const normalizeQueryText = query => query.trim().toLowerCase().replace(/[^\w\s]/g, '');
|
48 |
+
|
49 |
+
// Generate a cache key
|
50 |
+
const generateCacheKey = (normalizedQuery, voice, history, modelName) =>
|
51 |
+
`${normalizedQuery}-${voice}-${JSON.stringify(history)}-${modelName}`;
|
52 |
+
|
53 |
+
// Prefetch and cache the first TTS audio chunk
|
54 |
+
const prefetchFirstAudioChunk = (query, voice) => {
|
55 |
+
const normalizedQuery = normalizeQueryText(query);
|
56 |
+
const cacheKey = generateCacheKey(normalizedQuery, voice, conversationHistory, modelSelectionDropdown.value);
|
57 |
+
|
58 |
+
if (pendingPrefetchRequests.has(cacheKey) || prefetchCache.has(cacheKey)) return;
|
59 |
+
|
60 |
+
prefetchQueue.push({ query:query.trim(), voice, cacheKey });
|
61 |
+
processPrefetchQueue();
|
62 |
+
};
|
63 |
+
|
64 |
+
// Process the prefetch queue
|
65 |
+
const processPrefetchQueue = async () => {
|
66 |
+
while (prefetchQueue.length > 0 && pendingPrefetchRequests.size < MAX_PREFETCH_REQUESTS) {
|
67 |
+
const { query, voice, cacheKey } = prefetchQueue.shift();
|
68 |
+
const abortController = new AbortController();
|
69 |
+
pendingPrefetchRequests.set(cacheKey, abortController);
|
70 |
+
|
71 |
+
const url = '/stream_text';
|
72 |
+
const requestBody = {
|
73 |
+
query: query,
|
74 |
+
history: JSON.stringify(conversationHistory),
|
75 |
+
model: modelSelectionDropdown.value
|
76 |
+
};
|
77 |
+
|
78 |
+
try {
|
79 |
+
const response = await fetch(url, {
|
80 |
+
method: 'POST',
|
81 |
+
headers: {
|
82 |
+
'Accept': 'text/event-stream',
|
83 |
+
'Content-Type': 'application/json'
|
84 |
+
},
|
85 |
+
body: JSON.stringify(requestBody),
|
86 |
+
signal: abortController.signal
|
87 |
+
});
|
88 |
+
|
89 |
+
if (!response.ok) throw new Error('Network response was not ok');
|
90 |
+
|
91 |
+
const firstAudioUrl = await handleStreamingResponseForPrefetch(response.body, voice, abortController.signal);
|
92 |
+
|
93 |
+
if (firstAudioUrl) prefetchCache.set(cacheKey, { url: firstAudioUrl, timestamp: Date.now() });
|
94 |
+
|
95 |
+
} catch (error) {
|
96 |
+
if (error.name !== 'AbortError') console.error("Error prefetching audio:", error);
|
97 |
+
} finally {
|
98 |
+
pendingPrefetchRequests.delete(cacheKey);
|
99 |
+
processPrefetchQueue();
|
100 |
+
}
|
101 |
+
}
|
102 |
+
};
|
103 |
+
|
104 |
+
// Handle the streaming response for prefetching
|
105 |
+
const handleStreamingResponseForPrefetch = async (responseStream, voice, abortSignal) => {
|
106 |
+
const reader = responseStream.getReader();
|
107 |
+
const decoder = new TextDecoder("utf-8");
|
108 |
+
let buffer = "";
|
109 |
+
|
110 |
+
try {
|
111 |
+
while (true) {
|
112 |
+
const { done, value } = await reader.read();
|
113 |
+
if (done) break;
|
114 |
+
if (abortSignal.aborted) throw new DOMException('Request aborted', 'AbortError');
|
115 |
+
|
116 |
+
const chunk = decoder.decode(value, { stream: true });
|
117 |
+
buffer += chunk;
|
118 |
+
const lines = buffer.split('\n');
|
119 |
+
|
120 |
+
for (let i = 0; i < lines.length - 1; i++) {
|
121 |
+
const line = lines[i];
|
122 |
+
if (line.startsWith('data: ')) {
|
123 |
+
const textContent = line.substring(6).trim();
|
124 |
+
if (textContent) {
|
125 |
+
const audioUrl = await generateTextToSpeechAudio(textContent, voice);
|
126 |
+
return audioUrl;
|
127 |
+
}
|
128 |
+
}
|
129 |
+
}
|
130 |
+
|
131 |
+
buffer = lines[lines.length - 1];
|
132 |
+
}
|
133 |
+
} catch (error) {
|
134 |
+
console.error("Error in handleStreamingResponseForPrefetch:", error);
|
135 |
+
} finally {
|
136 |
+
reader.releaseLock();
|
137 |
+
}
|
138 |
+
|
139 |
+
return null;
|
140 |
+
};
|
141 |
+
|
142 |
+
// Play audio from the queue
|
143 |
+
const playNextAudio = async () => {
|
144 |
+
if (audioPlaybackQueue.length > 0) {
|
145 |
+
const audioData = audioPlaybackQueue.shift();
|
146 |
+
const audio = new Audio(audioData.url);
|
147 |
+
updateActivityIndicators();
|
148 |
+
|
149 |
+
const audioPromise = new Promise(resolve => {
|
150 |
+
audio.onended = resolve;
|
151 |
+
audio.onerror = resolve;
|
152 |
+
});
|
153 |
+
if (currentAudio) {
|
154 |
+
currentAudio.pause();
|
155 |
+
currentAudio.currentTime = 0;
|
156 |
+
}
|
157 |
+
|
158 |
+
currentAudio = audio;
|
159 |
+
await audio.play();
|
160 |
+
await audioPromise;
|
161 |
+
playNextAudio();
|
162 |
+
} else {
|
163 |
+
updateActivityIndicators();
|
164 |
+
}
|
165 |
+
};
|
166 |
+
|
167 |
+
// Generate Text-to-Speech audio with caching
|
168 |
+
const generateTextToSpeechAudio = async (text, voice) => {
|
169 |
+
const normalizedText = normalizeQueryText(text);
|
170 |
+
const cacheKey = `${normalizedText}-${voice}`;
|
171 |
+
|
172 |
+
if (audioCache.has(cacheKey)) {
|
173 |
+
const cachedData = audioCache.get(cacheKey);
|
174 |
+
if (Date.now() - cachedData.timestamp < audioCacheExpiration) {
|
175 |
+
return cachedData.url;
|
176 |
+
} else {
|
177 |
+
audioCache.delete(cacheKey);
|
178 |
+
}
|
179 |
+
}
|
180 |
+
|
181 |
+
try {
|
182 |
+
const response = await fetch(`${TEXT_TO_SPEECH_API_ENDPOINT}?voice=${voice}&text=${encodeURIComponent(text)}`, { method: 'GET' });
|
183 |
+
if (!response.ok) throw new Error('Network response was not ok');
|
184 |
+
const audioBlob = await response.blob();
|
185 |
+
const audioUrl = URL.createObjectURL(audioBlob);
|
186 |
+
|
187 |
+
audioCache.set(cacheKey, { url: audioUrl, timestamp: Date.now() });
|
188 |
+
return audioUrl;
|
189 |
+
} catch (error) {
|
190 |
+
console.error("Error generating TTS audio:", error);
|
191 |
+
return null;
|
192 |
+
}
|
193 |
+
};
|
194 |
+
|
195 |
+
// Send a query to the AI
|
196 |
+
const sendQueryToAI = async (query) => {
|
197 |
+
console.log("Sending query to AI:", query);
|
198 |
+
isRequestInProgress = true;
|
199 |
+
updateActivityIndicators();
|
200 |
+
queryStartTime = Date.now();
|
201 |
+
firstResponseTextTimestamp = null;
|
202 |
+
|
203 |
+
const normalizedQuery = normalizeQueryText(query);
|
204 |
+
const cacheKey = generateCacheKey(normalizedQuery, modelSelectionDropdown.value, conversationHistory, modelSelectionDropdown.value);
|
205 |
+
|
206 |
+
if (prefetchCache.has(cacheKey)) {
|
207 |
+
const cachedData = prefetchCache.get(cacheKey);
|
208 |
+
if (Date.now() - cachedData.timestamp < prefetchCacheExpiration) {
|
209 |
+
const prefetchedAudioUrl = cachedData.url;
|
210 |
+
audioPlaybackQueue.push({ url: prefetchedAudioUrl, isPrefetched: true });
|
211 |
+
playNextAudio();
|
212 |
+
} else {
|
213 |
+
prefetchCache.delete(cacheKey);
|
214 |
+
}
|
215 |
+
}
|
216 |
+
|
217 |
+
requestAbortController = new AbortController();
|
218 |
+
|
219 |
+
const url = '/stream_text';
|
220 |
+
const requestBody = {
|
221 |
+
query: query,
|
222 |
+
history: JSON.stringify(conversationHistory),
|
223 |
+
model: modelSelectionDropdown.value
|
224 |
+
};
|
225 |
+
|
226 |
+
try {
|
227 |
+
const response = await fetch(url, {
|
228 |
+
method: 'POST',
|
229 |
+
headers: {
|
230 |
+
'Accept': 'text/event-stream',
|
231 |
+
'Content-Type': 'application/json'
|
232 |
+
},
|
233 |
+
body: JSON.stringify(requestBody),
|
234 |
+
signal: requestAbortController.signal
|
235 |
+
});
|
236 |
+
|
237 |
+
if (!response.ok) {
|
238 |
+
if (response.status === 429) {
|
239 |
+
console.log("Rate limit hit, retrying in 1 second...");
|
240 |
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
241 |
+
await sendQueryToAI(query);
|
242 |
+
return;
|
243 |
+
}
|
244 |
+
throw new Error(`Network response was not ok: ${response.status}`);
|
245 |
+
}
|
246 |
+
|
247 |
+
console.log("Streaming audio response received");
|
248 |
+
await handleStreamingResponse(response.body, voiceSelectionDropdown.value, requestAbortController.signal);
|
249 |
+
} catch (error) {
|
250 |
+
if (error.name !== 'AbortError') {
|
251 |
+
console.error("Error sending query to AI:", error);
|
252 |
+
}
|
253 |
+
} finally {
|
254 |
+
isRequestInProgress = false;
|
255 |
+
updateActivityIndicators();
|
256 |
+
}
|
257 |
+
};
|
258 |
+
|
259 |
+
// Handle the streaming audio response
|
260 |
+
const handleStreamingResponse = async (responseStream, voice, abortSignal) => {
|
261 |
+
const reader = responseStream.getReader();
|
262 |
+
const decoder = new TextDecoder("utf-8");
|
263 |
+
let buffer = "";
|
264 |
+
let initialChunksSent = 0;
|
265 |
+
let fullResponseText = "";
|
266 |
+
let fullResponseText2 = "";
|
267 |
+
let textChunk = "";
|
268 |
+
let sentText = "";
|
269 |
+
|
270 |
+
try {
|
271 |
+
while (true) {
|
272 |
+
const { done, value } = await reader.read();
|
273 |
+
if (done) break;
|
274 |
+
if (abortSignal.aborted) throw new DOMException('Request aborted', 'AbortError');
|
275 |
+
|
276 |
+
if (isUserSpeaking) {
|
277 |
+
interruptAudioPlayback('user is speaking');
|
278 |
+
break;
|
279 |
+
}
|
280 |
+
|
281 |
+
const chunk = decoder.decode(value, { stream: true });
|
282 |
+
buffer += chunk;
|
283 |
+
const lines = buffer.split('\n');
|
284 |
+
|
285 |
+
for (let i = 0; i < lines.length - 1; i++) {
|
286 |
+
const line = lines[i];
|
287 |
+
if (line.startsWith('data: ')) {
|
288 |
+
const textContent = line.substring(6).trim();
|
289 |
+
if (textContent) {
|
290 |
+
if (!firstResponseTextTimestamp) firstResponseTextTimestamp = Date.now();
|
291 |
+
|
292 |
+
fullResponseText += textContent + " ";
|
293 |
+
fullResponseText2 += textContent + " ";
|
294 |
+
textChunk += textContent + " ";
|
295 |
+
transcriptDiv.textContent = fullResponseText2;
|
296 |
+
|
297 |
+
if (initialChunksSent < 2) {
|
298 |
+
const audioUrl = await generateTextToSpeechAudio(textContent, voice);
|
299 |
+
if (audioUrl) {
|
300 |
+
audioPlaybackQueue.push({ url: audioUrl, isPrefetched: false });
|
301 |
+
if (!currentAudio) playNextAudio();
|
302 |
+
}
|
303 |
+
sentText += textContent + " ";
|
304 |
+
initialChunksSent++;
|
305 |
+
} else {
|
306 |
+
let unsentTextChunk = textChunk.replace(sentText, '').trim();
|
307 |
+
|
308 |
+
if (unsentTextChunk.length >= CHUNK_SIZE) {
|
309 |
+
const audioUrl = await generateTextToSpeechAudio(unsentTextChunk, voice);
|
310 |
+
if (audioUrl) {
|
311 |
+
audioPlaybackQueue.push({ url: audioUrl, isPrefetched: false });
|
312 |
+
if (!currentAudio) playNextAudio();
|
313 |
+
}
|
314 |
+
textChunk = "";
|
315 |
+
}
|
316 |
+
}
|
317 |
+
|
318 |
+
if (fullResponseText !== '') {
|
319 |
+
fullResponseText = '';
|
320 |
+
}
|
321 |
+
}
|
322 |
+
}
|
323 |
+
}
|
324 |
+
|
325 |
+
buffer = lines[lines.length - 1];
|
326 |
+
}
|
327 |
+
} catch (error) {
|
328 |
+
console.error("Error in handleStreamingResponse:", error);
|
329 |
+
} finally {
|
330 |
+
reader.releaseLock();
|
331 |
+
|
332 |
+
let unsentTextChunk = textChunk.replace(sentText, '').trim();
|
333 |
+
if (unsentTextChunk !== "") {
|
334 |
+
const audioUrl = await generateTextToSpeechAudio(unsentTextChunk, voice);
|
335 |
+
if (audioUrl) {
|
336 |
+
audioPlaybackQueue.push({ url: audioUrl, isPrefetched: false });
|
337 |
+
if (!currentAudio) playNextAudio();
|
338 |
+
}
|
339 |
+
}
|
340 |
+
|
341 |
+
if (fullResponseText !== '') {
|
342 |
+
fullResponseText = '';
|
343 |
+
}
|
344 |
+
if (fullResponseText2 !== '') {
|
345 |
+
addToConversationHistory('assistant', fullResponseText2);
|
346 |
+
fullResponseText2 = '';
|
347 |
+
}
|
348 |
+
}
|
349 |
+
};
|
350 |
+
|
351 |
+
// Update activity indicators
|
352 |
+
const updateActivityIndicators = (state = null) => {
|
353 |
+
userActivityIndicator.textContent = isUserSpeaking ? "User: Speaking" : "User: Idle";
|
354 |
+
userActivityIndicator.className = isUserSpeaking
|
355 |
+
? "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-blue-400 to-blue-600 hover:bg-gradient-to-r from-blue-500 to-blue-700"
|
356 |
+
: "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-gray-300 to-gray-400 dark:from-gray-700 dark:to-gray-800 hover:bg-gradient-to-r from-gray-400 to-gray-500"; // Tailwind classes
|
357 |
+
|
358 |
+
if (isRequestInProgress && !currentAudio) {
|
359 |
+
aiActivityIndicator.textContent = "AI: Processing...";
|
360 |
+
aiActivityIndicator.className = "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-purple-400 to-purple-600 hover:bg-gradient-to-r from-purple-500 to-purple-700"; // Tailwind class for thinking
|
361 |
+
} else if (currentAudio && !isUserSpeaking) {
|
362 |
+
aiActivityIndicator.textContent = state || "AI: Speaking";
|
363 |
+
aiActivityIndicator.className = "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-green-400 to-green-600 hover:bg-gradient-to-r from-green-500 to-green-700"; // Tailwind class for speaking
|
364 |
+
} else if (isUserSpeaking) {
|
365 |
+
aiActivityIndicator.textContent = "AI: Listening";
|
366 |
+
aiActivityIndicator.className = "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-yellow-400 to-yellow-600 hover:bg-gradient-to-r from-yellow-500 to-yellow-700"; // Tailwind class for listening
|
367 |
+
} else {
|
368 |
+
aiActivityIndicator.textContent = "AI: Idle";
|
369 |
+
aiActivityIndicator.className = "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-gray-300 to-gray-400 dark:from-gray-700 dark:to-gray-800 hover:bg-gradient-to-r from-gray-400 to-gray-500"; // Tailwind classes
|
370 |
+
}
|
371 |
+
};
|
372 |
+
|
373 |
+
|
374 |
+
// Initialize speech recognition
|
375 |
+
if ('webkitSpeechRecognition' in window) {
|
376 |
+
speechRecognizer = new webkitSpeechRecognition();
|
377 |
+
Object.assign(speechRecognizer, {
|
378 |
+
continuous: true,
|
379 |
+
interimResults: true,
|
380 |
+
language: 'en-US',
|
381 |
+
maxAlternatives: 3
|
382 |
+
});
|
383 |
+
|
384 |
+
speechRecognizer.onstart = () => {
|
385 |
+
console.log("Speech recognition started");
|
386 |
+
completeTranscript = '';
|
387 |
+
isUserSpeaking = true;
|
388 |
+
lastUserSpeechTimestamp = Date.now();
|
389 |
+
updateActivityIndicators();
|
390 |
+
startStopButton.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M9 9h6v6h-6z"></path><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"></path><path d="M19 10v2a7 7 0 0 1-14 0v-2"></path><line x1="12" y1="19" x2="12" y2="23"></line><line x1="8" y1="23" x2="16" y2="23"></line></svg> Stop Listening';
|
391 |
+
};
|
392 |
+
|
393 |
+
speechRecognizer.onresult = (event) => {
|
394 |
+
let interimTranscript = '';
|
395 |
+
for (let i = event.resultIndex; i < event.results.length; i++) {
|
396 |
+
const transcript = event.results[i][0].transcript;
|
397 |
+
if (event.results[i].isFinal) {
|
398 |
+
completeTranscript += transcript;
|
399 |
+
interruptAudioPlayback('final');
|
400 |
+
processSpeechTranscript(completeTranscript);
|
401 |
+
completeTranscript = '';
|
402 |
+
isUserSpeaking = false;
|
403 |
+
updateActivityIndicators();
|
404 |
+
queryStartTime = Date.now();
|
405 |
+
} else {
|
406 |
+
interimTranscript += transcript;
|
407 |
+
isUserSpeaking = true;
|
408 |
+
lastUserSpeechTimestamp = Date.now();
|
409 |
+
updateActivityIndicators();
|
410 |
+
|
411 |
+
if (interimTranscript.length > prefetchTextQuery.length + 5) {
|
412 |
+
cancelPrefetchRequests(prefetchTextQuery);
|
413 |
+
}
|
414 |
+
prefetchTextQuery = interimTranscript;
|
415 |
+
prefetchFirstAudioChunk(interimTranscript, voiceSelectionDropdown.value);
|
416 |
+
|
417 |
+
if (isRequestInProgress && shouldInterruptAudioPlayback(interimTranscript)) {
|
418 |
+
interruptAudioPlayback('interim');
|
419 |
+
}
|
420 |
+
}
|
421 |
+
}
|
422 |
+
};
|
423 |
+
|
424 |
+
speechRecognizer.onerror = (event) => {
|
425 |
+
console.error('Speech recognition error:', event.error);
|
426 |
+
if (isSpeechRecognitionActive) speechRecognizer.start();
|
427 |
+
};
|
428 |
+
|
429 |
+
speechRecognizer.onend = () => {
|
430 |
+
isUserSpeaking = false;
|
431 |
+
updateActivityIndicators();
|
432 |
+
|
433 |
+
if (!isRequestInProgress && completeTranscript !== '') {
|
434 |
+
processSpeechTranscript(completeTranscript);
|
435 |
+
completeTranscript = '';
|
436 |
+
}
|
437 |
+
|
438 |
+
if (isSpeechRecognitionActive) speechRecognizer.start();
|
439 |
+
};
|
440 |
+
|
441 |
+
startStopButton.addEventListener('click', () => {
|
442 |
+
if (isSpeechRecognitionActive) {
|
443 |
+
speechRecognizer.stop();
|
444 |
+
isSpeechRecognitionActive = false;
|
445 |
+
startStopButton.innerHTML = '<svg id="microphoneIcon" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"></path><path d="M19 10v2a7 7 0 0 1-14 0v-2"></path><line x1="12" y1="19" x2="12" y2="23"></line><line x1="8" y1="23" x2="16" y2="23"></line></svg> Start Listening';
|
446 |
+
} else {
|
447 |
+
speechRecognizer.start();
|
448 |
+
isSpeechRecognitionActive = true;
|
449 |
+
startStopButton.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M9 9h6v6h-6z"></path><path d="M12 1a3 3 0 0 0-3 3v8a3 3 0 0 0 6 0V4a3 3 0 0 0-3-3z"></path><path d="M19 10v2a7 7 0 0 1-14 0v-2"></path><line x1="12" y1="19" x2="12" y2="23"></line><line x1="8" y1="23" x2="16" y2="23"></line></svg> Stop Listening';
|
450 |
+
}
|
451 |
+
});
|
452 |
+
} else {
|
453 |
+
alert('Your browser does not support the Web Speech API.');
|
454 |
+
}
|
455 |
+
|
456 |
+
// Add to conversation history
|
457 |
+
const addToConversationHistory = (role, content) => {
|
458 |
+
if (conversationHistory.length > 0 &&
|
459 |
+
conversationHistory[conversationHistory.length - 1].role === 'assistant' &&
|
460 |
+
conversationHistory[conversationHistory.length - 1].content === "") {
|
461 |
+
conversationHistory.pop();
|
462 |
+
}
|
463 |
+
|
464 |
+
conversationHistory.push({ role, content });
|
465 |
+
|
466 |
+
if (conversationHistory.length > 6) conversationHistory.splice(0, 2);
|
467 |
+
};
|
468 |
+
|
469 |
+
// Process the final speech transcript
|
470 |
+
const processSpeechTranscript = (transcript) => {
|
471 |
+
const trimmedTranscript = transcript.trimStart();
|
472 |
+
if (trimmedTranscript !== '' && !isRequestInProgress) {
|
473 |
+
activeQuery = trimmedTranscript;
|
474 |
+
sendQueryToAI(activeQuery);
|
475 |
+
addToConversationHistory('user', activeQuery);
|
476 |
+
}
|
477 |
+
};
|
478 |
+
|
479 |
+
// Check if audio playback should be interrupted
|
480 |
+
const shouldInterruptAudioPlayback = (interimTranscript) =>
|
481 |
+
Date.now() - lastUserSpeechTimestamp > USER_SPEECH_INTERRUPT_DELAY || interimTranscript.length > 5;
|
482 |
+
|
483 |
+
// Interrupt audio playback
|
484 |
+
const interruptAudioPlayback = (reason = 'unknown') => {
|
485 |
+
console.log(`Interrupting audio (reason: ${reason})...`);
|
486 |
+
if (currentAudio) {
|
487 |
+
currentAudio.pause();
|
488 |
+
currentAudio.currentTime = 0;
|
489 |
+
currentAudio = null;
|
490 |
+
}
|
491 |
+
|
492 |
+
audioPlaybackQueue.length = 0;
|
493 |
+
isRequestInProgress = false;
|
494 |
+
|
495 |
+
if (requestAbortController) {
|
496 |
+
requestAbortController.abort();
|
497 |
+
requestAbortController = null;
|
498 |
+
}
|
499 |
+
|
500 |
+
prefetchCache.clear();
|
501 |
+
prefetchQueue.length = 0;
|
502 |
+
updateActivityIndicators();
|
503 |
+
};
|
504 |
+
|
505 |
+
// Cancel pending prefetch requests
|
506 |
+
const cancelPrefetchRequests = (query) => {
|
507 |
+
const normalizedQuery = normalizeQueryText(query);
|
508 |
+
|
509 |
+
for (const [cacheKey, abortController] of pendingPrefetchRequests) {
|
510 |
+
if (cacheKey.startsWith(normalizedQuery)) {
|
511 |
+
abortController.abort();
|
512 |
+
pendingPrefetchRequests.delete(cacheKey);pendingPrefetchRequests.delete(cacheKey);
|
513 |
+
}
|
514 |
+
}
|
515 |
+
};
|
516 |
+
|
517 |
+
// Update latency display
|
518 |
+
const updateLatency = () => {
|
519 |
+
if (firstResponseTextTimestamp) {
|
520 |
+
const latency = firstResponseTextTimestamp - queryStartTime;
|
521 |
+
responseTimeDisplay.textContent = `Latency: ${latency}ms`;
|
522 |
+
} else {
|
523 |
+
responseTimeDisplay.textContent = "Latency: 0ms";
|
524 |
+
}
|
525 |
+
};
|
526 |
+
|
527 |
+
setInterval(updateLatency, 200);
|