jbilcke-hf HF staff commited on
Commit
77030e1
1 Parent(s): 89b09eb

work on the assistant

Browse files
Files changed (36) hide show
  1. package-lock.json +499 -438
  2. package.json +1 -0
  3. src/app/api/assistant/askAnyAssistant.ts +24 -5
  4. src/app/api/assistant/providers/google/README.md +11 -0
  5. src/app/api/assistant/providers/google/sample.txt +32 -0
  6. src/app/api/assistant/providers/openai/askAssistant.ts +0 -129
  7. src/app/api/assistant/templates.ts +5 -1
  8. src/app/api/resolve/providers/lumalabs/getGenerations.ts +2 -0
  9. src/app/api/resolve/providers/lumalabs/todo.ts +5 -0
  10. src/app/api/resolve/providers/lumalabs/types.ts +36 -0
  11. src/app/api/resolve/providers/openai/TODO.md +1 -0
  12. src/app/main.tsx +11 -4
  13. src/components/assistant/ChatBubble.tsx +3 -3
  14. src/components/forms/FormField.tsx +1 -1
  15. src/components/forms/FormInput.tsx +4 -6
  16. src/components/forms/FormSwitch.tsx +29 -0
  17. src/components/settings/constants.ts +12 -0
  18. src/components/settings/provider.tsx +50 -23
  19. src/components/settings/sound.tsx +1 -1
  20. src/components/toolbars/top-menu/assistant/index.tsx +5 -0
  21. src/components/toolbars/top-menu/view/index.tsx +9 -9
  22. src/controllers/assistant/types.ts +3 -1
  23. src/controllers/assistant/useAssistant.ts +62 -71
  24. src/controllers/audio/useAudio.ts +6 -1
  25. src/controllers/io/useIO.ts +6 -1
  26. src/controllers/metrics/useMetrics.ts +5 -0
  27. src/controllers/monitor/useMonitor.ts +7 -1
  28. src/controllers/renderer/useRenderer.ts +18 -4
  29. src/controllers/resolver/useResolver.ts +4 -0
  30. src/controllers/settings/getDefaultSettingsState.ts +5 -0
  31. src/controllers/settings/useSettings.ts +1 -1
  32. src/controllers/ui/getDefaultUIState.ts +6 -4
  33. src/controllers/ui/types.ts +6 -2
  34. src/controllers/ui/useUI.ts +9 -4
  35. src/lib/core/constants.ts +1 -1
  36. src/types.ts +3 -0
package-lock.json CHANGED
@@ -83,6 +83,7 @@
83
  "react-hotkeys-hook": "^4.5.0",
84
  "react-icons": "^5.2.1",
85
  "react-reflex": "^4.2.6",
 
86
  "replicate": "^0.30.2",
87
  "sharp": "^0.33.4",
88
  "sonner": "^1.4.41",
@@ -205,131 +206,180 @@
205
  "tslib": "^1.11.1"
206
  }
207
  },
208
- "node_modules/@aws-crypto/crc32/node_modules/tslib": {
209
- "version": "1.14.1",
210
- "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
211
- "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
212
- },
213
- "node_modules/@aws-crypto/ie11-detection": {
214
  "version": "3.0.0",
215
- "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz",
216
- "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==",
217
  "dependencies": {
 
 
218
  "tslib": "^1.11.1"
219
  }
220
  },
221
- "node_modules/@aws-crypto/ie11-detection/node_modules/tslib": {
222
  "version": "1.14.1",
223
  "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
224
  "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
225
  },
226
  "node_modules/@aws-crypto/sha256-browser": {
227
- "version": "3.0.0",
228
- "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz",
229
- "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==",
230
  "dependencies": {
231
- "@aws-crypto/ie11-detection": "^3.0.0",
232
- "@aws-crypto/sha256-js": "^3.0.0",
233
- "@aws-crypto/supports-web-crypto": "^3.0.0",
234
- "@aws-crypto/util": "^3.0.0",
235
  "@aws-sdk/types": "^3.222.0",
236
  "@aws-sdk/util-locate-window": "^3.0.0",
237
- "@aws-sdk/util-utf8-browser": "^3.0.0",
238
- "tslib": "^1.11.1"
239
  }
240
  },
241
- "node_modules/@aws-crypto/sha256-browser/node_modules/tslib": {
242
- "version": "1.14.1",
243
- "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
244
- "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
245
  },
246
  "node_modules/@aws-crypto/sha256-js": {
247
- "version": "3.0.0",
248
- "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz",
249
- "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==",
250
  "dependencies": {
251
- "@aws-crypto/util": "^3.0.0",
252
  "@aws-sdk/types": "^3.222.0",
253
- "tslib": "^1.11.1"
 
 
 
254
  }
255
  },
256
- "node_modules/@aws-crypto/sha256-js/node_modules/tslib": {
257
- "version": "1.14.1",
258
- "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
259
- "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
260
- },
261
  "node_modules/@aws-crypto/supports-web-crypto": {
262
- "version": "3.0.0",
263
- "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz",
264
- "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==",
265
  "dependencies": {
266
- "tslib": "^1.11.1"
267
  }
268
  },
269
- "node_modules/@aws-crypto/supports-web-crypto/node_modules/tslib": {
270
- "version": "1.14.1",
271
- "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
272
- "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
273
- },
274
  "node_modules/@aws-crypto/util": {
275
- "version": "3.0.0",
276
- "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz",
277
- "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==",
278
  "dependencies": {
279
  "@aws-sdk/types": "^3.222.0",
280
- "@aws-sdk/util-utf8-browser": "^3.0.0",
281
- "tslib": "^1.11.1"
282
  }
283
  },
284
- "node_modules/@aws-crypto/util/node_modules/tslib": {
285
- "version": "1.14.1",
286
- "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
287
- "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
288
  },
289
  "node_modules/@aws-sdk/client-cognito-identity": {
290
- "version": "3.596.0",
291
- "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.596.0.tgz",
292
- "integrity": "sha512-EnMebSL120H1V3CvxlSDu7xVg/c/U19J2pw5t3TbgWdP0bWR4gmaf2m7wczyi5XtPel0NIklnpPhlDJqr6T4Eg==",
293
- "dependencies": {
294
- "@aws-crypto/sha256-browser": "3.0.0",
295
- "@aws-crypto/sha256-js": "3.0.0",
296
- "@aws-sdk/client-sso-oidc": "3.596.0",
297
- "@aws-sdk/client-sts": "3.596.0",
298
- "@aws-sdk/core": "3.592.0",
299
- "@aws-sdk/credential-provider-node": "3.596.0",
300
- "@aws-sdk/middleware-host-header": "3.577.0",
301
- "@aws-sdk/middleware-logger": "3.577.0",
302
- "@aws-sdk/middleware-recursion-detection": "3.577.0",
303
- "@aws-sdk/middleware-user-agent": "3.587.0",
304
- "@aws-sdk/region-config-resolver": "3.587.0",
305
- "@aws-sdk/types": "3.577.0",
306
- "@aws-sdk/util-endpoints": "3.587.0",
307
- "@aws-sdk/util-user-agent-browser": "3.577.0",
308
- "@aws-sdk/util-user-agent-node": "3.587.0",
309
- "@smithy/config-resolver": "^3.0.1",
310
- "@smithy/core": "^2.2.0",
311
- "@smithy/fetch-http-handler": "^3.0.1",
312
- "@smithy/hash-node": "^3.0.0",
313
- "@smithy/invalid-dependency": "^3.0.0",
314
- "@smithy/middleware-content-length": "^3.0.0",
315
- "@smithy/middleware-endpoint": "^3.0.1",
316
- "@smithy/middleware-retry": "^3.0.3",
317
- "@smithy/middleware-serde": "^3.0.0",
318
- "@smithy/middleware-stack": "^3.0.0",
319
- "@smithy/node-config-provider": "^3.1.0",
320
- "@smithy/node-http-handler": "^3.0.0",
321
- "@smithy/protocol-http": "^4.0.0",
322
- "@smithy/smithy-client": "^3.1.1",
323
- "@smithy/types": "^3.0.0",
324
- "@smithy/url-parser": "^3.0.0",
325
  "@smithy/util-base64": "^3.0.0",
326
  "@smithy/util-body-length-browser": "^3.0.0",
327
  "@smithy/util-body-length-node": "^3.0.0",
328
- "@smithy/util-defaults-mode-browser": "^3.0.3",
329
- "@smithy/util-defaults-mode-node": "^3.0.3",
330
- "@smithy/util-endpoints": "^2.0.1",
331
- "@smithy/util-middleware": "^3.0.0",
332
- "@smithy/util-retry": "^3.0.0",
333
  "@smithy/util-utf8": "^3.0.0",
334
  "tslib": "^2.6.2"
335
  },
@@ -338,51 +388,51 @@
338
  }
339
  },
340
  "node_modules/@aws-sdk/client-sagemaker": {
341
- "version": "3.596.0",
342
- "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker/-/client-sagemaker-3.596.0.tgz",
343
- "integrity": "sha512-VDEo+pVw4fl9X18mEz0uj2rULke8QQ2QvpFWJ5oXGUqYCQRBqYOHaYtUvC+4kvj4vLmrmhTxYA6mO91tGbNadg==",
344
- "dependencies": {
345
- "@aws-crypto/sha256-browser": "3.0.0",
346
- "@aws-crypto/sha256-js": "3.0.0",
347
- "@aws-sdk/client-sso-oidc": "3.596.0",
348
- "@aws-sdk/client-sts": "3.596.0",
349
- "@aws-sdk/core": "3.592.0",
350
- "@aws-sdk/credential-provider-node": "3.596.0",
351
- "@aws-sdk/middleware-host-header": "3.577.0",
352
- "@aws-sdk/middleware-logger": "3.577.0",
353
- "@aws-sdk/middleware-recursion-detection": "3.577.0",
354
- "@aws-sdk/middleware-user-agent": "3.587.0",
355
- "@aws-sdk/region-config-resolver": "3.587.0",
356
- "@aws-sdk/types": "3.577.0",
357
- "@aws-sdk/util-endpoints": "3.587.0",
358
- "@aws-sdk/util-user-agent-browser": "3.577.0",
359
- "@aws-sdk/util-user-agent-node": "3.587.0",
360
- "@smithy/config-resolver": "^3.0.1",
361
- "@smithy/core": "^2.2.0",
362
- "@smithy/fetch-http-handler": "^3.0.1",
363
- "@smithy/hash-node": "^3.0.0",
364
- "@smithy/invalid-dependency": "^3.0.0",
365
- "@smithy/middleware-content-length": "^3.0.0",
366
- "@smithy/middleware-endpoint": "^3.0.1",
367
- "@smithy/middleware-retry": "^3.0.3",
368
- "@smithy/middleware-serde": "^3.0.0",
369
- "@smithy/middleware-stack": "^3.0.0",
370
- "@smithy/node-config-provider": "^3.1.0",
371
- "@smithy/node-http-handler": "^3.0.0",
372
- "@smithy/protocol-http": "^4.0.0",
373
- "@smithy/smithy-client": "^3.1.1",
374
- "@smithy/types": "^3.0.0",
375
- "@smithy/url-parser": "^3.0.0",
376
  "@smithy/util-base64": "^3.0.0",
377
  "@smithy/util-body-length-browser": "^3.0.0",
378
  "@smithy/util-body-length-node": "^3.0.0",
379
- "@smithy/util-defaults-mode-browser": "^3.0.3",
380
- "@smithy/util-defaults-mode-node": "^3.0.3",
381
- "@smithy/util-endpoints": "^2.0.1",
382
- "@smithy/util-middleware": "^3.0.0",
383
- "@smithy/util-retry": "^3.0.0",
384
  "@smithy/util-utf8": "^3.0.0",
385
- "@smithy/util-waiter": "^3.0.0",
386
  "tslib": "^2.6.2",
387
  "uuid": "^9.0.1"
388
  },
@@ -391,46 +441,46 @@
391
  }
392
  },
393
  "node_modules/@aws-sdk/client-sso": {
394
- "version": "3.592.0",
395
- "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.592.0.tgz",
396
- "integrity": "sha512-w+SuW47jQqvOC7fonyjFjsOh3yjqJ+VpWdVrmrl0E/KryBE7ho/Wn991Buf/EiHHeJikoWgHsAIPkBH29+ntdA==",
397
- "dependencies": {
398
- "@aws-crypto/sha256-browser": "3.0.0",
399
- "@aws-crypto/sha256-js": "3.0.0",
400
- "@aws-sdk/core": "3.592.0",
401
- "@aws-sdk/middleware-host-header": "3.577.0",
402
- "@aws-sdk/middleware-logger": "3.577.0",
403
- "@aws-sdk/middleware-recursion-detection": "3.577.0",
404
- "@aws-sdk/middleware-user-agent": "3.587.0",
405
- "@aws-sdk/region-config-resolver": "3.587.0",
406
- "@aws-sdk/types": "3.577.0",
407
- "@aws-sdk/util-endpoints": "3.587.0",
408
- "@aws-sdk/util-user-agent-browser": "3.577.0",
409
- "@aws-sdk/util-user-agent-node": "3.587.0",
410
- "@smithy/config-resolver": "^3.0.1",
411
- "@smithy/core": "^2.2.0",
412
- "@smithy/fetch-http-handler": "^3.0.1",
413
- "@smithy/hash-node": "^3.0.0",
414
- "@smithy/invalid-dependency": "^3.0.0",
415
- "@smithy/middleware-content-length": "^3.0.0",
416
- "@smithy/middleware-endpoint": "^3.0.1",
417
- "@smithy/middleware-retry": "^3.0.3",
418
- "@smithy/middleware-serde": "^3.0.0",
419
- "@smithy/middleware-stack": "^3.0.0",
420
- "@smithy/node-config-provider": "^3.1.0",
421
- "@smithy/node-http-handler": "^3.0.0",
422
- "@smithy/protocol-http": "^4.0.0",
423
- "@smithy/smithy-client": "^3.1.1",
424
- "@smithy/types": "^3.0.0",
425
- "@smithy/url-parser": "^3.0.0",
426
  "@smithy/util-base64": "^3.0.0",
427
  "@smithy/util-body-length-browser": "^3.0.0",
428
  "@smithy/util-body-length-node": "^3.0.0",
429
- "@smithy/util-defaults-mode-browser": "^3.0.3",
430
- "@smithy/util-defaults-mode-node": "^3.0.3",
431
- "@smithy/util-endpoints": "^2.0.1",
432
- "@smithy/util-middleware": "^3.0.0",
433
- "@smithy/util-retry": "^3.0.0",
434
  "@smithy/util-utf8": "^3.0.0",
435
  "tslib": "^2.6.2"
436
  },
@@ -439,48 +489,48 @@
439
  }
440
  },
441
  "node_modules/@aws-sdk/client-sso-oidc": {
442
- "version": "3.596.0",
443
- "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.596.0.tgz",
444
- "integrity": "sha512-KnTWtKzO0N+rMdIrVwbewFp4FAvVWBV/ekCAh5w7EN+uAvBHxMoFElE2RwlcRF/gH1/F715OspPMvOxPom6bMA==",
445
- "dependencies": {
446
- "@aws-crypto/sha256-browser": "3.0.0",
447
- "@aws-crypto/sha256-js": "3.0.0",
448
- "@aws-sdk/client-sts": "3.596.0",
449
- "@aws-sdk/core": "3.592.0",
450
- "@aws-sdk/credential-provider-node": "3.596.0",
451
- "@aws-sdk/middleware-host-header": "3.577.0",
452
- "@aws-sdk/middleware-logger": "3.577.0",
453
- "@aws-sdk/middleware-recursion-detection": "3.577.0",
454
- "@aws-sdk/middleware-user-agent": "3.587.0",
455
- "@aws-sdk/region-config-resolver": "3.587.0",
456
- "@aws-sdk/types": "3.577.0",
457
- "@aws-sdk/util-endpoints": "3.587.0",
458
- "@aws-sdk/util-user-agent-browser": "3.577.0",
459
- "@aws-sdk/util-user-agent-node": "3.587.0",
460
- "@smithy/config-resolver": "^3.0.1",
461
- "@smithy/core": "^2.2.0",
462
- "@smithy/fetch-http-handler": "^3.0.1",
463
- "@smithy/hash-node": "^3.0.0",
464
- "@smithy/invalid-dependency": "^3.0.0",
465
- "@smithy/middleware-content-length": "^3.0.0",
466
- "@smithy/middleware-endpoint": "^3.0.1",
467
- "@smithy/middleware-retry": "^3.0.3",
468
- "@smithy/middleware-serde": "^3.0.0",
469
- "@smithy/middleware-stack": "^3.0.0",
470
- "@smithy/node-config-provider": "^3.1.0",
471
- "@smithy/node-http-handler": "^3.0.0",
472
- "@smithy/protocol-http": "^4.0.0",
473
- "@smithy/smithy-client": "^3.1.1",
474
- "@smithy/types": "^3.0.0",
475
- "@smithy/url-parser": "^3.0.0",
476
  "@smithy/util-base64": "^3.0.0",
477
  "@smithy/util-body-length-browser": "^3.0.0",
478
  "@smithy/util-body-length-node": "^3.0.0",
479
- "@smithy/util-defaults-mode-browser": "^3.0.3",
480
- "@smithy/util-defaults-mode-node": "^3.0.3",
481
- "@smithy/util-endpoints": "^2.0.1",
482
- "@smithy/util-middleware": "^3.0.0",
483
- "@smithy/util-retry": "^3.0.0",
484
  "@smithy/util-utf8": "^3.0.0",
485
  "tslib": "^2.6.2"
486
  },
@@ -489,48 +539,48 @@
489
  }
490
  },
491
  "node_modules/@aws-sdk/client-sts": {
492
- "version": "3.596.0",
493
- "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.596.0.tgz",
494
- "integrity": "sha512-37+WQDjgmqS/YXj3vPzIVIrbXaFcZ1WXk715AMGIPBZn9Y2/wr2bmSTpX7bsMyn0G8+LxmoIxFcG7n1Gu0nvLg==",
495
- "dependencies": {
496
- "@aws-crypto/sha256-browser": "3.0.0",
497
- "@aws-crypto/sha256-js": "3.0.0",
498
- "@aws-sdk/client-sso-oidc": "3.596.0",
499
- "@aws-sdk/core": "3.592.0",
500
- "@aws-sdk/credential-provider-node": "3.596.0",
501
- "@aws-sdk/middleware-host-header": "3.577.0",
502
- "@aws-sdk/middleware-logger": "3.577.0",
503
- "@aws-sdk/middleware-recursion-detection": "3.577.0",
504
- "@aws-sdk/middleware-user-agent": "3.587.0",
505
- "@aws-sdk/region-config-resolver": "3.587.0",
506
- "@aws-sdk/types": "3.577.0",
507
- "@aws-sdk/util-endpoints": "3.587.0",
508
- "@aws-sdk/util-user-agent-browser": "3.577.0",
509
- "@aws-sdk/util-user-agent-node": "3.587.0",
510
- "@smithy/config-resolver": "^3.0.1",
511
- "@smithy/core": "^2.2.0",
512
- "@smithy/fetch-http-handler": "^3.0.1",
513
- "@smithy/hash-node": "^3.0.0",
514
- "@smithy/invalid-dependency": "^3.0.0",
515
- "@smithy/middleware-content-length": "^3.0.0",
516
- "@smithy/middleware-endpoint": "^3.0.1",
517
- "@smithy/middleware-retry": "^3.0.3",
518
- "@smithy/middleware-serde": "^3.0.0",
519
- "@smithy/middleware-stack": "^3.0.0",
520
- "@smithy/node-config-provider": "^3.1.0",
521
- "@smithy/node-http-handler": "^3.0.0",
522
- "@smithy/protocol-http": "^4.0.0",
523
- "@smithy/smithy-client": "^3.1.1",
524
- "@smithy/types": "^3.0.0",
525
- "@smithy/url-parser": "^3.0.0",
526
  "@smithy/util-base64": "^3.0.0",
527
  "@smithy/util-body-length-browser": "^3.0.0",
528
  "@smithy/util-body-length-node": "^3.0.0",
529
- "@smithy/util-defaults-mode-browser": "^3.0.3",
530
- "@smithy/util-defaults-mode-node": "^3.0.3",
531
- "@smithy/util-endpoints": "^2.0.1",
532
- "@smithy/util-middleware": "^3.0.0",
533
- "@smithy/util-retry": "^3.0.0",
534
  "@smithy/util-utf8": "^3.0.0",
535
  "tslib": "^2.6.2"
536
  },
@@ -539,15 +589,15 @@
539
  }
540
  },
541
  "node_modules/@aws-sdk/core": {
542
- "version": "3.592.0",
543
- "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.592.0.tgz",
544
- "integrity": "sha512-gLPMXR/HXDP+9gXAt58t7gaMTvRts9i6Q7NMISpkGF54wehskl5WGrbdtHJFylrlJ5BQo3XVY6i661o+EuR1wg==",
545
- "dependencies": {
546
- "@smithy/core": "^2.2.0",
547
- "@smithy/protocol-http": "^4.0.0",
548
- "@smithy/signature-v4": "^3.0.0",
549
- "@smithy/smithy-client": "^3.1.1",
550
- "@smithy/types": "^3.0.0",
551
  "fast-xml-parser": "4.2.5",
552
  "tslib": "^2.6.2"
553
  },
@@ -577,14 +627,14 @@
577
  }
578
  },
579
  "node_modules/@aws-sdk/credential-provider-cognito-identity": {
580
- "version": "3.596.0",
581
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.596.0.tgz",
582
- "integrity": "sha512-ps/1P+wwEbzOryIdnPGkfo83AH5+kFPe0UKTc6Lhsc4l4zhfvyU3WV/JzrCINEKqo3bEZdUt6tl/IpsyC+nggQ==",
583
- "dependencies": {
584
- "@aws-sdk/client-cognito-identity": "3.596.0",
585
- "@aws-sdk/types": "3.577.0",
586
- "@smithy/property-provider": "^3.1.0",
587
- "@smithy/types": "^3.0.0",
588
  "tslib": "^2.6.2"
589
  },
590
  "engines": {
@@ -592,13 +642,13 @@
592
  }
593
  },
594
  "node_modules/@aws-sdk/credential-provider-env": {
595
- "version": "3.587.0",
596
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz",
597
- "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==",
598
  "dependencies": {
599
- "@aws-sdk/types": "3.577.0",
600
- "@smithy/property-provider": "^3.1.0",
601
- "@smithy/types": "^3.0.0",
602
  "tslib": "^2.6.2"
603
  },
604
  "engines": {
@@ -606,18 +656,18 @@
606
  }
607
  },
608
  "node_modules/@aws-sdk/credential-provider-http": {
609
- "version": "3.596.0",
610
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.596.0.tgz",
611
- "integrity": "sha512-nnmvEsz1KJgRmfSZJPWuzbxPRXu8Y+/78Ifa1jY3fQKSKdEJfXMDsjPljJvMDBl4dZ8pf5Hwx+S/ONnMEDwYEA==",
612
- "dependencies": {
613
- "@aws-sdk/types": "3.577.0",
614
- "@smithy/fetch-http-handler": "^3.0.1",
615
- "@smithy/node-http-handler": "^3.0.0",
616
- "@smithy/property-provider": "^3.1.0",
617
- "@smithy/protocol-http": "^4.0.0",
618
- "@smithy/smithy-client": "^3.1.1",
619
- "@smithy/types": "^3.0.0",
620
- "@smithy/util-stream": "^3.0.1",
621
  "tslib": "^2.6.2"
622
  },
623
  "engines": {
@@ -625,45 +675,45 @@
625
  }
626
  },
627
  "node_modules/@aws-sdk/credential-provider-ini": {
628
- "version": "3.596.0",
629
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.596.0.tgz",
630
- "integrity": "sha512-c7PLtd7GbnOVAc5sk3sVlHxLvEsM8RF96rsBGlRo4AVpil/lXLKyNv9VarS4w/ZZZoRbJRyZ+m92PjNcLvpTDQ==",
631
- "dependencies": {
632
- "@aws-sdk/credential-provider-env": "3.587.0",
633
- "@aws-sdk/credential-provider-http": "3.596.0",
634
- "@aws-sdk/credential-provider-process": "3.587.0",
635
- "@aws-sdk/credential-provider-sso": "3.592.0",
636
- "@aws-sdk/credential-provider-web-identity": "3.587.0",
637
- "@aws-sdk/types": "3.577.0",
638
- "@smithy/credential-provider-imds": "^3.1.0",
639
- "@smithy/property-provider": "^3.1.0",
640
- "@smithy/shared-ini-file-loader": "^3.1.0",
641
- "@smithy/types": "^3.0.0",
642
  "tslib": "^2.6.2"
643
  },
644
  "engines": {
645
  "node": ">=16.0.0"
646
  },
647
  "peerDependencies": {
648
- "@aws-sdk/client-sts": "^3.596.0"
649
  }
650
  },
651
  "node_modules/@aws-sdk/credential-provider-node": {
652
- "version": "3.596.0",
653
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.596.0.tgz",
654
- "integrity": "sha512-F4MLyXpQyie1AnJS9n7TIRL0aF7YH8tKMIJXDsM5OXpSZi2en+yR6SzsxvHf5dwS2Ga8LUdEJyiyS2NoebaJGA==",
655
- "dependencies": {
656
- "@aws-sdk/credential-provider-env": "3.587.0",
657
- "@aws-sdk/credential-provider-http": "3.596.0",
658
- "@aws-sdk/credential-provider-ini": "3.596.0",
659
- "@aws-sdk/credential-provider-process": "3.587.0",
660
- "@aws-sdk/credential-provider-sso": "3.592.0",
661
- "@aws-sdk/credential-provider-web-identity": "3.587.0",
662
- "@aws-sdk/types": "3.577.0",
663
- "@smithy/credential-provider-imds": "^3.1.0",
664
- "@smithy/property-provider": "^3.1.0",
665
- "@smithy/shared-ini-file-loader": "^3.1.0",
666
- "@smithy/types": "^3.0.0",
667
  "tslib": "^2.6.2"
668
  },
669
  "engines": {
@@ -671,14 +721,14 @@
671
  }
672
  },
673
  "node_modules/@aws-sdk/credential-provider-process": {
674
- "version": "3.587.0",
675
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz",
676
- "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==",
677
- "dependencies": {
678
- "@aws-sdk/types": "3.577.0",
679
- "@smithy/property-provider": "^3.1.0",
680
- "@smithy/shared-ini-file-loader": "^3.1.0",
681
- "@smithy/types": "^3.0.0",
682
  "tslib": "^2.6.2"
683
  },
684
  "engines": {
@@ -686,16 +736,16 @@
686
  }
687
  },
688
  "node_modules/@aws-sdk/credential-provider-sso": {
689
- "version": "3.592.0",
690
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.592.0.tgz",
691
- "integrity": "sha512-fYFzAdDHKHvhtufPPtrLdSv8lO6GuW3em6n3erM5uFdpGytNpjXvr3XGokIsuXcNkETAY/Xihg+G9ksNE8WJxQ==",
692
- "dependencies": {
693
- "@aws-sdk/client-sso": "3.592.0",
694
- "@aws-sdk/token-providers": "3.587.0",
695
- "@aws-sdk/types": "3.577.0",
696
- "@smithy/property-provider": "^3.1.0",
697
- "@smithy/shared-ini-file-loader": "^3.1.0",
698
- "@smithy/types": "^3.0.0",
699
  "tslib": "^2.6.2"
700
  },
701
  "engines": {
@@ -703,42 +753,42 @@
703
  }
704
  },
705
  "node_modules/@aws-sdk/credential-provider-web-identity": {
706
- "version": "3.587.0",
707
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz",
708
- "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==",
709
  "dependencies": {
710
- "@aws-sdk/types": "3.577.0",
711
- "@smithy/property-provider": "^3.1.0",
712
- "@smithy/types": "^3.0.0",
713
  "tslib": "^2.6.2"
714
  },
715
  "engines": {
716
  "node": ">=16.0.0"
717
  },
718
  "peerDependencies": {
719
- "@aws-sdk/client-sts": "^3.587.0"
720
  }
721
  },
722
  "node_modules/@aws-sdk/credential-providers": {
723
- "version": "3.596.0",
724
- "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.596.0.tgz",
725
- "integrity": "sha512-EsbkylyO08P3alxXTpanKT1rPTh5/vXu7r/GoKbPl+7Laqheme41CYg0jtwAou/w7/6RFxqMn5ey5vg/qopNVA==",
726
- "dependencies": {
727
- "@aws-sdk/client-cognito-identity": "3.596.0",
728
- "@aws-sdk/client-sso": "3.592.0",
729
- "@aws-sdk/client-sts": "3.596.0",
730
- "@aws-sdk/credential-provider-cognito-identity": "3.596.0",
731
- "@aws-sdk/credential-provider-env": "3.587.0",
732
- "@aws-sdk/credential-provider-http": "3.596.0",
733
- "@aws-sdk/credential-provider-ini": "3.596.0",
734
- "@aws-sdk/credential-provider-node": "3.596.0",
735
- "@aws-sdk/credential-provider-process": "3.587.0",
736
- "@aws-sdk/credential-provider-sso": "3.592.0",
737
- "@aws-sdk/credential-provider-web-identity": "3.587.0",
738
- "@aws-sdk/types": "3.577.0",
739
- "@smithy/credential-provider-imds": "^3.1.0",
740
- "@smithy/property-provider": "^3.1.0",
741
- "@smithy/types": "^3.0.0",
742
  "tslib": "^2.6.2"
743
  },
744
  "engines": {
@@ -746,13 +796,13 @@
746
  }
747
  },
748
  "node_modules/@aws-sdk/middleware-host-header": {
749
- "version": "3.577.0",
750
- "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz",
751
- "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==",
752
  "dependencies": {
753
- "@aws-sdk/types": "3.577.0",
754
- "@smithy/protocol-http": "^4.0.0",
755
- "@smithy/types": "^3.0.0",
756
  "tslib": "^2.6.2"
757
  },
758
  "engines": {
@@ -760,12 +810,12 @@
760
  }
761
  },
762
  "node_modules/@aws-sdk/middleware-logger": {
763
- "version": "3.577.0",
764
- "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz",
765
- "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==",
766
  "dependencies": {
767
- "@aws-sdk/types": "3.577.0",
768
- "@smithy/types": "^3.0.0",
769
  "tslib": "^2.6.2"
770
  },
771
  "engines": {
@@ -773,13 +823,13 @@
773
  }
774
  },
775
  "node_modules/@aws-sdk/middleware-recursion-detection": {
776
- "version": "3.577.0",
777
- "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz",
778
- "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==",
779
  "dependencies": {
780
- "@aws-sdk/types": "3.577.0",
781
- "@smithy/protocol-http": "^4.0.0",
782
- "@smithy/types": "^3.0.0",
783
  "tslib": "^2.6.2"
784
  },
785
  "engines": {
@@ -787,14 +837,14 @@
787
  }
788
  },
789
  "node_modules/@aws-sdk/middleware-user-agent": {
790
- "version": "3.587.0",
791
- "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz",
792
- "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==",
793
- "dependencies": {
794
- "@aws-sdk/types": "3.577.0",
795
- "@aws-sdk/util-endpoints": "3.587.0",
796
- "@smithy/protocol-http": "^4.0.0",
797
- "@smithy/types": "^3.0.0",
798
  "tslib": "^2.6.2"
799
  },
800
  "engines": {
@@ -838,15 +888,15 @@
838
  }
839
  },
840
  "node_modules/@aws-sdk/region-config-resolver": {
841
- "version": "3.587.0",
842
- "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz",
843
- "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==",
844
  "dependencies": {
845
- "@aws-sdk/types": "3.577.0",
846
- "@smithy/node-config-provider": "^3.1.0",
847
- "@smithy/types": "^3.0.0",
848
  "@smithy/util-config-provider": "^3.0.0",
849
- "@smithy/util-middleware": "^3.0.0",
850
  "tslib": "^2.6.2"
851
  },
852
  "engines": {
@@ -964,29 +1014,29 @@
964
  }
965
  },
966
  "node_modules/@aws-sdk/token-providers": {
967
- "version": "3.587.0",
968
- "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz",
969
- "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==",
970
- "dependencies": {
971
- "@aws-sdk/types": "3.577.0",
972
- "@smithy/property-provider": "^3.1.0",
973
- "@smithy/shared-ini-file-loader": "^3.1.0",
974
- "@smithy/types": "^3.0.0",
975
  "tslib": "^2.6.2"
976
  },
977
  "engines": {
978
  "node": ">=16.0.0"
979
  },
980
  "peerDependencies": {
981
- "@aws-sdk/client-sso-oidc": "^3.587.0"
982
  }
983
  },
984
  "node_modules/@aws-sdk/types": {
985
- "version": "3.577.0",
986
- "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz",
987
- "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==",
988
  "dependencies": {
989
- "@smithy/types": "^3.0.0",
990
  "tslib": "^2.6.2"
991
  },
992
  "engines": {
@@ -994,13 +1044,13 @@
994
  }
995
  },
996
  "node_modules/@aws-sdk/util-endpoints": {
997
- "version": "3.587.0",
998
- "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz",
999
- "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==",
1000
  "dependencies": {
1001
- "@aws-sdk/types": "3.577.0",
1002
- "@smithy/types": "^3.0.0",
1003
- "@smithy/util-endpoints": "^2.0.1",
1004
  "tslib": "^2.6.2"
1005
  },
1006
  "engines": {
@@ -1019,24 +1069,24 @@
1019
  }
1020
  },
1021
  "node_modules/@aws-sdk/util-user-agent-browser": {
1022
- "version": "3.577.0",
1023
- "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz",
1024
- "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==",
1025
  "dependencies": {
1026
- "@aws-sdk/types": "3.577.0",
1027
- "@smithy/types": "^3.0.0",
1028
  "bowser": "^2.11.0",
1029
  "tslib": "^2.6.2"
1030
  }
1031
  },
1032
  "node_modules/@aws-sdk/util-user-agent-node": {
1033
- "version": "3.587.0",
1034
- "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz",
1035
- "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==",
1036
  "dependencies": {
1037
- "@aws-sdk/types": "3.577.0",
1038
- "@smithy/node-config-provider": "^3.1.0",
1039
- "@smithy/types": "^3.0.0",
1040
  "tslib": "^2.6.2"
1041
  },
1042
  "engines": {
@@ -3804,9 +3854,9 @@
3804
  "integrity": "sha512-Kpx/fQ/ZFX31OtlqVEFfgaD1ACzul4NksrvIgYfIFq9JpDHFwQkMVZ10tbo0FU/grje4rcL4EIrjekl3kYwgWw=="
3805
  },
3806
  "node_modules/@react-three/drei": {
3807
- "version": "9.106.0",
3808
- "resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-9.106.0.tgz",
3809
- "integrity": "sha512-P+X3Iz79ea+L4YTVPOIFAzqVU8hwrKdr/I6UgN7ZQkJ9qIQ7qa7ew8ss/+c3+46RT1tzxgo5ec89RnLjFIBmuw==",
3810
  "dependencies": {
3811
  "@babel/runtime": "^7.11.2",
3812
  "@mediapipe/tasks-vision": "0.10.8",
@@ -4337,9 +4387,9 @@
4337
  }
4338
  },
4339
  "node_modules/@smithy/signature-v4": {
4340
- "version": "3.0.1",
4341
- "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.1.tgz",
4342
- "integrity": "sha512-ARAmD+E7j6TIEhKLjSZxdzs7wceINTMJRN2BXPM09BiUmJhkXAF1ZZtDXH6fhlk7oehBZeh37wGiPOqtdKjLeg==",
4343
  "dependencies": {
4344
  "@smithy/is-array-buffer": "^3.0.0",
4345
  "@smithy/types": "^3.1.0",
@@ -4930,9 +4980,9 @@
4930
  }
4931
  },
4932
  "node_modules/acorn": {
4933
- "version": "8.11.3",
4934
- "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
4935
- "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
4936
  "bin": {
4937
  "acorn": "bin/acorn"
4938
  },
@@ -4949,9 +4999,12 @@
4949
  }
4950
  },
4951
  "node_modules/acorn-walk": {
4952
- "version": "8.3.2",
4953
- "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.2.tgz",
4954
- "integrity": "sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==",
 
 
 
4955
  "engines": {
4956
  "node": ">=0.4.0"
4957
  }
@@ -5568,9 +5621,9 @@
5568
  }
5569
  },
5570
  "node_modules/caniuse-lite": {
5571
- "version": "1.0.30001633",
5572
- "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001633.tgz",
5573
- "integrity": "sha512-6sT0yf/z5jqf8tISAgpJDrmwOpLsrpnyCdD/lOZKvKkkJK4Dn0X5i7KF7THEZhOq+30bmhwBlNEaqPUiHiKtZg==",
5574
  "funding": [
5575
  {
5576
  "type": "opencollective",
@@ -6481,9 +6534,9 @@
6481
  }
6482
  },
6483
  "node_modules/electron-to-chromium": {
6484
- "version": "1.4.802",
6485
- "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.802.tgz",
6486
- "integrity": "sha512-TnTMUATbgNdPXVSHsxvNVSG0uEd6cSZsANjm8c9HbvflZVVn1yTRcmVXYT1Ma95/ssB/Dcd30AHweH2TE+dNpA=="
6487
  },
6488
  "node_modules/emoji-regex": {
6489
  "version": "9.2.2",
@@ -7372,9 +7425,9 @@
7372
  }
7373
  },
7374
  "node_modules/foreground-child": {
7375
- "version": "3.2.0",
7376
- "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.0.tgz",
7377
- "integrity": "sha512-CrWQNaEl1/6WeZoarcM9LHupTo3RpZO2Pdk1vktwzPiQTsJnAKJmm3TACKeG5UZbWDfaH2AbvYxzP96y0MT7fA==",
7378
  "dependencies": {
7379
  "cross-spawn": "^7.0.0",
7380
  "signal-exit": "^4.0.1"
@@ -10138,6 +10191,14 @@
10138
  }
10139
  }
10140
  },
 
 
 
 
 
 
 
 
10141
  "node_modules/react-style-singleton": {
10142
  "version": "2.2.1",
10143
  "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
 
83
  "react-hotkeys-hook": "^4.5.0",
84
  "react-icons": "^5.2.1",
85
  "react-reflex": "^4.2.6",
86
+ "react-speakup": "^1.0.0",
87
  "replicate": "^0.30.2",
88
  "sharp": "^0.33.4",
89
  "sonner": "^1.4.41",
 
206
  "tslib": "^1.11.1"
207
  }
208
  },
209
+ "node_modules/@aws-crypto/crc32/node_modules/@aws-crypto/util": {
 
 
 
 
 
210
  "version": "3.0.0",
211
+ "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz",
212
+ "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==",
213
  "dependencies": {
214
+ "@aws-sdk/types": "^3.222.0",
215
+ "@aws-sdk/util-utf8-browser": "^3.0.0",
216
  "tslib": "^1.11.1"
217
  }
218
  },
219
+ "node_modules/@aws-crypto/crc32/node_modules/tslib": {
220
  "version": "1.14.1",
221
  "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
222
  "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
223
  },
224
  "node_modules/@aws-crypto/sha256-browser": {
225
+ "version": "5.2.0",
226
+ "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz",
227
+ "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==",
228
  "dependencies": {
229
+ "@aws-crypto/sha256-js": "^5.2.0",
230
+ "@aws-crypto/supports-web-crypto": "^5.2.0",
231
+ "@aws-crypto/util": "^5.2.0",
 
232
  "@aws-sdk/types": "^3.222.0",
233
  "@aws-sdk/util-locate-window": "^3.0.0",
234
+ "@smithy/util-utf8": "^2.0.0",
235
+ "tslib": "^2.6.2"
236
  }
237
  },
238
+ "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": {
239
+ "version": "2.2.0",
240
+ "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz",
241
+ "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==",
242
+ "dependencies": {
243
+ "tslib": "^2.6.2"
244
+ },
245
+ "engines": {
246
+ "node": ">=14.0.0"
247
+ }
248
+ },
249
+ "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": {
250
+ "version": "2.2.0",
251
+ "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz",
252
+ "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==",
253
+ "dependencies": {
254
+ "@smithy/is-array-buffer": "^2.2.0",
255
+ "tslib": "^2.6.2"
256
+ },
257
+ "engines": {
258
+ "node": ">=14.0.0"
259
+ }
260
+ },
261
+ "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": {
262
+ "version": "2.3.0",
263
+ "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz",
264
+ "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==",
265
+ "dependencies": {
266
+ "@smithy/util-buffer-from": "^2.2.0",
267
+ "tslib": "^2.6.2"
268
+ },
269
+ "engines": {
270
+ "node": ">=14.0.0"
271
+ }
272
  },
273
  "node_modules/@aws-crypto/sha256-js": {
274
+ "version": "5.2.0",
275
+ "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz",
276
+ "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==",
277
  "dependencies": {
278
+ "@aws-crypto/util": "^5.2.0",
279
  "@aws-sdk/types": "^3.222.0",
280
+ "tslib": "^2.6.2"
281
+ },
282
+ "engines": {
283
+ "node": ">=16.0.0"
284
  }
285
  },
 
 
 
 
 
286
  "node_modules/@aws-crypto/supports-web-crypto": {
287
+ "version": "5.2.0",
288
+ "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz",
289
+ "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==",
290
  "dependencies": {
291
+ "tslib": "^2.6.2"
292
  }
293
  },
 
 
 
 
 
294
  "node_modules/@aws-crypto/util": {
295
+ "version": "5.2.0",
296
+ "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz",
297
+ "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==",
298
  "dependencies": {
299
  "@aws-sdk/types": "^3.222.0",
300
+ "@smithy/util-utf8": "^2.0.0",
301
+ "tslib": "^2.6.2"
302
  }
303
  },
304
+ "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": {
305
+ "version": "2.2.0",
306
+ "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz",
307
+ "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==",
308
+ "dependencies": {
309
+ "tslib": "^2.6.2"
310
+ },
311
+ "engines": {
312
+ "node": ">=14.0.0"
313
+ }
314
+ },
315
+ "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": {
316
+ "version": "2.2.0",
317
+ "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz",
318
+ "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==",
319
+ "dependencies": {
320
+ "@smithy/is-array-buffer": "^2.2.0",
321
+ "tslib": "^2.6.2"
322
+ },
323
+ "engines": {
324
+ "node": ">=14.0.0"
325
+ }
326
+ },
327
+ "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": {
328
+ "version": "2.3.0",
329
+ "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz",
330
+ "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==",
331
+ "dependencies": {
332
+ "@smithy/util-buffer-from": "^2.2.0",
333
+ "tslib": "^2.6.2"
334
+ },
335
+ "engines": {
336
+ "node": ">=14.0.0"
337
+ }
338
  },
339
  "node_modules/@aws-sdk/client-cognito-identity": {
340
+ "version": "3.598.0",
341
+ "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.598.0.tgz",
342
+ "integrity": "sha512-N/1lnkhkzk1Il8WEZBWR713/7sDEqBtl/1AS6dfgw6Zh7NWUYSwBkZx6xdN8KogDu4CFExRHhilNOgI1JMug3w==",
343
+ "dependencies": {
344
+ "@aws-crypto/sha256-browser": "5.2.0",
345
+ "@aws-crypto/sha256-js": "5.2.0",
346
+ "@aws-sdk/client-sso-oidc": "3.598.0",
347
+ "@aws-sdk/client-sts": "3.598.0",
348
+ "@aws-sdk/core": "3.598.0",
349
+ "@aws-sdk/credential-provider-node": "3.598.0",
350
+ "@aws-sdk/middleware-host-header": "3.598.0",
351
+ "@aws-sdk/middleware-logger": "3.598.0",
352
+ "@aws-sdk/middleware-recursion-detection": "3.598.0",
353
+ "@aws-sdk/middleware-user-agent": "3.598.0",
354
+ "@aws-sdk/region-config-resolver": "3.598.0",
355
+ "@aws-sdk/types": "3.598.0",
356
+ "@aws-sdk/util-endpoints": "3.598.0",
357
+ "@aws-sdk/util-user-agent-browser": "3.598.0",
358
+ "@aws-sdk/util-user-agent-node": "3.598.0",
359
+ "@smithy/config-resolver": "^3.0.2",
360
+ "@smithy/core": "^2.2.1",
361
+ "@smithy/fetch-http-handler": "^3.0.2",
362
+ "@smithy/hash-node": "^3.0.1",
363
+ "@smithy/invalid-dependency": "^3.0.1",
364
+ "@smithy/middleware-content-length": "^3.0.1",
365
+ "@smithy/middleware-endpoint": "^3.0.2",
366
+ "@smithy/middleware-retry": "^3.0.4",
367
+ "@smithy/middleware-serde": "^3.0.1",
368
+ "@smithy/middleware-stack": "^3.0.1",
369
+ "@smithy/node-config-provider": "^3.1.1",
370
+ "@smithy/node-http-handler": "^3.0.1",
371
+ "@smithy/protocol-http": "^4.0.1",
372
+ "@smithy/smithy-client": "^3.1.2",
373
+ "@smithy/types": "^3.1.0",
374
+ "@smithy/url-parser": "^3.0.1",
375
  "@smithy/util-base64": "^3.0.0",
376
  "@smithy/util-body-length-browser": "^3.0.0",
377
  "@smithy/util-body-length-node": "^3.0.0",
378
+ "@smithy/util-defaults-mode-browser": "^3.0.4",
379
+ "@smithy/util-defaults-mode-node": "^3.0.4",
380
+ "@smithy/util-endpoints": "^2.0.2",
381
+ "@smithy/util-middleware": "^3.0.1",
382
+ "@smithy/util-retry": "^3.0.1",
383
  "@smithy/util-utf8": "^3.0.0",
384
  "tslib": "^2.6.2"
385
  },
 
388
  }
389
  },
390
  "node_modules/@aws-sdk/client-sagemaker": {
391
+ "version": "3.598.0",
392
+ "resolved": "https://registry.npmjs.org/@aws-sdk/client-sagemaker/-/client-sagemaker-3.598.0.tgz",
393
+ "integrity": "sha512-NIaDyRozT4L52R/jNJ6dcIoq3Cj737a6nG75VpSvOBUy8P+mjY1BedeWRAf0i5tu1bS0qMkLfrDLw3//zYixug==",
394
+ "dependencies": {
395
+ "@aws-crypto/sha256-browser": "5.2.0",
396
+ "@aws-crypto/sha256-js": "5.2.0",
397
+ "@aws-sdk/client-sso-oidc": "3.598.0",
398
+ "@aws-sdk/client-sts": "3.598.0",
399
+ "@aws-sdk/core": "3.598.0",
400
+ "@aws-sdk/credential-provider-node": "3.598.0",
401
+ "@aws-sdk/middleware-host-header": "3.598.0",
402
+ "@aws-sdk/middleware-logger": "3.598.0",
403
+ "@aws-sdk/middleware-recursion-detection": "3.598.0",
404
+ "@aws-sdk/middleware-user-agent": "3.598.0",
405
+ "@aws-sdk/region-config-resolver": "3.598.0",
406
+ "@aws-sdk/types": "3.598.0",
407
+ "@aws-sdk/util-endpoints": "3.598.0",
408
+ "@aws-sdk/util-user-agent-browser": "3.598.0",
409
+ "@aws-sdk/util-user-agent-node": "3.598.0",
410
+ "@smithy/config-resolver": "^3.0.2",
411
+ "@smithy/core": "^2.2.1",
412
+ "@smithy/fetch-http-handler": "^3.0.2",
413
+ "@smithy/hash-node": "^3.0.1",
414
+ "@smithy/invalid-dependency": "^3.0.1",
415
+ "@smithy/middleware-content-length": "^3.0.1",
416
+ "@smithy/middleware-endpoint": "^3.0.2",
417
+ "@smithy/middleware-retry": "^3.0.4",
418
+ "@smithy/middleware-serde": "^3.0.1",
419
+ "@smithy/middleware-stack": "^3.0.1",
420
+ "@smithy/node-config-provider": "^3.1.1",
421
+ "@smithy/node-http-handler": "^3.0.1",
422
+ "@smithy/protocol-http": "^4.0.1",
423
+ "@smithy/smithy-client": "^3.1.2",
424
+ "@smithy/types": "^3.1.0",
425
+ "@smithy/url-parser": "^3.0.1",
426
  "@smithy/util-base64": "^3.0.0",
427
  "@smithy/util-body-length-browser": "^3.0.0",
428
  "@smithy/util-body-length-node": "^3.0.0",
429
+ "@smithy/util-defaults-mode-browser": "^3.0.4",
430
+ "@smithy/util-defaults-mode-node": "^3.0.4",
431
+ "@smithy/util-endpoints": "^2.0.2",
432
+ "@smithy/util-middleware": "^3.0.1",
433
+ "@smithy/util-retry": "^3.0.1",
434
  "@smithy/util-utf8": "^3.0.0",
435
+ "@smithy/util-waiter": "^3.0.1",
436
  "tslib": "^2.6.2",
437
  "uuid": "^9.0.1"
438
  },
 
441
  }
442
  },
443
  "node_modules/@aws-sdk/client-sso": {
444
+ "version": "3.598.0",
445
+ "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.598.0.tgz",
446
+ "integrity": "sha512-nOI5lqPYa+YZlrrzwAJywJSw3MKVjvu6Ge2fCqQUNYMfxFB0NAaDFnl0EPjXi+sEbtCuz/uWE77poHbqiZ+7Iw==",
447
+ "dependencies": {
448
+ "@aws-crypto/sha256-browser": "5.2.0",
449
+ "@aws-crypto/sha256-js": "5.2.0",
450
+ "@aws-sdk/core": "3.598.0",
451
+ "@aws-sdk/middleware-host-header": "3.598.0",
452
+ "@aws-sdk/middleware-logger": "3.598.0",
453
+ "@aws-sdk/middleware-recursion-detection": "3.598.0",
454
+ "@aws-sdk/middleware-user-agent": "3.598.0",
455
+ "@aws-sdk/region-config-resolver": "3.598.0",
456
+ "@aws-sdk/types": "3.598.0",
457
+ "@aws-sdk/util-endpoints": "3.598.0",
458
+ "@aws-sdk/util-user-agent-browser": "3.598.0",
459
+ "@aws-sdk/util-user-agent-node": "3.598.0",
460
+ "@smithy/config-resolver": "^3.0.2",
461
+ "@smithy/core": "^2.2.1",
462
+ "@smithy/fetch-http-handler": "^3.0.2",
463
+ "@smithy/hash-node": "^3.0.1",
464
+ "@smithy/invalid-dependency": "^3.0.1",
465
+ "@smithy/middleware-content-length": "^3.0.1",
466
+ "@smithy/middleware-endpoint": "^3.0.2",
467
+ "@smithy/middleware-retry": "^3.0.4",
468
+ "@smithy/middleware-serde": "^3.0.1",
469
+ "@smithy/middleware-stack": "^3.0.1",
470
+ "@smithy/node-config-provider": "^3.1.1",
471
+ "@smithy/node-http-handler": "^3.0.1",
472
+ "@smithy/protocol-http": "^4.0.1",
473
+ "@smithy/smithy-client": "^3.1.2",
474
+ "@smithy/types": "^3.1.0",
475
+ "@smithy/url-parser": "^3.0.1",
476
  "@smithy/util-base64": "^3.0.0",
477
  "@smithy/util-body-length-browser": "^3.0.0",
478
  "@smithy/util-body-length-node": "^3.0.0",
479
+ "@smithy/util-defaults-mode-browser": "^3.0.4",
480
+ "@smithy/util-defaults-mode-node": "^3.0.4",
481
+ "@smithy/util-endpoints": "^2.0.2",
482
+ "@smithy/util-middleware": "^3.0.1",
483
+ "@smithy/util-retry": "^3.0.1",
484
  "@smithy/util-utf8": "^3.0.0",
485
  "tslib": "^2.6.2"
486
  },
 
489
  }
490
  },
491
  "node_modules/@aws-sdk/client-sso-oidc": {
492
+ "version": "3.598.0",
493
+ "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.598.0.tgz",
494
+ "integrity": "sha512-jfdH1pAO9Tt8Nkta/JJLoUnwl7jaRdxToQTJfUtE+o3+0JP5sA4LfC2rBkJSWcU5BdAA+kyOs5Lv776DlN04Vg==",
495
+ "dependencies": {
496
+ "@aws-crypto/sha256-browser": "5.2.0",
497
+ "@aws-crypto/sha256-js": "5.2.0",
498
+ "@aws-sdk/client-sts": "3.598.0",
499
+ "@aws-sdk/core": "3.598.0",
500
+ "@aws-sdk/credential-provider-node": "3.598.0",
501
+ "@aws-sdk/middleware-host-header": "3.598.0",
502
+ "@aws-sdk/middleware-logger": "3.598.0",
503
+ "@aws-sdk/middleware-recursion-detection": "3.598.0",
504
+ "@aws-sdk/middleware-user-agent": "3.598.0",
505
+ "@aws-sdk/region-config-resolver": "3.598.0",
506
+ "@aws-sdk/types": "3.598.0",
507
+ "@aws-sdk/util-endpoints": "3.598.0",
508
+ "@aws-sdk/util-user-agent-browser": "3.598.0",
509
+ "@aws-sdk/util-user-agent-node": "3.598.0",
510
+ "@smithy/config-resolver": "^3.0.2",
511
+ "@smithy/core": "^2.2.1",
512
+ "@smithy/fetch-http-handler": "^3.0.2",
513
+ "@smithy/hash-node": "^3.0.1",
514
+ "@smithy/invalid-dependency": "^3.0.1",
515
+ "@smithy/middleware-content-length": "^3.0.1",
516
+ "@smithy/middleware-endpoint": "^3.0.2",
517
+ "@smithy/middleware-retry": "^3.0.4",
518
+ "@smithy/middleware-serde": "^3.0.1",
519
+ "@smithy/middleware-stack": "^3.0.1",
520
+ "@smithy/node-config-provider": "^3.1.1",
521
+ "@smithy/node-http-handler": "^3.0.1",
522
+ "@smithy/protocol-http": "^4.0.1",
523
+ "@smithy/smithy-client": "^3.1.2",
524
+ "@smithy/types": "^3.1.0",
525
+ "@smithy/url-parser": "^3.0.1",
526
  "@smithy/util-base64": "^3.0.0",
527
  "@smithy/util-body-length-browser": "^3.0.0",
528
  "@smithy/util-body-length-node": "^3.0.0",
529
+ "@smithy/util-defaults-mode-browser": "^3.0.4",
530
+ "@smithy/util-defaults-mode-node": "^3.0.4",
531
+ "@smithy/util-endpoints": "^2.0.2",
532
+ "@smithy/util-middleware": "^3.0.1",
533
+ "@smithy/util-retry": "^3.0.1",
534
  "@smithy/util-utf8": "^3.0.0",
535
  "tslib": "^2.6.2"
536
  },
 
539
  }
540
  },
541
  "node_modules/@aws-sdk/client-sts": {
542
+ "version": "3.598.0",
543
+ "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.598.0.tgz",
544
+ "integrity": "sha512-bXhz/cHL0iB9UH9IFtMaJJf4F8mV+HzncETCRFzZ9SyUMt5rP9j8A7VZknqGYSx/6mI8SsB1XJQkWSbhn6FiSQ==",
545
+ "dependencies": {
546
+ "@aws-crypto/sha256-browser": "5.2.0",
547
+ "@aws-crypto/sha256-js": "5.2.0",
548
+ "@aws-sdk/client-sso-oidc": "3.598.0",
549
+ "@aws-sdk/core": "3.598.0",
550
+ "@aws-sdk/credential-provider-node": "3.598.0",
551
+ "@aws-sdk/middleware-host-header": "3.598.0",
552
+ "@aws-sdk/middleware-logger": "3.598.0",
553
+ "@aws-sdk/middleware-recursion-detection": "3.598.0",
554
+ "@aws-sdk/middleware-user-agent": "3.598.0",
555
+ "@aws-sdk/region-config-resolver": "3.598.0",
556
+ "@aws-sdk/types": "3.598.0",
557
+ "@aws-sdk/util-endpoints": "3.598.0",
558
+ "@aws-sdk/util-user-agent-browser": "3.598.0",
559
+ "@aws-sdk/util-user-agent-node": "3.598.0",
560
+ "@smithy/config-resolver": "^3.0.2",
561
+ "@smithy/core": "^2.2.1",
562
+ "@smithy/fetch-http-handler": "^3.0.2",
563
+ "@smithy/hash-node": "^3.0.1",
564
+ "@smithy/invalid-dependency": "^3.0.1",
565
+ "@smithy/middleware-content-length": "^3.0.1",
566
+ "@smithy/middleware-endpoint": "^3.0.2",
567
+ "@smithy/middleware-retry": "^3.0.4",
568
+ "@smithy/middleware-serde": "^3.0.1",
569
+ "@smithy/middleware-stack": "^3.0.1",
570
+ "@smithy/node-config-provider": "^3.1.1",
571
+ "@smithy/node-http-handler": "^3.0.1",
572
+ "@smithy/protocol-http": "^4.0.1",
573
+ "@smithy/smithy-client": "^3.1.2",
574
+ "@smithy/types": "^3.1.0",
575
+ "@smithy/url-parser": "^3.0.1",
576
  "@smithy/util-base64": "^3.0.0",
577
  "@smithy/util-body-length-browser": "^3.0.0",
578
  "@smithy/util-body-length-node": "^3.0.0",
579
+ "@smithy/util-defaults-mode-browser": "^3.0.4",
580
+ "@smithy/util-defaults-mode-node": "^3.0.4",
581
+ "@smithy/util-endpoints": "^2.0.2",
582
+ "@smithy/util-middleware": "^3.0.1",
583
+ "@smithy/util-retry": "^3.0.1",
584
  "@smithy/util-utf8": "^3.0.0",
585
  "tslib": "^2.6.2"
586
  },
 
589
  }
590
  },
591
  "node_modules/@aws-sdk/core": {
592
+ "version": "3.598.0",
593
+ "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.598.0.tgz",
594
+ "integrity": "sha512-HaSjt7puO5Cc7cOlrXFCW0rtA0BM9lvzjl56x0A20Pt+0wxXGeTOZZOkXQIepbrFkV2e/HYukuT9e99vXDm59g==",
595
+ "dependencies": {
596
+ "@smithy/core": "^2.2.1",
597
+ "@smithy/protocol-http": "^4.0.1",
598
+ "@smithy/signature-v4": "^3.1.0",
599
+ "@smithy/smithy-client": "^3.1.2",
600
+ "@smithy/types": "^3.1.0",
601
  "fast-xml-parser": "4.2.5",
602
  "tslib": "^2.6.2"
603
  },
 
627
  }
628
  },
629
  "node_modules/@aws-sdk/credential-provider-cognito-identity": {
630
+ "version": "3.598.0",
631
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.598.0.tgz",
632
+ "integrity": "sha512-u6oocRReswkA2mFlOwtCetgmEr9B+Yhle3K13x37rb1lQgq1wUuWUvHU7U9v26hUZIhfUpigV/Mgr/RQZB6+Yw==",
633
+ "dependencies": {
634
+ "@aws-sdk/client-cognito-identity": "3.598.0",
635
+ "@aws-sdk/types": "3.598.0",
636
+ "@smithy/property-provider": "^3.1.1",
637
+ "@smithy/types": "^3.1.0",
638
  "tslib": "^2.6.2"
639
  },
640
  "engines": {
 
642
  }
643
  },
644
  "node_modules/@aws-sdk/credential-provider-env": {
645
+ "version": "3.598.0",
646
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.598.0.tgz",
647
+ "integrity": "sha512-vi1khgn7yXzLCcgSIzQrrtd2ilUM0dWodxj3PQ6BLfP0O+q1imO3hG1nq7DVyJtq7rFHs6+9N8G4mYvTkxby2w==",
648
  "dependencies": {
649
+ "@aws-sdk/types": "3.598.0",
650
+ "@smithy/property-provider": "^3.1.1",
651
+ "@smithy/types": "^3.1.0",
652
  "tslib": "^2.6.2"
653
  },
654
  "engines": {
 
656
  }
657
  },
658
  "node_modules/@aws-sdk/credential-provider-http": {
659
+ "version": "3.598.0",
660
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.598.0.tgz",
661
+ "integrity": "sha512-N7cIafi4HVlQvEgvZSo1G4T9qb/JMLGMdBsDCT5XkeJrF0aptQWzTFH0jIdZcLrMYvzPcuEyO3yCBe6cy/ba0g==",
662
+ "dependencies": {
663
+ "@aws-sdk/types": "3.598.0",
664
+ "@smithy/fetch-http-handler": "^3.0.2",
665
+ "@smithy/node-http-handler": "^3.0.1",
666
+ "@smithy/property-provider": "^3.1.1",
667
+ "@smithy/protocol-http": "^4.0.1",
668
+ "@smithy/smithy-client": "^3.1.2",
669
+ "@smithy/types": "^3.1.0",
670
+ "@smithy/util-stream": "^3.0.2",
671
  "tslib": "^2.6.2"
672
  },
673
  "engines": {
 
675
  }
676
  },
677
  "node_modules/@aws-sdk/credential-provider-ini": {
678
+ "version": "3.598.0",
679
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.598.0.tgz",
680
+ "integrity": "sha512-/ppcIVUbRwDIwJDoYfp90X3+AuJo2mvE52Y1t2VSrvUovYn6N4v95/vXj6LS8CNDhz2jvEJYmu+0cTMHdhI6eA==",
681
+ "dependencies": {
682
+ "@aws-sdk/credential-provider-env": "3.598.0",
683
+ "@aws-sdk/credential-provider-http": "3.598.0",
684
+ "@aws-sdk/credential-provider-process": "3.598.0",
685
+ "@aws-sdk/credential-provider-sso": "3.598.0",
686
+ "@aws-sdk/credential-provider-web-identity": "3.598.0",
687
+ "@aws-sdk/types": "3.598.0",
688
+ "@smithy/credential-provider-imds": "^3.1.1",
689
+ "@smithy/property-provider": "^3.1.1",
690
+ "@smithy/shared-ini-file-loader": "^3.1.1",
691
+ "@smithy/types": "^3.1.0",
692
  "tslib": "^2.6.2"
693
  },
694
  "engines": {
695
  "node": ">=16.0.0"
696
  },
697
  "peerDependencies": {
698
+ "@aws-sdk/client-sts": "^3.598.0"
699
  }
700
  },
701
  "node_modules/@aws-sdk/credential-provider-node": {
702
+ "version": "3.598.0",
703
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.598.0.tgz",
704
+ "integrity": "sha512-sXTlqL5I/awlF9Dg2MQ17SfrEaABVnsj2mf4jF5qQrIRhfbvQOIYdEqdy8Rn1AWlJMz/N450SGzc0XJ5owxxqw==",
705
+ "dependencies": {
706
+ "@aws-sdk/credential-provider-env": "3.598.0",
707
+ "@aws-sdk/credential-provider-http": "3.598.0",
708
+ "@aws-sdk/credential-provider-ini": "3.598.0",
709
+ "@aws-sdk/credential-provider-process": "3.598.0",
710
+ "@aws-sdk/credential-provider-sso": "3.598.0",
711
+ "@aws-sdk/credential-provider-web-identity": "3.598.0",
712
+ "@aws-sdk/types": "3.598.0",
713
+ "@smithy/credential-provider-imds": "^3.1.1",
714
+ "@smithy/property-provider": "^3.1.1",
715
+ "@smithy/shared-ini-file-loader": "^3.1.1",
716
+ "@smithy/types": "^3.1.0",
717
  "tslib": "^2.6.2"
718
  },
719
  "engines": {
 
721
  }
722
  },
723
  "node_modules/@aws-sdk/credential-provider-process": {
724
+ "version": "3.598.0",
725
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.598.0.tgz",
726
+ "integrity": "sha512-rM707XbLW8huMk722AgjVyxu2tMZee++fNA8TJVNgs1Ma02Wx6bBrfIvlyK0rCcIRb0WdQYP6fe3Xhiu4e8IBA==",
727
+ "dependencies": {
728
+ "@aws-sdk/types": "3.598.0",
729
+ "@smithy/property-provider": "^3.1.1",
730
+ "@smithy/shared-ini-file-loader": "^3.1.1",
731
+ "@smithy/types": "^3.1.0",
732
  "tslib": "^2.6.2"
733
  },
734
  "engines": {
 
736
  }
737
  },
738
  "node_modules/@aws-sdk/credential-provider-sso": {
739
+ "version": "3.598.0",
740
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.598.0.tgz",
741
+ "integrity": "sha512-5InwUmrAuqQdOOgxTccRayMMkSmekdLk6s+az9tmikq0QFAHUCtofI+/fllMXSR9iL6JbGYi1940+EUmS4pHJA==",
742
+ "dependencies": {
743
+ "@aws-sdk/client-sso": "3.598.0",
744
+ "@aws-sdk/token-providers": "3.598.0",
745
+ "@aws-sdk/types": "3.598.0",
746
+ "@smithy/property-provider": "^3.1.1",
747
+ "@smithy/shared-ini-file-loader": "^3.1.1",
748
+ "@smithy/types": "^3.1.0",
749
  "tslib": "^2.6.2"
750
  },
751
  "engines": {
 
753
  }
754
  },
755
  "node_modules/@aws-sdk/credential-provider-web-identity": {
756
+ "version": "3.598.0",
757
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.598.0.tgz",
758
+ "integrity": "sha512-GV5GdiMbz5Tz9JO4NJtRoFXjW0GPEujA0j+5J/B723rTN+REHthJu48HdBKouHGhdzkDWkkh1bu52V02Wprw8w==",
759
  "dependencies": {
760
+ "@aws-sdk/types": "3.598.0",
761
+ "@smithy/property-provider": "^3.1.1",
762
+ "@smithy/types": "^3.1.0",
763
  "tslib": "^2.6.2"
764
  },
765
  "engines": {
766
  "node": ">=16.0.0"
767
  },
768
  "peerDependencies": {
769
+ "@aws-sdk/client-sts": "^3.598.0"
770
  }
771
  },
772
  "node_modules/@aws-sdk/credential-providers": {
773
+ "version": "3.598.0",
774
+ "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.598.0.tgz",
775
+ "integrity": "sha512-IFZwH3F2rA2WbpYFUxOeu/M3/9p4+oRbKVLDZlaaDtwwuZ9VHEbnkUm20zOgSXeVExa3qgYhJvg7H5JrqxP97A==",
776
+ "dependencies": {
777
+ "@aws-sdk/client-cognito-identity": "3.598.0",
778
+ "@aws-sdk/client-sso": "3.598.0",
779
+ "@aws-sdk/client-sts": "3.598.0",
780
+ "@aws-sdk/credential-provider-cognito-identity": "3.598.0",
781
+ "@aws-sdk/credential-provider-env": "3.598.0",
782
+ "@aws-sdk/credential-provider-http": "3.598.0",
783
+ "@aws-sdk/credential-provider-ini": "3.598.0",
784
+ "@aws-sdk/credential-provider-node": "3.598.0",
785
+ "@aws-sdk/credential-provider-process": "3.598.0",
786
+ "@aws-sdk/credential-provider-sso": "3.598.0",
787
+ "@aws-sdk/credential-provider-web-identity": "3.598.0",
788
+ "@aws-sdk/types": "3.598.0",
789
+ "@smithy/credential-provider-imds": "^3.1.1",
790
+ "@smithy/property-provider": "^3.1.1",
791
+ "@smithy/types": "^3.1.0",
792
  "tslib": "^2.6.2"
793
  },
794
  "engines": {
 
796
  }
797
  },
798
  "node_modules/@aws-sdk/middleware-host-header": {
799
+ "version": "3.598.0",
800
+ "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.598.0.tgz",
801
+ "integrity": "sha512-WiaG059YBQwQraNejLIi0gMNkX7dfPZ8hDIhvMr5aVPRbaHH8AYF3iNSsXYCHvA2Cfa1O9haYXsuMF9flXnCmA==",
802
  "dependencies": {
803
+ "@aws-sdk/types": "3.598.0",
804
+ "@smithy/protocol-http": "^4.0.1",
805
+ "@smithy/types": "^3.1.0",
806
  "tslib": "^2.6.2"
807
  },
808
  "engines": {
 
810
  }
811
  },
812
  "node_modules/@aws-sdk/middleware-logger": {
813
+ "version": "3.598.0",
814
+ "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.598.0.tgz",
815
+ "integrity": "sha512-bxBjf/VYiu3zfu8SYM2S9dQQc3tz5uBAOcPz/Bt8DyyK3GgOpjhschH/2XuUErsoUO1gDJqZSdGOmuHGZQn00Q==",
816
  "dependencies": {
817
+ "@aws-sdk/types": "3.598.0",
818
+ "@smithy/types": "^3.1.0",
819
  "tslib": "^2.6.2"
820
  },
821
  "engines": {
 
823
  }
824
  },
825
  "node_modules/@aws-sdk/middleware-recursion-detection": {
826
+ "version": "3.598.0",
827
+ "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.598.0.tgz",
828
+ "integrity": "sha512-vjT9BeFY9FeN0f8hm2l6F53tI0N5bUq6RcDkQXKNabXBnQxKptJRad6oP2X5y3FoVfBLOuDkQgiC2940GIPxtQ==",
829
  "dependencies": {
830
+ "@aws-sdk/types": "3.598.0",
831
+ "@smithy/protocol-http": "^4.0.1",
832
+ "@smithy/types": "^3.1.0",
833
  "tslib": "^2.6.2"
834
  },
835
  "engines": {
 
837
  }
838
  },
839
  "node_modules/@aws-sdk/middleware-user-agent": {
840
+ "version": "3.598.0",
841
+ "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.598.0.tgz",
842
+ "integrity": "sha512-4tjESlHG5B5MdjUaLK7tQs/miUtHbb6deauQx8ryqSBYOhfHVgb1ZnzvQR0bTrhpqUg0WlybSkDaZAICf9xctg==",
843
+ "dependencies": {
844
+ "@aws-sdk/types": "3.598.0",
845
+ "@aws-sdk/util-endpoints": "3.598.0",
846
+ "@smithy/protocol-http": "^4.0.1",
847
+ "@smithy/types": "^3.1.0",
848
  "tslib": "^2.6.2"
849
  },
850
  "engines": {
 
888
  }
889
  },
890
  "node_modules/@aws-sdk/region-config-resolver": {
891
+ "version": "3.598.0",
892
+ "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.598.0.tgz",
893
+ "integrity": "sha512-oYXhmTokSav4ytmWleCr3rs/1nyvZW/S0tdi6X7u+dLNL5Jee+uMxWGzgOrWK6wrQOzucLVjS4E/wA11Kv2GTw==",
894
  "dependencies": {
895
+ "@aws-sdk/types": "3.598.0",
896
+ "@smithy/node-config-provider": "^3.1.1",
897
+ "@smithy/types": "^3.1.0",
898
  "@smithy/util-config-provider": "^3.0.0",
899
+ "@smithy/util-middleware": "^3.0.1",
900
  "tslib": "^2.6.2"
901
  },
902
  "engines": {
 
1014
  }
1015
  },
1016
  "node_modules/@aws-sdk/token-providers": {
1017
+ "version": "3.598.0",
1018
+ "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.598.0.tgz",
1019
+ "integrity": "sha512-TKY1EVdHVBnZqpyxyTHdpZpa1tUpb6nxVeRNn1zWG8QB5MvH4ALLd/jR+gtmWDNQbIG4cVuBOZFVL8hIYicKTA==",
1020
+ "dependencies": {
1021
+ "@aws-sdk/types": "3.598.0",
1022
+ "@smithy/property-provider": "^3.1.1",
1023
+ "@smithy/shared-ini-file-loader": "^3.1.1",
1024
+ "@smithy/types": "^3.1.0",
1025
  "tslib": "^2.6.2"
1026
  },
1027
  "engines": {
1028
  "node": ">=16.0.0"
1029
  },
1030
  "peerDependencies": {
1031
+ "@aws-sdk/client-sso-oidc": "^3.598.0"
1032
  }
1033
  },
1034
  "node_modules/@aws-sdk/types": {
1035
+ "version": "3.598.0",
1036
+ "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.598.0.tgz",
1037
+ "integrity": "sha512-742uRl6z7u0LFmZwDrFP6r1wlZcgVPw+/TilluDJmCAR8BgRw3IR+743kUXKBGd8QZDRW2n6v/PYsi/AWCDDMQ==",
1038
  "dependencies": {
1039
+ "@smithy/types": "^3.1.0",
1040
  "tslib": "^2.6.2"
1041
  },
1042
  "engines": {
 
1044
  }
1045
  },
1046
  "node_modules/@aws-sdk/util-endpoints": {
1047
+ "version": "3.598.0",
1048
+ "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.598.0.tgz",
1049
+ "integrity": "sha512-Qo9UoiVVZxcOEdiOMZg3xb1mzkTxrhd4qSlg5QQrfWPJVx/QOg+Iy0NtGxPtHtVZNHZxohYwDwV/tfsnDSE2gQ==",
1050
  "dependencies": {
1051
+ "@aws-sdk/types": "3.598.0",
1052
+ "@smithy/types": "^3.1.0",
1053
+ "@smithy/util-endpoints": "^2.0.2",
1054
  "tslib": "^2.6.2"
1055
  },
1056
  "engines": {
 
1069
  }
1070
  },
1071
  "node_modules/@aws-sdk/util-user-agent-browser": {
1072
+ "version": "3.598.0",
1073
+ "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.598.0.tgz",
1074
+ "integrity": "sha512-36Sxo6F+ykElaL1mWzWjlg+1epMpSe8obwhCN1yGE7Js9ywy5U6k6l+A3q3YM9YRbm740sNxncbwLklMvuhTKw==",
1075
  "dependencies": {
1076
+ "@aws-sdk/types": "3.598.0",
1077
+ "@smithy/types": "^3.1.0",
1078
  "bowser": "^2.11.0",
1079
  "tslib": "^2.6.2"
1080
  }
1081
  },
1082
  "node_modules/@aws-sdk/util-user-agent-node": {
1083
+ "version": "3.598.0",
1084
+ "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.598.0.tgz",
1085
+ "integrity": "sha512-oyWGcOlfTdzkC6SVplyr0AGh54IMrDxbhg5RxJ5P+V4BKfcDoDcZV9xenUk9NsOi9MuUjxMumb9UJGkDhM1m0A==",
1086
  "dependencies": {
1087
+ "@aws-sdk/types": "3.598.0",
1088
+ "@smithy/node-config-provider": "^3.1.1",
1089
+ "@smithy/types": "^3.1.0",
1090
  "tslib": "^2.6.2"
1091
  },
1092
  "engines": {
 
3854
  "integrity": "sha512-Kpx/fQ/ZFX31OtlqVEFfgaD1ACzul4NksrvIgYfIFq9JpDHFwQkMVZ10tbo0FU/grje4rcL4EIrjekl3kYwgWw=="
3855
  },
3856
  "node_modules/@react-three/drei": {
3857
+ "version": "9.106.1",
3858
+ "resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-9.106.1.tgz",
3859
+ "integrity": "sha512-yl/XloQz1bBhLX+WVLpLYziiijyNxFz4xi48YuHOJuOjGbLcdL+v8czPLy1Y7kUhndmCJHw6yzuR1NJTfWJ4cQ==",
3860
  "dependencies": {
3861
  "@babel/runtime": "^7.11.2",
3862
  "@mediapipe/tasks-vision": "0.10.8",
 
4387
  }
4388
  },
4389
  "node_modules/@smithy/signature-v4": {
4390
+ "version": "3.1.0",
4391
+ "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.1.0.tgz",
4392
+ "integrity": "sha512-m0/6LW3IQ3/JBcdhqjpkpABPTPhcejqeAn0U877zxBdNLiWAnG2WmCe5MfkUyVuvpFTPQnQwCo/0ZBR4uF5kxg==",
4393
  "dependencies": {
4394
  "@smithy/is-array-buffer": "^3.0.0",
4395
  "@smithy/types": "^3.1.0",
 
4980
  }
4981
  },
4982
  "node_modules/acorn": {
4983
+ "version": "8.12.0",
4984
+ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.0.tgz",
4985
+ "integrity": "sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==",
4986
  "bin": {
4987
  "acorn": "bin/acorn"
4988
  },
 
4999
  }
5000
  },
5001
  "node_modules/acorn-walk": {
5002
+ "version": "8.3.3",
5003
+ "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.3.tgz",
5004
+ "integrity": "sha512-MxXdReSRhGO7VlFe1bRG/oI7/mdLV9B9JJT0N8vZOhF7gFRR5l3M8W9G8JxmKV+JC5mGqJ0QvqfSOLsCPa4nUw==",
5005
+ "dependencies": {
5006
+ "acorn": "^8.11.0"
5007
+ },
5008
  "engines": {
5009
  "node": ">=0.4.0"
5010
  }
 
5621
  }
5622
  },
5623
  "node_modules/caniuse-lite": {
5624
+ "version": "1.0.30001634",
5625
+ "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001634.tgz",
5626
+ "integrity": "sha512-fbBYXQ9q3+yp1q1gBk86tOFs4pyn/yxFm5ZNP18OXJDfA3txImOY9PhfxVggZ4vRHDqoU8NrKU81eN0OtzOgRA==",
5627
  "funding": [
5628
  {
5629
  "type": "opencollective",
 
6534
  }
6535
  },
6536
  "node_modules/electron-to-chromium": {
6537
+ "version": "1.4.803",
6538
+ "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.803.tgz",
6539
+ "integrity": "sha512-61H9mLzGOCLLVsnLiRzCbc63uldP0AniRYPV3hbGVtONA1pI7qSGILdbofR7A8TMbOypDocEAjH/e+9k1QIe3g=="
6540
  },
6541
  "node_modules/emoji-regex": {
6542
  "version": "9.2.2",
 
7425
  }
7426
  },
7427
  "node_modules/foreground-child": {
7428
+ "version": "3.2.1",
7429
+ "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.1.tgz",
7430
+ "integrity": "sha512-PXUUyLqrR2XCWICfv6ukppP96sdFwWbNEnfEMt7jNsISjMsvaLNinAHNDYyvkyU+SZG2BTSbT5NjG+vZslfGTA==",
7431
  "dependencies": {
7432
  "cross-spawn": "^7.0.0",
7433
  "signal-exit": "^4.0.1"
 
10191
  }
10192
  }
10193
  },
10194
+ "node_modules/react-speakup": {
10195
+ "version": "1.0.0",
10196
+ "resolved": "https://registry.npmjs.org/react-speakup/-/react-speakup-1.0.0.tgz",
10197
+ "integrity": "sha512-uZlLfBrWcdlw9fC6N5NY14GA8olnXh7DNaWNQya1osBPIE2nd/URv3nc6+yXeIlu55bwdGYSPoyAXEo5znN1hg==",
10198
+ "peerDependencies": {
10199
+ "react": ">=16"
10200
+ }
10201
+ },
10202
  "node_modules/react-style-singleton": {
10203
  "version": "2.2.1",
10204
  "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
package.json CHANGED
@@ -85,6 +85,7 @@
85
  "react-hotkeys-hook": "^4.5.0",
86
  "react-icons": "^5.2.1",
87
  "react-reflex": "^4.2.6",
 
88
  "replicate": "^0.30.2",
89
  "sharp": "^0.33.4",
90
  "sonner": "^1.4.41",
 
85
  "react-hotkeys-hook": "^4.5.0",
86
  "react-icons": "^5.2.1",
87
  "react-reflex": "^4.2.6",
88
+ "react-speakup": "^1.0.0",
89
  "replicate": "^0.30.2",
90
  "sharp": "^0.33.4",
91
  "sonner": "^1.4.41",
src/app/api/assistant/askAnyAssistant.ts CHANGED
@@ -3,8 +3,8 @@
3
  import { ClapSegmentCategory } from "@aitube/clap"
4
  import { RunnableLike } from "@langchain/core/runnables"
5
  import { ChatPromptValueInterface } from "@langchain/core/dist/prompt_values"
6
- import { AIMessageChunk } from "@langchain/core/messages"
7
- import { ChatPromptTemplate } from "@langchain/core/prompts"
8
  import { StructuredOutputParser } from "@langchain/core/output_parsers"
9
  import { ChatOpenAI } from "@langchain/openai"
10
  import { ChatGroq } from "@langchain/groq"
@@ -47,7 +47,9 @@ export async function askAnyAssistant({
47
  entities = {},
48
 
49
  // used to provide more context
50
- projectInfo = ""
 
 
51
  }: AssistantRequest): Promise<AssistantResponse> {
52
 
53
  const provider = settings.assistantProvider
@@ -98,6 +100,7 @@ export async function askAnyAssistant({
98
  const chatPrompt = ChatPromptTemplate.fromMessages(
99
  [
100
  ["system", systemTemplate],
 
101
  ["human", humanTemplate],
102
  ]
103
  )
@@ -109,7 +112,7 @@ export async function askAnyAssistant({
109
  category: segment.category,
110
  } as SimplifiedSegmentData))
111
 
112
- // console.log("INPUT:", JSON.stringify(inputData, null, 2))
113
 
114
  const chain = chatPrompt.pipe(coerceable).pipe(parser)
115
 
@@ -121,7 +124,23 @@ export async function askAnyAssistant({
121
  fullScene,
122
  actionLine,
123
  userPrompt: prompt,
124
- inputData,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  })
126
 
127
  console.log("OUTPUT:", JSON.stringify(result, null, 2))
 
3
  import { ClapSegmentCategory } from "@aitube/clap"
4
  import { RunnableLike } from "@langchain/core/runnables"
5
  import { ChatPromptValueInterface } from "@langchain/core/dist/prompt_values"
6
+ import { AIMessage, AIMessageChunk, HumanMessage } from "@langchain/core/messages"
7
+ import { ChatPromptTemplate, MessagesPlaceholder } from "@langchain/core/prompts"
8
  import { StructuredOutputParser } from "@langchain/core/output_parsers"
9
  import { ChatOpenAI } from "@langchain/openai"
10
  import { ChatGroq } from "@langchain/groq"
 
47
  entities = {},
48
 
49
  // used to provide more context
50
+ projectInfo = "",
51
+
52
+ history = [],
53
  }: AssistantRequest): Promise<AssistantResponse> {
54
 
55
  const provider = settings.assistantProvider
 
100
  const chatPrompt = ChatPromptTemplate.fromMessages(
101
  [
102
  ["system", systemTemplate],
103
+ new MessagesPlaceholder("chatHistory"),
104
  ["human", humanTemplate],
105
  ]
106
  )
 
112
  category: segment.category,
113
  } as SimplifiedSegmentData))
114
 
115
+ console.log("INPUT:", JSON.stringify(inputData, null, 2))
116
 
117
  const chain = chatPrompt.pipe(coerceable).pipe(parser)
118
 
 
124
  fullScene,
125
  actionLine,
126
  userPrompt: prompt,
127
+ chatHistory: history.map(({
128
+ eventId,
129
+ senderId,
130
+ senderName,
131
+ roomId,
132
+ roomName,
133
+ sentAt,
134
+ message,
135
+ isCurrentUser,
136
+ }) => {
137
+ if (isCurrentUser) {
138
+ return new HumanMessage(message)
139
+ } else {
140
+ return new AIMessage(message)
141
+ }
142
+ }),
143
+ inputData: JSON.stringify(inputData),
144
  })
145
 
146
  console.log("OUTPUT:", JSON.stringify(result, null, 2))
src/app/api/assistant/providers/google/README.md ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ We can use Gemini 1.5 to analyze a video:
3
+
4
+ https://cloud.google.com/vertex-ai/generative-ai/docs/samples/generativeaionvertexai-gemini-video-with-audio
5
+
6
+
7
+ Important note: not all models do everything!
8
+
9
+ https://firebase.google.com/docs/vertex-ai/gemini-models#capabilities-features-comparison
10
+
11
+ Gemini 1.5 Pro has more capabilities than the Flash version for instance
src/app/api/assistant/providers/google/sample.txt ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ const {VertexAI} = require('@google-cloud/vertexai');
2
+
3
+ /**
4
+ * TODO(developer): Update these variables before running the sample.
5
+ */
6
+ async function analyze_video_with_audio(projectId = 'PROJECT_ID') {
7
+ const vertexAI = new VertexAI({project: projectId, location: 'us-central1'});
8
+
9
+ const generativeModel = vertexAI.getGenerativeModel({
10
+ model: 'gemini-1.5-flash-001',
11
+ });
12
+
13
+ const filePart = {
14
+ file_data: {
15
+ file_uri: 'gs://cloud-samples-data/generative-ai/video/pixel8.mp4',
16
+ mime_type: 'video/mp4',
17
+ },
18
+ };
19
+ const textPart = {
20
+ text: `
21
+ Provide a description of the video.
22
+ The description should also contain anything important which people say in the video.`,
23
+ };
24
+
25
+ const request = {
26
+ contents: [{role: 'user', parts: [filePart, textPart]}],
27
+ };
28
+
29
+ const resp = await generativeModel.generateContent(request);
30
+ const contentResponse = await resp.response;
31
+ console.log(JSON.stringify(contentResponse));
32
+ }
src/app/api/assistant/providers/openai/askAssistant.ts DELETED
@@ -1,129 +0,0 @@
1
- "use server"
2
-
3
- import { ClapSegmentCategory, ClapScene, } from "@aitube/clap"
4
- import { ChatOpenAI } from "@langchain/openai"
5
-
6
- import { ChatPromptTemplate } from "@langchain/core/prompts"
7
- import { StructuredOutputParser } from "@langchain/core/output_parsers"
8
- import { AssistantRequest, AssistantResponse } from "@/types"
9
- import { SimplifiedSegmentData, simplifiedSegmentDataZ } from "../../types"
10
- import { examples, humanTemplate, systemTemplate } from "../../templates"
11
-
12
- const parser = StructuredOutputParser.fromZodSchema(simplifiedSegmentDataZ)
13
-
14
- const formatInstructions = parser.getFormatInstructions()
15
-
16
- /**
17
- * Query the preferred language model on the user prompt + the segments of the current scene
18
- * @param userPrompt
19
- * @param segments
20
- * @returns
21
- */
22
- export async function askAssistant({
23
- settings,
24
-
25
- prompt,
26
-
27
- // the slice to edit
28
- segments = [],
29
-
30
- fullScene = "",
31
-
32
- actionLine = "",
33
-
34
- // used to provide more context
35
- entities = {},
36
-
37
- // used to provide more context
38
- projectInfo = ""
39
- }: AssistantRequest): Promise<AssistantResponse> {
40
-
41
- // TODO:
42
- const model = new ChatOpenAI({
43
- openAIApiKey: settings.openaiApiKey,
44
- modelName: settings.openaiModelForAssistant,
45
- temperature: 0.7,
46
- })
47
-
48
- const chatPrompt = ChatPromptTemplate.fromMessages(
49
- [
50
- ["system", systemTemplate],
51
- ["human", humanTemplate],
52
- ]
53
- )
54
-
55
- // we don't give the whole thing to the LLM as to not confuse it,
56
- // and also to keep things tight and performant
57
- const inputData: SimplifiedSegmentData[] = segments.map((segment) => ({
58
- prompt: segment.prompt,
59
- category: segment.category,
60
- } as SimplifiedSegmentData))
61
-
62
- // console.log("INPUT:", JSON.stringify(inputData, null, 2))
63
-
64
- const chain = chatPrompt.pipe(model).pipe(parser)
65
-
66
- try {
67
- const result = await chain.invoke({
68
- formatInstructions,
69
- examples,
70
- projectInfo,
71
- fullScene,
72
- actionLine,
73
- userPrompt: prompt,
74
- inputData,
75
- })
76
-
77
- console.log("OUTPUT:", JSON.stringify(result, null, 2))
78
-
79
- /*
80
- this whole code doesn't work well actually..
81
-
82
- let match: SegmentData | undefined = segments[result.index] || undefined
83
-
84
- // LLM gave an object, but the index is wrong
85
- if (!match) {
86
- match = segments.find(s => s.category === result.category) || undefined
87
- }
88
- */
89
-
90
-
91
- // let's create a new segment then!
92
- const categoryName: ClapSegmentCategory =
93
- result?.category && Object.keys(ClapSegmentCategory).includes(result.category.toUpperCase())
94
- ? (result.category as ClapSegmentCategory)
95
- : ClapSegmentCategory.GENERIC
96
-
97
- return {
98
- prompt: result?.prompt || "",
99
- categoryName,
100
- llmOutput: "",
101
- }
102
- } catch (err1) {
103
-
104
- // a common scenario is when the output from the LLM is just not a JSON
105
- // this can happen quite often, for instance if the user tried to bypass
106
- // our prompt, or if they are just asking generic questions
107
- const errObj = err1 as any
108
- try {
109
- const keys = Object.keys(errObj)
110
- // console.log("keys:", keys)
111
- if (errObj.llmOutput) {
112
- return {
113
- prompt: "",
114
- categoryName: ClapSegmentCategory.GENERIC,
115
- llmOutput: `${errObj.llmOutput || ""}`,
116
- }
117
- }
118
- } catch (err2) {
119
- // err2 is just the error for when the LLM failed to reply
120
- console.error(`----<${err1}>----`)
121
- }
122
-
123
- return {
124
- prompt: "",
125
- categoryName: ClapSegmentCategory.GENERIC,
126
- llmOutput: ""
127
- }
128
- }
129
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/app/api/assistant/templates.ts CHANGED
@@ -23,7 +23,7 @@ DO NOT UNDER ANY CIRCUMSTANCES reply using natural language, instead either retu
23
  ## Output response schema
24
  {formatInstructions}
25
 
26
- # Current project
27
 
28
  ## Meta-information about the current video project and/or movie
29
  \`\`\`
@@ -46,6 +46,8 @@ DO NOT UNDER ANY CIRCUMSTANCES reply using natural language, instead either retu
46
  \`\`\`
47
 
48
  # Final warning and guidelines
 
 
49
  - Remember, if the director is asking to edit the video project data structure, you MUST only return the item object, in JSON format.
50
  - If you don't understand how to modify, it's okay to say you don't understand and politely ask for clarification.
51
  - When you edit a JSON list, sure to recopy the id for each field exactly like it is in the original, otherwise it breaks everything.
@@ -102,6 +104,8 @@ Now you understand the format, here are some more simplified examples:
102
  - "who is elizabeth" on a pirate scene would give "Elizabeth is a <replace with your short character analysis>" (not in JSON!)
103
  - "what do you think of this scene?" and then you reply with a short analysis (not in JSON!)
104
  etc.. you see the idea! you need to write in English.
 
 
105
  `
106
 
107
  export const humanTemplate = `{userPrompt}`
 
23
  ## Output response schema
24
  {formatInstructions}
25
 
26
+ # Information about the current project
27
 
28
  ## Meta-information about the current video project and/or movie
29
  \`\`\`
 
46
  \`\`\`
47
 
48
  # Final warning and guidelines
49
+ - Always give responses related to the current project, not the examples
50
+ - don't say introduction sentences like "Based on the provided JSON data" (the director doesn't need to be told that this is in JSON)
51
  - Remember, if the director is asking to edit the video project data structure, you MUST only return the item object, in JSON format.
52
  - If you don't understand how to modify, it's okay to say you don't understand and politely ask for clarification.
53
  - When you edit a JSON list, sure to recopy the id for each field exactly like it is in the original, otherwise it breaks everything.
 
104
  - "who is elizabeth" on a pirate scene would give "Elizabeth is a <replace with your short character analysis>" (not in JSON!)
105
  - "what do you think of this scene?" and then you reply with a short analysis (not in JSON!)
106
  etc.. you see the idea! you need to write in English.
107
+
108
+ One more thing: you will be provided a chat history, use that to contextualize and better understand the conversation!
109
  `
110
 
111
  export const humanTemplate = `{userPrompt}`
src/app/api/resolve/providers/lumalabs/getGenerations.ts ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+
2
+ // /api/photon/v1/user/generations/?offset=0&limit=10
src/app/api/resolve/providers/lumalabs/todo.ts ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // the API is not officially released yet, so we aren't gonna touch it
2
+ // however, it is interesting to note that it will most likely sit at:
3
+ //
4
+ // POST /api/photon/v1/generations/
5
+
src/app/api/resolve/providers/lumalabs/types.ts ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // interestingly, the API / models has multiple public and internal names:
2
+
3
+ // photon
4
+ // dream machine
5
+ // lit_lite_inference_text2vid_v1.0
6
+
7
+ enum LumaLabsPhotonGenerationState {
8
+ PENDING = "pending",
9
+ PROCESSING = "processing",
10
+ COMPLETED = "completed"
11
+ }
12
+
13
+ type LumaLabsPhotonGenerationVideo = {
14
+ url: string // url to to CDN
15
+ width: number // 512 by default
16
+ height: number // 512 by default
17
+ thumbnail: string | null // url to CDN
18
+ }
19
+
20
+ type LumaLabsPhotonGenerationRequest = {
21
+ fspect_ratio: "16:9" | "1:1" // "16:9", "1:1" what are the others?
22
+ expand_prompt: boolean
23
+ user_prompt: string
24
+ }
25
+
26
+ type LumaLabsPhotonGeneration = {
27
+ id: string
28
+ prompt: string
29
+ state: LumaLabsPhotonGenerationState
30
+ created_at: string // iso date
31
+ video: LumaLabsPhotonGenerationVideo | null
32
+ liked: null
33
+ estimate_wait_seconds: null | number
34
+ }
35
+
36
+ type LumaLabsPhotonGenerations = LumaLabsPhotonGeneration[]
src/app/api/resolve/providers/openai/TODO.md ADDED
@@ -0,0 +1 @@
 
 
1
+ put functions to generate images using OpenAI
src/app/main.tsx CHANGED
@@ -1,6 +1,6 @@
1
  "use client"
2
 
3
- import React, { useRef } from "react"
4
  import {
5
  ReflexContainer,
6
  ReflexSplitter,
@@ -22,6 +22,7 @@ import { TopBar } from "@/components/toolbars/top-bar"
22
  import { Timeline } from "@/components/core/timeline"
23
  import { useIO } from "@/controllers/io/useIO"
24
  import { ChatView } from "@/components/assistant/ChatView"
 
25
 
26
  type DroppableThing = { files: File[] }
27
 
@@ -29,7 +30,7 @@ function MainContent() {
29
  const ref = useRef<HTMLDivElement>(null)
30
  const isEmpty = useTimeline(s => s.isEmpty)
31
  const showTimeline = useUI((s) => s.showTimeline)
32
- const showChat = useUI((s) => s.showChat)
33
 
34
  const openFiles = useIO(s => s.openFiles)
35
 
@@ -49,6 +50,12 @@ function MainContent() {
49
 
50
  connectFileDrop(ref)
51
 
 
 
 
 
 
 
52
  return (
53
  <div
54
  ref={ref}
@@ -92,8 +99,8 @@ function MainContent() {
92
  </ReflexContainer>
93
  </ReflexElement>
94
 
95
- {showChat && <ReflexSplitter />}
96
- {showChat && <ReflexElement size={300}><ChatView /></ReflexElement>}
97
 
98
  </ReflexContainer>
99
 
 
1
  "use client"
2
 
3
+ import React, { useEffect, useRef } from "react"
4
  import {
5
  ReflexContainer,
6
  ReflexSplitter,
 
22
  import { Timeline } from "@/components/core/timeline"
23
  import { useIO } from "@/controllers/io/useIO"
24
  import { ChatView } from "@/components/assistant/ChatView"
25
+ import { useSearchParams } from "next/navigation"
26
 
27
  type DroppableThing = { files: File[] }
28
 
 
30
  const ref = useRef<HTMLDivElement>(null)
31
  const isEmpty = useTimeline(s => s.isEmpty)
32
  const showTimeline = useUI((s) => s.showTimeline)
33
+ const showAssistant = useUI((s) => s.showAssistant)
34
 
35
  const openFiles = useIO(s => s.openFiles)
36
 
 
50
 
51
  connectFileDrop(ref)
52
 
53
+ const setHasBetaAccess = useUI(s => s.setHasBetaAccess)
54
+
55
+ const searchParams = useSearchParams()
56
+ const hasBetaAccess = searchParams.get("beta") === "true"
57
+ useEffect(() => { setHasBetaAccess(hasBetaAccess) }, [hasBetaAccess])
58
+
59
  return (
60
  <div
61
  ref={ref}
 
99
  </ReflexContainer>
100
  </ReflexElement>
101
 
102
+ {showAssistant && <ReflexSplitter />}
103
+ {showAssistant && <ReflexElement size={300}><ChatView /></ReflexElement>}
104
 
105
  </ReflexContainer>
106
 
src/components/assistant/ChatBubble.tsx CHANGED
@@ -23,13 +23,13 @@ export function ChatBubble({
23
  )}>
24
  <div className={cn(
25
  `flex flex-col p-4`,
26
- isCurrentUser ? `bg-sky-700/60 text-sky-100/80` : `bg-indigo-700/60 text-indigo-100/80`,
27
  `rounded-lg`
28
  )}>
29
- <p>{message}</p>
30
  </div>
31
  <div className={cn(
32
- `text-sm`,
33
  isCurrentUser ? ` text-sky-100/60` : `text-indigo-100/60`,
34
  )}>{senderName}</div>
35
  </div>
 
23
  )}>
24
  <div className={cn(
25
  `flex flex-col p-4`,
26
+ isCurrentUser ? `bg-sky-800 text-sky-200` : `bg-indigo-800 text-indigo-200`,
27
  `rounded-lg`
28
  )}>
29
+ <p className={cn(`text-sm select-text`)}>{message}</p>
30
  </div>
31
  <div className={cn(
32
+ `text-sm select-text`,
33
  isCurrentUser ? ` text-sky-100/60` : `text-indigo-100/60`,
34
  )}>{senderName}</div>
35
  </div>
src/components/forms/FormField.tsx CHANGED
@@ -5,7 +5,7 @@ import { cn } from "@/lib/utils"
5
  import { FormLabel } from "./FormLabel"
6
 
7
  export function FormField({ label, children, className, horizontal = false }: {
8
- label?: string
9
  children?: ReactNode
10
  className?: string
11
  horizontal?: boolean
 
5
  import { FormLabel } from "./FormLabel"
6
 
7
  export function FormField({ label, children, className, horizontal = false }: {
8
+ label?: ReactNode
9
  children?: ReactNode
10
  className?: string
11
  horizontal?: boolean
src/components/forms/FormInput.tsx CHANGED
@@ -1,4 +1,4 @@
1
- import { ChangeEvent, HTMLInputTypeAttribute, useMemo, useRef } from "react"
2
 
3
  import { cn, getValidNumber, isValidNumber } from "@/lib/utils"
4
 
@@ -20,9 +20,9 @@ export function FormInput<T>({
20
  type,
21
  // ...props
22
  }: {
23
- label?: string
24
  className?: string
25
- placeholder?: string
26
  value?: T
27
  minValue?: T
28
  maxValue?: T
@@ -85,9 +85,7 @@ export function FormInput<T>({
85
 
86
  return (
87
  <FormField
88
- label={
89
- `${label}:`
90
- }
91
  horizontal={horizontal}
92
  >
93
  <Input
 
1
+ import { ChangeEvent, HTMLInputTypeAttribute, ReactNode, useMemo, useRef } from "react"
2
 
3
  import { cn, getValidNumber, isValidNumber } from "@/lib/utils"
4
 
 
20
  type,
21
  // ...props
22
  }: {
23
+ label?: ReactNode
24
  className?: string
25
+ placeholder?: ReactNode
26
  value?: T
27
  minValue?: T
28
  maxValue?: T
 
85
 
86
  return (
87
  <FormField
88
+ label={<>{label}:</>}
 
 
89
  horizontal={horizontal}
90
  >
91
  <Input
src/components/forms/FormSwitch.tsx ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { MdOutlineCheckCircle, MdRadioButtonUnchecked } from "react-icons/md"
2
+
3
+ import { cn } from "@/lib/utils"
4
+
5
+ import { FormField } from "./FormField"
6
+ import { Switch } from "../ui/switch"
7
+
8
+ export function FormSwitch({ label, className, checked, onCheckedChange, horizontal }: {
9
+ label?: string
10
+ className?: string
11
+ checked?: boolean
12
+ onCheckedChange: (checked: boolean) => void
13
+ horizontal?: boolean
14
+ }) {
15
+ return (
16
+ <FormField
17
+ label={label}
18
+ className={cn(`flex-row space-x-5`, className)}
19
+ horizontal={horizontal}>
20
+ <Switch
21
+ checked={checked}
22
+ onCheckedChange={(checked) => {
23
+ console.log("onCheckedChange: " + checked)
24
+ onCheckedChange(!checked)
25
+ }}
26
+ />
27
+ </FormField>
28
+ )
29
+ }
src/components/settings/constants.ts CHANGED
@@ -32,6 +32,7 @@ export const availableComputeProvidersForAssistant = [
32
  ComputeProvider.HUGGINGFACE,
33
  ComputeProvider.GROQ,
34
  ComputeProvider.OPENAI,
 
35
  ComputeProvider.ANTHROPIC,
36
  ComputeProvider.FIREWORKSAI,
37
  ]
@@ -119,11 +120,22 @@ export const availableModelsForAssistant: Partial<Record<ComputeProvider, string
119
  "Llama3-8b-8192",
120
  ],
121
  [ComputeProvider.ANTHROPIC]: [
 
122
  "claude-3-opus-20240229",
123
  "claude-3-sonnet-20240229",
124
  "claude-3-haiku-20240307"
125
  ],
126
  [ComputeProvider.GOOGLE]: [
 
 
 
 
 
 
 
 
 
 
127
  "claude-3-opus@20240229",
128
  "claude-3-sonnet@20240229",
129
  "claude-3-haiku@20240307"
 
32
  ComputeProvider.HUGGINGFACE,
33
  ComputeProvider.GROQ,
34
  ComputeProvider.OPENAI,
35
+ ComputeProvider.GOOGLE,
36
  ComputeProvider.ANTHROPIC,
37
  ComputeProvider.FIREWORKSAI,
38
  ]
 
120
  "Llama3-8b-8192",
121
  ],
122
  [ComputeProvider.ANTHROPIC]: [
123
+ // you can find this list here: https://docs.anthropic.com/en/docs/models-overview
124
  "claude-3-opus-20240229",
125
  "claude-3-sonnet-20240229",
126
  "claude-3-haiku-20240307"
127
  ],
128
  [ComputeProvider.GOOGLE]: [
129
+ // you can find this list here: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/model-versioning
130
+ "gemini-1.5-flash-001",
131
+ "gemini-1.5-pro-001",
132
+ "gemini-1.0-pro-vision-001",
133
+ "gemini-1.0-pro-002",
134
+
135
+ // The auto-updated version of a Gemini model points to the most recent stable version
136
+
137
+
138
+ // you can find this list here: https://docs.anthropic.com/en/docs/models-overview
139
  "claude-3-opus@20240229",
140
  "claude-3-sonnet@20240229",
141
  "claude-3-haiku@20240307"
src/components/settings/provider.tsx CHANGED
@@ -1,14 +1,32 @@
 
 
1
  import { FormSection } from "@/components/forms/FormSection"
2
  import { getDefaultSettingsState, useSettings } from "@/controllers/settings"
3
- import { FormSelect } from "../forms/FormSelect"
4
  import { ComfyIcuAccelerator } from "@/types"
5
- import { FormInput } from "../forms/FormInput"
6
  import { APP_NAME } from "@/lib/core/constants"
 
7
  import { availableComfyIcuAccelerators } from "./constants"
8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  export function SettingsSectionProvider() {
10
  const defaultSettings = getDefaultSettingsState()
11
 
 
 
 
12
  const replicateApiKey = useSettings(s => s.replicateApiKey)
13
  const setReplicateApiKey = useSettings(s => s.setReplicateApiKey)
14
 
@@ -54,20 +72,29 @@ export function SettingsSectionProvider() {
54
  const kitsAiApiKey = useSettings(s => s.kitsAiApiKey)
55
  const setKitsAiApiKey = useSettings(s => s.setKitsAiApiKey)
56
 
 
 
57
  return (
58
  <div className="flex flex-col space-y-6 justify-between">
59
  <FormSection label="Compute providers">
60
 
61
- <p className="italic text-sm text-stone-500 max-w-80">
62
  Note: preferences and credentials are stored inside your browser local storage.<br/>{APP_NAME} uses them to perform API calls on your behalf, but forgets them immediately.
63
  </p>
64
 
 
 
 
 
 
 
 
65
  <FormInput
66
  label="Hugging Face API key"
67
  value={huggingFaceApiKey}
68
  defaultValue={""}
69
  onChange={setHuggingFaceApiKey}
70
- type="password"
71
  />
72
 
73
  <FormInput
@@ -75,14 +102,14 @@ export function SettingsSectionProvider() {
75
  value={replicateApiKey}
76
  defaultValue={defaultSettings.replicateApiKey}
77
  onChange={setReplicateApiKey}
78
- type="password"
79
  />
80
  <FormInput
81
  label="Comfy.icu API key"
82
  value={comfyIcuApiKey}
83
  defaultValue={defaultSettings.comfyIcuApiKey}
84
  onChange={setComfyIcuApiKey}
85
- type="password"
86
  />
87
 
88
  <FormSelect<ComfyIcuAccelerator>
@@ -103,11 +130,11 @@ export function SettingsSectionProvider() {
103
  />
104
 
105
  <FormInput
106
- label="Fal.ai API Key"
107
  value={falAiApiKey}
108
  defaultValue={defaultSettings.falAiApiKey}
109
  onChange={setFalAiApiKey}
110
- type="password"
111
  />
112
 
113
  <FormInput
@@ -115,15 +142,15 @@ export function SettingsSectionProvider() {
115
  value={modelsLabApiKey}
116
  defaultValue={defaultSettings.modelsLabApiKey}
117
  onChange={setModelsLabApiKey}
118
- type="password"
119
  />
120
 
121
  <FormInput
122
- label="OpenAI API Key"
123
  value={openaiApiKey}
124
  defaultValue={defaultSettings.openaiApiKey}
125
  onChange={setOpenaiApiKey}
126
- type="password"
127
  />
128
 
129
  <FormInput
@@ -131,31 +158,31 @@ export function SettingsSectionProvider() {
131
  value={groqApiKey}
132
  defaultValue={defaultSettings.groqApiKey}
133
  onChange={setGroqApiKey}
134
- type="password"
135
  />
136
 
137
  <FormInput
138
- label="Google API Key"
139
  value={googleApiKey}
140
  defaultValue={defaultSettings.googleApiKey}
141
  onChange={setGoogleApiKey}
142
- type="password"
143
  />
144
 
145
  <FormInput
146
- label="Anthropic API Key"
147
  value={anthropicApiKey}
148
  defaultValue={defaultSettings.anthropicApiKey}
149
  onChange={setAnthropicApiKey}
150
- type="password"
151
  />
152
 
153
  <FormInput
154
- label="Cohere API Key"
155
  value={cohereApiKey}
156
  defaultValue={defaultSettings.cohereApiKey}
157
  onChange={setCohereApiKey}
158
- type="password"
159
  />
160
 
161
  <FormInput
@@ -163,15 +190,15 @@ export function SettingsSectionProvider() {
163
  value={mistralAiApiKey}
164
  defaultValue={defaultSettings.mistralAiApiKey}
165
  onChange={setMistralAiApiKey}
166
- type="password"
167
  />
168
 
169
  <FormInput
170
- label="StabilityAI API Key"
171
  value={stabilityAiApiKey}
172
  defaultValue={defaultSettings.stabilityAiApiKey}
173
  onChange={setStabilityAiApiKey}
174
- type="password"
175
  />
176
 
177
  <FormInput
@@ -179,7 +206,7 @@ export function SettingsSectionProvider() {
179
  value={elevenLabsApiKey}
180
  defaultValue={defaultSettings.elevenLabsApiKey}
181
  onChange={setElevenLabsApiKey}
182
- type="password"
183
  />
184
 
185
  <FormInput
@@ -187,7 +214,7 @@ export function SettingsSectionProvider() {
187
  value={kitsAiApiKey}
188
  defaultValue={defaultSettings.kitsAiApiKey}
189
  onChange={setKitsAiApiKey}
190
- type="password"
191
  />
192
  </FormSection>
193
  </div>
 
1
+ import { ReactNode } from "react"
2
+
3
  import { FormSection } from "@/components/forms/FormSection"
4
  import { getDefaultSettingsState, useSettings } from "@/controllers/settings"
 
5
  import { ComfyIcuAccelerator } from "@/types"
 
6
  import { APP_NAME } from "@/lib/core/constants"
7
+
8
  import { availableComfyIcuAccelerators } from "./constants"
9
 
10
+ import { FormSelect } from "../forms/FormSelect"
11
+ import { FormInput } from "../forms/FormInput"
12
+ import { useUI } from "@/controllers/ui"
13
+ import { FormSwitch } from "../forms/FormSwitch"
14
+
15
+ function GetItHere({ href, children }: { href: string; children: ReactNode }) {
16
+ return (
17
+ <span>{children} (<a
18
+ className="underline text-stone-500 hover:text-stone-400"
19
+ href={href}
20
+ target="_blank"
21
+ >get one</a>)</span>
22
+ )
23
+ }
24
  export function SettingsSectionProvider() {
25
  const defaultSettings = getDefaultSettingsState()
26
 
27
+ const showApiKeys = useUI(s => s.showApiKeys)
28
+ const setShowApiKeys = useUI(s => s.setShowApiKeys)
29
+
30
  const replicateApiKey = useSettings(s => s.replicateApiKey)
31
  const setReplicateApiKey = useSettings(s => s.setReplicateApiKey)
32
 
 
72
  const kitsAiApiKey = useSettings(s => s.kitsAiApiKey)
73
  const setKitsAiApiKey = useSettings(s => s.setKitsAiApiKey)
74
 
75
+ const apiKeyType = showApiKeys ? "text" : "password"
76
+
77
  return (
78
  <div className="flex flex-col space-y-6 justify-between">
79
  <FormSection label="Compute providers">
80
 
81
+ <p className="italic text-sm text-stone-500 max-w-80">
82
  Note: preferences and credentials are stored inside your browser local storage.<br/>{APP_NAME} uses them to perform API calls on your behalf, but forgets them immediately.
83
  </p>
84
 
85
+ <FormSwitch
86
+ label="Hide API Keys"
87
+ checked={!showApiKeys}
88
+ onCheckedChange={setShowApiKeys}
89
+ horizontal
90
+ />
91
+
92
  <FormInput
93
  label="Hugging Face API key"
94
  value={huggingFaceApiKey}
95
  defaultValue={""}
96
  onChange={setHuggingFaceApiKey}
97
+ type={apiKeyType}
98
  />
99
 
100
  <FormInput
 
102
  value={replicateApiKey}
103
  defaultValue={defaultSettings.replicateApiKey}
104
  onChange={setReplicateApiKey}
105
+ type={apiKeyType}
106
  />
107
  <FormInput
108
  label="Comfy.icu API key"
109
  value={comfyIcuApiKey}
110
  defaultValue={defaultSettings.comfyIcuApiKey}
111
  onChange={setComfyIcuApiKey}
112
+ type={apiKeyType}
113
  />
114
 
115
  <FormSelect<ComfyIcuAccelerator>
 
130
  />
131
 
132
  <FormInput
133
+ label={<GetItHere href="https://fal.ai/dashboard/keys">Fal.ai API Key</GetItHere>}
134
  value={falAiApiKey}
135
  defaultValue={defaultSettings.falAiApiKey}
136
  onChange={setFalAiApiKey}
137
+ type={apiKeyType}
138
  />
139
 
140
  <FormInput
 
142
  value={modelsLabApiKey}
143
  defaultValue={defaultSettings.modelsLabApiKey}
144
  onChange={setModelsLabApiKey}
145
+ type={apiKeyType}
146
  />
147
 
148
  <FormInput
149
+ label={<GetItHere href="https://platform.openai.com/api-keys">OpenAI API Key</GetItHere>}
150
  value={openaiApiKey}
151
  defaultValue={defaultSettings.openaiApiKey}
152
  onChange={setOpenaiApiKey}
153
+ type={apiKeyType}
154
  />
155
 
156
  <FormInput
 
158
  value={groqApiKey}
159
  defaultValue={defaultSettings.groqApiKey}
160
  onChange={setGroqApiKey}
161
+ type={apiKeyType}
162
  />
163
 
164
  <FormInput
165
+ label={<GetItHere href="https://aistudio.google.com/app/apikey">Google API Key</GetItHere>}
166
  value={googleApiKey}
167
  defaultValue={defaultSettings.googleApiKey}
168
  onChange={setGoogleApiKey}
169
+ type={apiKeyType}
170
  />
171
 
172
  <FormInput
173
+ label={<GetItHere href="https://console.anthropic.com/settings/keys">Anthropic API Key</GetItHere>}
174
  value={anthropicApiKey}
175
  defaultValue={defaultSettings.anthropicApiKey}
176
  onChange={setAnthropicApiKey}
177
+ type={apiKeyType}
178
  />
179
 
180
  <FormInput
181
+ label={<GetItHere href="https://dashboard.cohere.com/api-keys">Cohere API Key</GetItHere>}
182
  value={cohereApiKey}
183
  defaultValue={defaultSettings.cohereApiKey}
184
  onChange={setCohereApiKey}
185
+ type={apiKeyType}
186
  />
187
 
188
  <FormInput
 
190
  value={mistralAiApiKey}
191
  defaultValue={defaultSettings.mistralAiApiKey}
192
  onChange={setMistralAiApiKey}
193
+ type={apiKeyType}
194
  />
195
 
196
  <FormInput
197
+ label={<GetItHere href="https://platform.stability.ai/account/keys">StabilityAI API Key</GetItHere>}
198
  value={stabilityAiApiKey}
199
  defaultValue={defaultSettings.stabilityAiApiKey}
200
  onChange={setStabilityAiApiKey}
201
+ type={apiKeyType}
202
  />
203
 
204
  <FormInput
 
206
  value={elevenLabsApiKey}
207
  defaultValue={defaultSettings.elevenLabsApiKey}
208
  onChange={setElevenLabsApiKey}
209
+ type={apiKeyType}
210
  />
211
 
212
  <FormInput
 
214
  value={kitsAiApiKey}
215
  defaultValue={defaultSettings.kitsAiApiKey}
216
  onChange={setKitsAiApiKey}
217
+ type={apiKeyType}
218
  />
219
  </FormSection>
220
  </div>
src/components/settings/sound.tsx CHANGED
@@ -40,7 +40,7 @@ export function SettingsSectionSound() {
40
  }
41
  items={availableComputeProvidersForSound.map(provider => ({
42
  id: provider,
43
- label: computeProviderShortNames[soundProvider] || "(missing name)",
44
  disabled: false,
45
  value: provider,
46
  }))}
 
40
  }
41
  items={availableComputeProvidersForSound.map(provider => ({
42
  id: provider,
43
+ label: computeProviderShortNames[provider] || "(missing name)",
44
  disabled: false,
45
  value: provider,
46
  }))}
src/components/toolbars/top-menu/assistant/index.tsx CHANGED
@@ -18,6 +18,7 @@ import { ProviderList } from "../lists/ProviderList"
18
  import { availableComputeProvidersForAssistant } from "@/components/settings/constants"
19
  import { ComputeProvider, SettingsCategory } from "@/types"
20
  import { AssistantModelList } from "../lists/AssistantModelList"
 
21
 
22
  export function TopMenuAssistant() {
23
  const setShowSettings = useUI(s => s.setShowSettings)
@@ -25,6 +26,10 @@ export function TopMenuAssistant() {
25
  const setAssistantProvider = useSettings(s => s.setAssistantProvider)
26
  const assistantModel = useSettings(s => s.assistantModel)
27
  const setAssistantModel = useSettings(s => s.setAssistantModel)
 
 
 
 
28
  return (
29
  <MenubarMenu>
30
  <MenubarTrigger>Assistant</MenubarTrigger>
 
18
  import { availableComputeProvidersForAssistant } from "@/components/settings/constants"
19
  import { ComputeProvider, SettingsCategory } from "@/types"
20
  import { AssistantModelList } from "../lists/AssistantModelList"
21
+ import { useInitAssistant } from "@/controllers/assistant/useAssistant"
22
 
23
  export function TopMenuAssistant() {
24
  const setShowSettings = useUI(s => s.setShowSettings)
 
26
  const setAssistantProvider = useSettings(s => s.setAssistantProvider)
27
  const assistantModel = useSettings(s => s.assistantModel)
28
  const setAssistantModel = useSettings(s => s.setAssistantModel)
29
+
30
+ // this should only be called on and at only one place in the project!
31
+ useInitAssistant()
32
+
33
  return (
34
  <MenubarMenu>
35
  <MenubarTrigger>Assistant</MenubarTrigger>
src/components/toolbars/top-menu/view/index.tsx CHANGED
@@ -29,12 +29,14 @@ export function TopMenuView() {
29
  const showExplorer = useUI((s) => s.showExplorer)
30
  const setShowExplorer = useUI((s) => s.setShowExplorer)
31
 
32
- const showChat = useUI((s) => s.showChat)
33
- const setShowChat = useUI((s) => s.setShowChat)
34
 
35
  const showVideoPlayer = useUI((s) => s.showVideoPlayer)
36
  const setShowVideoPlayer = useUI((s) => s.setShowVideoPlayer)
37
 
 
 
38
  return (
39
  <MenubarMenu>
40
  <MenubarTrigger>View</MenubarTrigger>
@@ -78,19 +80,17 @@ export function TopMenuView() {
78
  }}
79
  >Show asset explorer</MenubarCheckboxItem>
80
  */}
81
-
82
  <MenubarCheckboxItem
83
- checked={showChat}
84
- disabled
85
- /*
86
  onClick={(e) => {
87
- setShowChat(!showChat)
88
  e.stopPropagation()
89
  e.preventDefault()
90
  return false
91
  }}
92
- */
93
- >Show chat assistant</MenubarCheckboxItem>
94
  {/*
95
  <MenubarCheckboxItem
96
  checked={showVideoPlayer}
 
29
  const showExplorer = useUI((s) => s.showExplorer)
30
  const setShowExplorer = useUI((s) => s.setShowExplorer)
31
 
32
+ const showAssistant = useUI((s) => s.showAssistant)
33
+ const setShowAssistant = useUI((s) => s.setShowAssistant)
34
 
35
  const showVideoPlayer = useUI((s) => s.showVideoPlayer)
36
  const setShowVideoPlayer = useUI((s) => s.setShowVideoPlayer)
37
 
38
+ const hasBetaAccess = useUI(s => s.hasBetaAccess)
39
+
40
  return (
41
  <MenubarMenu>
42
  <MenubarTrigger>View</MenubarTrigger>
 
80
  }}
81
  >Show asset explorer</MenubarCheckboxItem>
82
  */}
83
+ {hasBetaAccess &&
84
  <MenubarCheckboxItem
85
+ checked={showAssistant}
 
 
86
  onClick={(e) => {
87
+ setShowAssistant(!showAssistant)
88
  e.stopPropagation()
89
  e.preventDefault()
90
  return false
91
  }}
92
+ >Show assistant</MenubarCheckboxItem>
93
+ }
94
  {/*
95
  <MenubarCheckboxItem
96
  checked={showVideoPlayer}
src/controllers/assistant/types.ts CHANGED
@@ -40,7 +40,9 @@ export type AssistantControls = {
40
  *
41
  * @returns true if operation succeeded, false otherwise
42
  */
43
- // toggleVoice: () => Promise<boolean>
 
 
44
 
45
  /**
46
  * Run a prompt command (which can come from a transcript or somewhere else)
 
40
  *
41
  * @returns true if operation succeeded, false otherwise
42
  */
43
+ toggleVoice: () => boolean
44
+
45
+ setVoiceTranscript: (transcript: string) => void
46
 
47
  /**
48
  * Run a prompt command (which can come from a transcript or somewhere else)
src/controllers/assistant/useAssistant.ts CHANGED
@@ -1,5 +1,6 @@
1
  "use client"
2
 
 
3
  import { create } from "zustand"
4
  import { ClapOutputType, ClapProject, ClapSegment, ClapSegmentCategory, newSegment, UUID } from "@aitube/clap"
5
 
@@ -9,6 +10,8 @@ import { DEFAULT_DURATION_IN_MS_PER_STEP, findFreeTrack, TimelineStore, useTimel
9
  import { useSettings } from "../settings"
10
  import { AssistantRequest } from "@/types"
11
  import { askAssistant } from "./askAssistant"
 
 
12
 
13
  // URL to the speech to text websocket server
14
  export const STT_API_URL = process.env.NEXT_PUBLIC_SPEECH_TO_TEXT_API_URL || ""
@@ -17,8 +20,8 @@ const enableTextToSpeech = false
17
 
18
  export const useAssistant = create<AssistantStore>((set, get) => ({
19
  ...getDefaultAssistantState(),
20
- /*
21
- toggleVoice: async (): Promise<boolean> => {
22
 
23
  if (!navigator?.mediaDevices?.getUserMedia || !MediaRecorder.isTypeSupported("audio/webm")) {
24
  console.error("This environment doesn't support microphone recording")
@@ -34,68 +37,12 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
34
 
35
  set({ isVoiceEnabled })
36
 
37
- if (isVoiceEnabled) {
38
-
39
- const stream = get().stream || (await navigator.mediaDevices.getUserMedia({ audio: true }))
40
- set({ stream })
41
-
42
- const recorder = get().recorder || new MediaRecorder(stream, { mimeType: "audio/webm" })
43
- set({ recorder })
44
-
45
- if (!get().socket) {
46
- //create a websocket connection
47
- const socket = new WebSocket(STT_API_URL)
48
- set({ socket })
49
-
50
- socket.onopen = () => {
51
- console.log({ event: "onopen" });
52
- recorder.addEventListener("dataavailable", async (event) => {
53
- if (event.data.size > 0 && socket.readyState === 1) {
54
- socket.send(event.data)
55
- }
56
- })
57
- recorder.start(1000)
58
- }
59
-
60
- socket.onmessage = (message) => {
61
- let transcript = ""
62
- try {
63
- const received = JSON.parse(message.data)
64
- transcript = received.channel.alternatives[0].transcript || ""
65
- } catch (err) {
66
- console.error(err)
67
- }
68
- set({
69
- transcript,
70
- })
71
- }
72
-
73
- socket.onclose = () => {
74
- console.log({ event: "onclose" });
75
- }
76
-
77
- socket.onerror = (error) => {
78
- console.log({ event: "onerror", error });
79
- }
80
- }
81
- } else {
82
- // voice recognition is expensive, so we make sure we removed everything
83
- const { socket, recorder, stream } = get()
84
-
85
- // stop existing socket
86
- socket?.close()
87
- recorder?.stop()
88
-
89
- set({
90
- socket: undefined,
91
- recorder: undefined,
92
- })
93
- }
94
-
95
- return true
96
  },
97
- */
98
 
 
 
 
99
  runCommand: (prompt: string) => {
100
 
101
  const query = prompt
@@ -179,9 +126,12 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
179
  }
180
 
181
  console.log(`TODO @julian: restore the concept of "addSegment()", "updateSegment()", "active segment" and "cursor position" inside @aitube-timeline`)
182
- // const { addSegment, activeSegments, cursorInSteps, } = useTimeline.getState()
 
 
 
 
183
 
184
- const activeSegments: ClapSegment[] = []
185
  const cursorInSteps = 0
186
 
187
  const referenceSegment: ClapSegment | undefined = activeSegments.at(0)
@@ -205,12 +155,21 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
205
 
206
  const segments: ClapSegment[] = activeSegments
207
  .filter(s =>
 
208
  s.category === ClapSegmentCategory.LOCATION ||
209
  s.category === ClapSegmentCategory.TIME ||
210
  s.category === ClapSegmentCategory.LIGHTING ||
211
  s.category === ClapSegmentCategory.ACTION ||
212
  s.category === ClapSegmentCategory.DIALOGUE ||
213
- s.category === ClapSegmentCategory.WEATHER
 
 
 
 
 
 
 
 
214
  )
215
 
216
  console.log(`TODO @julian: provide both contextual segments and editable ones to the LLM?`)
@@ -223,6 +182,7 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
223
  actionLine,
224
  entities,
225
  projectInfo,
 
226
  }
227
 
228
  const { prompt, categoryName, llmOutput } = await askAssistant(request)
@@ -234,6 +194,8 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
234
  })
235
  return
236
  }
 
 
237
 
238
  let match = segments.find(s => s.category === categoryName) || undefined
239
  if (!match) {
@@ -271,18 +233,20 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
271
  ...match,
272
  prompt,
273
  label: prompt,
 
274
  })
275
 
276
  console.log(`TODO Julian: update the segment!!`)
277
- // addSegment(newSeg)
278
 
279
- /*
280
- updateSegment({
281
- ...match,
282
  prompt,
283
  label: prompt,
284
  })
285
- */
 
 
286
 
287
  addEventToHistory({
288
  senderId: "assistant",
@@ -291,4 +255,31 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
291
  })
292
  }
293
  }
294
- }))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  "use client"
2
 
3
+ import { useVoiceToText } from "react-speakup"
4
  import { create } from "zustand"
5
  import { ClapOutputType, ClapProject, ClapSegment, ClapSegmentCategory, newSegment, UUID } from "@aitube/clap"
6
 
 
10
  import { useSettings } from "../settings"
11
  import { AssistantRequest } from "@/types"
12
  import { askAssistant } from "./askAssistant"
13
+ import { useRenderer } from "../renderer"
14
+ import { useEffect } from "react"
15
 
16
  // URL to the speech to text websocket server
17
  export const STT_API_URL = process.env.NEXT_PUBLIC_SPEECH_TO_TEXT_API_URL || ""
 
20
 
21
  export const useAssistant = create<AssistantStore>((set, get) => ({
22
  ...getDefaultAssistantState(),
23
+
24
+ toggleVoice: (): boolean => {
25
 
26
  if (!navigator?.mediaDevices?.getUserMedia || !MediaRecorder.isTypeSupported("audio/webm")) {
27
  console.error("This environment doesn't support microphone recording")
 
37
 
38
  set({ isVoiceEnabled })
39
 
40
+ return isVoiceEnabled
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
41
  },
 
42
 
43
+ setVoiceTranscript: (transcript: string) => {
44
+ set({ transcript })
45
+ },
46
  runCommand: (prompt: string) => {
47
 
48
  const query = prompt
 
126
  }
127
 
128
  console.log(`TODO @julian: restore the concept of "addSegment()", "updateSegment()", "active segment" and "cursor position" inside @aitube-timeline`)
129
+ const {
130
+ bufferedSegments: {
131
+ activeSegments
132
+ }
133
+ } = useRenderer.getState()
134
 
 
135
  const cursorInSteps = 0
136
 
137
  const referenceSegment: ClapSegment | undefined = activeSegments.at(0)
 
155
 
156
  const segments: ClapSegment[] = activeSegments
157
  .filter(s =>
158
+ s.category === ClapSegmentCategory.CAMERA ||
159
  s.category === ClapSegmentCategory.LOCATION ||
160
  s.category === ClapSegmentCategory.TIME ||
161
  s.category === ClapSegmentCategory.LIGHTING ||
162
  s.category === ClapSegmentCategory.ACTION ||
163
  s.category === ClapSegmentCategory.DIALOGUE ||
164
+ s.category === ClapSegmentCategory.WEATHER ||
165
+ s.category === ClapSegmentCategory.ERA ||
166
+ s.category === ClapSegmentCategory.INTERFACE ||
167
+ s.category === ClapSegmentCategory.MUSIC ||
168
+ s.category === ClapSegmentCategory.SOUND ||
169
+ s.category === ClapSegmentCategory.STORYBOARD ||
170
+ s.category === ClapSegmentCategory.STYLE ||
171
+ s.category === ClapSegmentCategory.TRANSITION ||
172
+ s.category === ClapSegmentCategory.GENERIC
173
  )
174
 
175
  console.log(`TODO @julian: provide both contextual segments and editable ones to the LLM?`)
 
182
  actionLine,
183
  entities,
184
  projectInfo,
185
+ history: get().history
186
  }
187
 
188
  const { prompt, categoryName, llmOutput } = await askAssistant(request)
 
194
  })
195
  return
196
  }
197
+
198
+ console.log(`askAssistant response: `, { prompt, categoryName, llmOutput })
199
 
200
  let match = segments.find(s => s.category === categoryName) || undefined
201
  if (!match) {
 
233
  ...match,
234
  prompt,
235
  label: prompt,
236
+ categoryName,
237
  })
238
 
239
  console.log(`TODO Julian: update the segment!!`)
 
240
 
241
+ // const segments: ClapSegment[] = useTimeline.getState().segments
242
+ // const segment = segments.find(s => s.id === newSeg.id)
243
+ Object.assign(match, {
244
  prompt,
245
  label: prompt,
246
  })
247
+
248
+ // tell the timeline that this individual segment should be redrawn
249
+ timelineState.trackSilentChangeInSegment(match.id)
250
 
251
  addEventToHistory({
252
  senderId: "assistant",
 
255
  })
256
  }
257
  }
258
+ }))
259
+
260
+ export function useInitAssistant() {
261
+ const isVoiceEnabled = useAssistant(s => s.isVoiceEnabled)
262
+ const toggleVoice = useAssistant(s => s.toggleVoice)
263
+ const setVoiceTranscript = useAssistant(s => s.setVoiceTranscript)
264
+ const { startListening, stopListening, transcript } = useVoiceToText({
265
+ continuous: true
266
+ })
267
+
268
+ useEffect(() => {
269
+ if (isVoiceEnabled) {
270
+ console.log(`TODO: startListening`)
271
+
272
+ startListening()
273
+ } else {
274
+ stopListening()
275
+ }
276
+ }, [isVoiceEnabled])
277
+
278
+ useEffect(() => {
279
+ setVoiceTranscript(transcript)
280
+ }, [transcript])
281
+ }
282
+
283
+ if (typeof window !== "undefined") {
284
+ (window as any).useAssistant = useAssistant
285
+ }
src/controllers/audio/useAudio.ts CHANGED
@@ -101,4 +101,9 @@ export const useAudio = create<AudioStore>((set, get) => ({
101
  ]
102
  })
103
  },
104
- }))
 
 
 
 
 
 
101
  ]
102
  })
103
  },
104
+ }))
105
+
106
+
107
+ if (typeof window !== "undefined") {
108
+ (window as any).useAudio = useAudio
109
+ }
src/controllers/io/useIO.ts CHANGED
@@ -501,4 +501,9 @@ export const useIO = create<IOStore>((set, get) => ({
501
  saveOpenTimelineIO: async () => {
502
 
503
  }
504
- }))
 
 
 
 
 
 
501
  saveOpenTimelineIO: async () => {
502
 
503
  }
504
+ }))
505
+
506
+
507
+ if (typeof window !== "undefined") {
508
+ (window as any).useIO = useIO
509
+ }
src/controllers/metrics/useMetrics.ts CHANGED
@@ -10,3 +10,8 @@ export const useMetrics = create<MetricsStore>((set, get) => ({
10
 
11
  // TODO: add a track metric callback
12
  }))
 
 
 
 
 
 
10
 
11
  // TODO: add a track metric callback
12
  }))
13
+
14
+
15
+ if (typeof window !== "undefined") {
16
+ (window as any).useMetrics = useMetrics
17
+ }
src/controllers/monitor/useMonitor.ts CHANGED
@@ -126,6 +126,7 @@ export const useMonitor = create<MonitorStore>((set, get) => ({
126
  staticVideoRef.currentTime = timeInMs / 1000
127
  } else if (mode === MonitoringMode.DYNAMIC) {
128
  // we force a state update
 
129
  renderLoop()
130
  }
131
  },
@@ -140,4 +141,9 @@ setTimeout(() => {
140
  if (typeof document !== "undefined") {
141
  useMonitor.getState().bindShortcuts()
142
  }
143
- }, 0)
 
 
 
 
 
 
126
  staticVideoRef.currentTime = timeInMs / 1000
127
  } else if (mode === MonitoringMode.DYNAMIC) {
128
  // we force a state update
129
+ // console.log(`forcing a state update`)
130
  renderLoop()
131
  }
132
  },
 
141
  if (typeof document !== "undefined") {
142
  useMonitor.getState().bindShortcuts()
143
  }
144
+ }, 0)
145
+
146
+
147
+ if (typeof window !== "undefined") {
148
+ (window as any).useMonitor = useMonitor
149
+ }
src/controllers/renderer/useRenderer.ts CHANGED
@@ -24,7 +24,6 @@ export const useRenderer = create<RendererStore>((set, get) => ({
24
  // this will be called at 60 FPS - and yes, it is expensive
25
  // we could probably improve things by using a temporal tree index
26
  renderLoop: (): BufferedSegments => {
27
-
28
  const { computeBufferedSegments, bufferedSegments } = get()
29
 
30
 
@@ -68,6 +67,7 @@ export const useRenderer = create<RendererStore>((set, get) => ({
68
  results.activeSegments.push(segment)
69
  results.activeVideoSegment = segment
70
  results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
 
71
  }
72
 
73
  const isActiveAudio =
@@ -81,6 +81,7 @@ export const useRenderer = create<RendererStore>((set, get) => ({
81
  results.activeSegments.push(segment)
82
  results.activeAudioSegments.push(segment)
83
  results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
 
84
  }
85
 
86
  const isActiveStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
@@ -88,8 +89,12 @@ export const useRenderer = create<RendererStore>((set, get) => ({
88
  results.activeSegments.push(segment)
89
  results.activeStoryboardSegment = segment
90
  results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
 
91
  }
92
-
 
 
 
93
  }
94
 
95
  const inUpcomingShot =
@@ -104,6 +109,7 @@ export const useRenderer = create<RendererStore>((set, get) => ({
104
  results.upcomingSegments.push(segment)
105
  results.upcomingVideoSegment = segment
106
  results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
 
107
  }
108
 
109
  const isUpcomingAudio =
@@ -117,6 +123,7 @@ export const useRenderer = create<RendererStore>((set, get) => ({
117
  results.upcomingSegments.push(segment)
118
  results.upcomingAudioSegments.push(segment)
119
  results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
 
120
  }
121
 
122
  const isUpcomingStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
@@ -124,8 +131,11 @@ export const useRenderer = create<RendererStore>((set, get) => ({
124
  results.upcomingSegments.push(segment)
125
  results.upcomingStoryboardSegment = segment
126
  results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
 
127
  }
128
-
 
 
129
  }
130
 
131
  }
@@ -134,4 +144,8 @@ export const useRenderer = create<RendererStore>((set, get) => ({
134
 
135
  return results
136
  },
137
- }))
 
 
 
 
 
24
  // this will be called at 60 FPS - and yes, it is expensive
25
  // we could probably improve things by using a temporal tree index
26
  renderLoop: (): BufferedSegments => {
 
27
  const { computeBufferedSegments, bufferedSegments } = get()
28
 
29
 
 
67
  results.activeSegments.push(segment)
68
  results.activeVideoSegment = segment
69
  results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
70
+ continue
71
  }
72
 
73
  const isActiveAudio =
 
81
  results.activeSegments.push(segment)
82
  results.activeAudioSegments.push(segment)
83
  results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
84
+ continue
85
  }
86
 
87
  const isActiveStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
 
89
  results.activeSegments.push(segment)
90
  results.activeStoryboardSegment = segment
91
  results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
92
+ continue
93
  }
94
+
95
+ results.activeSegments.push(segment)
96
+ results.activeSegmentsCacheKey = getSegmentCacheKey(segment, results.activeSegmentsCacheKey)
97
+ continue
98
  }
99
 
100
  const inUpcomingShot =
 
109
  results.upcomingSegments.push(segment)
110
  results.upcomingVideoSegment = segment
111
  results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
112
+ continue
113
  }
114
 
115
  const isUpcomingAudio =
 
123
  results.upcomingSegments.push(segment)
124
  results.upcomingAudioSegments.push(segment)
125
  results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
126
+ continue
127
  }
128
 
129
  const isUpcomingStoryboard = segment.category === ClapSegmentCategory.STORYBOARD && segment.assetUrl
 
131
  results.upcomingSegments.push(segment)
132
  results.upcomingStoryboardSegment = segment
133
  results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
134
+ continue
135
  }
136
+ results.upcomingSegments.push(segment)
137
+ results.upcomingSegmentsCacheKey = getSegmentCacheKey(segment, results.upcomingSegmentsCacheKey)
138
+ continue
139
  }
140
 
141
  }
 
144
 
145
  return results
146
  },
147
+ }))
148
+
149
+ if (typeof window !== "undefined") {
150
+ (window as any).useRenderer = useRenderer
151
+ }
src/controllers/resolver/useResolver.ts CHANGED
@@ -362,3 +362,7 @@ export const useResolver = create<ResolverStore>((set, get) => ({
362
  }
363
 
364
  }))
 
 
 
 
 
362
  }
363
 
364
  }))
365
+
366
+ if (typeof window !== "undefined") {
367
+ (window as any).useResolver = useResolver
368
+ }
src/controllers/settings/getDefaultSettingsState.ts CHANGED
@@ -130,6 +130,11 @@ export function getDefaultSettingsState(): SettingsState {
130
  googleModelForVoice: "v1",
131
  googleModelForMusic: "MusicLM",
132
 
 
 
 
 
 
133
  anthropicModelForAssistant: "claude-3-opus-20240229",
134
 
135
  elevenLabsModelForVoice: "v1",
 
130
  googleModelForVoice: "v1",
131
  googleModelForMusic: "MusicLM",
132
 
133
+
134
+ // Claud 3 stats:
135
+ // Context window 200K*
136
+ // *~150K words, ~680K unicode characters
137
+ // Max output 4096 tokens
138
  anthropicModelForAssistant: "claude-3-opus-20240229",
139
 
140
  elevenLabsModelForVoice: "v1",
src/controllers/settings/useSettings.ts CHANGED
@@ -498,7 +498,7 @@ export const useSettings = create<SettingsStore>()(
498
  },
499
  }),
500
  {
501
- name: 'CLAPPER_REVISION_0_SETTINGS_RENDERING'
502
  },
503
  ),
504
  )
 
498
  },
499
  }),
500
  {
501
+ name: 'CLAPPER_REVISION_0_CONTROLLERS_USE_SETTINGS'
502
  },
503
  ),
504
  )
src/controllers/ui/getDefaultUIState.ts CHANGED
@@ -3,14 +3,16 @@ import { UIState } from "./types"
3
 
4
  export function getDefaultUIState(): UIState {
5
  const state: UIState = {
 
 
6
  showSettings: SettingsCategory.NONE,
7
  showImporter: false,
8
  showTimeline: true,
9
- showExplorer: true,
10
  showVideoPlayer: true,
11
- showChat: true,
12
- showFPS: true,
13
- followCursor: true,
14
  }
15
 
16
  return state
 
3
 
4
  export function getDefaultUIState(): UIState {
5
  const state: UIState = {
6
+ hasBetaAccess: false,
7
+ showApiKeys: false,
8
  showSettings: SettingsCategory.NONE,
9
  showImporter: false,
10
  showTimeline: true,
11
+ showExplorer: false,
12
  showVideoPlayer: true,
13
+ showAssistant: false,
14
+ showFPS: false,
15
+ followCursor: false,
16
  }
17
 
18
  return state
src/controllers/ui/types.ts CHANGED
@@ -1,23 +1,27 @@
1
  import { SettingsCategory } from "@/types"
2
 
3
  export type UIState = {
 
 
4
  showSettings: SettingsCategory
5
  showImporter: boolean
6
  showTimeline: boolean
7
  showExplorer: boolean
8
  showVideoPlayer: boolean
9
- showChat: boolean
10
  showFPS: boolean
11
  followCursor: boolean
12
  }
13
 
14
  export type UIControls = {
 
 
15
  setShowSettings: (showSettings: SettingsCategory) => void
16
  setShowImporter: (showImporter: boolean) => void
17
  setShowTimeline: (showTimeline: boolean) => void
18
  setShowExplorer: (showExplorer: boolean) => void
19
  setShowVideoPlayer: (showVideoPlayer: boolean) => void
20
- setShowChat: (showChat: boolean) => void
21
  setShowFPS: (showFPS: boolean) => void
22
  setFollowCursor: (followCursor: boolean) => void
23
  }
 
1
  import { SettingsCategory } from "@/types"
2
 
3
  export type UIState = {
4
+ hasBetaAccess: boolean
5
+ showApiKeys: boolean
6
  showSettings: SettingsCategory
7
  showImporter: boolean
8
  showTimeline: boolean
9
  showExplorer: boolean
10
  showVideoPlayer: boolean
11
+ showAssistant: boolean
12
  showFPS: boolean
13
  followCursor: boolean
14
  }
15
 
16
  export type UIControls = {
17
+ setHasBetaAccess: (hasBetaAccess: boolean) => void
18
+ setShowApiKeys: (showApiKeys: boolean) => void
19
  setShowSettings: (showSettings: SettingsCategory) => void
20
  setShowImporter: (showImporter: boolean) => void
21
  setShowTimeline: (showTimeline: boolean) => void
22
  setShowExplorer: (showExplorer: boolean) => void
23
  setShowVideoPlayer: (showVideoPlayer: boolean) => void
24
+ setShowAssistant: (showAssistant: boolean) => void
25
  setShowFPS: (showFPS: boolean) => void
26
  setFollowCursor: (followCursor: boolean) => void
27
  }
src/controllers/ui/useUI.ts CHANGED
@@ -11,7 +11,12 @@ export const useUI = create<UIStore>()(
11
  persist(
12
  (set, get) => ({
13
  ...getDefaultUIState(),
14
-
 
 
 
 
 
15
  setShowSettings: (showSettings: SettingsCategory) => {
16
  set({ showSettings })
17
  },
@@ -27,8 +32,8 @@ export const useUI = create<UIStore>()(
27
  setShowVideoPlayer: (showVideoPlayer: boolean) => {
28
  set({ showVideoPlayer })
29
  },
30
- setShowChat: (showChat: boolean) => {
31
- set({ showChat })
32
  },
33
  setShowFPS: (showFPS: boolean) => {
34
  set({ showFPS })
@@ -38,7 +43,7 @@ export const useUI = create<UIStore>()(
38
  },
39
  }),
40
  {
41
- name: 'CLAPPER_REVISION_0_SETTINGS_VIEW'
42
  },
43
  ),
44
  )
 
11
  persist(
12
  (set, get) => ({
13
  ...getDefaultUIState(),
14
+ setHasBetaAccess: (hasBetaAccess: boolean) => {
15
+ set({ hasBetaAccess })
16
+ },
17
+ setShowApiKeys: (showApiKeys: boolean) => {
18
+ set({ showApiKeys })
19
+ },
20
  setShowSettings: (showSettings: SettingsCategory) => {
21
  set({ showSettings })
22
  },
 
32
  setShowVideoPlayer: (showVideoPlayer: boolean) => {
33
  set({ showVideoPlayer })
34
  },
35
+ setShowAssistant: (showAssistant: boolean) => {
36
+ set({ showAssistant })
37
  },
38
  setShowFPS: (showFPS: boolean) => {
39
  set({ showFPS })
 
43
  },
44
  }),
45
  {
46
+ name: 'CLAPPER_REVISION_0_CONTROLLERS_UI'
47
  },
48
  ),
49
  )
src/lib/core/constants.ts CHANGED
@@ -4,7 +4,7 @@
4
  export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
5
 
6
  export const APP_NAME = "Clapper AI"
7
- export const APP_REVISION = "r20240613-1555"
8
 
9
  export const APP_DOMAIN = "Clapper.app"
10
  export const APP_LINK = "https://clapper.app"
 
4
  export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
5
 
6
  export const APP_NAME = "Clapper AI"
7
+ export const APP_REVISION = "r20240615-0202"
8
 
9
  export const APP_DOMAIN = "Clapper.app"
10
  export const APP_LINK = "https://clapper.app"
src/types.ts CHANGED
@@ -1,5 +1,6 @@
1
  import { ClapEntity, ClapMeta, ClapSegment, ClapSegmentCategory, ClapSegmentStatus } from "@aitube/clap"
2
  import { SettingsState } from "./controllers/settings"
 
3
 
4
  export enum SettingsCategory {
5
  NONE = "NONE",
@@ -87,6 +88,8 @@ export type AssistantRequest = {
87
 
88
  // used to provide more context
89
  projectInfo: string
 
 
90
  }
91
 
92
  export type AssistantResponse = {
 
1
  import { ClapEntity, ClapMeta, ClapSegment, ClapSegmentCategory, ClapSegmentStatus } from "@aitube/clap"
2
  import { SettingsState } from "./controllers/settings"
3
+ import { ChatEvent } from "./controllers/assistant/types"
4
 
5
  export enum SettingsCategory {
6
  NONE = "NONE",
 
88
 
89
  // used to provide more context
90
  projectInfo: string
91
+
92
+ history: ChatEvent[]
93
  }
94
 
95
  export type AssistantResponse = {