jbilcke-hf HF staff commited on
Commit
7b5fc2b
β€’
1 Parent(s): c1e4aec

added images

Browse files
.env CHANGED
@@ -2,6 +2,9 @@
2
  AI_BEDTIME_STORY_API_GRADIO_URL="https://jbilcke-hf-ai-bedtime-story-server.hf.space"
3
  AI_BEDTIME_STORY_API_SECRET_TOKEN=
4
 
 
 
 
5
  # ----------- CENSORSHIP -------
6
  ENABLE_CENSORSHIP=
7
  FINGERPRINT_KEY=
 
2
  AI_BEDTIME_STORY_API_GRADIO_URL="https://jbilcke-hf-ai-bedtime-story-server.hf.space"
3
  AI_BEDTIME_STORY_API_SECRET_TOKEN=
4
 
5
+ FAST_IMAGE_SERVER_API_GRADIO_URL="https://jbilcke-hf-fast-image-server.hf.space"
6
+ FAST_IMAGE_SERVER_API_SECRET_TOKEN=
7
+
8
  # ----------- CENSORSHIP -------
9
  ENABLE_CENSORSHIP=
10
  FINGERPRINT_KEY=
package-lock.json CHANGED
@@ -46,6 +46,7 @@
46
  "react": "18.2.0",
47
  "react-circular-progressbar": "^2.1.0",
48
  "react-dom": "18.2.0",
 
49
  "react-snowfall": "^1.2.1",
50
  "react-virtualized-auto-sizer": "^1.0.20",
51
  "replicate": "^0.17.0",
@@ -5531,6 +5532,18 @@
5531
  }
5532
  }
5533
  },
 
 
 
 
 
 
 
 
 
 
 
 
5534
  "node_modules/react-snowfall": {
5535
  "version": "1.2.1",
5536
  "resolved": "https://registry.npmjs.org/react-snowfall/-/react-snowfall-1.2.1.tgz",
 
46
  "react": "18.2.0",
47
  "react-circular-progressbar": "^2.1.0",
48
  "react-dom": "18.2.0",
49
+ "react-smooth-scroll-hook": "^1.3.4",
50
  "react-snowfall": "^1.2.1",
51
  "react-virtualized-auto-sizer": "^1.0.20",
52
  "replicate": "^0.17.0",
 
5532
  }
5533
  }
5534
  },
5535
+ "node_modules/react-smooth-scroll-hook": {
5536
+ "version": "1.3.4",
5537
+ "resolved": "https://registry.npmjs.org/react-smooth-scroll-hook/-/react-smooth-scroll-hook-1.3.4.tgz",
5538
+ "integrity": "sha512-nPNSQStr8gz1ogQbWTmeXiOEValKKr7hImucipoUlP7mK1n54qOBoFpO1aGE9yZEB7vSnkRx3mTH9zwO4nj8MQ==",
5539
+ "engines": {
5540
+ "node": ">=10"
5541
+ },
5542
+ "peerDependencies": {
5543
+ "react": ">=16.8.0",
5544
+ "react-dom": ">=16.8.0"
5545
+ }
5546
+ },
5547
  "node_modules/react-snowfall": {
5548
  "version": "1.2.1",
5549
  "resolved": "https://registry.npmjs.org/react-snowfall/-/react-snowfall-1.2.1.tgz",
package.json CHANGED
@@ -47,6 +47,7 @@
47
  "react": "18.2.0",
48
  "react-circular-progressbar": "^2.1.0",
49
  "react-dom": "18.2.0",
 
50
  "react-snowfall": "^1.2.1",
51
  "react-virtualized-auto-sizer": "^1.0.20",
52
  "replicate": "^0.17.0",
 
47
  "react": "18.2.0",
48
  "react-circular-progressbar": "^2.1.0",
49
  "react-dom": "18.2.0",
50
+ "react-smooth-scroll-hook": "^1.3.4",
51
  "react-snowfall": "^1.2.1",
52
  "react-virtualized-auto-sizer": "^1.0.20",
53
  "replicate": "^0.17.0",
src/app/interface/generate/index.tsx CHANGED
@@ -3,6 +3,7 @@
3
  import { useEffect, useRef, useState, useTransition } from "react"
4
  import { useSpring, animated } from "@react-spring/web"
5
  import { usePathname, useRouter, useSearchParams } from "next/navigation"
 
6
  import { split } from "sentence-splitter"
7
 
8
  import { useToast } from "@/components/ui/use-toast"
@@ -16,6 +17,7 @@ import { useCountdown } from "@/lib/useCountdown"
16
  import { useAudio } from "@/lib/useAudio"
17
 
18
  import { Countdown } from "../countdown"
 
19
 
20
  type Stage = "generate" | "finished"
21
 
@@ -32,6 +34,7 @@ export function Generate() {
32
  const [promptDraft, setPromptDraft] = useState("")
33
  const [assetUrl, setAssetUrl] = useState("")
34
  const [isOverSubmitButton, setOverSubmitButton] = useState(false)
 
35
 
36
  const [runs, setRuns] = useState(0)
37
  const runsRef = useRef(0)
@@ -39,13 +42,30 @@ export function Generate() {
39
  const currentLineIndexRef = useRef(0)
40
  const [currentLineIndex, setCurrentLineIndex] = useState(0)
41
 
 
 
 
 
 
 
 
 
 
42
  useEffect(() => {
43
  currentLineIndexRef.current = currentLineIndex
44
  }, [currentLineIndex])
45
-
46
  const [storyLines, setStoryLines] = useState<StoryLine[]>([])
47
 
 
 
 
 
 
 
 
48
  // computing those is cheap
 
49
  const wholeStory = storyLines.map(line => line.text).join("\n")
50
  const currentLine = storyLines.at(currentLineIndex)
51
  const currentLineText = currentLine?.text || ""
@@ -60,30 +80,12 @@ export function Generate() {
60
 
61
  const { toast } = useToast()
62
 
63
- const audio = useAudio()
64
-
65
- /*
66
- // to simulate a "typing" effect
67
- however.. we don't need this as we already have an audio player!
68
 
69
- const [typedStoryText, setTypedStoryText] = useState("")
70
- const [typedStoryCharacterIndex, setTypedStoryCharacterIndex] = useState(0)
71
-
72
- useEffect(() => {
73
- if (storyText && typedStoryCharacterIndex < storyText.length) {
74
- setTimeout(() => {
75
- setTypedStoryText(typedStoryText + story.text[typedStoryCharacterIndex])
76
- setTypedStoryCharacterIndex(typedStoryCharacterIndex + 1)
77
- console.log("boom")
78
- }, 40)
79
- }
80
- }, [storyText, typedStoryCharacterIndex])
81
- */
82
-
83
  const { progressPercent, remainingTimeInSec } = useCountdown({
84
  isActive: isLocked,
85
  timerId: runs, // everytime we change this, the timer will reset
86
- durationInSec: /*stage === "interpolate" ? 30 :*/ 35, // it usually takes 40 seconds, but there might be lag
87
  onEnd: () => {}
88
  })
89
 
@@ -107,6 +109,20 @@ export function Generate() {
107
  },
108
  })
109
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110
  const handleSubmit = () => {
111
  if (isLocked) { return }
112
  if (!promptDraft) { return }
@@ -187,11 +203,47 @@ export function Generate() {
187
  console.log("story audio changed!")
188
 
189
  try {
190
- console.log("playing audio!")
191
- await audio(currentLineAudio) // play
192
- console.log("audio has ended, I think? let's go next!")
193
- setCurrentLineIndex(currentLineIndexRef.current += 1)
194
- // TODO change the line
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
195
  } catch (err) {
196
  console.error(err)
197
  }
@@ -199,9 +251,9 @@ export function Generate() {
199
  fn()
200
 
201
  return () => {
202
- audio() // stop
203
  }
204
- }, [currentLineAudio])
205
 
206
  return (
207
  <div
@@ -254,7 +306,7 @@ export function Generate() {
254
 
255
  <div className={cn(
256
  `flex flex-col md:flex-row`,
257
- `space-y-3 md:space-y-0 md:space-x-3`,
258
  ` w-full md:max-w-[1024px]`,
259
  `items-center justify-between`
260
  )}>
@@ -270,13 +322,16 @@ export function Generate() {
270
  `input input-bordered rounded-full`,
271
  `transition-all duration-300 ease-in-out`,
272
  `backdrop-blur-md `,
273
- `placeholder:text-gray-400`,
274
- `disabled:bg-gray-500 disabled:text-yellow-300 disabled:border-transparent`,
275
  isLocked
276
- ? `bg-white/10 text-yellow-400/60 selection:bg-yellow-200/60 selection:text-yellow-200/60 border-transparent`
277
  : `bg-white/10 text-yellow-400/100 selection:bg-yellow-200/100 selection:text-yellow-200/100`,
278
  `text-left`,
279
- `text-2xl leading-10 px-6 h-16 pt-1`,
 
 
 
280
  )}
281
  value={promptDraft}
282
  onChange={e => setPromptDraft(e.target.value)}
@@ -306,39 +361,82 @@ export function Generate() {
306
  <span>{nbCharsLimits}</span>
307
  </div>
308
  </div>
309
- <div className="flex flex-row w-44">
310
- <animated.button
311
- style={{
312
- textShadow: "0px 0px 1px #000000ab",
313
- ...submitButtonBouncer
314
- }}
315
- onMouseEnter={() => setOverSubmitButton(true)}
316
- onMouseLeave={() => setOverSubmitButton(false)}
317
- className={cn(
318
- `px-4 h-16`,
319
- `rounded-full`,
320
- `transition-all duration-300 ease-in-out`,
321
- `backdrop-blur-sm`,
322
- isLocked
323
- ? `bg-orange-200/50 text-sky-50/80 border-yellow-600/10`
324
- : `bg-yellow-400/70 text-sky-50 border-yellow-800/20 hover:bg-yellow-400/80`,
325
- `text-center`,
326
- `w-full`,
327
- `text-2xl `,
328
- `border`,
329
- headingFont.className,
330
- // `transition-all duration-300`,
331
- // `hover:animate-bounce`
332
- )}
333
- disabled={isLocked}
334
- onClick={handleSubmit}
335
- >
336
- {isLocked
337
- ? `Dreaming..`
338
- : "Dream"
339
- }
340
- </animated.button>
341
- </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
342
  </div>
343
  </div>
344
 
@@ -350,6 +448,7 @@ export function Generate() {
350
  `space-y-8`,
351
  // `transition-all duration-300 ease-in-out`,
352
  )}>
 
353
 
354
  <div
355
  className={cn(
@@ -379,6 +478,7 @@ export function Generate() {
379
  />}
380
  </div> : null}
381
 
 
382
  <div className={cn(
383
  `flex flex-col md:flex-row`,
384
  `space-y-3 md:space-y-0 md:space-x-3`,
@@ -386,17 +486,41 @@ export function Generate() {
386
  `items-center justify-between`
387
  )}>
388
  <div className={cn(
389
- `flex flex-col flex-grow w-full space-y-2 text-2xl text-blue-200/90`
390
  )}>
391
  {storyLines.map((line, i) =>
392
  <div
393
- key={`${line.text}_${i}`}
 
 
394
 
395
  // TODO change a color if we have progressed at the current index (i)
396
- className={cn()}
397
- >{
398
- line.text
399
- }</div>)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
400
  </div>
401
  </div>
402
  </div>
 
3
  import { useEffect, useRef, useState, useTransition } from "react"
4
  import { useSpring, animated } from "@react-spring/web"
5
  import { usePathname, useRouter, useSearchParams } from "next/navigation"
6
+ import useSmoothScroll from "react-smooth-scroll-hook"
7
  import { split } from "sentence-splitter"
8
 
9
  import { useToast } from "@/components/ui/use-toast"
 
17
  import { useAudio } from "@/lib/useAudio"
18
 
19
  import { Countdown } from "../countdown"
20
+ import { generateImage } from "@/app/server/actions/generateImage"
21
 
22
  type Stage = "generate" | "finished"
23
 
 
34
  const [promptDraft, setPromptDraft] = useState("")
35
  const [assetUrl, setAssetUrl] = useState("")
36
  const [isOverSubmitButton, setOverSubmitButton] = useState(false)
37
+ const [isOverPauseButton, setOverPauseButton] = useState(false)
38
 
39
  const [runs, setRuns] = useState(0)
40
  const runsRef = useRef(0)
 
42
  const currentLineIndexRef = useRef(0)
43
  const [currentLineIndex, setCurrentLineIndex] = useState(0)
44
 
45
+ const voices: TTSVoice[] = ["CloΓ©e", "Julian"]
46
+ const [voice, setVoice] = useState<TTSVoice>("CloΓ©e")
47
+
48
+ const { scrollTo } = useSmoothScroll({
49
+ ref: scrollRef,
50
+ speed: 2000,
51
+ direction: 'y',
52
+ });
53
+
54
  useEffect(() => {
55
  currentLineIndexRef.current = currentLineIndex
56
  }, [currentLineIndex])
57
+
58
  const [storyLines, setStoryLines] = useState<StoryLine[]>([])
59
 
60
+ const [images, setImages] = useState<string[]>([])
61
+ const imagesRef = useRef<string[]>([])
62
+ const imageListKey = images.join("")
63
+ useEffect(() => {
64
+ imagesRef.current = images
65
+ }, [imageListKey])
66
+
67
  // computing those is cheap
68
+
69
  const wholeStory = storyLines.map(line => line.text).join("\n")
70
  const currentLine = storyLines.at(currentLineIndex)
71
  const currentLineText = currentLine?.text || ""
 
80
 
81
  const { toast } = useToast()
82
 
83
+ const { playback, isPlaying, isSwitchingTracks, isLoaded, progress, togglePause } = useAudio()
 
 
 
 
84
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
  const { progressPercent, remainingTimeInSec } = useCountdown({
86
  isActive: isLocked,
87
  timerId: runs, // everytime we change this, the timer will reset
88
+ durationInSec: /*stage === "interpolate" ? 30 :*/ 50, // it usually takes 40 seconds, but there might be lag
89
  onEnd: () => {}
90
  })
91
 
 
109
  },
110
  })
111
 
112
+ const pauseButtonBouncer = useSpring({
113
+ transform: isOverPauseButton
114
+ ? 'scale(1.05)'
115
+ : 'scale(1.0)',
116
+ boxShadow: isOverPauseButton
117
+ ? `0px 5px 15px 0px rgba(0, 0, 0, 0.05)`
118
+ : `0px 0px 0px 0px rgba(0, 0, 0, 0.05)`,
119
+ loop: true,
120
+ config: {
121
+ tension: 300,
122
+ friction: 10,
123
+ },
124
+ })
125
+
126
  const handleSubmit = () => {
127
  if (isLocked) { return }
128
  if (!promptDraft) { return }
 
203
  console.log("story audio changed!")
204
 
205
  try {
206
+ const isLastLine =
207
+ (storyLines.length === 0) ||
208
+ (currentLineIndexRef.current === (storyLines.length - 1))
209
+
210
+ scrollTo(`#story-line-${currentLineIndexRef.current}`)
211
+
212
+ const nextLineIndex = (currentLineIndexRef.current += 1)
213
+ const nextLineText = storyLines[nextLineIndex]?.text || ""
214
+
215
+ if (nextLineText) {
216
+ setTimeout(() => {
217
+ startTransition(async () => {
218
+ try {
219
+ const newImage = await generateImage({
220
+ positivePrompt: [
221
+ "bedtime story illustration",
222
+ "painting illustration",
223
+ promptDraft,
224
+ nextLineText,
225
+ ].join(", "),
226
+ width: 1024,
227
+ height: 800
228
+ })
229
+ // console.log("newImage:", newImage.slice(0, 50))
230
+ setImages(imagesRef.current.concat(newImage))
231
+ } catch (err) {
232
+ setImages(imagesRef.current.concat(""))
233
+ }
234
+ })
235
+ }, 100)
236
+ } else {
237
+ setImages(imagesRef.current.concat(""))
238
+ }
239
+
240
+ await playback(currentLineAudio, isLastLine) // play
241
+
242
+ if (!isLastLine && nextLineText) {
243
+ setTimeout(() => {
244
+ setCurrentLineIndex(nextLineIndex)
245
+ }, 1000)
246
+ }
247
  } catch (err) {
248
  console.error(err)
249
  }
 
251
  fn()
252
 
253
  return () => {
254
+ playback() // stop
255
  }
256
+ }, [currentLineText, currentLineAudio])
257
 
258
  return (
259
  <div
 
306
 
307
  <div className={cn(
308
  `flex flex-col md:flex-row`,
309
+ `space-y-4 md:space-y-0 md:space-x-4`,
310
  ` w-full md:max-w-[1024px]`,
311
  `items-center justify-between`
312
  )}>
 
322
  `input input-bordered rounded-full`,
323
  `transition-all duration-300 ease-in-out`,
324
  `backdrop-blur-md `,
325
+ `placeholder:text-gray-400/90`,
326
+ `disabled:bg-blue-900/70 disabled:text-blue-300/60 disabled:border-transparent`,
327
  isLocked
328
+ ? `bg-blue-100/80 text-yellow-400/60 selection:bg-yellow-200/60 selection:text-yellow-200/60 border-transparent`
329
  : `bg-white/10 text-yellow-400/100 selection:bg-yellow-200/100 selection:text-yellow-200/100`,
330
  `text-left`,
331
+ ``,
332
+ storyLines?.length
333
+ ? `text-2xl leading-10 px-6 h-16 pt-1`
334
+ : `text-3xl leading-14 px-8 h-[70px] pt-1`
335
  )}
336
  value={promptDraft}
337
  onChange={e => setPromptDraft(e.target.value)}
 
361
  <span>{nbCharsLimits}</span>
362
  </div>
363
  </div>
364
+ <div className="flex flex-row w-full md:w-auto justify-center">
365
+ <div className="flex flex-row w-1/2 md:w-52">
366
+ <animated.button
367
+ style={{
368
+ textShadow: "0px 0px 1px #000000ab",
369
+ ...submitButtonBouncer
370
+ }}
371
+ onMouseEnter={() => setOverSubmitButton(true)}
372
+ onMouseLeave={() => setOverSubmitButton(false)}
373
+ className={cn(
374
+ storyLines?.length
375
+ ? `text-2xl leading-10 px-4 h-16`
376
+ : `text-3xl leading-14 px-6 h-[70px]`,
377
+ `rounded-full`,
378
+ `transition-all duration-300 ease-in-out`,
379
+ `backdrop-blur-sm`,
380
+ isLocked
381
+ ? `bg-blue-900/70 text-sky-50/80 border-yellow-600/10`
382
+ : `bg-yellow-400/70 text-sky-50 border-yellow-800/20 hover:bg-yellow-400/80`,
383
+ `text-center`,
384
+ `w-full`,
385
+ `border`,
386
+ headingFont.className,
387
+ // `transition-all duration-300`,
388
+ // `hover:animate-bounce`
389
+ )}
390
+ disabled={isLocked}
391
+ onClick={handleSubmit}
392
+ >
393
+ {isLocked
394
+ ? `Dreaming..`
395
+ : "Dream πŸŒ™"
396
+ }
397
+ </animated.button>
398
+ </div>
399
+ {
400
+ /*
401
+ !!storyLines.length && <div className={cn(
402
+ `flex flex-row w-1/2 md:w-44`,
403
+ `transition-all duration-300 ease-in-out`,
404
+ isLoaded ? 'scale-100' : 'scale-0'
405
+ )}>
406
+ <animated.button
407
+ style={{
408
+ textShadow: "0px 0px 1px #000000ab",
409
+ ...pauseButtonBouncer
410
+ }}
411
+ onMouseEnter={() => setOverPauseButton(true)}
412
+ onMouseLeave={() => setOverPauseButton(false)}
413
+ className={cn(
414
+ `px-4 h-16`,
415
+ `rounded-full`,
416
+ `transition-all duration-300 ease-in-out`,
417
+ `backdrop-blur-sm`,
418
+ isLocked
419
+ ? `bg-orange-200/30 text-sky-50/60 border-yellow-600/10`
420
+ : `bg-yellow-400/50 text-sky-50 border-yellow-800/20 hover:bg-yellow-400/60`,
421
+ `text-center`,
422
+ `w-full`,
423
+ `text-2xl `,
424
+ `border`,
425
+ headingFont.className,
426
+ // `transition-all duration-300`,
427
+ // `hover:animate-bounce`
428
+ )}
429
+ disabled={isLocked}
430
+ onClick={togglePause}
431
+ >
432
+ {isPlaying || isSwitchingTracks
433
+ ? "Pause πŸ”Š"
434
+ : "Play πŸ”Š"
435
+ }
436
+ </animated.button>
437
+ </div>
438
+ */
439
+ }</div>
440
  </div>
441
  </div>
442
 
 
448
  `space-y-8`,
449
  // `transition-all duration-300 ease-in-out`,
450
  )}>
451
+
452
 
453
  <div
454
  className={cn(
 
478
  />}
479
  </div> : null}
480
 
481
+
482
  <div className={cn(
483
  `flex flex-col md:flex-row`,
484
  `space-y-3 md:space-y-0 md:space-x-3`,
 
486
  `items-center justify-between`
487
  )}>
488
  <div className={cn(
489
+ `flex flex-col flex-grow w-full items-center space-y-2 text-2xl text-blue-200/60`
490
  )}>
491
  {storyLines.map((line, i) =>
492
  <div
493
+ id={`story-line-${i}`}
494
+ key={`${line.text}_${i}`}>
495
+ <div
496
 
497
  // TODO change a color if we have progressed at the current index (i)
498
+ className={cn(
499
+ "flex flex-col items-center w-full "
500
+ //i < currentLineIndex
501
+ //? 'text-yellow-200'
502
+ //: 'text-blue-200/80'
503
+ )}
504
+ style={{}}
505
+ >
506
+ <div className="w-full md:w-2/3 text-center"> {
507
+ line.text.split("").map((c, j, arr) => <span
508
+ key={`${c}_${j}`}
509
+ className={cn(
510
+ `transition-all duration-100 ease-in-out`,
511
+ i < currentLineIndex || (isLoaded && i === currentLineIndex && j <= (progress * 1.3 * arr.length))
512
+ ? 'text-yellow-400/90'
513
+ : ''
514
+ )}>{c || " "}</span>)
515
+ }</div>
516
+ <div className="flex flex-col items-center justify-center w-full p-8">
517
+ {images.at(i) ? <img
518
+ className="h-[400px] rounded-lg overflow-hidden"
519
+ src={images.at(i)}
520
+ /> : null}
521
+ </div>
522
+ </div>
523
+ </div>)}
524
  </div>
525
  </div>
526
  </div>
src/app/server/actions/generateImage.ts ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use server"
2
+
3
+ // TODO add a system to mark failed instances as "unavailable" for a couple of minutes
4
+ // console.log("process.env:", process.env)
5
+
6
+ import { generateSeed } from "@/lib/generateSeed";
7
+ import { getValidNumber } from "@/lib/getValidNumber";
8
+
9
+ // note: to reduce costs I use the small A10s (not the large)
10
+ // anyway, we will soon not need to use this cloud anymore
11
+ // since we will be able to leverage the Inference API
12
+ const instance = `${process.env.FAST_IMAGE_SERVER_API_GRADIO_URL || ""}`
13
+ const secretToken = `${process.env.FAST_IMAGE_SERVER_API_SECRET_TOKEN || ""}`
14
+
15
+ // console.log("DEBUG:", JSON.stringify({ instances, secretToken }, null, 2))
16
+
17
+ export async function generateImage(options: {
18
+ positivePrompt: string;
19
+ negativePrompt?: string;
20
+ seed?: number;
21
+ width?: number;
22
+ height?: number;
23
+ nbSteps?: number;
24
+ }): Promise<string> {
25
+
26
+ // console.log("querying " + instance)
27
+ const positivePrompt = options?.positivePrompt || ""
28
+ if (!positivePrompt) {
29
+ throw new Error("missing prompt")
30
+ }
31
+
32
+ // the negative prompt CAN be missing, since we use a trick
33
+ // where we make the interface mandatory in the TS doc,
34
+ // but browsers might send something partial
35
+ const negativePrompt = options?.negativePrompt || ""
36
+
37
+ // we treat 0 as meaning "random seed"
38
+ const seed = (options?.seed ? options.seed : 0) || generateSeed()
39
+
40
+ const width = getValidNumber(options?.width, 256, 1024, 512)
41
+ const height = getValidNumber(options?.height, 256, 1024, 512)
42
+ const nbSteps = getValidNumber(options?.nbSteps, 1, 8, 4)
43
+ // console.log("SEED:", seed)
44
+
45
+ const positive = [
46
+
47
+ // oh well.. is it too late to move this to the bottom?
48
+ "beautiful",
49
+
50
+ // too opinionated, so let's remove it
51
+ // "intricate details",
52
+
53
+ positivePrompt,
54
+
55
+ "award winning",
56
+ "high resolution"
57
+ ].filter(word => word)
58
+ .join(", ")
59
+
60
+ const negative = [
61
+ negativePrompt,
62
+ "watermark",
63
+ "copyright",
64
+ "blurry",
65
+ // "artificial",
66
+ // "cropped",
67
+ "low quality",
68
+ "ugly"
69
+ ].filter(word => word)
70
+ .join(", ")
71
+
72
+ const res = await fetch(instance + (instance.endsWith("/") ? "" : "/") + "api/predict", {
73
+ method: "POST",
74
+ headers: {
75
+ "Content-Type": "application/json",
76
+ // Authorization: `Bearer ${token}`,
77
+ },
78
+ body: JSON.stringify({
79
+ fn_index: 0, // <- important!
80
+ data: [
81
+ positive, // string in 'Prompt' Textbox component
82
+ negative, // string in 'Negative prompt' Textbox component
83
+ seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
84
+ width, // number (numeric value between 256 and 1024) in 'Width' Slider component
85
+ height, // number (numeric value between 256 and 1024) in 'Height' Slider component
86
+ 0.0, // can be disabled for LCM SDXL
87
+ nbSteps, // number (numeric value between 2 and 8) in 'Number of inference steps for base' Slider component
88
+ secretToken
89
+ ]
90
+ }),
91
+ cache: "no-store",
92
+ })
93
+
94
+ const { data } = await res.json()
95
+
96
+ if (res.status !== 200 || !Array.isArray(data)) {
97
+ // This will activate the closest `error.js` Error Boundary
98
+ throw new Error(`Failed to fetch data (status: ${res.status})`)
99
+ }
100
+
101
+ if (!data[0]) {
102
+ throw new Error(`the returned image was empty`)
103
+ }
104
+
105
+ return data[0] as string
106
+ }
src/lib/getValidNumber.ts ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ export const getValidNumber = (something: any, minValue: number, maxValue: number, defaultValue: number) => {
2
+ const strValue = `${something || defaultValue}`
3
+ const numValue = Number(strValue)
4
+ const isValid = !isNaN(numValue) && isFinite(numValue)
5
+ if (!isValid) {
6
+ return defaultValue
7
+ }
8
+ return Math.max(minValue, Math.min(maxValue, numValue))
9
+
10
+ }
src/lib/useAudio.ts CHANGED
@@ -1,11 +1,37 @@
1
- import { useCallback, useEffect, useRef } from 'react';
2
 
3
- export function useAudio() {
4
- const audioContextRef = useRef<AudioContext | null>(null);
 
 
 
 
 
 
 
5
 
 
 
 
 
 
 
 
 
 
 
6
  const stopAudio = useCallback(() => {
7
- audioContextRef.current?.close();
8
- audioContextRef.current = null;
 
 
 
 
 
 
 
 
 
9
  }, []);
10
 
11
  // Helper function to handle conversion from Base64 to an ArrayBuffer
@@ -14,13 +40,13 @@ export function useAudio() {
14
  return response.arrayBuffer();
15
  }
16
 
17
- const playAudio = useCallback(
18
- async (base64Data?: string) => {
19
  stopAudio(); // Stop any playing audio first
20
 
21
  // If no base64 data provided, we don't attempt to play any audio
22
  if (!base64Data) {
23
- return;
24
  }
25
 
26
  // Initialize AudioContext
@@ -52,13 +78,41 @@ export function useAudio() {
52
  source.connect(gainNode);
53
  gainNode.connect(audioContext.destination);
54
 
55
- // Start playback and handle finishing
56
- source.start();
 
 
57
 
58
- source.onended = () => {
59
- stopAudio();
60
- resolve(true);
61
- };
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  }, (error) => {
63
  console.error('Error decoding audio data:', error);
64
  reject(error);
@@ -68,6 +122,25 @@ export function useAudio() {
68
  [stopAudio]
69
  );
70
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  // Effect to handle cleanup on component unmount
72
  useEffect(() => {
73
  return () => {
@@ -75,6 +148,5 @@ export function useAudio() {
75
  };
76
  }, [stopAudio]);
77
 
78
- // Return the playAudio function from the hook
79
- return playAudio;
80
  }
 
1
+ import { useCallback, useEffect, useRef, useState } from 'react';
2
 
3
+ // Helper Types
4
+ type UseAudioResponse = {
5
+ playback: (base64Data?: string, isLastTrackOfPlaylist?: boolean) => Promise<boolean>;
6
+ progress: number;
7
+ isLoaded: boolean;
8
+ isPlaying: boolean;
9
+ isSwitchingTracks: boolean; // when audio is temporary cut (but it's not a real pause)
10
+ togglePause: () => void;
11
+ };
12
 
13
+ export function useAudio(): UseAudioResponse {
14
+ const audioContextRef = useRef<AudioContext | null>(null);
15
+ const sourceNodeRef = useRef<AudioBufferSourceNode | null>(null);
16
+ const [progress, setProgress] = useState(0.0);
17
+ const [isPlaying, setIsPlaying] = useState(false);
18
+ const [isLoaded, setIsLoaded] = useState(false);
19
+ const [isSwitchingTracks, setSwitchingTracks] = useState(false);
20
+ const startTimeRef = useRef(0);
21
+ const pauseTimeRef = useRef(0);
22
+
23
  const stopAudio = useCallback(() => {
24
+ try {
25
+ audioContextRef.current?.close();
26
+ } catch (err) {
27
+ // already closed probably
28
+ }
29
+ setSwitchingTracks(false);
30
+
31
+ sourceNodeRef.current = null;
32
+ sourceNodeRef.current = null;
33
+
34
+ // setProgress(0); // Reset progress
35
  }, []);
36
 
37
  // Helper function to handle conversion from Base64 to an ArrayBuffer
 
40
  return response.arrayBuffer();
41
  }
42
 
43
+ const playback = useCallback(
44
+ async (base64Data?: string, isLastTrackOfPlaylist?: boolean): Promise<boolean> => {
45
  stopAudio(); // Stop any playing audio first
46
 
47
  // If no base64 data provided, we don't attempt to play any audio
48
  if (!base64Data) {
49
+ return false;
50
  }
51
 
52
  // Initialize AudioContext
 
78
  source.connect(gainNode);
79
  gainNode.connect(audioContext.destination);
80
 
81
+ // Assign source node to ref for progress tracking
82
+ sourceNodeRef.current = source;
83
+ source.start(0, pauseTimeRef.current % audioBuffer.duration); // Start at the correct offset if paused previously
84
+ startTimeRef.current = audioContextRef.current!.currentTime - pauseTimeRef.current;
85
 
86
+ setSwitchingTracks(false);
87
+ setProgress(0);
88
+ setIsLoaded(true);
89
+ setIsPlaying(true);
90
+
91
+ // Set up progress interval
92
+ const totalDuration = audioBuffer.duration;
93
+ const updateProgressInterval = setInterval(() => {
94
+ if (sourceNodeRef.current && audioContextRef.current) {
95
+ const currentTime = audioContextRef.current.currentTime;
96
+ const currentProgress = currentTime / totalDuration;
97
+ setProgress(currentProgress);
98
+ if (currentProgress >= 1.0) {
99
+ clearInterval(updateProgressInterval);
100
+ }
101
+ }
102
+ }, 50); // Update every 50ms
103
+
104
+ if (source) {
105
+ source.onended = () => {
106
+ // used to indicate a temporary stop, while we switch tracks
107
+ if (!isLastTrackOfPlaylist) {
108
+ setSwitchingTracks(true);
109
+ }
110
+ setIsPlaying(false);
111
+ clearInterval(updateProgressInterval);
112
+ stopAudio();
113
+ resolve(true);
114
+ };
115
+ }
116
  }, (error) => {
117
  console.error('Error decoding audio data:', error);
118
  reject(error);
 
122
  [stopAudio]
123
  );
124
 
125
+ const togglePause = useCallback(() => {
126
+ if (!audioContextRef.current || !sourceNodeRef.current) {
127
+ return; // Do nothing if audio is not initialized
128
+ }
129
+
130
+ if (isPlaying) {
131
+ // Pause the audio
132
+ pauseTimeRef.current += audioContextRef.current.currentTime - startTimeRef.current;
133
+ sourceNodeRef.current.stop(); // This effectively "pauses" the audio, but it also means the sourceNode will be unusable
134
+ sourceNodeRef.current = null; // As the node is now unusable, we nullify it
135
+ setIsPlaying(false);
136
+ } else {
137
+ // Resume playing
138
+ audioContextRef.current.resume().then(() => {
139
+ playback(); // This will pick up where we left off due to pauseTimeRef
140
+ });
141
+ }
142
+ }, [audioContextRef, sourceNodeRef, isPlaying, playback]);
143
+
144
  // Effect to handle cleanup on component unmount
145
  useEffect(() => {
146
  return () => {
 
148
  };
149
  }, [stopAudio]);
150
 
151
+ return { playback, isPlaying, isSwitchingTracks, isLoaded, progress, togglePause };
 
152
  }