Commit
β’
7b5fc2b
1
Parent(s):
c1e4aec
added images
Browse files- .env +3 -0
- package-lock.json +13 -0
- package.json +1 -0
- src/app/interface/generate/index.tsx +196 -72
- src/app/server/actions/generateImage.ts +106 -0
- src/lib/getValidNumber.ts +10 -0
- src/lib/useAudio.ts +88 -16
.env
CHANGED
@@ -2,6 +2,9 @@
|
|
2 |
AI_BEDTIME_STORY_API_GRADIO_URL="https://jbilcke-hf-ai-bedtime-story-server.hf.space"
|
3 |
AI_BEDTIME_STORY_API_SECRET_TOKEN=
|
4 |
|
|
|
|
|
|
|
5 |
# ----------- CENSORSHIP -------
|
6 |
ENABLE_CENSORSHIP=
|
7 |
FINGERPRINT_KEY=
|
|
|
2 |
AI_BEDTIME_STORY_API_GRADIO_URL="https://jbilcke-hf-ai-bedtime-story-server.hf.space"
|
3 |
AI_BEDTIME_STORY_API_SECRET_TOKEN=
|
4 |
|
5 |
+
FAST_IMAGE_SERVER_API_GRADIO_URL="https://jbilcke-hf-fast-image-server.hf.space"
|
6 |
+
FAST_IMAGE_SERVER_API_SECRET_TOKEN=
|
7 |
+
|
8 |
# ----------- CENSORSHIP -------
|
9 |
ENABLE_CENSORSHIP=
|
10 |
FINGERPRINT_KEY=
|
package-lock.json
CHANGED
@@ -46,6 +46,7 @@
|
|
46 |
"react": "18.2.0",
|
47 |
"react-circular-progressbar": "^2.1.0",
|
48 |
"react-dom": "18.2.0",
|
|
|
49 |
"react-snowfall": "^1.2.1",
|
50 |
"react-virtualized-auto-sizer": "^1.0.20",
|
51 |
"replicate": "^0.17.0",
|
@@ -5531,6 +5532,18 @@
|
|
5531 |
}
|
5532 |
}
|
5533 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5534 |
"node_modules/react-snowfall": {
|
5535 |
"version": "1.2.1",
|
5536 |
"resolved": "https://registry.npmjs.org/react-snowfall/-/react-snowfall-1.2.1.tgz",
|
|
|
46 |
"react": "18.2.0",
|
47 |
"react-circular-progressbar": "^2.1.0",
|
48 |
"react-dom": "18.2.0",
|
49 |
+
"react-smooth-scroll-hook": "^1.3.4",
|
50 |
"react-snowfall": "^1.2.1",
|
51 |
"react-virtualized-auto-sizer": "^1.0.20",
|
52 |
"replicate": "^0.17.0",
|
|
|
5532 |
}
|
5533 |
}
|
5534 |
},
|
5535 |
+
"node_modules/react-smooth-scroll-hook": {
|
5536 |
+
"version": "1.3.4",
|
5537 |
+
"resolved": "https://registry.npmjs.org/react-smooth-scroll-hook/-/react-smooth-scroll-hook-1.3.4.tgz",
|
5538 |
+
"integrity": "sha512-nPNSQStr8gz1ogQbWTmeXiOEValKKr7hImucipoUlP7mK1n54qOBoFpO1aGE9yZEB7vSnkRx3mTH9zwO4nj8MQ==",
|
5539 |
+
"engines": {
|
5540 |
+
"node": ">=10"
|
5541 |
+
},
|
5542 |
+
"peerDependencies": {
|
5543 |
+
"react": ">=16.8.0",
|
5544 |
+
"react-dom": ">=16.8.0"
|
5545 |
+
}
|
5546 |
+
},
|
5547 |
"node_modules/react-snowfall": {
|
5548 |
"version": "1.2.1",
|
5549 |
"resolved": "https://registry.npmjs.org/react-snowfall/-/react-snowfall-1.2.1.tgz",
|
package.json
CHANGED
@@ -47,6 +47,7 @@
|
|
47 |
"react": "18.2.0",
|
48 |
"react-circular-progressbar": "^2.1.0",
|
49 |
"react-dom": "18.2.0",
|
|
|
50 |
"react-snowfall": "^1.2.1",
|
51 |
"react-virtualized-auto-sizer": "^1.0.20",
|
52 |
"replicate": "^0.17.0",
|
|
|
47 |
"react": "18.2.0",
|
48 |
"react-circular-progressbar": "^2.1.0",
|
49 |
"react-dom": "18.2.0",
|
50 |
+
"react-smooth-scroll-hook": "^1.3.4",
|
51 |
"react-snowfall": "^1.2.1",
|
52 |
"react-virtualized-auto-sizer": "^1.0.20",
|
53 |
"replicate": "^0.17.0",
|
src/app/interface/generate/index.tsx
CHANGED
@@ -3,6 +3,7 @@
|
|
3 |
import { useEffect, useRef, useState, useTransition } from "react"
|
4 |
import { useSpring, animated } from "@react-spring/web"
|
5 |
import { usePathname, useRouter, useSearchParams } from "next/navigation"
|
|
|
6 |
import { split } from "sentence-splitter"
|
7 |
|
8 |
import { useToast } from "@/components/ui/use-toast"
|
@@ -16,6 +17,7 @@ import { useCountdown } from "@/lib/useCountdown"
|
|
16 |
import { useAudio } from "@/lib/useAudio"
|
17 |
|
18 |
import { Countdown } from "../countdown"
|
|
|
19 |
|
20 |
type Stage = "generate" | "finished"
|
21 |
|
@@ -32,6 +34,7 @@ export function Generate() {
|
|
32 |
const [promptDraft, setPromptDraft] = useState("")
|
33 |
const [assetUrl, setAssetUrl] = useState("")
|
34 |
const [isOverSubmitButton, setOverSubmitButton] = useState(false)
|
|
|
35 |
|
36 |
const [runs, setRuns] = useState(0)
|
37 |
const runsRef = useRef(0)
|
@@ -39,13 +42,30 @@ export function Generate() {
|
|
39 |
const currentLineIndexRef = useRef(0)
|
40 |
const [currentLineIndex, setCurrentLineIndex] = useState(0)
|
41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
42 |
useEffect(() => {
|
43 |
currentLineIndexRef.current = currentLineIndex
|
44 |
}, [currentLineIndex])
|
45 |
-
|
46 |
const [storyLines, setStoryLines] = useState<StoryLine[]>([])
|
47 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
48 |
// computing those is cheap
|
|
|
49 |
const wholeStory = storyLines.map(line => line.text).join("\n")
|
50 |
const currentLine = storyLines.at(currentLineIndex)
|
51 |
const currentLineText = currentLine?.text || ""
|
@@ -60,30 +80,12 @@ export function Generate() {
|
|
60 |
|
61 |
const { toast } = useToast()
|
62 |
|
63 |
-
const
|
64 |
-
|
65 |
-
/*
|
66 |
-
// to simulate a "typing" effect
|
67 |
-
however.. we don't need this as we already have an audio player!
|
68 |
|
69 |
-
const [typedStoryText, setTypedStoryText] = useState("")
|
70 |
-
const [typedStoryCharacterIndex, setTypedStoryCharacterIndex] = useState(0)
|
71 |
-
|
72 |
-
useEffect(() => {
|
73 |
-
if (storyText && typedStoryCharacterIndex < storyText.length) {
|
74 |
-
setTimeout(() => {
|
75 |
-
setTypedStoryText(typedStoryText + story.text[typedStoryCharacterIndex])
|
76 |
-
setTypedStoryCharacterIndex(typedStoryCharacterIndex + 1)
|
77 |
-
console.log("boom")
|
78 |
-
}, 40)
|
79 |
-
}
|
80 |
-
}, [storyText, typedStoryCharacterIndex])
|
81 |
-
*/
|
82 |
-
|
83 |
const { progressPercent, remainingTimeInSec } = useCountdown({
|
84 |
isActive: isLocked,
|
85 |
timerId: runs, // everytime we change this, the timer will reset
|
86 |
-
durationInSec: /*stage === "interpolate" ? 30 :*/
|
87 |
onEnd: () => {}
|
88 |
})
|
89 |
|
@@ -107,6 +109,20 @@ export function Generate() {
|
|
107 |
},
|
108 |
})
|
109 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
110 |
const handleSubmit = () => {
|
111 |
if (isLocked) { return }
|
112 |
if (!promptDraft) { return }
|
@@ -187,11 +203,47 @@ export function Generate() {
|
|
187 |
console.log("story audio changed!")
|
188 |
|
189 |
try {
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
|
194 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
195 |
} catch (err) {
|
196 |
console.error(err)
|
197 |
}
|
@@ -199,9 +251,9 @@ export function Generate() {
|
|
199 |
fn()
|
200 |
|
201 |
return () => {
|
202 |
-
|
203 |
}
|
204 |
-
}, [currentLineAudio])
|
205 |
|
206 |
return (
|
207 |
<div
|
@@ -254,7 +306,7 @@ export function Generate() {
|
|
254 |
|
255 |
<div className={cn(
|
256 |
`flex flex-col md:flex-row`,
|
257 |
-
`space-y-
|
258 |
` w-full md:max-w-[1024px]`,
|
259 |
`items-center justify-between`
|
260 |
)}>
|
@@ -270,13 +322,16 @@ export function Generate() {
|
|
270 |
`input input-bordered rounded-full`,
|
271 |
`transition-all duration-300 ease-in-out`,
|
272 |
`backdrop-blur-md `,
|
273 |
-
`placeholder:text-gray-400`,
|
274 |
-
`disabled:bg-
|
275 |
isLocked
|
276 |
-
? `bg-
|
277 |
: `bg-white/10 text-yellow-400/100 selection:bg-yellow-200/100 selection:text-yellow-200/100`,
|
278 |
`text-left`,
|
279 |
-
|
|
|
|
|
|
|
280 |
)}
|
281 |
value={promptDraft}
|
282 |
onChange={e => setPromptDraft(e.target.value)}
|
@@ -306,39 +361,82 @@ export function Generate() {
|
|
306 |
<span>{nbCharsLimits}</span>
|
307 |
</div>
|
308 |
</div>
|
309 |
-
<div className="flex flex-row w-
|
310 |
-
<
|
311 |
-
|
312 |
-
|
313 |
-
|
314 |
-
|
315 |
-
|
316 |
-
|
317 |
-
|
318 |
-
|
319 |
-
|
320 |
-
|
321 |
-
|
322 |
-
|
323 |
-
|
324 |
-
|
325 |
-
|
326 |
-
|
327 |
-
|
328 |
-
|
329 |
-
|
330 |
-
|
331 |
-
|
332 |
-
|
333 |
-
|
334 |
-
|
335 |
-
|
336 |
-
|
337 |
-
|
338 |
-
|
339 |
-
|
340 |
-
|
341 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
342 |
</div>
|
343 |
</div>
|
344 |
|
@@ -350,6 +448,7 @@ export function Generate() {
|
|
350 |
`space-y-8`,
|
351 |
// `transition-all duration-300 ease-in-out`,
|
352 |
)}>
|
|
|
353 |
|
354 |
<div
|
355 |
className={cn(
|
@@ -379,6 +478,7 @@ export function Generate() {
|
|
379 |
/>}
|
380 |
</div> : null}
|
381 |
|
|
|
382 |
<div className={cn(
|
383 |
`flex flex-col md:flex-row`,
|
384 |
`space-y-3 md:space-y-0 md:space-x-3`,
|
@@ -386,17 +486,41 @@ export function Generate() {
|
|
386 |
`items-center justify-between`
|
387 |
)}>
|
388 |
<div className={cn(
|
389 |
-
`flex flex-col flex-grow w-full space-y-2 text-2xl text-blue-200/
|
390 |
)}>
|
391 |
{storyLines.map((line, i) =>
|
392 |
<div
|
393 |
-
|
|
|
|
|
394 |
|
395 |
// TODO change a color if we have progressed at the current index (i)
|
396 |
-
className={cn(
|
397 |
-
|
398 |
-
|
399 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
400 |
</div>
|
401 |
</div>
|
402 |
</div>
|
|
|
3 |
import { useEffect, useRef, useState, useTransition } from "react"
|
4 |
import { useSpring, animated } from "@react-spring/web"
|
5 |
import { usePathname, useRouter, useSearchParams } from "next/navigation"
|
6 |
+
import useSmoothScroll from "react-smooth-scroll-hook"
|
7 |
import { split } from "sentence-splitter"
|
8 |
|
9 |
import { useToast } from "@/components/ui/use-toast"
|
|
|
17 |
import { useAudio } from "@/lib/useAudio"
|
18 |
|
19 |
import { Countdown } from "../countdown"
|
20 |
+
import { generateImage } from "@/app/server/actions/generateImage"
|
21 |
|
22 |
type Stage = "generate" | "finished"
|
23 |
|
|
|
34 |
const [promptDraft, setPromptDraft] = useState("")
|
35 |
const [assetUrl, setAssetUrl] = useState("")
|
36 |
const [isOverSubmitButton, setOverSubmitButton] = useState(false)
|
37 |
+
const [isOverPauseButton, setOverPauseButton] = useState(false)
|
38 |
|
39 |
const [runs, setRuns] = useState(0)
|
40 |
const runsRef = useRef(0)
|
|
|
42 |
const currentLineIndexRef = useRef(0)
|
43 |
const [currentLineIndex, setCurrentLineIndex] = useState(0)
|
44 |
|
45 |
+
const voices: TTSVoice[] = ["CloΓ©e", "Julian"]
|
46 |
+
const [voice, setVoice] = useState<TTSVoice>("CloΓ©e")
|
47 |
+
|
48 |
+
const { scrollTo } = useSmoothScroll({
|
49 |
+
ref: scrollRef,
|
50 |
+
speed: 2000,
|
51 |
+
direction: 'y',
|
52 |
+
});
|
53 |
+
|
54 |
useEffect(() => {
|
55 |
currentLineIndexRef.current = currentLineIndex
|
56 |
}, [currentLineIndex])
|
57 |
+
|
58 |
const [storyLines, setStoryLines] = useState<StoryLine[]>([])
|
59 |
|
60 |
+
const [images, setImages] = useState<string[]>([])
|
61 |
+
const imagesRef = useRef<string[]>([])
|
62 |
+
const imageListKey = images.join("")
|
63 |
+
useEffect(() => {
|
64 |
+
imagesRef.current = images
|
65 |
+
}, [imageListKey])
|
66 |
+
|
67 |
// computing those is cheap
|
68 |
+
|
69 |
const wholeStory = storyLines.map(line => line.text).join("\n")
|
70 |
const currentLine = storyLines.at(currentLineIndex)
|
71 |
const currentLineText = currentLine?.text || ""
|
|
|
80 |
|
81 |
const { toast } = useToast()
|
82 |
|
83 |
+
const { playback, isPlaying, isSwitchingTracks, isLoaded, progress, togglePause } = useAudio()
|
|
|
|
|
|
|
|
|
84 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
85 |
const { progressPercent, remainingTimeInSec } = useCountdown({
|
86 |
isActive: isLocked,
|
87 |
timerId: runs, // everytime we change this, the timer will reset
|
88 |
+
durationInSec: /*stage === "interpolate" ? 30 :*/ 50, // it usually takes 40 seconds, but there might be lag
|
89 |
onEnd: () => {}
|
90 |
})
|
91 |
|
|
|
109 |
},
|
110 |
})
|
111 |
|
112 |
+
const pauseButtonBouncer = useSpring({
|
113 |
+
transform: isOverPauseButton
|
114 |
+
? 'scale(1.05)'
|
115 |
+
: 'scale(1.0)',
|
116 |
+
boxShadow: isOverPauseButton
|
117 |
+
? `0px 5px 15px 0px rgba(0, 0, 0, 0.05)`
|
118 |
+
: `0px 0px 0px 0px rgba(0, 0, 0, 0.05)`,
|
119 |
+
loop: true,
|
120 |
+
config: {
|
121 |
+
tension: 300,
|
122 |
+
friction: 10,
|
123 |
+
},
|
124 |
+
})
|
125 |
+
|
126 |
const handleSubmit = () => {
|
127 |
if (isLocked) { return }
|
128 |
if (!promptDraft) { return }
|
|
|
203 |
console.log("story audio changed!")
|
204 |
|
205 |
try {
|
206 |
+
const isLastLine =
|
207 |
+
(storyLines.length === 0) ||
|
208 |
+
(currentLineIndexRef.current === (storyLines.length - 1))
|
209 |
+
|
210 |
+
scrollTo(`#story-line-${currentLineIndexRef.current}`)
|
211 |
+
|
212 |
+
const nextLineIndex = (currentLineIndexRef.current += 1)
|
213 |
+
const nextLineText = storyLines[nextLineIndex]?.text || ""
|
214 |
+
|
215 |
+
if (nextLineText) {
|
216 |
+
setTimeout(() => {
|
217 |
+
startTransition(async () => {
|
218 |
+
try {
|
219 |
+
const newImage = await generateImage({
|
220 |
+
positivePrompt: [
|
221 |
+
"bedtime story illustration",
|
222 |
+
"painting illustration",
|
223 |
+
promptDraft,
|
224 |
+
nextLineText,
|
225 |
+
].join(", "),
|
226 |
+
width: 1024,
|
227 |
+
height: 800
|
228 |
+
})
|
229 |
+
// console.log("newImage:", newImage.slice(0, 50))
|
230 |
+
setImages(imagesRef.current.concat(newImage))
|
231 |
+
} catch (err) {
|
232 |
+
setImages(imagesRef.current.concat(""))
|
233 |
+
}
|
234 |
+
})
|
235 |
+
}, 100)
|
236 |
+
} else {
|
237 |
+
setImages(imagesRef.current.concat(""))
|
238 |
+
}
|
239 |
+
|
240 |
+
await playback(currentLineAudio, isLastLine) // play
|
241 |
+
|
242 |
+
if (!isLastLine && nextLineText) {
|
243 |
+
setTimeout(() => {
|
244 |
+
setCurrentLineIndex(nextLineIndex)
|
245 |
+
}, 1000)
|
246 |
+
}
|
247 |
} catch (err) {
|
248 |
console.error(err)
|
249 |
}
|
|
|
251 |
fn()
|
252 |
|
253 |
return () => {
|
254 |
+
playback() // stop
|
255 |
}
|
256 |
+
}, [currentLineText, currentLineAudio])
|
257 |
|
258 |
return (
|
259 |
<div
|
|
|
306 |
|
307 |
<div className={cn(
|
308 |
`flex flex-col md:flex-row`,
|
309 |
+
`space-y-4 md:space-y-0 md:space-x-4`,
|
310 |
` w-full md:max-w-[1024px]`,
|
311 |
`items-center justify-between`
|
312 |
)}>
|
|
|
322 |
`input input-bordered rounded-full`,
|
323 |
`transition-all duration-300 ease-in-out`,
|
324 |
`backdrop-blur-md `,
|
325 |
+
`placeholder:text-gray-400/90`,
|
326 |
+
`disabled:bg-blue-900/70 disabled:text-blue-300/60 disabled:border-transparent`,
|
327 |
isLocked
|
328 |
+
? `bg-blue-100/80 text-yellow-400/60 selection:bg-yellow-200/60 selection:text-yellow-200/60 border-transparent`
|
329 |
: `bg-white/10 text-yellow-400/100 selection:bg-yellow-200/100 selection:text-yellow-200/100`,
|
330 |
`text-left`,
|
331 |
+
``,
|
332 |
+
storyLines?.length
|
333 |
+
? `text-2xl leading-10 px-6 h-16 pt-1`
|
334 |
+
: `text-3xl leading-14 px-8 h-[70px] pt-1`
|
335 |
)}
|
336 |
value={promptDraft}
|
337 |
onChange={e => setPromptDraft(e.target.value)}
|
|
|
361 |
<span>{nbCharsLimits}</span>
|
362 |
</div>
|
363 |
</div>
|
364 |
+
<div className="flex flex-row w-full md:w-auto justify-center">
|
365 |
+
<div className="flex flex-row w-1/2 md:w-52">
|
366 |
+
<animated.button
|
367 |
+
style={{
|
368 |
+
textShadow: "0px 0px 1px #000000ab",
|
369 |
+
...submitButtonBouncer
|
370 |
+
}}
|
371 |
+
onMouseEnter={() => setOverSubmitButton(true)}
|
372 |
+
onMouseLeave={() => setOverSubmitButton(false)}
|
373 |
+
className={cn(
|
374 |
+
storyLines?.length
|
375 |
+
? `text-2xl leading-10 px-4 h-16`
|
376 |
+
: `text-3xl leading-14 px-6 h-[70px]`,
|
377 |
+
`rounded-full`,
|
378 |
+
`transition-all duration-300 ease-in-out`,
|
379 |
+
`backdrop-blur-sm`,
|
380 |
+
isLocked
|
381 |
+
? `bg-blue-900/70 text-sky-50/80 border-yellow-600/10`
|
382 |
+
: `bg-yellow-400/70 text-sky-50 border-yellow-800/20 hover:bg-yellow-400/80`,
|
383 |
+
`text-center`,
|
384 |
+
`w-full`,
|
385 |
+
`border`,
|
386 |
+
headingFont.className,
|
387 |
+
// `transition-all duration-300`,
|
388 |
+
// `hover:animate-bounce`
|
389 |
+
)}
|
390 |
+
disabled={isLocked}
|
391 |
+
onClick={handleSubmit}
|
392 |
+
>
|
393 |
+
{isLocked
|
394 |
+
? `Dreaming..`
|
395 |
+
: "Dream π"
|
396 |
+
}
|
397 |
+
</animated.button>
|
398 |
+
</div>
|
399 |
+
{
|
400 |
+
/*
|
401 |
+
!!storyLines.length && <div className={cn(
|
402 |
+
`flex flex-row w-1/2 md:w-44`,
|
403 |
+
`transition-all duration-300 ease-in-out`,
|
404 |
+
isLoaded ? 'scale-100' : 'scale-0'
|
405 |
+
)}>
|
406 |
+
<animated.button
|
407 |
+
style={{
|
408 |
+
textShadow: "0px 0px 1px #000000ab",
|
409 |
+
...pauseButtonBouncer
|
410 |
+
}}
|
411 |
+
onMouseEnter={() => setOverPauseButton(true)}
|
412 |
+
onMouseLeave={() => setOverPauseButton(false)}
|
413 |
+
className={cn(
|
414 |
+
`px-4 h-16`,
|
415 |
+
`rounded-full`,
|
416 |
+
`transition-all duration-300 ease-in-out`,
|
417 |
+
`backdrop-blur-sm`,
|
418 |
+
isLocked
|
419 |
+
? `bg-orange-200/30 text-sky-50/60 border-yellow-600/10`
|
420 |
+
: `bg-yellow-400/50 text-sky-50 border-yellow-800/20 hover:bg-yellow-400/60`,
|
421 |
+
`text-center`,
|
422 |
+
`w-full`,
|
423 |
+
`text-2xl `,
|
424 |
+
`border`,
|
425 |
+
headingFont.className,
|
426 |
+
// `transition-all duration-300`,
|
427 |
+
// `hover:animate-bounce`
|
428 |
+
)}
|
429 |
+
disabled={isLocked}
|
430 |
+
onClick={togglePause}
|
431 |
+
>
|
432 |
+
{isPlaying || isSwitchingTracks
|
433 |
+
? "Pause π"
|
434 |
+
: "Play π"
|
435 |
+
}
|
436 |
+
</animated.button>
|
437 |
+
</div>
|
438 |
+
*/
|
439 |
+
}</div>
|
440 |
</div>
|
441 |
</div>
|
442 |
|
|
|
448 |
`space-y-8`,
|
449 |
// `transition-all duration-300 ease-in-out`,
|
450 |
)}>
|
451 |
+
|
452 |
|
453 |
<div
|
454 |
className={cn(
|
|
|
478 |
/>}
|
479 |
</div> : null}
|
480 |
|
481 |
+
|
482 |
<div className={cn(
|
483 |
`flex flex-col md:flex-row`,
|
484 |
`space-y-3 md:space-y-0 md:space-x-3`,
|
|
|
486 |
`items-center justify-between`
|
487 |
)}>
|
488 |
<div className={cn(
|
489 |
+
`flex flex-col flex-grow w-full items-center space-y-2 text-2xl text-blue-200/60`
|
490 |
)}>
|
491 |
{storyLines.map((line, i) =>
|
492 |
<div
|
493 |
+
id={`story-line-${i}`}
|
494 |
+
key={`${line.text}_${i}`}>
|
495 |
+
<div
|
496 |
|
497 |
// TODO change a color if we have progressed at the current index (i)
|
498 |
+
className={cn(
|
499 |
+
"flex flex-col items-center w-full "
|
500 |
+
//i < currentLineIndex
|
501 |
+
//? 'text-yellow-200'
|
502 |
+
//: 'text-blue-200/80'
|
503 |
+
)}
|
504 |
+
style={{}}
|
505 |
+
>
|
506 |
+
<div className="w-full md:w-2/3 text-center"> {
|
507 |
+
line.text.split("").map((c, j, arr) => <span
|
508 |
+
key={`${c}_${j}`}
|
509 |
+
className={cn(
|
510 |
+
`transition-all duration-100 ease-in-out`,
|
511 |
+
i < currentLineIndex || (isLoaded && i === currentLineIndex && j <= (progress * 1.3 * arr.length))
|
512 |
+
? 'text-yellow-400/90'
|
513 |
+
: ''
|
514 |
+
)}>{c || " "}</span>)
|
515 |
+
}</div>
|
516 |
+
<div className="flex flex-col items-center justify-center w-full p-8">
|
517 |
+
{images.at(i) ? <img
|
518 |
+
className="h-[400px] rounded-lg overflow-hidden"
|
519 |
+
src={images.at(i)}
|
520 |
+
/> : null}
|
521 |
+
</div>
|
522 |
+
</div>
|
523 |
+
</div>)}
|
524 |
</div>
|
525 |
</div>
|
526 |
</div>
|
src/app/server/actions/generateImage.ts
ADDED
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"use server"
|
2 |
+
|
3 |
+
// TODO add a system to mark failed instances as "unavailable" for a couple of minutes
|
4 |
+
// console.log("process.env:", process.env)
|
5 |
+
|
6 |
+
import { generateSeed } from "@/lib/generateSeed";
|
7 |
+
import { getValidNumber } from "@/lib/getValidNumber";
|
8 |
+
|
9 |
+
// note: to reduce costs I use the small A10s (not the large)
|
10 |
+
// anyway, we will soon not need to use this cloud anymore
|
11 |
+
// since we will be able to leverage the Inference API
|
12 |
+
const instance = `${process.env.FAST_IMAGE_SERVER_API_GRADIO_URL || ""}`
|
13 |
+
const secretToken = `${process.env.FAST_IMAGE_SERVER_API_SECRET_TOKEN || ""}`
|
14 |
+
|
15 |
+
// console.log("DEBUG:", JSON.stringify({ instances, secretToken }, null, 2))
|
16 |
+
|
17 |
+
export async function generateImage(options: {
|
18 |
+
positivePrompt: string;
|
19 |
+
negativePrompt?: string;
|
20 |
+
seed?: number;
|
21 |
+
width?: number;
|
22 |
+
height?: number;
|
23 |
+
nbSteps?: number;
|
24 |
+
}): Promise<string> {
|
25 |
+
|
26 |
+
// console.log("querying " + instance)
|
27 |
+
const positivePrompt = options?.positivePrompt || ""
|
28 |
+
if (!positivePrompt) {
|
29 |
+
throw new Error("missing prompt")
|
30 |
+
}
|
31 |
+
|
32 |
+
// the negative prompt CAN be missing, since we use a trick
|
33 |
+
// where we make the interface mandatory in the TS doc,
|
34 |
+
// but browsers might send something partial
|
35 |
+
const negativePrompt = options?.negativePrompt || ""
|
36 |
+
|
37 |
+
// we treat 0 as meaning "random seed"
|
38 |
+
const seed = (options?.seed ? options.seed : 0) || generateSeed()
|
39 |
+
|
40 |
+
const width = getValidNumber(options?.width, 256, 1024, 512)
|
41 |
+
const height = getValidNumber(options?.height, 256, 1024, 512)
|
42 |
+
const nbSteps = getValidNumber(options?.nbSteps, 1, 8, 4)
|
43 |
+
// console.log("SEED:", seed)
|
44 |
+
|
45 |
+
const positive = [
|
46 |
+
|
47 |
+
// oh well.. is it too late to move this to the bottom?
|
48 |
+
"beautiful",
|
49 |
+
|
50 |
+
// too opinionated, so let's remove it
|
51 |
+
// "intricate details",
|
52 |
+
|
53 |
+
positivePrompt,
|
54 |
+
|
55 |
+
"award winning",
|
56 |
+
"high resolution"
|
57 |
+
].filter(word => word)
|
58 |
+
.join(", ")
|
59 |
+
|
60 |
+
const negative = [
|
61 |
+
negativePrompt,
|
62 |
+
"watermark",
|
63 |
+
"copyright",
|
64 |
+
"blurry",
|
65 |
+
// "artificial",
|
66 |
+
// "cropped",
|
67 |
+
"low quality",
|
68 |
+
"ugly"
|
69 |
+
].filter(word => word)
|
70 |
+
.join(", ")
|
71 |
+
|
72 |
+
const res = await fetch(instance + (instance.endsWith("/") ? "" : "/") + "api/predict", {
|
73 |
+
method: "POST",
|
74 |
+
headers: {
|
75 |
+
"Content-Type": "application/json",
|
76 |
+
// Authorization: `Bearer ${token}`,
|
77 |
+
},
|
78 |
+
body: JSON.stringify({
|
79 |
+
fn_index: 0, // <- important!
|
80 |
+
data: [
|
81 |
+
positive, // string in 'Prompt' Textbox component
|
82 |
+
negative, // string in 'Negative prompt' Textbox component
|
83 |
+
seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component
|
84 |
+
width, // number (numeric value between 256 and 1024) in 'Width' Slider component
|
85 |
+
height, // number (numeric value between 256 and 1024) in 'Height' Slider component
|
86 |
+
0.0, // can be disabled for LCM SDXL
|
87 |
+
nbSteps, // number (numeric value between 2 and 8) in 'Number of inference steps for base' Slider component
|
88 |
+
secretToken
|
89 |
+
]
|
90 |
+
}),
|
91 |
+
cache: "no-store",
|
92 |
+
})
|
93 |
+
|
94 |
+
const { data } = await res.json()
|
95 |
+
|
96 |
+
if (res.status !== 200 || !Array.isArray(data)) {
|
97 |
+
// This will activate the closest `error.js` Error Boundary
|
98 |
+
throw new Error(`Failed to fetch data (status: ${res.status})`)
|
99 |
+
}
|
100 |
+
|
101 |
+
if (!data[0]) {
|
102 |
+
throw new Error(`the returned image was empty`)
|
103 |
+
}
|
104 |
+
|
105 |
+
return data[0] as string
|
106 |
+
}
|
src/lib/getValidNumber.ts
ADDED
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
export const getValidNumber = (something: any, minValue: number, maxValue: number, defaultValue: number) => {
|
2 |
+
const strValue = `${something || defaultValue}`
|
3 |
+
const numValue = Number(strValue)
|
4 |
+
const isValid = !isNaN(numValue) && isFinite(numValue)
|
5 |
+
if (!isValid) {
|
6 |
+
return defaultValue
|
7 |
+
}
|
8 |
+
return Math.max(minValue, Math.min(maxValue, numValue))
|
9 |
+
|
10 |
+
}
|
src/lib/useAudio.ts
CHANGED
@@ -1,11 +1,37 @@
|
|
1 |
-
import { useCallback, useEffect, useRef } from 'react';
|
2 |
|
3 |
-
|
4 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
6 |
const stopAudio = useCallback(() => {
|
7 |
-
|
8 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
9 |
}, []);
|
10 |
|
11 |
// Helper function to handle conversion from Base64 to an ArrayBuffer
|
@@ -14,13 +40,13 @@ export function useAudio() {
|
|
14 |
return response.arrayBuffer();
|
15 |
}
|
16 |
|
17 |
-
const
|
18 |
-
async (base64Data?: string) => {
|
19 |
stopAudio(); // Stop any playing audio first
|
20 |
|
21 |
// If no base64 data provided, we don't attempt to play any audio
|
22 |
if (!base64Data) {
|
23 |
-
return;
|
24 |
}
|
25 |
|
26 |
// Initialize AudioContext
|
@@ -52,13 +78,41 @@ export function useAudio() {
|
|
52 |
source.connect(gainNode);
|
53 |
gainNode.connect(audioContext.destination);
|
54 |
|
55 |
-
//
|
56 |
-
source
|
|
|
|
|
57 |
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
62 |
}, (error) => {
|
63 |
console.error('Error decoding audio data:', error);
|
64 |
reject(error);
|
@@ -68,6 +122,25 @@ export function useAudio() {
|
|
68 |
[stopAudio]
|
69 |
);
|
70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
71 |
// Effect to handle cleanup on component unmount
|
72 |
useEffect(() => {
|
73 |
return () => {
|
@@ -75,6 +148,5 @@ export function useAudio() {
|
|
75 |
};
|
76 |
}, [stopAudio]);
|
77 |
|
78 |
-
|
79 |
-
return playAudio;
|
80 |
}
|
|
|
1 |
+
import { useCallback, useEffect, useRef, useState } from 'react';
|
2 |
|
3 |
+
// Helper Types
|
4 |
+
type UseAudioResponse = {
|
5 |
+
playback: (base64Data?: string, isLastTrackOfPlaylist?: boolean) => Promise<boolean>;
|
6 |
+
progress: number;
|
7 |
+
isLoaded: boolean;
|
8 |
+
isPlaying: boolean;
|
9 |
+
isSwitchingTracks: boolean; // when audio is temporary cut (but it's not a real pause)
|
10 |
+
togglePause: () => void;
|
11 |
+
};
|
12 |
|
13 |
+
export function useAudio(): UseAudioResponse {
|
14 |
+
const audioContextRef = useRef<AudioContext | null>(null);
|
15 |
+
const sourceNodeRef = useRef<AudioBufferSourceNode | null>(null);
|
16 |
+
const [progress, setProgress] = useState(0.0);
|
17 |
+
const [isPlaying, setIsPlaying] = useState(false);
|
18 |
+
const [isLoaded, setIsLoaded] = useState(false);
|
19 |
+
const [isSwitchingTracks, setSwitchingTracks] = useState(false);
|
20 |
+
const startTimeRef = useRef(0);
|
21 |
+
const pauseTimeRef = useRef(0);
|
22 |
+
|
23 |
const stopAudio = useCallback(() => {
|
24 |
+
try {
|
25 |
+
audioContextRef.current?.close();
|
26 |
+
} catch (err) {
|
27 |
+
// already closed probably
|
28 |
+
}
|
29 |
+
setSwitchingTracks(false);
|
30 |
+
|
31 |
+
sourceNodeRef.current = null;
|
32 |
+
sourceNodeRef.current = null;
|
33 |
+
|
34 |
+
// setProgress(0); // Reset progress
|
35 |
}, []);
|
36 |
|
37 |
// Helper function to handle conversion from Base64 to an ArrayBuffer
|
|
|
40 |
return response.arrayBuffer();
|
41 |
}
|
42 |
|
43 |
+
const playback = useCallback(
|
44 |
+
async (base64Data?: string, isLastTrackOfPlaylist?: boolean): Promise<boolean> => {
|
45 |
stopAudio(); // Stop any playing audio first
|
46 |
|
47 |
// If no base64 data provided, we don't attempt to play any audio
|
48 |
if (!base64Data) {
|
49 |
+
return false;
|
50 |
}
|
51 |
|
52 |
// Initialize AudioContext
|
|
|
78 |
source.connect(gainNode);
|
79 |
gainNode.connect(audioContext.destination);
|
80 |
|
81 |
+
// Assign source node to ref for progress tracking
|
82 |
+
sourceNodeRef.current = source;
|
83 |
+
source.start(0, pauseTimeRef.current % audioBuffer.duration); // Start at the correct offset if paused previously
|
84 |
+
startTimeRef.current = audioContextRef.current!.currentTime - pauseTimeRef.current;
|
85 |
|
86 |
+
setSwitchingTracks(false);
|
87 |
+
setProgress(0);
|
88 |
+
setIsLoaded(true);
|
89 |
+
setIsPlaying(true);
|
90 |
+
|
91 |
+
// Set up progress interval
|
92 |
+
const totalDuration = audioBuffer.duration;
|
93 |
+
const updateProgressInterval = setInterval(() => {
|
94 |
+
if (sourceNodeRef.current && audioContextRef.current) {
|
95 |
+
const currentTime = audioContextRef.current.currentTime;
|
96 |
+
const currentProgress = currentTime / totalDuration;
|
97 |
+
setProgress(currentProgress);
|
98 |
+
if (currentProgress >= 1.0) {
|
99 |
+
clearInterval(updateProgressInterval);
|
100 |
+
}
|
101 |
+
}
|
102 |
+
}, 50); // Update every 50ms
|
103 |
+
|
104 |
+
if (source) {
|
105 |
+
source.onended = () => {
|
106 |
+
// used to indicate a temporary stop, while we switch tracks
|
107 |
+
if (!isLastTrackOfPlaylist) {
|
108 |
+
setSwitchingTracks(true);
|
109 |
+
}
|
110 |
+
setIsPlaying(false);
|
111 |
+
clearInterval(updateProgressInterval);
|
112 |
+
stopAudio();
|
113 |
+
resolve(true);
|
114 |
+
};
|
115 |
+
}
|
116 |
}, (error) => {
|
117 |
console.error('Error decoding audio data:', error);
|
118 |
reject(error);
|
|
|
122 |
[stopAudio]
|
123 |
);
|
124 |
|
125 |
+
const togglePause = useCallback(() => {
|
126 |
+
if (!audioContextRef.current || !sourceNodeRef.current) {
|
127 |
+
return; // Do nothing if audio is not initialized
|
128 |
+
}
|
129 |
+
|
130 |
+
if (isPlaying) {
|
131 |
+
// Pause the audio
|
132 |
+
pauseTimeRef.current += audioContextRef.current.currentTime - startTimeRef.current;
|
133 |
+
sourceNodeRef.current.stop(); // This effectively "pauses" the audio, but it also means the sourceNode will be unusable
|
134 |
+
sourceNodeRef.current = null; // As the node is now unusable, we nullify it
|
135 |
+
setIsPlaying(false);
|
136 |
+
} else {
|
137 |
+
// Resume playing
|
138 |
+
audioContextRef.current.resume().then(() => {
|
139 |
+
playback(); // This will pick up where we left off due to pauseTimeRef
|
140 |
+
});
|
141 |
+
}
|
142 |
+
}, [audioContextRef, sourceNodeRef, isPlaying, playback]);
|
143 |
+
|
144 |
// Effect to handle cleanup on component unmount
|
145 |
useEffect(() => {
|
146 |
return () => {
|
|
|
148 |
};
|
149 |
}, [stopAudio]);
|
150 |
|
151 |
+
return { playback, isPlaying, isSwitchingTracks, isLoaded, progress, togglePause };
|
|
|
152 |
}
|