Commit
•
82d1e90
1
Parent(s):
68ec2c8
experimental support for images
Browse files- src/app/config.ts +3 -1
- src/app/main.tsx +39 -872
- src/components/interface/bottom-bar.tsx +61 -0
- src/components/interface/character-button.tsx +77 -0
- src/components/interface/characters.tsx +19 -0
- src/components/interface/index.ts +2 -0
- src/components/interface/load-clap-button.tsx +42 -0
- src/components/interface/main-title.tsx +56 -0
- src/components/interface/save-clap-button.tsx +43 -0
- src/components/interface/video-preview.tsx +164 -0
- src/lib/hooks/index.tsx +6 -0
- src/lib/hooks/useEntityPicture.ts +28 -0
- src/lib/hooks/useImportClap.ts +112 -0
- src/lib/hooks/useIsBusy.ts +11 -0
- src/lib/hooks/useOpenPictureFile.ts +19 -0
- src/lib/hooks/useOrientation.ts +24 -0
- src/lib/hooks/useProcessors.ts +447 -0
- src/lib/hooks/useProgressTimer.ts +45 -0
- src/lib/hooks/useQueryStringParams.ts +51 -0
- src/lib/hooks/useStoryPromptDraft.ts +16 -0
- src/lib/utils/index.ts +3 -0
src/app/config.ts
CHANGED
@@ -3,4 +3,6 @@ export const defaultPrompt =
|
|
3 |
// "underwater footage of the loch ness"
|
4 |
// "beautiful underwater footage of a clownfish swimming around coral" // who discovers a secret pirate treasure chest"
|
5 |
// "beautiful footage of a Caribbean fishing village and bay, sail ships, during golden hour, no captions"
|
6 |
-
"videogame gameplay footage, first person, exploring some mysterious ruins, no commentary"
|
|
|
|
|
|
3 |
// "underwater footage of the loch ness"
|
4 |
// "beautiful underwater footage of a clownfish swimming around coral" // who discovers a secret pirate treasure chest"
|
5 |
// "beautiful footage of a Caribbean fishing village and bay, sail ships, during golden hour, no captions"
|
6 |
+
"videogame gameplay footage, first person, exploring some mysterious ruins, no commentary"
|
7 |
+
|
8 |
+
export const localStorageStoryDraftKey = "AI_STORIES_FACTORY_STORY_PROMPT_DRAFT"
|
src/app/main.tsx
CHANGED
@@ -1,596 +1,32 @@
|
|
1 |
"use client"
|
2 |
|
3 |
-
import React
|
4 |
import { IoMdPhonePortrait } from "react-icons/io"
|
5 |
import { GiRollingDices } from "react-icons/gi"
|
6 |
-
import {
|
7 |
-
import { GiSpellBook } from "react-icons/gi"
|
8 |
-
import { useLocalStorage } from "usehooks-ts"
|
9 |
-
import { ClapProject, ClapMediaOrientation, ClapSegmentCategory, updateClap } from "@aitube/clap"
|
10 |
-
import Image from "next/image"
|
11 |
-
import { useSearchParams } from "next/navigation"
|
12 |
-
import { useFilePicker } from "use-file-picker"
|
13 |
-
import { DeviceFrameset } from "react-device-frameset"
|
14 |
-
import "react-device-frameset/styles/marvel-devices.min.css"
|
15 |
|
16 |
import { Card, CardContent, CardHeader } from "@/components/ui/card"
|
17 |
import { Button } from "@/components/ui/button"
|
18 |
import { Toaster } from "@/components/ui/sonner"
|
19 |
import { TextareaField } from "@/components/form/textarea-field"
|
20 |
-
import { cn } from "@/lib/utils/cn"
|
21 |
|
22 |
-
import {
|
23 |
-
import { editClapEntities } from "./server/aitube/editClapEntities"
|
24 |
-
import { editClapDialogues } from "./server/aitube/editClapDialogues"
|
25 |
-
import { editClapStoryboards } from "./server/aitube/editClapStoryboards"
|
26 |
-
import { editClapSounds } from "./server/aitube/editClapSounds"
|
27 |
-
import { editClapMusic } from "./server/aitube/editClapMusic"
|
28 |
-
import { editClapVideos } from "./server/aitube/editClapVideos"
|
29 |
-
import { exportClapToVideo } from "./server/aitube/exportClapToVideo"
|
30 |
-
|
31 |
-
import { useStore } from "./store"
|
32 |
-
import HFLogo from "./hf-logo.svg"
|
33 |
-
import { Input } from "@/components/ui/input"
|
34 |
-
import { Field } from "@/components/form/field"
|
35 |
-
import { Label } from "@/components/form/label"
|
36 |
-
import { getParam } from "@/lib/utils/getParam"
|
37 |
-
import { GenerationStage } from "@/types"
|
38 |
-
import { FileContent } from "use-file-picker/dist/interfaces"
|
39 |
-
import { generateRandomStory } from "@/lib/utils/generateRandomStory"
|
40 |
-
import { logImage } from "@/lib/utils/logImage"
|
41 |
import { defaultPrompt } from "./config"
|
42 |
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
|
44 |
export function Main() {
|
45 |
-
const
|
46 |
-
|
47 |
-
|
48 |
-
)
|
49 |
-
|
50 |
-
promptDraftRef.current = storyPromptDraft
|
51 |
-
|
52 |
-
const [_isPending, startTransition] = useTransition()
|
53 |
-
|
54 |
-
const storyPrompt = useStore(s => s.storyPrompt)
|
55 |
-
const mainCharacterImage = useStore(s => s.mainCharacterImage)
|
56 |
-
const mainCharacterVoice = useStore(s => s.mainCharacterVoice)
|
57 |
-
const orientation = useStore(s => s.orientation)
|
58 |
-
const setOrientation = useStore(s => s.setOrientation)
|
59 |
-
const status = useStore(s => s.status)
|
60 |
-
const parseGenerationStatus = useStore(s => s.parseGenerationStatus)
|
61 |
-
const storyGenerationStatus = useStore(s => s.storyGenerationStatus)
|
62 |
-
const assetGenerationStatus = useStore(s => s.assetGenerationStatus)
|
63 |
-
const soundGenerationStatus = useStore(s => s.soundGenerationStatus)
|
64 |
-
const musicGenerationStatus = useStore(s => s.musicGenerationStatus)
|
65 |
-
const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus)
|
66 |
-
const imageGenerationStatus = useStore(s => s.imageGenerationStatus)
|
67 |
-
const videoGenerationStatus = useStore(s => s.videoGenerationStatus)
|
68 |
-
const finalGenerationStatus = useStore(s => s.finalGenerationStatus)
|
69 |
-
const currentClap = useStore(s => s.currentClap)
|
70 |
-
const currentVideo = useStore(s => s.currentVideo)
|
71 |
-
const currentVideoOrientation = useStore(s => s.currentVideoOrientation)
|
72 |
-
const setStoryPrompt = useStore(s => s.setStoryPrompt)
|
73 |
-
const setMainCharacterImage = useStore(s => s.setMainCharacterImage)
|
74 |
-
const setMainCharacterVoice = useStore(s => s.setMainCharacterVoice)
|
75 |
-
const setStatus = useStore(s => s.setStatus)
|
76 |
-
const toggleOrientation = useStore(s => s.toggleOrientation)
|
77 |
-
const error = useStore(s => s.error)
|
78 |
-
const setError = useStore(s => s.setError)
|
79 |
-
const setParseGenerationStatus = useStore(s => s.setParseGenerationStatus)
|
80 |
-
const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus)
|
81 |
-
const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus)
|
82 |
-
const setSoundGenerationStatus = useStore(s => s.setSoundGenerationStatus)
|
83 |
-
const setMusicGenerationStatus = useStore(s => s.setMusicGenerationStatus)
|
84 |
-
const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus)
|
85 |
-
const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus)
|
86 |
-
const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus)
|
87 |
-
const setFinalGenerationStatus = useStore(s => s.setFinalGenerationStatus)
|
88 |
-
const setCurrentClap = useStore(s => s.setCurrentClap)
|
89 |
-
const setCurrentVideo = useStore(s => s.setCurrentVideo)
|
90 |
-
const progress = useStore(s => s.progress)
|
91 |
-
const setProgress = useStore(s => s.setProgress)
|
92 |
-
const saveVideo = useStore(s => s.saveVideo)
|
93 |
-
const saveClap = useStore(s => s.saveClap)
|
94 |
-
const loadClap = useStore(s => s.loadClap)
|
95 |
-
|
96 |
-
// let's disable this for now
|
97 |
-
const canSeeBetaFeatures = true // getParam<boolean>("beta", false)
|
98 |
-
|
99 |
-
const isBusy = useStore(s => s.isBusy)
|
100 |
-
const busyRef = useRef(isBusy)
|
101 |
-
busyRef.current = isBusy
|
102 |
-
|
103 |
-
const importStory = async (fileData: FileContent<ArrayBuffer>): Promise<{
|
104 |
-
clap: ClapProject
|
105 |
-
regenerateVideo: boolean
|
106 |
-
}> => {
|
107 |
-
if (!fileData?.name) { throw new Error(`invalid file (missing file name)`) }
|
108 |
-
|
109 |
-
const {
|
110 |
-
setParseGenerationStatus,
|
111 |
-
} = useStore.getState()
|
112 |
-
|
113 |
-
setParseGenerationStatus("generating")
|
114 |
-
|
115 |
-
try {
|
116 |
-
const blob = new Blob([fileData.content])
|
117 |
-
const res = await loadClap(blob, fileData.name)
|
118 |
-
|
119 |
-
if (!res?.clap) { throw new Error(`failed to load the clap file`) }
|
120 |
-
|
121 |
-
setParseGenerationStatus("finished")
|
122 |
-
return res
|
123 |
-
} catch (err) {
|
124 |
-
console.error("failed to load the Clap file:", err)
|
125 |
-
setParseGenerationStatus("error")
|
126 |
-
throw err
|
127 |
-
}
|
128 |
-
}
|
129 |
-
|
130 |
-
|
131 |
-
const generateStory = async (): Promise<ClapProject> => {
|
132 |
-
|
133 |
-
let clap: ClapProject | undefined = undefined
|
134 |
-
try {
|
135 |
-
setProgress(0)
|
136 |
-
|
137 |
-
setStatus("generating")
|
138 |
-
setStoryGenerationStatus("generating")
|
139 |
-
setStoryPrompt(promptDraftRef.current)
|
140 |
-
|
141 |
-
clap = await createClap({
|
142 |
-
prompt: promptDraftRef.current,
|
143 |
-
orientation: useStore.getState().orientation,
|
144 |
-
|
145 |
-
turbo: true,
|
146 |
-
})
|
147 |
-
|
148 |
-
if (!clap) { throw new Error(`failed to create the clap`) }
|
149 |
-
|
150 |
-
if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) }
|
151 |
-
|
152 |
-
console.log(`handleSubmit(): received a clap = `, clap)
|
153 |
-
setCurrentClap(clap)
|
154 |
-
setStoryGenerationStatus("finished")
|
155 |
-
|
156 |
-
console.log("---------------- GENERATED STORY ----------------")
|
157 |
-
console.table(clap.segments, [
|
158 |
-
// 'startTimeInMs',
|
159 |
-
'endTimeInMs',
|
160 |
-
// 'track',
|
161 |
-
'category',
|
162 |
-
'prompt'
|
163 |
-
])
|
164 |
-
return clap
|
165 |
-
} catch (err) {
|
166 |
-
setStoryGenerationStatus("error")
|
167 |
-
throw err
|
168 |
-
}
|
169 |
-
}
|
170 |
-
|
171 |
-
const generateEntities = async (clap: ClapProject): Promise<ClapProject> => {
|
172 |
-
try {
|
173 |
-
// setProgress(20)
|
174 |
-
setAssetGenerationStatus("generating")
|
175 |
-
clap = await editClapEntities({
|
176 |
-
clap,
|
177 |
-
|
178 |
-
// generating entities requires a "smart" LLM
|
179 |
-
turbo: false,
|
180 |
-
// turbo: true,
|
181 |
-
}).then(r => r.promise)
|
182 |
-
|
183 |
-
if (!clap) { throw new Error(`failed to edit the entities`) }
|
184 |
-
|
185 |
-
console.log(`handleSubmit(): received a clap with entities = `, clap)
|
186 |
-
setAssetGenerationStatus("finished")
|
187 |
-
console.log("---------------- GENERATED ENTITIES ----------------")
|
188 |
-
console.table(clap.entities, [
|
189 |
-
'category',
|
190 |
-
'label',
|
191 |
-
'imagePrompt',
|
192 |
-
'appearance'
|
193 |
-
])
|
194 |
-
return clap
|
195 |
-
} catch (err) {
|
196 |
-
setAssetGenerationStatus("error")
|
197 |
-
throw err
|
198 |
-
}
|
199 |
-
}
|
200 |
-
|
201 |
-
const generateSounds = async (clap: ClapProject): Promise<ClapProject> => {
|
202 |
-
try {
|
203 |
-
// setProgress(30)
|
204 |
-
setSoundGenerationStatus("generating")
|
205 |
-
|
206 |
-
clap = await editClapSounds({
|
207 |
-
clap,
|
208 |
-
turbo: true
|
209 |
-
}).then(r => r.promise)
|
210 |
-
|
211 |
-
if (!clap) { throw new Error(`failed to edit the sound`) }
|
212 |
-
|
213 |
-
console.log(`handleSubmit(): received a clap with sound = `, clap)
|
214 |
-
setSoundGenerationStatus("finished")
|
215 |
-
console.log("---------------- GENERATED SOUND ----------------")
|
216 |
-
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.SOUND), [
|
217 |
-
'endTimeInMs',
|
218 |
-
'prompt',
|
219 |
-
'entityId',
|
220 |
-
])
|
221 |
-
return clap
|
222 |
-
} catch (err) {
|
223 |
-
setSoundGenerationStatus("error")
|
224 |
-
throw err
|
225 |
-
}
|
226 |
-
}
|
227 |
-
|
228 |
-
const generateMusic = async (clap: ClapProject): Promise<ClapProject> => {
|
229 |
-
try {
|
230 |
-
// setProgress(30)
|
231 |
-
setMusicGenerationStatus("generating")
|
232 |
-
|
233 |
-
clap = await editClapMusic({
|
234 |
-
clap,
|
235 |
-
turbo: true
|
236 |
-
}).then(r => r.promise)
|
237 |
-
|
238 |
-
if (!clap) { throw new Error(`failed to edit the music`) }
|
239 |
-
|
240 |
-
console.log(`handleSubmit(): received a clap with music = `, clap)
|
241 |
-
setMusicGenerationStatus("finished")
|
242 |
-
console.log("---------------- GENERATED MUSIC ----------------")
|
243 |
-
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC), [
|
244 |
-
'endTimeInMs',
|
245 |
-
'prompt',
|
246 |
-
'entityId',
|
247 |
-
])
|
248 |
-
return clap
|
249 |
-
} catch (err) {
|
250 |
-
setMusicGenerationStatus("error")
|
251 |
-
throw err
|
252 |
-
}
|
253 |
-
}
|
254 |
-
|
255 |
-
const generateStoryboards = async (clap: ClapProject): Promise<ClapProject> => {
|
256 |
-
try {
|
257 |
-
// setProgress(40)
|
258 |
-
setImageGenerationStatus("generating")
|
259 |
-
clap = await editClapStoryboards({
|
260 |
-
clap,
|
261 |
-
// if we use entities, then we MUST use turbo
|
262 |
-
// that's because turbo uses PulID,
|
263 |
-
// but SDXL doesn't
|
264 |
-
turbo: true,
|
265 |
-
}).then(r => r.promise)
|
266 |
-
|
267 |
-
if (!clap) { throw new Error(`failed to edit the storyboards`) }
|
268 |
-
|
269 |
-
// const fusion =
|
270 |
-
console.log(`handleSubmit(): received a clap with images = `, clap)
|
271 |
-
setImageGenerationStatus("finished")
|
272 |
-
console.log("---------------- GENERATED STORYBOARDS ----------------")
|
273 |
-
clap.segments
|
274 |
-
.filter(s => s.category === ClapSegmentCategory.STORYBOARD)
|
275 |
-
.forEach((s, i) => {
|
276 |
-
if (s.status === "completed" && s.assetUrl) {
|
277 |
-
// console.log(` [${i}] storyboard: ${s.prompt}`)
|
278 |
-
logImage(s.assetUrl, 0.35)
|
279 |
-
} else {
|
280 |
-
console.log(` [${i}] failed to generate storyboard`)
|
281 |
-
}
|
282 |
-
// console.log(`------------------`)
|
283 |
-
})
|
284 |
-
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD), [
|
285 |
-
'endTimeInMs',
|
286 |
-
'prompt',
|
287 |
-
'assetUrl'
|
288 |
-
])
|
289 |
-
return clap
|
290 |
-
} catch (err) {
|
291 |
-
setImageGenerationStatus("error")
|
292 |
-
throw err
|
293 |
-
}
|
294 |
-
}
|
295 |
-
|
296 |
-
const generateVideos = async (clap: ClapProject): Promise<ClapProject> => {
|
297 |
-
try {
|
298 |
-
// setProgress(50)
|
299 |
-
setVideoGenerationStatus("generating")
|
300 |
-
|
301 |
-
clap = await editClapVideos({
|
302 |
-
clap,
|
303 |
-
turbo: false
|
304 |
-
}).then(r => r.promise)
|
305 |
-
|
306 |
-
if (!clap) { throw new Error(`failed to edit the videos`) }
|
307 |
-
|
308 |
-
console.log(`handleSubmit(): received a clap with videos = `, clap)
|
309 |
-
setVideoGenerationStatus("finished")
|
310 |
-
console.log("---------------- GENERATED VIDEOS ----------------")
|
311 |
-
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO), [
|
312 |
-
'endTimeInMs',
|
313 |
-
'prompt',
|
314 |
-
'entityId',
|
315 |
-
])
|
316 |
-
return clap
|
317 |
-
} catch (err) {
|
318 |
-
setVideoGenerationStatus("error")
|
319 |
-
throw err
|
320 |
-
}
|
321 |
-
}
|
322 |
-
|
323 |
-
const generateStoryboardsThenVideos = async (clap: ClapProject): Promise<ClapProject> => {
|
324 |
-
clap = await generateStoryboards(clap)
|
325 |
-
clap = await generateVideos(clap)
|
326 |
-
return clap
|
327 |
-
}
|
328 |
-
|
329 |
-
|
330 |
-
const generateDialogues = async (clap: ClapProject): Promise<ClapProject> => {
|
331 |
-
try {
|
332 |
-
// setProgress(70)
|
333 |
-
setVoiceGenerationStatus("generating")
|
334 |
-
clap = await editClapDialogues({
|
335 |
-
clap,
|
336 |
-
turbo: true
|
337 |
-
}).then(r => r.promise)
|
338 |
-
|
339 |
-
if (!clap) { throw new Error(`failed to edit the dialogues`) }
|
340 |
-
|
341 |
-
console.log(`handleSubmit(): received a clap with dialogues = `, clap)
|
342 |
-
setVoiceGenerationStatus("finished")
|
343 |
-
console.log("---------------- GENERATED DIALOGUES ----------------")
|
344 |
-
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.DIALOGUE), [
|
345 |
-
'endTimeInMs',
|
346 |
-
'prompt',
|
347 |
-
'entityId',
|
348 |
-
])
|
349 |
-
return clap
|
350 |
-
} catch (err) {
|
351 |
-
setVoiceGenerationStatus("error")
|
352 |
-
throw err
|
353 |
-
}
|
354 |
-
}
|
355 |
-
|
356 |
-
const generateFinalVideo = async (clap: ClapProject): Promise<string> => {
|
357 |
-
|
358 |
-
let assetUrl = ""
|
359 |
-
try {
|
360 |
-
// setProgress(85)
|
361 |
-
setFinalGenerationStatus("generating")
|
362 |
-
assetUrl = await exportClapToVideo({
|
363 |
-
clap,
|
364 |
-
turbo: true
|
365 |
-
})
|
366 |
-
|
367 |
-
setCurrentVideo(assetUrl)
|
368 |
-
|
369 |
-
if (assetUrl.length < 128) { throw new Error(`generateFinalVideo(): the generated video is too small, so we failed`) }
|
370 |
-
|
371 |
-
console.log(`generateFinalVideo(): received a video: ${assetUrl.slice(0, 120)}...`)
|
372 |
-
setFinalGenerationStatus("finished")
|
373 |
-
return assetUrl
|
374 |
-
} catch (err) {
|
375 |
-
setFinalGenerationStatus("error")
|
376 |
-
throw err
|
377 |
-
}
|
378 |
-
}
|
379 |
-
|
380 |
-
const handleSubmit = async () => {
|
381 |
-
setStatus("generating")
|
382 |
-
busyRef.current = true
|
383 |
-
|
384 |
-
startTransition(async () => {
|
385 |
-
setStatus("generating")
|
386 |
-
busyRef.current = true
|
387 |
-
|
388 |
-
console.log(`handleSubmit(): generating a clap using prompt = "${promptDraftRef.current}" `)
|
389 |
-
|
390 |
-
try {
|
391 |
-
let clap = await generateStory()
|
392 |
-
|
393 |
-
setCurrentClap(clap)
|
394 |
-
|
395 |
-
const tasks = [
|
396 |
-
generateMusic(clap),
|
397 |
-
generateStoryboardsThenVideos(clap)
|
398 |
-
]
|
399 |
-
|
400 |
-
const claps = await Promise.all(tasks)
|
401 |
-
|
402 |
-
console.log(`finished processing ${tasks.length} tasks in parallel`)
|
403 |
-
|
404 |
-
for (const newerClap of claps) {
|
405 |
-
clap = await updateClap(clap, newerClap, {
|
406 |
-
overwriteMeta: false,
|
407 |
-
inlineReplace: true,
|
408 |
-
})
|
409 |
-
setCurrentClap(clap)
|
410 |
-
}
|
411 |
-
|
412 |
-
/*
|
413 |
-
clap = await claps.reduce(async (existingClap, newerClap) =>
|
414 |
-
updateClap(existingClap, newerClap, {
|
415 |
-
overwriteMeta: false,
|
416 |
-
inlineReplace: true,
|
417 |
-
})
|
418 |
-
, Promise.resolve(clap)
|
419 |
-
*/
|
420 |
-
|
421 |
-
|
422 |
-
// We can't have consistent characters with video (yet)
|
423 |
-
// clap = await generateEntities(clap)
|
424 |
-
|
425 |
-
/*
|
426 |
-
if (mainCharacterImage) {
|
427 |
-
console.log("handleSubmit(): User specified a main character image")
|
428 |
-
// various strategies here, for instance we can assume that the first character is the main character,
|
429 |
-
// or maybe a more reliable way is to count the number of occurrences.
|
430 |
-
// there is a risk of misgendering, so ideally we should add some kind of UI to do this,
|
431 |
-
// such as a list of characters.
|
432 |
-
}
|
433 |
-
*/
|
434 |
-
|
435 |
-
// let's skip storyboards for now
|
436 |
-
// clap = await generateStoryboards(clap)
|
437 |
-
|
438 |
-
// clap = await generateVideos(clap)
|
439 |
-
// clap = await generateDialogues(clap)
|
440 |
-
|
441 |
-
|
442 |
-
|
443 |
-
console.log("final clap: ", clap)
|
444 |
-
setCurrentClap(clap)
|
445 |
-
await generateFinalVideo(clap)
|
446 |
-
|
447 |
-
setStatus("finished")
|
448 |
-
setError("")
|
449 |
-
} catch (err) {
|
450 |
-
console.error(`failed to generate: `, err)
|
451 |
-
setStatus("error")
|
452 |
-
setError(`Error, please contact an admin on Discord (${err})`)
|
453 |
-
}
|
454 |
-
})
|
455 |
-
}
|
456 |
-
|
457 |
-
|
458 |
-
const { openFilePicker, filesContent } = useFilePicker({
|
459 |
-
accept: '.clap',
|
460 |
-
readAs: "ArrayBuffer"
|
461 |
-
})
|
462 |
-
const fileData = filesContent[0]
|
463 |
-
|
464 |
-
useEffect(() => {
|
465 |
-
const fn = async () => {
|
466 |
-
if (!fileData?.name) { return }
|
467 |
-
|
468 |
-
const { setStatus, setProgress } = useStore.getState()
|
469 |
-
|
470 |
-
setProgress(0)
|
471 |
-
setStatus("generating")
|
472 |
-
|
473 |
-
try {
|
474 |
-
let { clap, regenerateVideo } = await importStory(fileData)
|
475 |
-
|
476 |
-
// clap = await generateSounds(clap)
|
477 |
-
|
478 |
-
// setCurrentClap(clap)
|
479 |
-
|
480 |
-
console.log("loadClap(): clap = ", clap)
|
481 |
-
|
482 |
-
// it is important to skip regeneration if we already have a video
|
483 |
-
if (regenerateVideo) {
|
484 |
-
console.log(`regenerating music and videos..`)
|
485 |
-
const claps = await Promise.all([
|
486 |
-
generateMusic(clap),
|
487 |
-
generateVideos(clap)
|
488 |
-
])
|
489 |
-
|
490 |
-
// console.log("finished processing the 2 tasks in parallel")
|
491 |
-
|
492 |
-
for (const newerClap of claps) {
|
493 |
-
clap = await updateClap(clap, newerClap, {
|
494 |
-
overwriteMeta: false,
|
495 |
-
inlineReplace: true,
|
496 |
-
})
|
497 |
-
}
|
498 |
-
|
499 |
-
|
500 |
-
setCurrentClap(clap)
|
501 |
-
|
502 |
-
await generateFinalVideo(clap)
|
503 |
-
|
504 |
-
} else {
|
505 |
-
console.log(`skipping music and video regeneration`)
|
506 |
-
}
|
507 |
-
|
508 |
-
setStatus("finished")
|
509 |
-
setProgress(100)
|
510 |
-
setError("")
|
511 |
-
} catch (err) {
|
512 |
-
console.error(`failed to import: `, err)
|
513 |
-
setStatus("error")
|
514 |
-
setError(`${err}`)
|
515 |
-
}
|
516 |
-
|
517 |
-
}
|
518 |
-
fn()
|
519 |
-
}, [fileData?.name])
|
520 |
-
|
521 |
-
// note: we are interested in the *current* video orientation,
|
522 |
-
// not the requested video orientation requested for the next video
|
523 |
-
const isLandscape = currentVideoOrientation === ClapMediaOrientation.LANDSCAPE
|
524 |
-
const isPortrait = currentVideoOrientation === ClapMediaOrientation.PORTRAIT
|
525 |
-
const isSquare = currentVideoOrientation === ClapMediaOrientation.SQUARE
|
526 |
-
|
527 |
-
const runningRef = useRef(false)
|
528 |
-
const timerRef = useRef<NodeJS.Timeout>()
|
529 |
-
|
530 |
-
const timerFn = async () => {
|
531 |
-
const { isBusy, progress, stage } = useStore.getState()
|
532 |
-
|
533 |
-
clearTimeout(timerRef.current)
|
534 |
-
if (!isBusy || stage === "idle") {
|
535 |
-
return
|
536 |
-
}
|
537 |
-
|
538 |
-
/*
|
539 |
-
console.log("progress function:", {
|
540 |
-
stage,
|
541 |
-
delay: progressDelayInMsPerStage[stage],
|
542 |
-
progress,
|
543 |
-
})
|
544 |
-
*/
|
545 |
-
useStore.setState({
|
546 |
-
// progress: Math.min(maxProgressPerStage[stage], progress + 1)
|
547 |
-
progress: Math.min(100, progress + 1)
|
548 |
-
})
|
549 |
-
|
550 |
-
// timerRef.current = setTimeout(timerFn, progressDelayInMsPerStage[stage])
|
551 |
-
timerRef.current = setTimeout(timerFn, 1200)
|
552 |
-
}
|
553 |
-
|
554 |
-
useEffect(() => {
|
555 |
-
timerFn()
|
556 |
-
clearTimeout(timerRef.current)
|
557 |
-
if (!isBusy) { return }
|
558 |
-
timerRef.current = setTimeout(timerFn, 0)
|
559 |
-
}, [isBusy])
|
560 |
-
|
561 |
-
// this is how we support query string parameters
|
562 |
-
// ?prompt=hello <- set a default prompt
|
563 |
-
// ?prompt=hello&autorun=true <- automatically run the app
|
564 |
-
// ?orientation=landscape <- can be "landscape" or "portrait" (default)
|
565 |
-
const searchParams = useSearchParams()
|
566 |
-
const queryStringPrompt = (searchParams?.get('prompt') as string) || ""
|
567 |
-
const queryStringAutorun = (searchParams?.get('autorun') as string) || ""
|
568 |
-
const queryStringOrientation = (searchParams?.get('orientation') as string) || ""
|
569 |
-
useEffect(() => {
|
570 |
-
if (queryStringOrientation?.length > 1) {
|
571 |
-
console.log(`orientation = "${queryStringOrientation}"`)
|
572 |
-
const orientation =
|
573 |
-
queryStringOrientation.trim().toLowerCase() === "landscape"
|
574 |
-
? ClapMediaOrientation.LANDSCAPE
|
575 |
-
: ClapMediaOrientation.PORTRAIT
|
576 |
-
setOrientation(orientation)
|
577 |
-
}
|
578 |
-
if (queryStringPrompt?.length > 1) {
|
579 |
-
console.log(`prompt = "${queryStringPrompt}"`)
|
580 |
-
if (queryStringPrompt !== promptDraftRef.current) {
|
581 |
-
setStoryPromptDraft(queryStringPrompt)
|
582 |
-
}
|
583 |
-
const maybeAutorun = queryStringAutorun.trim().toLowerCase()
|
584 |
-
console.log(`autorun = "${maybeAutorun}"`)
|
585 |
-
|
586 |
-
// note: during development we will be called twice,
|
587 |
-
// which is why we have a guard on busyRef.current
|
588 |
-
if (maybeAutorun === "true" || maybeAutorun === "1" && !busyRef.current) {
|
589 |
-
handleSubmit()
|
590 |
-
}
|
591 |
-
}
|
592 |
-
}, [queryStringPrompt, queryStringAutorun, queryStringOrientation])
|
593 |
-
|
594 |
return (
|
595 |
<TooltipProvider>
|
596 |
<div className={cn(
|
@@ -659,60 +95,7 @@ export function Main() {
|
|
659 |
|
660 |
)}>
|
661 |
<CardHeader>
|
662 |
-
|
663 |
-
<div className="
|
664 |
-
flex flex-row
|
665 |
-
items-center justify-center
|
666 |
-
transition-all duration-200 ease-in-out
|
667 |
-
px-3
|
668 |
-
|
669 |
-
rounded-full">
|
670 |
-
<div
|
671 |
-
className="
|
672 |
-
flex
|
673 |
-
transition-all duration-200 ease-in-out
|
674 |
-
items-center justify-center text-center
|
675 |
-
w-10 h-10 md:w-12 md:h-12 lg:w-16 lg:h-16
|
676 |
-
text-3xl md:text-4xl lg:text-5xl
|
677 |
-
rounded-lg
|
678 |
-
mr-2
|
679 |
-
font-sans
|
680 |
-
bg-amber-400 dark:bg-amber-400
|
681 |
-
|
682 |
-
text-stone-950/80 dark:text-stone-950/80 font-bold
|
683 |
-
"
|
684 |
-
>AI</div>
|
685 |
-
<div
|
686 |
-
className="
|
687 |
-
transition-all duration-200 ease-in-out
|
688 |
-
text-amber-400 dark:text-amber-400
|
689 |
-
text-3xl md:text-4xl lg:text-5xl
|
690 |
-
"
|
691 |
-
|
692 |
-
style={{ textShadow: "#00000035 0px 0px 2px" }}
|
693 |
-
|
694 |
-
/*
|
695 |
-
className="
|
696 |
-
text-5xl
|
697 |
-
bg-gradient-to-br from-yellow-300 to-yellow-500
|
698 |
-
inline-block text-transparent bg-clip-text
|
699 |
-
py-6
|
700 |
-
"
|
701 |
-
*/
|
702 |
-
>Stories Factory</div>
|
703 |
-
</div>
|
704 |
-
|
705 |
-
<p
|
706 |
-
className="
|
707 |
-
transition-all duration-200 ease-in-out
|
708 |
-
text-stone-900/90 dark:text-stone-900/90
|
709 |
-
text-lg md:text-xl lg:text-2xl
|
710 |
-
text-center
|
711 |
-
pt-2 md:pt-4
|
712 |
-
"
|
713 |
-
style={{ textShadow: "rgb(255 255 255 / 19%) 0px 0px 2px" }}
|
714 |
-
>Make video stories using AI ✨</p>
|
715 |
-
</div>
|
716 |
</CardHeader>
|
717 |
<CardContent
|
718 |
className="flex flex-col space-y-3"
|
@@ -806,75 +189,24 @@ export function Main() {
|
|
806 |
*/}
|
807 |
|
808 |
<div className="
|
809 |
-
|
810 |
-
|
811 |
-
|
812 |
-
|
813 |
-
|
814 |
-
|
815 |
-
{canSeeBetaFeatures &&
|
816 |
-
<Tooltip>
|
817 |
-
<TooltipTrigger asChild><Button
|
818 |
-
|
819 |
-
onClick={openFilePicker}
|
820 |
-
disabled={isBusy}
|
821 |
-
// variant="ghost"
|
822 |
-
className={cn(
|
823 |
-
`text-xs md:text-sm lg:text-base`,
|
824 |
-
`bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
|
825 |
-
`font-bold`,
|
826 |
-
`hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
|
827 |
-
storyPromptDraft ? "opacity-100" : "opacity-80"
|
828 |
-
)}
|
829 |
-
>
|
830 |
-
<span className="hidden xl:inline mr-1">Load .clap</span>
|
831 |
-
<span className="inline xl:hidden mr-1">Load .clap</span>
|
832 |
-
</Button></TooltipTrigger>
|
833 |
-
<TooltipContent side="top">
|
834 |
-
<p className="text-xs font-normal text-stone-100/90 text-center">
|
835 |
-
Clap is a new AI format,<br/>check out the academy<br/>to learn more about it.
|
836 |
-
</p>
|
837 |
-
</TooltipContent>
|
838 |
-
</Tooltip>}
|
839 |
-
|
840 |
-
|
841 |
-
|
842 |
-
{canSeeBetaFeatures &&
|
843 |
-
<Tooltip>
|
844 |
-
<TooltipTrigger asChild><Button
|
845 |
-
onClick={() => saveClap()}
|
846 |
-
disabled={!currentClap || isBusy}
|
847 |
-
// variant="ghost"
|
848 |
-
className={cn(
|
849 |
-
`text-xs md:text-sm lg:text-base`,
|
850 |
-
`bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
|
851 |
-
`font-bold`,
|
852 |
-
`hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
|
853 |
-
storyPromptDraft ? "opacity-100" : "opacity-80"
|
854 |
-
)}
|
855 |
-
>
|
856 |
-
<span className="hidden xl:inline mr-1">Save .clap</span>
|
857 |
-
<span className="inline xl:hidden mr-1">Save .clap</span>
|
858 |
-
</Button></TooltipTrigger>
|
859 |
-
<TooltipContent side="top">
|
860 |
-
<p className="text-xs font-normal text-stone-100/90 text-center">
|
861 |
-
Clap is a new AI format,<br/>check out the academy<br/>to learn more about it.
|
862 |
-
</p>
|
863 |
-
</TooltipContent>
|
864 |
-
</Tooltip>
|
865 |
-
}
|
866 |
</div>
|
867 |
|
868 |
<div className="
|
869 |
-
|
870 |
-
|
871 |
-
|
872 |
-
|
873 |
-
|
874 |
|
875 |
-
|
876 |
-
|
877 |
-
<div className="
|
878 |
flex flex-row
|
879 |
justify-between items-center
|
880 |
cursor-pointer
|
@@ -901,10 +233,15 @@ export function Main() {
|
|
901 |
>
|
902 |
<GiRollingDices size={24} />
|
903 |
</div>
|
904 |
-
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
905 |
{/* END OF RANDOMNESS SWITCH */}
|
906 |
|
907 |
-
|
908 |
{/* ORIENTATION SWITCH */}
|
909 |
<div className="
|
910 |
flex flex-row
|
@@ -965,180 +302,10 @@ export function Main() {
|
|
965 |
|
966 |
`-mt-[20px] -mb-[90px] md:-mt-0 md:-mb-0`,
|
967 |
)}>
|
968 |
-
|
969 |
-
<div className={cn(`
|
970 |
-
-mt-8 md:mt-0
|
971 |
-
transition-all duration-200 ease-in-out
|
972 |
-
`,
|
973 |
-
isLandscape
|
974 |
-
? `scale-[0.9] md:scale-[0.75] lg:scale-[0.9] xl:scale-[1.0] 2xl:scale-[1.1]`
|
975 |
-
: `scale-[0.8] md:scale-[0.9] lg:scale-[1.1]`
|
976 |
-
)}>
|
977 |
-
<DeviceFrameset
|
978 |
-
device="Nexus 5"
|
979 |
-
// color="black"
|
980 |
-
|
981 |
-
landscape={isLandscape}
|
982 |
-
|
983 |
-
// note 1: videos are generated in 1024x576 or 576x1024
|
984 |
-
// so we need to keep the same ratio here
|
985 |
-
|
986 |
-
// note 2: width and height are fixed, if width always stays 512
|
987 |
-
// that's because the landscape={} parameter will do the switch for us
|
988 |
-
|
989 |
-
width={288}
|
990 |
-
height={512}
|
991 |
-
>
|
992 |
-
<div className="
|
993 |
-
flex flex-col items-center justify-center
|
994 |
-
w-full h-full
|
995 |
-
bg-black text-white
|
996 |
-
">
|
997 |
-
{isBusy ? <div className="
|
998 |
-
flex flex-col
|
999 |
-
items-center justify-center
|
1000 |
-
text-center space-y-1.5">
|
1001 |
-
<p className="text-2xl font-bold">{progress}%</p>
|
1002 |
-
<p className="text-base text-white/70">{isBusy
|
1003 |
-
? (
|
1004 |
-
// note: some of those tasks are running in parallel,
|
1005 |
-
// and some are super-slow (like music or video)
|
1006 |
-
// by carefully selecting in which order we set the ternaries,
|
1007 |
-
// we can create the illusion that we just have a succession of reasonably-sized tasks
|
1008 |
-
storyGenerationStatus === "generating" ? "Writing story.."
|
1009 |
-
: parseGenerationStatus === "generating" ? "Loading the project.."
|
1010 |
-
: assetGenerationStatus === "generating" ? "Casting characters.."
|
1011 |
-
: imageGenerationStatus === "generating" ? "Creating storyboards.."
|
1012 |
-
: soundGenerationStatus === "generating" ? "Recording sounds.."
|
1013 |
-
: videoGenerationStatus === "generating" ? "Filming shots.."
|
1014 |
-
: musicGenerationStatus === "generating" ? "Producing music.."
|
1015 |
-
: voiceGenerationStatus === "generating" ? "Recording dialogues.."
|
1016 |
-
: finalGenerationStatus === "generating" ? "Editing final cut.."
|
1017 |
-
: "Please wait.."
|
1018 |
-
)
|
1019 |
-
: status === "error"
|
1020 |
-
? <span>{error || ""}</span>
|
1021 |
-
: <span>{error ? error : <span> </span>}</span> // to prevent layout changes
|
1022 |
-
}</p>
|
1023 |
-
</div>
|
1024 |
-
: (currentVideo && currentVideo?.length > 128) ? <video
|
1025 |
-
src={currentVideo}
|
1026 |
-
controls
|
1027 |
-
playsInline
|
1028 |
-
// I think we can't autoplay with sound,
|
1029 |
-
// so let's disable auto-play
|
1030 |
-
// autoPlay
|
1031 |
-
// muted
|
1032 |
-
loop
|
1033 |
-
className="object-cover"
|
1034 |
-
style={{
|
1035 |
-
}}
|
1036 |
-
/> : <div className="
|
1037 |
-
flex flex-col
|
1038 |
-
items-center justify-center
|
1039 |
-
text-lg text-center"></div>}
|
1040 |
-
</div>
|
1041 |
-
|
1042 |
-
<div className={cn(`
|
1043 |
-
fixed
|
1044 |
-
flex flex-row items-center justify-center
|
1045 |
-
bg-transparent
|
1046 |
-
font-sans
|
1047 |
-
-mb-0
|
1048 |
-
`,
|
1049 |
-
isLandscape ? 'h-4' : 'h-14'
|
1050 |
-
)}
|
1051 |
-
style={{ width: isPortrait ? 288 : 512 }}>
|
1052 |
-
<span className="text-stone-100/50 text-4xs"
|
1053 |
-
style={{ textShadow: "rgb(0 0 0 / 80%) 0px 0px 2px" }}>
|
1054 |
-
Powered by
|
1055 |
-
</span>
|
1056 |
-
<span className="ml-1 mr-0.5">
|
1057 |
-
<Image src={HFLogo} alt="Hugging Face" width={14} height={13} />
|
1058 |
-
</span>
|
1059 |
-
<span className="text-stone-100/80 text-3xs font-semibold"
|
1060 |
-
style={{ textShadow: "rgb(0 0 0 / 80%) 0px 0px 2px" }}>Hugging Face</span>
|
1061 |
-
|
1062 |
-
</div>
|
1063 |
-
</DeviceFrameset>
|
1064 |
-
|
1065 |
-
{(currentVideo && currentVideo.length > 128) ? <div
|
1066 |
-
className={cn(`
|
1067 |
-
w-full
|
1068 |
-
flex flex-row
|
1069 |
-
items-center justify-center
|
1070 |
-
transition-all duration-150 ease-in-out
|
1071 |
-
|
1072 |
-
text-stone-800
|
1073 |
-
|
1074 |
-
group
|
1075 |
-
pt-2 md:pt-4
|
1076 |
-
`,
|
1077 |
-
isBusy ? 'opacity-50' : 'cursor-pointer opacity-100 hover:scale-110 active:scale-150 hover:text-stone-950 active:text-black'
|
1078 |
-
)}
|
1079 |
-
style={{ textShadow: "rgb(255 255 255 / 19%) 0px 0px 2px" }}
|
1080 |
-
onClick={isBusy ? undefined : saveVideo}
|
1081 |
-
>
|
1082 |
-
<div className="
|
1083 |
-
text-base md:text-lg lg:text-xl
|
1084 |
-
transition-all duration-150 ease-out
|
1085 |
-
group-hover:animate-swing
|
1086 |
-
"><FaCloudDownloadAlt /></div>
|
1087 |
-
<div className="text-xs md:text-sm lg:text-base"> Download</div>
|
1088 |
-
</div> : null}
|
1089 |
-
</div>
|
1090 |
</div>
|
1091 |
</div>
|
1092 |
-
<
|
1093 |
-
className="
|
1094 |
-
fixed
|
1095 |
-
|
1096 |
-
left-4 md:left-8
|
1097 |
-
bottom-4
|
1098 |
-
flex flex-col md:flex-row
|
1099 |
-
md:items-center justify-center
|
1100 |
-
space-y-4 md:space-x-4 md:space-y-0
|
1101 |
-
|
1102 |
-
">
|
1103 |
-
<a
|
1104 |
-
className="
|
1105 |
-
flex
|
1106 |
-
no-underline
|
1107 |
-
animation-all duration-150 ease-in-out
|
1108 |
-
group
|
1109 |
-
text-stone-950/60 hover:text-stone-950/80 scale-95 hover:scale-100"
|
1110 |
-
href="https://discord.gg/AEruz9B92B"
|
1111 |
-
target="_blank">
|
1112 |
-
<div className="
|
1113 |
-
text-base md:text-lg lg:text-xl
|
1114 |
-
transition-all duration-150 ease-out
|
1115 |
-
group-hover:animate-swing
|
1116 |
-
"><FaDiscord /></div>
|
1117 |
-
<div className="text-xs md:text-sm lg:text-base ml-1.5">
|
1118 |
-
<span className="hidden md:block">Chat on Discord</span>
|
1119 |
-
<span className="block md:hidden">Discord</span>
|
1120 |
-
</div>
|
1121 |
-
</a>
|
1122 |
-
<a
|
1123 |
-
className="
|
1124 |
-
flex
|
1125 |
-
no-underline
|
1126 |
-
animation-all duration-150 ease-in-out
|
1127 |
-
group
|
1128 |
-
text-stone-950/60 hover:text-stone-950/80 scale-95 hover:scale-100"
|
1129 |
-
href="https://latent-store.notion.site/AI-Stories-Academy-8e3ce6ff2d5946ffadc94193619dd5cd"
|
1130 |
-
target="_blank">
|
1131 |
-
<div className="
|
1132 |
-
text-base md:text-lg lg:text-xl
|
1133 |
-
transition-all duration-150 ease-out
|
1134 |
-
group-hover:animate-swing
|
1135 |
-
"><GiSpellBook /></div>
|
1136 |
-
<div className="text-xs md:text-sm lg:text-base ml-1.5">
|
1137 |
-
<span className="hidden md:block">Prompt academy</span>
|
1138 |
-
<span className="block md:hidden">Academy</span>
|
1139 |
-
</div>
|
1140 |
-
</a>
|
1141 |
-
</div>
|
1142 |
</div>
|
1143 |
<Toaster />
|
1144 |
</div>
|
|
|
1 |
"use client"
|
2 |
|
3 |
+
import React from "react"
|
4 |
import { IoMdPhonePortrait } from "react-icons/io"
|
5 |
import { GiRollingDices } from "react-icons/gi"
|
6 |
+
import { ClapMediaOrientation } from "@aitube/clap"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
|
8 |
import { Card, CardContent, CardHeader } from "@/components/ui/card"
|
9 |
import { Button } from "@/components/ui/button"
|
10 |
import { Toaster } from "@/components/ui/sonner"
|
11 |
import { TextareaField } from "@/components/form/textarea-field"
|
|
|
12 |
|
13 |
+
import { cn, generateRandomStory } from "@/lib/utils"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
14 |
import { defaultPrompt } from "./config"
|
15 |
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"
|
16 |
+
import { useOrientation, useProgressTimer, useQueryStringParams, useStoryPromptDraft } from "@/lib/hooks"
|
17 |
+
import { BottomBar, VideoPreview } from "@/components/interface"
|
18 |
+
import { MainTitle } from "@/components/interface/main-title"
|
19 |
+
import { LoadClapButton } from "@/components/interface/load-clap-button"
|
20 |
+
import { SaveClapButton } from "@/components/interface/save-clap-button"
|
21 |
+
import { useProcessors } from "@/lib/hooks/useProcessors"
|
22 |
+
import { Characters } from "@/components/interface/characters"
|
23 |
|
24 |
export function Main() {
|
25 |
+
const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
|
26 |
+
const { isBusy } = useProgressTimer()
|
27 |
+
const { orientation, toggleOrientation } = useOrientation()
|
28 |
+
const { handleSubmit } = useProcessors()
|
29 |
+
useQueryStringParams()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
return (
|
31 |
<TooltipProvider>
|
32 |
<div className={cn(
|
|
|
95 |
|
96 |
)}>
|
97 |
<CardHeader>
|
98 |
+
<MainTitle />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
99 |
</CardHeader>
|
100 |
<CardContent
|
101 |
className="flex flex-col space-y-3"
|
|
|
189 |
*/}
|
190 |
|
191 |
<div className="
|
192 |
+
flex flex-row
|
193 |
+
justify-between items-center
|
194 |
+
space-x-3">
|
195 |
+
<LoadClapButton />
|
196 |
+
<SaveClapButton />
|
197 |
+
<Characters />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
198 |
</div>
|
199 |
|
200 |
<div className="
|
201 |
+
flex flex-row
|
202 |
+
justify-between items-center
|
203 |
+
space-x-3
|
204 |
+
select-none
|
205 |
+
">
|
206 |
|
207 |
+
{/* RANDOMNESS SWITCH */}
|
208 |
+
<Tooltip>
|
209 |
+
<TooltipTrigger asChild><div className="
|
210 |
flex flex-row
|
211 |
justify-between items-center
|
212 |
cursor-pointer
|
|
|
233 |
>
|
234 |
<GiRollingDices size={24} />
|
235 |
</div>
|
236 |
+
</div></TooltipTrigger>
|
237 |
+
<TooltipContent side="top">
|
238 |
+
<p className="text-xs font-normal text-stone-100/90 text-center">
|
239 |
+
Generate a random prompt.
|
240 |
+
</p>
|
241 |
+
</TooltipContent>
|
242 |
+
</Tooltip>
|
243 |
{/* END OF RANDOMNESS SWITCH */}
|
244 |
|
|
|
245 |
{/* ORIENTATION SWITCH */}
|
246 |
<div className="
|
247 |
flex flex-row
|
|
|
302 |
|
303 |
`-mt-[20px] -mb-[90px] md:-mt-0 md:-mb-0`,
|
304 |
)}>
|
305 |
+
<VideoPreview />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
306 |
</div>
|
307 |
</div>
|
308 |
+
<BottomBar />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
309 |
</div>
|
310 |
<Toaster />
|
311 |
</div>
|
src/components/interface/bottom-bar.tsx
ADDED
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"use client"
|
2 |
+
|
3 |
+
import React from "react"
|
4 |
+
import { FaDiscord } from "react-icons/fa"
|
5 |
+
import { GiSpellBook } from "react-icons/gi"
|
6 |
+
|
7 |
+
export function BottomBar() {
|
8 |
+
|
9 |
+
return (
|
10 |
+
<div
|
11 |
+
className="
|
12 |
+
fixed
|
13 |
+
|
14 |
+
left-4 md:left-8
|
15 |
+
bottom-4
|
16 |
+
flex flex-col md:flex-row
|
17 |
+
md:items-center justify-center
|
18 |
+
space-y-4 md:space-x-4 md:space-y-0
|
19 |
+
|
20 |
+
">
|
21 |
+
<a
|
22 |
+
className="
|
23 |
+
flex
|
24 |
+
no-underline
|
25 |
+
animation-all duration-150 ease-in-out
|
26 |
+
group
|
27 |
+
text-stone-950/60 hover:text-stone-950/80 scale-95 hover:scale-100"
|
28 |
+
href="https://discord.gg/AEruz9B92B"
|
29 |
+
target="_blank">
|
30 |
+
<div className="
|
31 |
+
text-base md:text-lg lg:text-xl
|
32 |
+
transition-all duration-150 ease-out
|
33 |
+
group-hover:animate-swing
|
34 |
+
"><FaDiscord /></div>
|
35 |
+
<div className="text-xs md:text-sm lg:text-base ml-1.5">
|
36 |
+
<span className="hidden md:block">Chat on Discord</span>
|
37 |
+
<span className="block md:hidden">Discord</span>
|
38 |
+
</div>
|
39 |
+
</a>
|
40 |
+
<a
|
41 |
+
className="
|
42 |
+
flex
|
43 |
+
no-underline
|
44 |
+
animation-all duration-150 ease-in-out
|
45 |
+
group
|
46 |
+
text-stone-950/60 hover:text-stone-950/80 scale-95 hover:scale-100"
|
47 |
+
href="https://latent-store.notion.site/AI-Stories-Academy-8e3ce6ff2d5946ffadc94193619dd5cd"
|
48 |
+
target="_blank">
|
49 |
+
<div className="
|
50 |
+
text-base md:text-lg lg:text-xl
|
51 |
+
transition-all duration-150 ease-out
|
52 |
+
group-hover:animate-swing
|
53 |
+
"><GiSpellBook /></div>
|
54 |
+
<div className="text-xs md:text-sm lg:text-base ml-1.5">
|
55 |
+
<span className="hidden md:block">Prompt academy</span>
|
56 |
+
<span className="block md:hidden">Academy</span>
|
57 |
+
</div>
|
58 |
+
</a>
|
59 |
+
</div>
|
60 |
+
);
|
61 |
+
}
|
src/components/interface/character-button.tsx
ADDED
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { ClapEntity } from "@aitube/clap"
|
2 |
+
import { HiCog6Tooth } from "react-icons/hi2"
|
3 |
+
import { MdFace2 } from "react-icons/md"
|
4 |
+
import { FaWrench } from "react-icons/fa6"
|
5 |
+
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
|
6 |
+
import { useEntityPicture } from "@/lib/hooks/useEntityPicture"
|
7 |
+
import { cn } from "@/lib/utils"
|
8 |
+
|
9 |
+
export function CharacterButton({
|
10 |
+
entity
|
11 |
+
}: {
|
12 |
+
entity?: ClapEntity
|
13 |
+
}) {
|
14 |
+
const { picture, openFilePicker } = useEntityPicture(entity)
|
15 |
+
|
16 |
+
return (
|
17 |
+
<Tooltip>
|
18 |
+
<TooltipTrigger asChild><div className={cn(`
|
19 |
+
flex flex-row
|
20 |
+
|
21 |
+
justify-between items-center
|
22 |
+
cursor-pointer
|
23 |
+
transition-all duration-150 ease-in-out
|
24 |
+
hover:scale-110 active:scale-150
|
25 |
+
text-stone-800
|
26 |
+
hover:text-stone-950
|
27 |
+
active:text-black
|
28 |
+
group
|
29 |
+
`,
|
30 |
+
// picture ? ` opacity-100 ` : `opacity-60`
|
31 |
+
)}
|
32 |
+
onClick={openFilePicker}>
|
33 |
+
|
34 |
+
{picture
|
35 |
+
? <div className={cn(`
|
36 |
+
flex
|
37 |
+
w-10 h-10 rounded-full
|
38 |
+
overflow-hidden
|
39 |
+
`,
|
40 |
+
picture
|
41 |
+
? `
|
42 |
+
border-2 group-hover:border
|
43 |
+
border-stone-950/70 dark:border-stone-950/70
|
44 |
+
group-hover:shadow-md
|
45 |
+
`
|
46 |
+
: ``
|
47 |
+
)}>
|
48 |
+
<img
|
49 |
+
className="object-cover"
|
50 |
+
src={picture}
|
51 |
+
/>
|
52 |
+
</div>: <MdFace2 size={24} />}
|
53 |
+
|
54 |
+
<div className="
|
55 |
+
-ml-2 mt-10
|
56 |
+
flex flex-row items-center justify-center
|
57 |
+
transition-all duration-150 ease-out
|
58 |
+
group-hover:animate-swing
|
59 |
+
opacity-0 group-hover:opacity-100
|
60 |
+
scale-0 group-hover:scale-100
|
61 |
+
"
|
62 |
+
>
|
63 |
+
{picture
|
64 |
+
? <FaWrench size={16} />
|
65 |
+
: null
|
66 |
+
}
|
67 |
+
</div>
|
68 |
+
</div></TooltipTrigger>
|
69 |
+
<TooltipContent side="top">
|
70 |
+
<p className="text-xs font-normal text-stone-100/90 text-center">
|
71 |
+
Using this experimental<br/>feature may reduce<br/>
|
72 |
+
the quality of images.
|
73 |
+
</p>
|
74 |
+
</TooltipContent>
|
75 |
+
</Tooltip>
|
76 |
+
)
|
77 |
+
}
|
src/components/interface/characters.tsx
ADDED
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useStore } from "@/app/store"
|
2 |
+
import { CharacterButton } from "./character-button"
|
3 |
+
|
4 |
+
export function Characters() {
|
5 |
+
const currentClap = useStore(s => s.currentClap)
|
6 |
+
|
7 |
+
|
8 |
+
return (
|
9 |
+
<div className="flex flex-row space-x-0">
|
10 |
+
{currentClap && currentClap.entities?.length > 0
|
11 |
+
// now: we only support displaying ONE entity for now
|
12 |
+
? currentClap.entities.slice(0, 1).map(entity =>
|
13 |
+
<CharacterButton key={entity.id} entity={entity} />
|
14 |
+
)
|
15 |
+
: <CharacterButton />
|
16 |
+
}
|
17 |
+
</div>
|
18 |
+
)
|
19 |
+
}
|
src/components/interface/index.ts
ADDED
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
1 |
+
export { VideoPreview } from "./video-preview"
|
2 |
+
export { BottomBar } from "./bottom-bar"
|
src/components/interface/load-clap-button.tsx
ADDED
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"use client"
|
2 |
+
|
3 |
+
import React from "react"
|
4 |
+
|
5 |
+
import { Button } from "@/components/ui/button"
|
6 |
+
|
7 |
+
import { cn } from "@/lib/utils"
|
8 |
+
|
9 |
+
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
|
10 |
+
import { useImportClap, useIsBusy, useStoryPromptDraft } from "@/lib/hooks"
|
11 |
+
|
12 |
+
export function LoadClapButton() {
|
13 |
+
const { isBusy } = useIsBusy()
|
14 |
+
const { openClapFilePicker } = useImportClap()
|
15 |
+
const { storyPromptDraft } = useStoryPromptDraft()
|
16 |
+
|
17 |
+
return (
|
18 |
+
<Tooltip>
|
19 |
+
<TooltipTrigger asChild><Button
|
20 |
+
|
21 |
+
onClick={openClapFilePicker}
|
22 |
+
disabled={isBusy}
|
23 |
+
// variant="ghost"
|
24 |
+
className={cn(
|
25 |
+
`text-xs md:text-sm lg:text-base`,
|
26 |
+
`bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
|
27 |
+
`font-bold`,
|
28 |
+
`hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
|
29 |
+
storyPromptDraft ? "opacity-100" : "opacity-80"
|
30 |
+
)}
|
31 |
+
>
|
32 |
+
<span className="hidden xl:inline mr-1">Load .clap</span>
|
33 |
+
<span className="inline xl:hidden mr-1">Load .clap</span>
|
34 |
+
</Button></TooltipTrigger>
|
35 |
+
<TooltipContent side="top">
|
36 |
+
<p className="text-xs font-normal text-stone-100/90 text-center">
|
37 |
+
Clap is a new AI format,<br/>check out the academy<br/>to learn more about it.
|
38 |
+
</p>
|
39 |
+
</TooltipContent>
|
40 |
+
</Tooltip>
|
41 |
+
)
|
42 |
+
}
|
src/components/interface/main-title.tsx
ADDED
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
export function MainTitle() {
|
2 |
+
return (
|
3 |
+
<div className="flex flex-col justify-start">
|
4 |
+
<div className="
|
5 |
+
flex flex-row
|
6 |
+
items-center justify-center
|
7 |
+
transition-all duration-200 ease-in-out
|
8 |
+
px-3
|
9 |
+
|
10 |
+
rounded-full">
|
11 |
+
<div
|
12 |
+
className="
|
13 |
+
flex
|
14 |
+
transition-all duration-200 ease-in-out
|
15 |
+
items-center justify-center text-center
|
16 |
+
w-10 h-10 md:w-12 md:h-12 lg:w-16 lg:h-16
|
17 |
+
text-3xl md:text-4xl lg:text-5xl
|
18 |
+
rounded-lg
|
19 |
+
mr-2
|
20 |
+
font-sans
|
21 |
+
bg-amber-400 dark:bg-amber-400
|
22 |
+
|
23 |
+
text-stone-950/80 dark:text-stone-950/80 font-bold
|
24 |
+
"
|
25 |
+
>AI</div>
|
26 |
+
<div
|
27 |
+
className="
|
28 |
+
transition-all duration-200 ease-in-out
|
29 |
+
text-amber-400 dark:text-amber-400
|
30 |
+
text-3xl md:text-4xl lg:text-5xl
|
31 |
+
"
|
32 |
+
style={{ textShadow: "#00000035 0px 0px 2px" }}
|
33 |
+
|
34 |
+
/*
|
35 |
+
className="
|
36 |
+
text-5xl
|
37 |
+
bg-gradient-to-br from-yellow-300 to-yellow-500
|
38 |
+
inline-block text-transparent bg-clip-text
|
39 |
+
py-6
|
40 |
+
"
|
41 |
+
*/
|
42 |
+
>Stories Factory</div>
|
43 |
+
</div>
|
44 |
+
<p
|
45 |
+
className="
|
46 |
+
transition-all duration-200 ease-in-out
|
47 |
+
text-stone-900/90 dark:text-stone-900/90
|
48 |
+
text-lg md:text-xl lg:text-2xl
|
49 |
+
text-center
|
50 |
+
pt-2 md:pt-4
|
51 |
+
"
|
52 |
+
style={{ textShadow: "rgb(255 255 255 / 19%) 0px 0px 2px" }}
|
53 |
+
>Make video stories using AI ✨</p>
|
54 |
+
</div>
|
55 |
+
)
|
56 |
+
}
|
src/components/interface/save-clap-button.tsx
ADDED
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"use client"
|
2 |
+
|
3 |
+
import React from "react"
|
4 |
+
|
5 |
+
import { Button } from "@/components/ui/button"
|
6 |
+
|
7 |
+
import { useStore } from "../../app/store"
|
8 |
+
import { cn } from "@/lib/utils"
|
9 |
+
|
10 |
+
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
|
11 |
+
import { useIsBusy, useStoryPromptDraft } from "@/lib/hooks"
|
12 |
+
|
13 |
+
export function SaveClapButton() {
|
14 |
+
const { isBusy } = useIsBusy()
|
15 |
+
const currentClap = useStore(s => s.currentClap)
|
16 |
+
const saveClap = useStore(s => s.saveClap)
|
17 |
+
const { storyPromptDraft } = useStoryPromptDraft()
|
18 |
+
|
19 |
+
return (
|
20 |
+
<Tooltip>
|
21 |
+
<TooltipTrigger asChild><Button
|
22 |
+
onClick={() => saveClap()}
|
23 |
+
disabled={!currentClap || isBusy}
|
24 |
+
// variant="ghost"
|
25 |
+
className={cn(
|
26 |
+
`text-xs md:text-sm lg:text-base`,
|
27 |
+
`bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
|
28 |
+
`font-bold`,
|
29 |
+
`hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
|
30 |
+
storyPromptDraft ? "opacity-100" : "opacity-80"
|
31 |
+
)}
|
32 |
+
>
|
33 |
+
<span className="hidden xl:inline mr-1">Save .clap</span>
|
34 |
+
<span className="inline xl:hidden mr-1">Save .clap</span>
|
35 |
+
</Button></TooltipTrigger>
|
36 |
+
<TooltipContent side="top">
|
37 |
+
<p className="text-xs font-normal text-stone-100/90 text-center">
|
38 |
+
Clap is a new AI format,<br/>check out the academy<br/>to learn more about it.
|
39 |
+
</p>
|
40 |
+
</TooltipContent>
|
41 |
+
</Tooltip>
|
42 |
+
)
|
43 |
+
}
|
src/components/interface/video-preview.tsx
ADDED
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"use client"
|
2 |
+
|
3 |
+
import React from "react"
|
4 |
+
import { FaCloudDownloadAlt } from "react-icons/fa"
|
5 |
+
import Image from "next/image"
|
6 |
+
import { DeviceFrameset } from "react-device-frameset"
|
7 |
+
import "react-device-frameset/styles/marvel-devices.min.css"
|
8 |
+
|
9 |
+
import { useOrientation } from "@/lib/hooks/useOrientation"
|
10 |
+
import { useProgressTimer } from "@/lib/hooks/useProgressTimer"
|
11 |
+
import { cn } from "@/lib/utils/cn"
|
12 |
+
|
13 |
+
import { useStore } from "../../app/store"
|
14 |
+
import HFLogo from "../../app/hf-logo.svg"
|
15 |
+
|
16 |
+
export function VideoPreview() {
|
17 |
+
const status = useStore(s => s.status)
|
18 |
+
const parseGenerationStatus = useStore(s => s.parseGenerationStatus)
|
19 |
+
const storyGenerationStatus = useStore(s => s.storyGenerationStatus)
|
20 |
+
const assetGenerationStatus = useStore(s => s.assetGenerationStatus)
|
21 |
+
const soundGenerationStatus = useStore(s => s.soundGenerationStatus)
|
22 |
+
const musicGenerationStatus = useStore(s => s.musicGenerationStatus)
|
23 |
+
const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus)
|
24 |
+
const imageGenerationStatus = useStore(s => s.imageGenerationStatus)
|
25 |
+
const videoGenerationStatus = useStore(s => s.videoGenerationStatus)
|
26 |
+
const finalGenerationStatus = useStore(s => s.finalGenerationStatus)
|
27 |
+
const currentVideo = useStore(s => s.currentVideo)
|
28 |
+
|
29 |
+
const error = useStore(s => s.error)
|
30 |
+
|
31 |
+
const saveVideo = useStore(s => s.saveVideo)
|
32 |
+
|
33 |
+
|
34 |
+
const { isBusy, progress } = useProgressTimer()
|
35 |
+
|
36 |
+
const {
|
37 |
+
isLandscape,
|
38 |
+
isPortrait,
|
39 |
+
} = useOrientation()
|
40 |
+
|
41 |
+
return (
|
42 |
+
<div className={cn(`
|
43 |
+
-mt-8 md:mt-0
|
44 |
+
transition-all duration-200 ease-in-out
|
45 |
+
`,
|
46 |
+
isLandscape
|
47 |
+
? `scale-[0.9] md:scale-[0.75] lg:scale-[0.9] xl:scale-[1.0] 2xl:scale-[1.1]`
|
48 |
+
: `scale-[0.8] md:scale-[0.9] lg:scale-[1.1]`
|
49 |
+
)}>
|
50 |
+
<DeviceFrameset
|
51 |
+
device="Nexus 5"
|
52 |
+
// color="black"
|
53 |
+
|
54 |
+
landscape={isLandscape}
|
55 |
+
|
56 |
+
// note 1: videos are generated in 1024x576 or 576x1024
|
57 |
+
// so we need to keep the same ratio here
|
58 |
+
|
59 |
+
// note 2: width and height are fixed, if width always stays 512
|
60 |
+
// that's because the landscape={} parameter will do the switch for us
|
61 |
+
|
62 |
+
width={288}
|
63 |
+
height={512}
|
64 |
+
>
|
65 |
+
<div className="
|
66 |
+
flex flex-col items-center justify-center
|
67 |
+
w-full h-full
|
68 |
+
bg-black text-white
|
69 |
+
">
|
70 |
+
{isBusy ? <div className="
|
71 |
+
flex flex-col
|
72 |
+
items-center justify-center
|
73 |
+
text-center space-y-1.5">
|
74 |
+
<p className="text-2xl font-bold">{progress}%</p>
|
75 |
+
<p className="text-base text-white/70">{isBusy
|
76 |
+
? (
|
77 |
+
// note: some of those tasks are running in parallel,
|
78 |
+
// and some are super-slow (like music or video)
|
79 |
+
// by carefully selecting in which order we set the ternaries,
|
80 |
+
// we can create the illusion that we just have a succession of reasonably-sized tasks
|
81 |
+
storyGenerationStatus === "generating" ? "Writing story.."
|
82 |
+
: parseGenerationStatus === "generating" ? "Loading the project.."
|
83 |
+
: assetGenerationStatus === "generating" ? "Casting characters.."
|
84 |
+
: imageGenerationStatus === "generating" ? "Creating storyboards.."
|
85 |
+
: soundGenerationStatus === "generating" ? "Recording sounds.."
|
86 |
+
: videoGenerationStatus === "generating" ? "Filming shots.."
|
87 |
+
: musicGenerationStatus === "generating" ? "Producing music.."
|
88 |
+
: voiceGenerationStatus === "generating" ? "Recording dialogues.."
|
89 |
+
: finalGenerationStatus === "generating" ? "Editing final cut.."
|
90 |
+
: "Please wait.."
|
91 |
+
)
|
92 |
+
: status === "error"
|
93 |
+
? <span>{error || ""}</span>
|
94 |
+
: <span>{error ? error : <span> </span>}</span> // to prevent layout changes
|
95 |
+
}</p>
|
96 |
+
</div>
|
97 |
+
: (currentVideo && currentVideo?.length > 128) ? <video
|
98 |
+
src={currentVideo}
|
99 |
+
controls
|
100 |
+
playsInline
|
101 |
+
// I think we can't autoplay with sound,
|
102 |
+
// so let's disable auto-play
|
103 |
+
// autoPlay
|
104 |
+
// muted
|
105 |
+
loop
|
106 |
+
className="object-cover"
|
107 |
+
style={{
|
108 |
+
}}
|
109 |
+
/> : <div className="
|
110 |
+
flex flex-col
|
111 |
+
items-center justify-center
|
112 |
+
text-lg text-center"></div>}
|
113 |
+
</div>
|
114 |
+
|
115 |
+
<div className={cn(`
|
116 |
+
fixed
|
117 |
+
flex flex-row items-center justify-center
|
118 |
+
bg-transparent
|
119 |
+
font-sans
|
120 |
+
-mb-0
|
121 |
+
`,
|
122 |
+
isLandscape ? 'h-4' : 'h-14'
|
123 |
+
)}
|
124 |
+
style={{ width: isPortrait ? 288 : 512 }}>
|
125 |
+
<span className="text-stone-100/50 text-4xs"
|
126 |
+
style={{ textShadow: "rgb(0 0 0 / 80%) 0px 0px 2px" }}>
|
127 |
+
Powered by
|
128 |
+
</span>
|
129 |
+
<span className="ml-1 mr-0.5">
|
130 |
+
<Image src={HFLogo} alt="Hugging Face" width={14} height={13} />
|
131 |
+
</span>
|
132 |
+
<span className="text-stone-100/80 text-3xs font-semibold"
|
133 |
+
style={{ textShadow: "rgb(0 0 0 / 80%) 0px 0px 2px" }}>Hugging Face</span>
|
134 |
+
|
135 |
+
</div>
|
136 |
+
</DeviceFrameset>
|
137 |
+
|
138 |
+
{(currentVideo && currentVideo.length > 128) ? <div
|
139 |
+
className={cn(`
|
140 |
+
w-full
|
141 |
+
flex flex-row
|
142 |
+
items-center justify-center
|
143 |
+
transition-all duration-150 ease-in-out
|
144 |
+
|
145 |
+
text-stone-800
|
146 |
+
|
147 |
+
group
|
148 |
+
pt-2 md:pt-4
|
149 |
+
`,
|
150 |
+
isBusy ? 'opacity-50' : 'cursor-pointer opacity-100 hover:scale-110 active:scale-150 hover:text-stone-950 active:text-black'
|
151 |
+
)}
|
152 |
+
style={{ textShadow: "rgb(255 255 255 / 19%) 0px 0px 2px" }}
|
153 |
+
onClick={isBusy ? undefined : saveVideo}
|
154 |
+
>
|
155 |
+
<div className="
|
156 |
+
text-base md:text-lg lg:text-xl
|
157 |
+
transition-all duration-150 ease-out
|
158 |
+
group-hover:animate-swing
|
159 |
+
"><FaCloudDownloadAlt /></div>
|
160 |
+
<div className="text-xs md:text-sm lg:text-base"> Download</div>
|
161 |
+
</div> : null}
|
162 |
+
</div>
|
163 |
+
)
|
164 |
+
}
|
src/lib/hooks/index.tsx
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
export { useImportClap } from "./useImportClap"
|
2 |
+
export { useIsBusy } from "./useIsBusy"
|
3 |
+
export { useOrientation } from "./useOrientation"
|
4 |
+
export { useProgressTimer } from "./useProgressTimer"
|
5 |
+
export { useQueryStringParams } from "./useQueryStringParams"
|
6 |
+
export { useStoryPromptDraft } from "./useStoryPromptDraft"
|
src/lib/hooks/useEntityPicture.ts
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useEffect, useState } from "react"
|
2 |
+
import { useFilePicker } from "use-file-picker"
|
3 |
+
|
4 |
+
import { useStore } from "../../app/store"
|
5 |
+
import { ClapEntity, getClapAssetSourceType } from "@aitube/clap"
|
6 |
+
import { useOpenPictureFile } from "./useOpenPictureFile"
|
7 |
+
|
8 |
+
export function useEntityPicture(entity?: ClapEntity) {
|
9 |
+
const defaultPicture = entity?.imageId
|
10 |
+
const [currentPicture, setCurrentPicture] = useState(defaultPicture)
|
11 |
+
const { file, openFilePicker } = useOpenPictureFile()
|
12 |
+
const setMainCharacterImage = useStore(s => s.setMainCharacterImage)
|
13 |
+
|
14 |
+
const newPicture = file || currentPicture || defaultPicture
|
15 |
+
|
16 |
+
useEffect(() => {
|
17 |
+
setCurrentPicture(newPicture)
|
18 |
+
if (!newPicture) { return }
|
19 |
+
|
20 |
+
if (entity) {
|
21 |
+
entity.imageId = newPicture
|
22 |
+
} else {
|
23 |
+
setMainCharacterImage(newPicture)
|
24 |
+
}
|
25 |
+
}, [newPicture, JSON.stringify(entity)])
|
26 |
+
|
27 |
+
return { picture: newPicture, openFilePicker }
|
28 |
+
}
|
src/lib/hooks/useImportClap.ts
ADDED
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useEffect, useRef } from "react"
|
2 |
+
import { useFilePicker } from "use-file-picker"
|
3 |
+
import { FileContent } from "use-file-picker/dist/interfaces"
|
4 |
+
import { ClapProject, updateClap } from "@aitube/clap"
|
5 |
+
|
6 |
+
import { useStore } from "../../app/store"
|
7 |
+
import { useProcessors } from "./useProcessors"
|
8 |
+
|
9 |
+
export function useImportClap() {
|
10 |
+
|
11 |
+
const setError = useStore(s => s.setError)
|
12 |
+
const setCurrentClap = useStore(s => s.setCurrentClap)
|
13 |
+
const loadClap = useStore(s => s.loadClap)
|
14 |
+
|
15 |
+
const {
|
16 |
+
generateMusic,
|
17 |
+
generateVideos,
|
18 |
+
generateFinalVideo,
|
19 |
+
} = useProcessors()
|
20 |
+
|
21 |
+
const importStory = async (fileData: FileContent<ArrayBuffer>): Promise<{
|
22 |
+
clap: ClapProject
|
23 |
+
regenerateVideo: boolean
|
24 |
+
}> => {
|
25 |
+
if (!fileData?.name) { throw new Error(`invalid file (missing file name)`) }
|
26 |
+
|
27 |
+
const {
|
28 |
+
setParseGenerationStatus,
|
29 |
+
} = useStore.getState()
|
30 |
+
|
31 |
+
setParseGenerationStatus("generating")
|
32 |
+
|
33 |
+
try {
|
34 |
+
const blob = new Blob([fileData.content])
|
35 |
+
const res = await loadClap(blob, fileData.name)
|
36 |
+
|
37 |
+
if (!res?.clap) { throw new Error(`failed to load the clap file`) }
|
38 |
+
|
39 |
+
setParseGenerationStatus("finished")
|
40 |
+
return res
|
41 |
+
} catch (err) {
|
42 |
+
console.error("failed to load the Clap file:", err)
|
43 |
+
setParseGenerationStatus("error")
|
44 |
+
throw err
|
45 |
+
}
|
46 |
+
}
|
47 |
+
|
48 |
+
const { openFilePicker: openClapFilePicker, filesContent } = useFilePicker({
|
49 |
+
accept: '.clap',
|
50 |
+
readAs: "ArrayBuffer"
|
51 |
+
})
|
52 |
+
const fileData = filesContent[0]
|
53 |
+
|
54 |
+
useEffect(() => {
|
55 |
+
const fn = async () => {
|
56 |
+
if (!fileData?.name) { return }
|
57 |
+
|
58 |
+
const { setStatus, setProgress } = useStore.getState()
|
59 |
+
|
60 |
+
setProgress(0)
|
61 |
+
setStatus("generating")
|
62 |
+
|
63 |
+
try {
|
64 |
+
let { clap, regenerateVideo } = await importStory(fileData)
|
65 |
+
|
66 |
+
// clap = await generateSounds(clap)
|
67 |
+
|
68 |
+
// setCurrentClap(clap)
|
69 |
+
|
70 |
+
console.log("loadClap(): clap = ", clap)
|
71 |
+
|
72 |
+
// it is important to skip regeneration if we already have a video
|
73 |
+
if (regenerateVideo) {
|
74 |
+
console.log(`regenerating music and videos..`)
|
75 |
+
const claps = await Promise.all([
|
76 |
+
generateMusic(clap),
|
77 |
+
generateVideos(clap)
|
78 |
+
])
|
79 |
+
|
80 |
+
// console.log("finished processing the 2 tasks in parallel")
|
81 |
+
|
82 |
+
for (const newerClap of claps) {
|
83 |
+
clap = await updateClap(clap, newerClap, {
|
84 |
+
overwriteMeta: false,
|
85 |
+
inlineReplace: true,
|
86 |
+
})
|
87 |
+
}
|
88 |
+
|
89 |
+
|
90 |
+
setCurrentClap(clap)
|
91 |
+
|
92 |
+
await generateFinalVideo(clap)
|
93 |
+
|
94 |
+
} else {
|
95 |
+
console.log(`skipping music and video regeneration`)
|
96 |
+
}
|
97 |
+
|
98 |
+
setStatus("finished")
|
99 |
+
setProgress(100)
|
100 |
+
setError("")
|
101 |
+
} catch (err) {
|
102 |
+
console.error(`failed to import: `, err)
|
103 |
+
setStatus("error")
|
104 |
+
setError(`${err}`)
|
105 |
+
}
|
106 |
+
|
107 |
+
}
|
108 |
+
fn()
|
109 |
+
}, [fileData?.name])
|
110 |
+
|
111 |
+
return { openClapFilePicker }
|
112 |
+
}
|
src/lib/hooks/useIsBusy.ts
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useRef } from "react"
|
2 |
+
|
3 |
+
import { useStore } from "@/app/store"
|
4 |
+
|
5 |
+
export function useIsBusy() {
|
6 |
+
const isBusy = useStore(s => s.isBusy)
|
7 |
+
const busyRef = useRef(isBusy)
|
8 |
+
busyRef.current = isBusy
|
9 |
+
|
10 |
+
return {isBusy, busyRef }
|
11 |
+
}
|
src/lib/hooks/useOpenPictureFile.ts
ADDED
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useEffect, useState } from "react"
|
2 |
+
import { useFilePicker } from "use-file-picker"
|
3 |
+
|
4 |
+
export function useOpenPictureFile() {
|
5 |
+
const [picture, setPicture] = useState("")
|
6 |
+
|
7 |
+
const { openFilePicker, filesContent } = useFilePicker({
|
8 |
+
readAs: 'DataURL',
|
9 |
+
accept: 'image/*',
|
10 |
+
})
|
11 |
+
const fileData = filesContent[0]
|
12 |
+
|
13 |
+
useEffect(() => {
|
14 |
+
if (!fileData?.name) { return }
|
15 |
+
setPicture(fileData.content)
|
16 |
+
}, [fileData?.name])
|
17 |
+
|
18 |
+
return { file: picture, openFilePicker }
|
19 |
+
}
|
src/lib/hooks/useOrientation.ts
ADDED
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useStore } from "@/app/store"
|
2 |
+
import { ClapMediaOrientation } from "@aitube/clap"
|
3 |
+
|
4 |
+
export function useOrientation() {
|
5 |
+
const orientation = useStore(s => s.orientation)
|
6 |
+
const setOrientation = useStore(s => s.setOrientation)
|
7 |
+
const currentVideoOrientation = useStore(s => s.currentVideoOrientation)
|
8 |
+
const toggleOrientation = useStore(s => s.toggleOrientation)
|
9 |
+
// note: we are interested in the *current* video orientation,
|
10 |
+
// not the requested video orientation requested for the next video
|
11 |
+
const isLandscape = currentVideoOrientation === ClapMediaOrientation.LANDSCAPE
|
12 |
+
const isPortrait = currentVideoOrientation === ClapMediaOrientation.PORTRAIT
|
13 |
+
const isSquare = currentVideoOrientation === ClapMediaOrientation.SQUARE
|
14 |
+
|
15 |
+
return {
|
16 |
+
orientation,
|
17 |
+
setOrientation,
|
18 |
+
currentVideoOrientation,
|
19 |
+
toggleOrientation,
|
20 |
+
isLandscape,
|
21 |
+
isPortrait,
|
22 |
+
isSquare
|
23 |
+
}
|
24 |
+
}
|
src/lib/hooks/useProcessors.ts
ADDED
@@ -0,0 +1,447 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"use client"
|
2 |
+
|
3 |
+
import React, { useTransition } from "react"
|
4 |
+
import { ClapProject, ClapSegmentCategory, getClapAssetSourceType, newEntity, updateClap } from "@aitube/clap"
|
5 |
+
|
6 |
+
import { logImage } from "@/lib/utils"
|
7 |
+
import { useIsBusy, useStoryPromptDraft } from "@/lib/hooks"
|
8 |
+
|
9 |
+
import { createClap } from "@/app/server/aitube/createClap"
|
10 |
+
import { editClapEntities } from "@/app/server/aitube/editClapEntities"
|
11 |
+
import { editClapDialogues } from "@/app/server/aitube/editClapDialogues"
|
12 |
+
import { editClapStoryboards } from "@/app/server/aitube/editClapStoryboards"
|
13 |
+
import { editClapSounds } from "@/app/server/aitube/editClapSounds"
|
14 |
+
import { editClapMusic } from "@/app/server/aitube/editClapMusic"
|
15 |
+
import { editClapVideos } from "@/app/server/aitube/editClapVideos"
|
16 |
+
import { exportClapToVideo } from "@/app/server/aitube/exportClapToVideo"
|
17 |
+
|
18 |
+
import { useStore } from "../../app/store"
|
19 |
+
|
20 |
+
export function useProcessors() {
|
21 |
+
const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
|
22 |
+
|
23 |
+
const [_isPending, startTransition] = useTransition()
|
24 |
+
|
25 |
+
const mainCharacterImage = useStore(s => s.mainCharacterImage)
|
26 |
+
const mainCharacterVoice = useStore(s => s.mainCharacterVoice)
|
27 |
+
|
28 |
+
const currentClap = useStore(s => s.currentClap)
|
29 |
+
const setStoryPrompt = useStore(s => s.setStoryPrompt)
|
30 |
+
const setMainCharacterImage = useStore(s => s.setMainCharacterImage)
|
31 |
+
const setMainCharacterVoice = useStore(s => s.setMainCharacterVoice)
|
32 |
+
const setStatus = useStore(s => s.setStatus)
|
33 |
+
|
34 |
+
const error = useStore(s => s.error)
|
35 |
+
const setError = useStore(s => s.setError)
|
36 |
+
const setParseGenerationStatus = useStore(s => s.setParseGenerationStatus)
|
37 |
+
const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus)
|
38 |
+
const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus)
|
39 |
+
const setSoundGenerationStatus = useStore(s => s.setSoundGenerationStatus)
|
40 |
+
const setMusicGenerationStatus = useStore(s => s.setMusicGenerationStatus)
|
41 |
+
const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus)
|
42 |
+
const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus)
|
43 |
+
const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus)
|
44 |
+
const setFinalGenerationStatus = useStore(s => s.setFinalGenerationStatus)
|
45 |
+
const setCurrentClap = useStore(s => s.setCurrentClap)
|
46 |
+
const setCurrentVideo = useStore(s => s.setCurrentVideo)
|
47 |
+
const setProgress = useStore(s => s.setProgress)
|
48 |
+
|
49 |
+
const { isBusy, busyRef } = useIsBusy()
|
50 |
+
|
51 |
+
const generateStory = async (): Promise<ClapProject> => {
|
52 |
+
|
53 |
+
let clap: ClapProject | undefined = undefined
|
54 |
+
try {
|
55 |
+
setProgress(0)
|
56 |
+
|
57 |
+
setStatus("generating")
|
58 |
+
setStoryGenerationStatus("generating")
|
59 |
+
setStoryPrompt(promptDraftRef.current)
|
60 |
+
|
61 |
+
clap = await createClap({
|
62 |
+
prompt: promptDraftRef.current,
|
63 |
+
orientation: useStore.getState().orientation,
|
64 |
+
|
65 |
+
turbo: false,
|
66 |
+
})
|
67 |
+
|
68 |
+
if (!clap) { throw new Error(`failed to create the clap`) }
|
69 |
+
|
70 |
+
if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) }
|
71 |
+
|
72 |
+
console.log(`handleSubmit(): received a clap = `, clap)
|
73 |
+
|
74 |
+
console.log(`handleSubmit(): copying over entities from the previous clap`)
|
75 |
+
|
76 |
+
console.log(`handleSubmit(): later we can add button(s) to clear the project and/or the character(s)`)
|
77 |
+
const { currentClap } = useStore.getState()
|
78 |
+
|
79 |
+
clap.entities = Array.isArray(currentClap?.entities) ? currentClap.entities : []
|
80 |
+
|
81 |
+
setCurrentClap(clap)
|
82 |
+
setStoryGenerationStatus("finished")
|
83 |
+
|
84 |
+
console.log("---------------- GENERATED STORY ----------------")
|
85 |
+
console.table(clap.segments, [
|
86 |
+
// 'startTimeInMs',
|
87 |
+
'endTimeInMs',
|
88 |
+
// 'track',
|
89 |
+
'category',
|
90 |
+
'prompt'
|
91 |
+
])
|
92 |
+
return clap
|
93 |
+
} catch (err) {
|
94 |
+
setStoryGenerationStatus("error")
|
95 |
+
throw err
|
96 |
+
}
|
97 |
+
}
|
98 |
+
|
99 |
+
const generateEntities = async (clap: ClapProject): Promise<ClapProject> => {
|
100 |
+
try {
|
101 |
+
// setProgress(20)
|
102 |
+
setAssetGenerationStatus("generating")
|
103 |
+
clap = await editClapEntities({
|
104 |
+
clap,
|
105 |
+
|
106 |
+
// generating entities requires a "smart" LLM
|
107 |
+
turbo: false,
|
108 |
+
// turbo: true,
|
109 |
+
}).then(r => r.promise)
|
110 |
+
|
111 |
+
if (!clap) { throw new Error(`failed to edit the entities`) }
|
112 |
+
|
113 |
+
console.log(`generateEntities(): received entities = `, clap)
|
114 |
+
|
115 |
+
if (mainCharacterImage) {
|
116 |
+
// we assume the main entity is the first one
|
117 |
+
let mainEntity = clap.entities.at(0)
|
118 |
+
|
119 |
+
if (mainEntity) {
|
120 |
+
console.log(`generateEntities(): replacing the main character's picture..`)
|
121 |
+
mainEntity.thumbnailUrl = mainCharacterImage
|
122 |
+
mainEntity.imagePrompt = ""
|
123 |
+
mainEntity.imageSourceType = getClapAssetSourceType(mainCharacterImage)
|
124 |
+
mainEntity.imageEngine = ""
|
125 |
+
mainEntity.imageId = mainCharacterImage
|
126 |
+
}
|
127 |
+
}
|
128 |
+
|
129 |
+
setAssetGenerationStatus("finished")
|
130 |
+
console.log("---------------- GENERATED ENTITIES ----------------")
|
131 |
+
console.table(clap.entities, [
|
132 |
+
'category',
|
133 |
+
'label',
|
134 |
+
'imagePrompt',
|
135 |
+
'appearance'
|
136 |
+
])
|
137 |
+
return clap
|
138 |
+
} catch (err) {
|
139 |
+
setAssetGenerationStatus("error")
|
140 |
+
throw err
|
141 |
+
}
|
142 |
+
}
|
143 |
+
|
144 |
+
const generateSounds = async (clap: ClapProject): Promise<ClapProject> => {
|
145 |
+
try {
|
146 |
+
// setProgress(30)
|
147 |
+
setSoundGenerationStatus("generating")
|
148 |
+
|
149 |
+
clap = await editClapSounds({
|
150 |
+
clap,
|
151 |
+
turbo: false,
|
152 |
+
}).then(r => r.promise)
|
153 |
+
|
154 |
+
if (!clap) { throw new Error(`failed to edit the sound`) }
|
155 |
+
|
156 |
+
console.log(`handleSubmit(): received a clap with sound = `, clap)
|
157 |
+
setSoundGenerationStatus("finished")
|
158 |
+
console.log("---------------- GENERATED SOUND ----------------")
|
159 |
+
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.SOUND), [
|
160 |
+
'endTimeInMs',
|
161 |
+
'prompt',
|
162 |
+
'entityId',
|
163 |
+
])
|
164 |
+
return clap
|
165 |
+
} catch (err) {
|
166 |
+
setSoundGenerationStatus("error")
|
167 |
+
throw err
|
168 |
+
}
|
169 |
+
}
|
170 |
+
|
171 |
+
const generateMusic = async (clap: ClapProject): Promise<ClapProject> => {
|
172 |
+
try {
|
173 |
+
// setProgress(30)
|
174 |
+
setMusicGenerationStatus("generating")
|
175 |
+
|
176 |
+
clap = await editClapMusic({
|
177 |
+
clap,
|
178 |
+
turbo: false,
|
179 |
+
}).then(r => r.promise)
|
180 |
+
|
181 |
+
if (!clap) { throw new Error(`failed to edit the music`) }
|
182 |
+
|
183 |
+
console.log(`handleSubmit(): received a clap with music = `, clap)
|
184 |
+
setMusicGenerationStatus("finished")
|
185 |
+
console.log("---------------- GENERATED MUSIC ----------------")
|
186 |
+
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC), [
|
187 |
+
'endTimeInMs',
|
188 |
+
'prompt',
|
189 |
+
'entityId',
|
190 |
+
])
|
191 |
+
return clap
|
192 |
+
} catch (err) {
|
193 |
+
setMusicGenerationStatus("error")
|
194 |
+
throw err
|
195 |
+
}
|
196 |
+
}
|
197 |
+
|
198 |
+
const generateStoryboards = async (clap: ClapProject): Promise<ClapProject> => {
|
199 |
+
try {
|
200 |
+
// setProgress(40)
|
201 |
+
setImageGenerationStatus("generating")
|
202 |
+
clap = await editClapStoryboards({
|
203 |
+
clap,
|
204 |
+
// if we use entities, then we MUST use turbo
|
205 |
+
// that's because turbo uses PulID,
|
206 |
+
// but SDXL doesn't
|
207 |
+
turbo: false,
|
208 |
+
}).then(r => r.promise)
|
209 |
+
|
210 |
+
if (!clap) { throw new Error(`failed to edit the storyboards`) }
|
211 |
+
|
212 |
+
// const fusion =
|
213 |
+
console.log(`handleSubmit(): received storyboards = `, clap)
|
214 |
+
|
215 |
+
setImageGenerationStatus("finished")
|
216 |
+
console.log("---------------- GENERATED STORYBOARDS ----------------")
|
217 |
+
clap.segments
|
218 |
+
.filter(s => s.category === ClapSegmentCategory.STORYBOARD)
|
219 |
+
.forEach((s, i) => {
|
220 |
+
if (s.status === "completed" && s.assetUrl) {
|
221 |
+
// console.log(` [${i}] storyboard: ${s.prompt}`)
|
222 |
+
logImage(s.assetUrl, 0.35)
|
223 |
+
} else {
|
224 |
+
console.log(` [${i}] failed to generate storyboard`)
|
225 |
+
}
|
226 |
+
// console.log(`------------------`)
|
227 |
+
})
|
228 |
+
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD), [
|
229 |
+
'endTimeInMs',
|
230 |
+
'prompt',
|
231 |
+
'assetUrl'
|
232 |
+
])
|
233 |
+
return clap
|
234 |
+
} catch (err) {
|
235 |
+
setImageGenerationStatus("error")
|
236 |
+
throw err
|
237 |
+
}
|
238 |
+
}
|
239 |
+
|
240 |
+
const generateVideos = async (clap: ClapProject): Promise<ClapProject> => {
|
241 |
+
try {
|
242 |
+
// setProgress(50)
|
243 |
+
setVideoGenerationStatus("generating")
|
244 |
+
|
245 |
+
clap = await editClapVideos({
|
246 |
+
clap,
|
247 |
+
turbo: false
|
248 |
+
}).then(r => r.promise)
|
249 |
+
|
250 |
+
if (!clap) { throw new Error(`failed to edit the videos`) }
|
251 |
+
|
252 |
+
console.log(`handleSubmit(): received individual video clips = `, clap)
|
253 |
+
setVideoGenerationStatus("finished")
|
254 |
+
console.log("---------------- GENERATED VIDEOS ----------------")
|
255 |
+
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO), [
|
256 |
+
'endTimeInMs',
|
257 |
+
'prompt',
|
258 |
+
'entityId',
|
259 |
+
])
|
260 |
+
return clap
|
261 |
+
} catch (err) {
|
262 |
+
setVideoGenerationStatus("error")
|
263 |
+
throw err
|
264 |
+
}
|
265 |
+
}
|
266 |
+
|
267 |
+
const generateStoryboardsThenVideos = async (clap: ClapProject): Promise<ClapProject> => {
|
268 |
+
clap = await generateStoryboards(clap)
|
269 |
+
clap = await generateVideos(clap)
|
270 |
+
return clap
|
271 |
+
}
|
272 |
+
|
273 |
+
|
274 |
+
const generateDialogues = async (clap: ClapProject): Promise<ClapProject> => {
|
275 |
+
try {
|
276 |
+
// setProgress(70)
|
277 |
+
setVoiceGenerationStatus("generating")
|
278 |
+
clap = await editClapDialogues({
|
279 |
+
clap,
|
280 |
+
turbo: false,
|
281 |
+
}).then(r => r.promise)
|
282 |
+
|
283 |
+
if (!clap) { throw new Error(`failed to edit the dialogues`) }
|
284 |
+
|
285 |
+
console.log(`handleSubmit(): received dialogues = `, clap)
|
286 |
+
setVoiceGenerationStatus("finished")
|
287 |
+
console.log("---------------- GENERATED DIALOGUES ----------------")
|
288 |
+
console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.DIALOGUE), [
|
289 |
+
'endTimeInMs',
|
290 |
+
'prompt',
|
291 |
+
'entityId',
|
292 |
+
])
|
293 |
+
return clap
|
294 |
+
} catch (err) {
|
295 |
+
setVoiceGenerationStatus("error")
|
296 |
+
throw err
|
297 |
+
}
|
298 |
+
}
|
299 |
+
|
300 |
+
const generateFinalVideo = async (clap: ClapProject): Promise<string> => {
|
301 |
+
|
302 |
+
let assetUrl = ""
|
303 |
+
try {
|
304 |
+
// setProgress(85)
|
305 |
+
setFinalGenerationStatus("generating")
|
306 |
+
assetUrl = await exportClapToVideo({
|
307 |
+
clap,
|
308 |
+
turbo: false,
|
309 |
+
})
|
310 |
+
|
311 |
+
setCurrentVideo(assetUrl)
|
312 |
+
|
313 |
+
if (assetUrl.length < 128) { throw new Error(`generateFinalVideo(): the generated video is too small, so we failed`) }
|
314 |
+
|
315 |
+
console.log(`generateFinalVideo(): received a video: ${assetUrl.slice(0, 120)}...`)
|
316 |
+
setFinalGenerationStatus("finished")
|
317 |
+
return assetUrl
|
318 |
+
} catch (err) {
|
319 |
+
setFinalGenerationStatus("error")
|
320 |
+
throw err
|
321 |
+
}
|
322 |
+
}
|
323 |
+
|
324 |
+
const handleSubmit = async () => {
|
325 |
+
setStatus("generating")
|
326 |
+
busyRef.current = true
|
327 |
+
|
328 |
+
startTransition(async () => {
|
329 |
+
setStatus("generating")
|
330 |
+
busyRef.current = true
|
331 |
+
|
332 |
+
console.log(`handleSubmit(): generating a clap using prompt = "${promptDraftRef.current}" `)
|
333 |
+
|
334 |
+
try {
|
335 |
+
let clap = await generateStory()
|
336 |
+
setCurrentClap(clap)
|
337 |
+
|
338 |
+
const storyboards = clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD)
|
339 |
+
|
340 |
+
let mainCharacter = clap.entities.at(0)
|
341 |
+
|
342 |
+
// let's do something basic for now: we only support 1 entity (character)
|
343 |
+
// and we apply it to *all* the storyboards (we can always improve this later)
|
344 |
+
if (mainCharacter) {
|
345 |
+
console.log(`handleSubmit(): we use the clap's main character's face on all storyboards`)
|
346 |
+
storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
|
347 |
+
logImage(mainCharacter.imageId, 0.35)
|
348 |
+
} else if (mainCharacterImage) {
|
349 |
+
console.log(`handleSubmit(): declaring a new entity for our main character`)
|
350 |
+
const entityName = "person"
|
351 |
+
mainCharacter = newEntity({
|
352 |
+
category: ClapSegmentCategory.CHARACTER,
|
353 |
+
triggerName: entityName,
|
354 |
+
label: entityName,
|
355 |
+
description: entityName,
|
356 |
+
author: "auto",
|
357 |
+
thumbnailUrl: mainCharacterImage,
|
358 |
+
|
359 |
+
imagePrompt: "",
|
360 |
+
imageSourceType: getClapAssetSourceType(mainCharacterImage),
|
361 |
+
imageEngine: "",
|
362 |
+
imageId: mainCharacterImage,
|
363 |
+
audioPrompt: "",
|
364 |
+
})
|
365 |
+
|
366 |
+
clap.entities.push(mainCharacter!)
|
367 |
+
console.log(`handleSubmit(): we use the main character's face on all storyboards`)
|
368 |
+
|
369 |
+
storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
|
370 |
+
logImage(mainCharacterImage, 0.35)
|
371 |
+
}
|
372 |
+
|
373 |
+
const tasks = [
|
374 |
+
generateMusic(clap),
|
375 |
+
generateStoryboardsThenVideos(clap)
|
376 |
+
]
|
377 |
+
|
378 |
+
const claps = await Promise.all(tasks)
|
379 |
+
|
380 |
+
console.log(`finished processing ${tasks.length} tasks in parallel`)
|
381 |
+
|
382 |
+
for (const newerClap of claps) {
|
383 |
+
clap = await updateClap(clap, newerClap, {
|
384 |
+
overwriteMeta: false,
|
385 |
+
inlineReplace: true,
|
386 |
+
})
|
387 |
+
setCurrentClap(clap)
|
388 |
+
}
|
389 |
+
|
390 |
+
/*
|
391 |
+
clap = await claps.reduce(async (existingClap, newerClap) =>
|
392 |
+
updateClap(existingClap, newerClap, {
|
393 |
+
overwriteMeta: false,
|
394 |
+
inlineReplace: true,
|
395 |
+
})
|
396 |
+
, Promise.resolve(clap)
|
397 |
+
*/
|
398 |
+
|
399 |
+
|
400 |
+
// We can't have consistent characters with video (yet)
|
401 |
+
// clap = await generateEntities(clap)
|
402 |
+
|
403 |
+
/*
|
404 |
+
if (mainCharacterImage) {
|
405 |
+
console.log("handleSubmit(): User specified a main character image")
|
406 |
+
// various strategies here, for instance we can assume that the first character is the main character,
|
407 |
+
// or maybe a more reliable way is to count the number of occurrences.
|
408 |
+
// there is a risk of misgendering, so ideally we should add some kind of UI to do this,
|
409 |
+
// such as a list of characters.
|
410 |
+
}
|
411 |
+
*/
|
412 |
+
|
413 |
+
// let's skip storyboards for now
|
414 |
+
// clap = await generateStoryboards(clap)
|
415 |
+
|
416 |
+
// clap = await generateVideos(clap)
|
417 |
+
// clap = await generateDialogues(clap)
|
418 |
+
|
419 |
+
|
420 |
+
|
421 |
+
console.log("final clap: ", clap)
|
422 |
+
setCurrentClap(clap)
|
423 |
+
await generateFinalVideo(clap)
|
424 |
+
|
425 |
+
setStatus("finished")
|
426 |
+
setError("")
|
427 |
+
} catch (err) {
|
428 |
+
console.error(`failed to generate: `, err)
|
429 |
+
setStatus("error")
|
430 |
+
setError(`Error, please contact an admin on Discord (${err})`)
|
431 |
+
}
|
432 |
+
})
|
433 |
+
}
|
434 |
+
|
435 |
+
return {
|
436 |
+
generateDialogues,
|
437 |
+
generateEntities,
|
438 |
+
generateFinalVideo,
|
439 |
+
generateMusic,
|
440 |
+
generateSounds,
|
441 |
+
generateStory,
|
442 |
+
generateStoryboards,
|
443 |
+
generateStoryboardsThenVideos,
|
444 |
+
generateVideos,
|
445 |
+
handleSubmit,
|
446 |
+
}
|
447 |
+
}
|
src/lib/hooks/useProgressTimer.ts
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useStore } from "@/app/store"
|
2 |
+
import { useEffect, useRef } from "react"
|
3 |
+
import { useIsBusy } from "./useIsBusy"
|
4 |
+
|
5 |
+
export function useProgressTimer() {
|
6 |
+
const runningRef = useRef(false)
|
7 |
+
const timerRef = useRef<NodeJS.Timeout>()
|
8 |
+
|
9 |
+
const progress = useStore(s => s.progress)
|
10 |
+
const stage = useStore(s => s.stage)
|
11 |
+
const { isBusy, busyRef } = useIsBusy()
|
12 |
+
|
13 |
+
const timerFn = async () => {
|
14 |
+
const { isBusy, progress, stage } = useStore.getState()
|
15 |
+
|
16 |
+
clearTimeout(timerRef.current)
|
17 |
+
if (!isBusy || stage === "idle") {
|
18 |
+
return
|
19 |
+
}
|
20 |
+
|
21 |
+
/*
|
22 |
+
console.log("progress function:", {
|
23 |
+
stage,
|
24 |
+
delay: progressDelayInMsPerStage[stage],
|
25 |
+
progress,
|
26 |
+
})
|
27 |
+
*/
|
28 |
+
useStore.setState({
|
29 |
+
// progress: Math.min(maxProgressPerStage[stage], progress + 1)
|
30 |
+
progress: Math.min(100, progress + 1)
|
31 |
+
})
|
32 |
+
|
33 |
+
// timerRef.current = setTimeout(timerFn, progressDelayInMsPerStage[stage])
|
34 |
+
timerRef.current = setTimeout(timerFn, 1200)
|
35 |
+
}
|
36 |
+
|
37 |
+
useEffect(() => {
|
38 |
+
timerFn()
|
39 |
+
clearTimeout(timerRef.current)
|
40 |
+
if (!isBusy) { return }
|
41 |
+
timerRef.current = setTimeout(timerFn, 0)
|
42 |
+
}, [isBusy])
|
43 |
+
|
44 |
+
return { isBusy, busyRef, progress, stage }
|
45 |
+
}
|
src/lib/hooks/useQueryStringParams.ts
ADDED
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useEffect } from "react"
|
2 |
+
import { useSearchParams } from "next/navigation"
|
3 |
+
import { ClapMediaOrientation } from "@aitube/clap"
|
4 |
+
|
5 |
+
import { useStore } from "@/app/store"
|
6 |
+
|
7 |
+
import { useStoryPromptDraft } from "./useStoryPromptDraft"
|
8 |
+
import { useIsBusy } from "./useIsBusy"
|
9 |
+
import { useProcessors } from "./useProcessors"
|
10 |
+
|
11 |
+
export function useQueryStringParams() {
|
12 |
+
const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
|
13 |
+
const { busyRef } = useIsBusy()
|
14 |
+
const { handleSubmit } = useProcessors()
|
15 |
+
|
16 |
+
const setOrientation = useStore(s => s.setOrientation)
|
17 |
+
// this is how we support query string parameters
|
18 |
+
// ?prompt=hello <- set a default prompt
|
19 |
+
// ?prompt=hello&autorun=true <- automatically run the app
|
20 |
+
// ?orientation=landscape <- can be "landscape" or "portrait" (default)
|
21 |
+
const searchParams = useSearchParams()
|
22 |
+
const queryStringPrompt = (searchParams?.get('prompt') as string) || ""
|
23 |
+
const queryStringAutorun = (searchParams?.get('autorun') as string) || ""
|
24 |
+
const queryStringOrientation = (searchParams?.get('orientation') as string) || ""
|
25 |
+
|
26 |
+
useEffect(() => {
|
27 |
+
if (queryStringOrientation?.length > 1) {
|
28 |
+
console.log(`orientation = "${queryStringOrientation}"`)
|
29 |
+
const orientation =
|
30 |
+
queryStringOrientation.trim().toLowerCase() === "landscape"
|
31 |
+
? ClapMediaOrientation.LANDSCAPE
|
32 |
+
: ClapMediaOrientation.PORTRAIT
|
33 |
+
setOrientation(orientation)
|
34 |
+
}
|
35 |
+
if (queryStringPrompt?.length > 1) {
|
36 |
+
console.log(`prompt = "${queryStringPrompt}"`)
|
37 |
+
if (queryStringPrompt !== promptDraftRef.current) {
|
38 |
+
setStoryPromptDraft(queryStringPrompt)
|
39 |
+
}
|
40 |
+
const maybeAutorun = queryStringAutorun.trim().toLowerCase()
|
41 |
+
console.log(`autorun = "${maybeAutorun}"`)
|
42 |
+
|
43 |
+
// note: during development we will be called twice,
|
44 |
+
// which is why we have a guard on busyRef.current
|
45 |
+
if (maybeAutorun === "true" || maybeAutorun === "1" && !busyRef.current) {
|
46 |
+
handleSubmit()
|
47 |
+
}
|
48 |
+
}
|
49 |
+
}, [queryStringPrompt, queryStringAutorun, queryStringOrientation])
|
50 |
+
|
51 |
+
}
|
src/lib/hooks/useStoryPromptDraft.ts
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
import { useRef } from "react"
|
3 |
+
import { useLocalStorage } from "usehooks-ts"
|
4 |
+
|
5 |
+
import { defaultPrompt, localStorageStoryDraftKey } from "@/app/config"
|
6 |
+
|
7 |
+
export function useStoryPromptDraft() {
|
8 |
+
const [storyPromptDraft, setStoryPromptDraft] = useLocalStorage<string>(
|
9 |
+
localStorageStoryDraftKey,
|
10 |
+
defaultPrompt
|
11 |
+
)
|
12 |
+
const promptDraftRef = useRef("")
|
13 |
+
promptDraftRef.current = storyPromptDraft
|
14 |
+
|
15 |
+
return { storyPromptDraft, setStoryPromptDraft, promptDraftRef }
|
16 |
+
}
|
src/lib/utils/index.ts
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
export { cn } from "./cn"
|
2 |
+
export { generateRandomStory } from "./generateRandomStory"
|
3 |
+
export { logImage } from "./logImage"
|