jbilcke-hf HF staff commited on
Commit
82d1e90
1 Parent(s): 68ec2c8

experimental support for images

Browse files
src/app/config.ts CHANGED
@@ -3,4 +3,6 @@ export const defaultPrompt =
3
  // "underwater footage of the loch ness"
4
  // "beautiful underwater footage of a clownfish swimming around coral" // who discovers a secret pirate treasure chest"
5
  // "beautiful footage of a Caribbean fishing village and bay, sail ships, during golden hour, no captions"
6
- "videogame gameplay footage, first person, exploring some mysterious ruins, no commentary"
 
 
 
3
  // "underwater footage of the loch ness"
4
  // "beautiful underwater footage of a clownfish swimming around coral" // who discovers a secret pirate treasure chest"
5
  // "beautiful footage of a Caribbean fishing village and bay, sail ships, during golden hour, no captions"
6
+ "videogame gameplay footage, first person, exploring some mysterious ruins, no commentary"
7
+
8
+ export const localStorageStoryDraftKey = "AI_STORIES_FACTORY_STORY_PROMPT_DRAFT"
src/app/main.tsx CHANGED
@@ -1,596 +1,32 @@
1
  "use client"
2
 
3
- import React, { useEffect, useRef, useTransition } from "react"
4
  import { IoMdPhonePortrait } from "react-icons/io"
5
  import { GiRollingDices } from "react-icons/gi"
6
- import { FaCloudDownloadAlt, FaDiscord } from "react-icons/fa"
7
- import { GiSpellBook } from "react-icons/gi"
8
- import { useLocalStorage } from "usehooks-ts"
9
- import { ClapProject, ClapMediaOrientation, ClapSegmentCategory, updateClap } from "@aitube/clap"
10
- import Image from "next/image"
11
- import { useSearchParams } from "next/navigation"
12
- import { useFilePicker } from "use-file-picker"
13
- import { DeviceFrameset } from "react-device-frameset"
14
- import "react-device-frameset/styles/marvel-devices.min.css"
15
 
16
  import { Card, CardContent, CardHeader } from "@/components/ui/card"
17
  import { Button } from "@/components/ui/button"
18
  import { Toaster } from "@/components/ui/sonner"
19
  import { TextareaField } from "@/components/form/textarea-field"
20
- import { cn } from "@/lib/utils/cn"
21
 
22
- import { createClap } from "./server/aitube/createClap"
23
- import { editClapEntities } from "./server/aitube/editClapEntities"
24
- import { editClapDialogues } from "./server/aitube/editClapDialogues"
25
- import { editClapStoryboards } from "./server/aitube/editClapStoryboards"
26
- import { editClapSounds } from "./server/aitube/editClapSounds"
27
- import { editClapMusic } from "./server/aitube/editClapMusic"
28
- import { editClapVideos } from "./server/aitube/editClapVideos"
29
- import { exportClapToVideo } from "./server/aitube/exportClapToVideo"
30
-
31
- import { useStore } from "./store"
32
- import HFLogo from "./hf-logo.svg"
33
- import { Input } from "@/components/ui/input"
34
- import { Field } from "@/components/form/field"
35
- import { Label } from "@/components/form/label"
36
- import { getParam } from "@/lib/utils/getParam"
37
- import { GenerationStage } from "@/types"
38
- import { FileContent } from "use-file-picker/dist/interfaces"
39
- import { generateRandomStory } from "@/lib/utils/generateRandomStory"
40
- import { logImage } from "@/lib/utils/logImage"
41
  import { defaultPrompt } from "./config"
42
  import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"
 
 
 
 
 
 
 
43
 
44
  export function Main() {
45
- const [storyPromptDraft, setStoryPromptDraft] = useLocalStorage<string>(
46
- "AI_STORIES_FACTORY_STORY_PROMPT_DRAFT",
47
- defaultPrompt
48
- )
49
- const promptDraftRef = useRef("")
50
- promptDraftRef.current = storyPromptDraft
51
-
52
- const [_isPending, startTransition] = useTransition()
53
-
54
- const storyPrompt = useStore(s => s.storyPrompt)
55
- const mainCharacterImage = useStore(s => s.mainCharacterImage)
56
- const mainCharacterVoice = useStore(s => s.mainCharacterVoice)
57
- const orientation = useStore(s => s.orientation)
58
- const setOrientation = useStore(s => s.setOrientation)
59
- const status = useStore(s => s.status)
60
- const parseGenerationStatus = useStore(s => s.parseGenerationStatus)
61
- const storyGenerationStatus = useStore(s => s.storyGenerationStatus)
62
- const assetGenerationStatus = useStore(s => s.assetGenerationStatus)
63
- const soundGenerationStatus = useStore(s => s.soundGenerationStatus)
64
- const musicGenerationStatus = useStore(s => s.musicGenerationStatus)
65
- const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus)
66
- const imageGenerationStatus = useStore(s => s.imageGenerationStatus)
67
- const videoGenerationStatus = useStore(s => s.videoGenerationStatus)
68
- const finalGenerationStatus = useStore(s => s.finalGenerationStatus)
69
- const currentClap = useStore(s => s.currentClap)
70
- const currentVideo = useStore(s => s.currentVideo)
71
- const currentVideoOrientation = useStore(s => s.currentVideoOrientation)
72
- const setStoryPrompt = useStore(s => s.setStoryPrompt)
73
- const setMainCharacterImage = useStore(s => s.setMainCharacterImage)
74
- const setMainCharacterVoice = useStore(s => s.setMainCharacterVoice)
75
- const setStatus = useStore(s => s.setStatus)
76
- const toggleOrientation = useStore(s => s.toggleOrientation)
77
- const error = useStore(s => s.error)
78
- const setError = useStore(s => s.setError)
79
- const setParseGenerationStatus = useStore(s => s.setParseGenerationStatus)
80
- const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus)
81
- const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus)
82
- const setSoundGenerationStatus = useStore(s => s.setSoundGenerationStatus)
83
- const setMusicGenerationStatus = useStore(s => s.setMusicGenerationStatus)
84
- const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus)
85
- const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus)
86
- const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus)
87
- const setFinalGenerationStatus = useStore(s => s.setFinalGenerationStatus)
88
- const setCurrentClap = useStore(s => s.setCurrentClap)
89
- const setCurrentVideo = useStore(s => s.setCurrentVideo)
90
- const progress = useStore(s => s.progress)
91
- const setProgress = useStore(s => s.setProgress)
92
- const saveVideo = useStore(s => s.saveVideo)
93
- const saveClap = useStore(s => s.saveClap)
94
- const loadClap = useStore(s => s.loadClap)
95
-
96
- // let's disable this for now
97
- const canSeeBetaFeatures = true // getParam<boolean>("beta", false)
98
-
99
- const isBusy = useStore(s => s.isBusy)
100
- const busyRef = useRef(isBusy)
101
- busyRef.current = isBusy
102
-
103
- const importStory = async (fileData: FileContent<ArrayBuffer>): Promise<{
104
- clap: ClapProject
105
- regenerateVideo: boolean
106
- }> => {
107
- if (!fileData?.name) { throw new Error(`invalid file (missing file name)`) }
108
-
109
- const {
110
- setParseGenerationStatus,
111
- } = useStore.getState()
112
-
113
- setParseGenerationStatus("generating")
114
-
115
- try {
116
- const blob = new Blob([fileData.content])
117
- const res = await loadClap(blob, fileData.name)
118
-
119
- if (!res?.clap) { throw new Error(`failed to load the clap file`) }
120
-
121
- setParseGenerationStatus("finished")
122
- return res
123
- } catch (err) {
124
- console.error("failed to load the Clap file:", err)
125
- setParseGenerationStatus("error")
126
- throw err
127
- }
128
- }
129
-
130
-
131
- const generateStory = async (): Promise<ClapProject> => {
132
-
133
- let clap: ClapProject | undefined = undefined
134
- try {
135
- setProgress(0)
136
-
137
- setStatus("generating")
138
- setStoryGenerationStatus("generating")
139
- setStoryPrompt(promptDraftRef.current)
140
-
141
- clap = await createClap({
142
- prompt: promptDraftRef.current,
143
- orientation: useStore.getState().orientation,
144
-
145
- turbo: true,
146
- })
147
-
148
- if (!clap) { throw new Error(`failed to create the clap`) }
149
-
150
- if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) }
151
-
152
- console.log(`handleSubmit(): received a clap = `, clap)
153
- setCurrentClap(clap)
154
- setStoryGenerationStatus("finished")
155
-
156
- console.log("---------------- GENERATED STORY ----------------")
157
- console.table(clap.segments, [
158
- // 'startTimeInMs',
159
- 'endTimeInMs',
160
- // 'track',
161
- 'category',
162
- 'prompt'
163
- ])
164
- return clap
165
- } catch (err) {
166
- setStoryGenerationStatus("error")
167
- throw err
168
- }
169
- }
170
-
171
- const generateEntities = async (clap: ClapProject): Promise<ClapProject> => {
172
- try {
173
- // setProgress(20)
174
- setAssetGenerationStatus("generating")
175
- clap = await editClapEntities({
176
- clap,
177
-
178
- // generating entities requires a "smart" LLM
179
- turbo: false,
180
- // turbo: true,
181
- }).then(r => r.promise)
182
-
183
- if (!clap) { throw new Error(`failed to edit the entities`) }
184
-
185
- console.log(`handleSubmit(): received a clap with entities = `, clap)
186
- setAssetGenerationStatus("finished")
187
- console.log("---------------- GENERATED ENTITIES ----------------")
188
- console.table(clap.entities, [
189
- 'category',
190
- 'label',
191
- 'imagePrompt',
192
- 'appearance'
193
- ])
194
- return clap
195
- } catch (err) {
196
- setAssetGenerationStatus("error")
197
- throw err
198
- }
199
- }
200
-
201
- const generateSounds = async (clap: ClapProject): Promise<ClapProject> => {
202
- try {
203
- // setProgress(30)
204
- setSoundGenerationStatus("generating")
205
-
206
- clap = await editClapSounds({
207
- clap,
208
- turbo: true
209
- }).then(r => r.promise)
210
-
211
- if (!clap) { throw new Error(`failed to edit the sound`) }
212
-
213
- console.log(`handleSubmit(): received a clap with sound = `, clap)
214
- setSoundGenerationStatus("finished")
215
- console.log("---------------- GENERATED SOUND ----------------")
216
- console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.SOUND), [
217
- 'endTimeInMs',
218
- 'prompt',
219
- 'entityId',
220
- ])
221
- return clap
222
- } catch (err) {
223
- setSoundGenerationStatus("error")
224
- throw err
225
- }
226
- }
227
-
228
- const generateMusic = async (clap: ClapProject): Promise<ClapProject> => {
229
- try {
230
- // setProgress(30)
231
- setMusicGenerationStatus("generating")
232
-
233
- clap = await editClapMusic({
234
- clap,
235
- turbo: true
236
- }).then(r => r.promise)
237
-
238
- if (!clap) { throw new Error(`failed to edit the music`) }
239
-
240
- console.log(`handleSubmit(): received a clap with music = `, clap)
241
- setMusicGenerationStatus("finished")
242
- console.log("---------------- GENERATED MUSIC ----------------")
243
- console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC), [
244
- 'endTimeInMs',
245
- 'prompt',
246
- 'entityId',
247
- ])
248
- return clap
249
- } catch (err) {
250
- setMusicGenerationStatus("error")
251
- throw err
252
- }
253
- }
254
-
255
- const generateStoryboards = async (clap: ClapProject): Promise<ClapProject> => {
256
- try {
257
- // setProgress(40)
258
- setImageGenerationStatus("generating")
259
- clap = await editClapStoryboards({
260
- clap,
261
- // if we use entities, then we MUST use turbo
262
- // that's because turbo uses PulID,
263
- // but SDXL doesn't
264
- turbo: true,
265
- }).then(r => r.promise)
266
-
267
- if (!clap) { throw new Error(`failed to edit the storyboards`) }
268
-
269
- // const fusion =
270
- console.log(`handleSubmit(): received a clap with images = `, clap)
271
- setImageGenerationStatus("finished")
272
- console.log("---------------- GENERATED STORYBOARDS ----------------")
273
- clap.segments
274
- .filter(s => s.category === ClapSegmentCategory.STORYBOARD)
275
- .forEach((s, i) => {
276
- if (s.status === "completed" && s.assetUrl) {
277
- // console.log(` [${i}] storyboard: ${s.prompt}`)
278
- logImage(s.assetUrl, 0.35)
279
- } else {
280
- console.log(` [${i}] failed to generate storyboard`)
281
- }
282
- // console.log(`------------------`)
283
- })
284
- console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD), [
285
- 'endTimeInMs',
286
- 'prompt',
287
- 'assetUrl'
288
- ])
289
- return clap
290
- } catch (err) {
291
- setImageGenerationStatus("error")
292
- throw err
293
- }
294
- }
295
-
296
- const generateVideos = async (clap: ClapProject): Promise<ClapProject> => {
297
- try {
298
- // setProgress(50)
299
- setVideoGenerationStatus("generating")
300
-
301
- clap = await editClapVideos({
302
- clap,
303
- turbo: false
304
- }).then(r => r.promise)
305
-
306
- if (!clap) { throw new Error(`failed to edit the videos`) }
307
-
308
- console.log(`handleSubmit(): received a clap with videos = `, clap)
309
- setVideoGenerationStatus("finished")
310
- console.log("---------------- GENERATED VIDEOS ----------------")
311
- console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO), [
312
- 'endTimeInMs',
313
- 'prompt',
314
- 'entityId',
315
- ])
316
- return clap
317
- } catch (err) {
318
- setVideoGenerationStatus("error")
319
- throw err
320
- }
321
- }
322
-
323
- const generateStoryboardsThenVideos = async (clap: ClapProject): Promise<ClapProject> => {
324
- clap = await generateStoryboards(clap)
325
- clap = await generateVideos(clap)
326
- return clap
327
- }
328
-
329
-
330
- const generateDialogues = async (clap: ClapProject): Promise<ClapProject> => {
331
- try {
332
- // setProgress(70)
333
- setVoiceGenerationStatus("generating")
334
- clap = await editClapDialogues({
335
- clap,
336
- turbo: true
337
- }).then(r => r.promise)
338
-
339
- if (!clap) { throw new Error(`failed to edit the dialogues`) }
340
-
341
- console.log(`handleSubmit(): received a clap with dialogues = `, clap)
342
- setVoiceGenerationStatus("finished")
343
- console.log("---------------- GENERATED DIALOGUES ----------------")
344
- console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.DIALOGUE), [
345
- 'endTimeInMs',
346
- 'prompt',
347
- 'entityId',
348
- ])
349
- return clap
350
- } catch (err) {
351
- setVoiceGenerationStatus("error")
352
- throw err
353
- }
354
- }
355
-
356
- const generateFinalVideo = async (clap: ClapProject): Promise<string> => {
357
-
358
- let assetUrl = ""
359
- try {
360
- // setProgress(85)
361
- setFinalGenerationStatus("generating")
362
- assetUrl = await exportClapToVideo({
363
- clap,
364
- turbo: true
365
- })
366
-
367
- setCurrentVideo(assetUrl)
368
-
369
- if (assetUrl.length < 128) { throw new Error(`generateFinalVideo(): the generated video is too small, so we failed`) }
370
-
371
- console.log(`generateFinalVideo(): received a video: ${assetUrl.slice(0, 120)}...`)
372
- setFinalGenerationStatus("finished")
373
- return assetUrl
374
- } catch (err) {
375
- setFinalGenerationStatus("error")
376
- throw err
377
- }
378
- }
379
-
380
- const handleSubmit = async () => {
381
- setStatus("generating")
382
- busyRef.current = true
383
-
384
- startTransition(async () => {
385
- setStatus("generating")
386
- busyRef.current = true
387
-
388
- console.log(`handleSubmit(): generating a clap using prompt = "${promptDraftRef.current}" `)
389
-
390
- try {
391
- let clap = await generateStory()
392
-
393
- setCurrentClap(clap)
394
-
395
- const tasks = [
396
- generateMusic(clap),
397
- generateStoryboardsThenVideos(clap)
398
- ]
399
-
400
- const claps = await Promise.all(tasks)
401
-
402
- console.log(`finished processing ${tasks.length} tasks in parallel`)
403
-
404
- for (const newerClap of claps) {
405
- clap = await updateClap(clap, newerClap, {
406
- overwriteMeta: false,
407
- inlineReplace: true,
408
- })
409
- setCurrentClap(clap)
410
- }
411
-
412
- /*
413
- clap = await claps.reduce(async (existingClap, newerClap) =>
414
- updateClap(existingClap, newerClap, {
415
- overwriteMeta: false,
416
- inlineReplace: true,
417
- })
418
- , Promise.resolve(clap)
419
- */
420
-
421
-
422
- // We can't have consistent characters with video (yet)
423
- // clap = await generateEntities(clap)
424
-
425
- /*
426
- if (mainCharacterImage) {
427
- console.log("handleSubmit(): User specified a main character image")
428
- // various strategies here, for instance we can assume that the first character is the main character,
429
- // or maybe a more reliable way is to count the number of occurrences.
430
- // there is a risk of misgendering, so ideally we should add some kind of UI to do this,
431
- // such as a list of characters.
432
- }
433
- */
434
-
435
- // let's skip storyboards for now
436
- // clap = await generateStoryboards(clap)
437
-
438
- // clap = await generateVideos(clap)
439
- // clap = await generateDialogues(clap)
440
-
441
-
442
-
443
- console.log("final clap: ", clap)
444
- setCurrentClap(clap)
445
- await generateFinalVideo(clap)
446
-
447
- setStatus("finished")
448
- setError("")
449
- } catch (err) {
450
- console.error(`failed to generate: `, err)
451
- setStatus("error")
452
- setError(`Error, please contact an admin on Discord (${err})`)
453
- }
454
- })
455
- }
456
-
457
-
458
- const { openFilePicker, filesContent } = useFilePicker({
459
- accept: '.clap',
460
- readAs: "ArrayBuffer"
461
- })
462
- const fileData = filesContent[0]
463
-
464
- useEffect(() => {
465
- const fn = async () => {
466
- if (!fileData?.name) { return }
467
-
468
- const { setStatus, setProgress } = useStore.getState()
469
-
470
- setProgress(0)
471
- setStatus("generating")
472
-
473
- try {
474
- let { clap, regenerateVideo } = await importStory(fileData)
475
-
476
- // clap = await generateSounds(clap)
477
-
478
- // setCurrentClap(clap)
479
-
480
- console.log("loadClap(): clap = ", clap)
481
-
482
- // it is important to skip regeneration if we already have a video
483
- if (regenerateVideo) {
484
- console.log(`regenerating music and videos..`)
485
- const claps = await Promise.all([
486
- generateMusic(clap),
487
- generateVideos(clap)
488
- ])
489
-
490
- // console.log("finished processing the 2 tasks in parallel")
491
-
492
- for (const newerClap of claps) {
493
- clap = await updateClap(clap, newerClap, {
494
- overwriteMeta: false,
495
- inlineReplace: true,
496
- })
497
- }
498
-
499
-
500
- setCurrentClap(clap)
501
-
502
- await generateFinalVideo(clap)
503
-
504
- } else {
505
- console.log(`skipping music and video regeneration`)
506
- }
507
-
508
- setStatus("finished")
509
- setProgress(100)
510
- setError("")
511
- } catch (err) {
512
- console.error(`failed to import: `, err)
513
- setStatus("error")
514
- setError(`${err}`)
515
- }
516
-
517
- }
518
- fn()
519
- }, [fileData?.name])
520
-
521
- // note: we are interested in the *current* video orientation,
522
- // not the requested video orientation requested for the next video
523
- const isLandscape = currentVideoOrientation === ClapMediaOrientation.LANDSCAPE
524
- const isPortrait = currentVideoOrientation === ClapMediaOrientation.PORTRAIT
525
- const isSquare = currentVideoOrientation === ClapMediaOrientation.SQUARE
526
-
527
- const runningRef = useRef(false)
528
- const timerRef = useRef<NodeJS.Timeout>()
529
-
530
- const timerFn = async () => {
531
- const { isBusy, progress, stage } = useStore.getState()
532
-
533
- clearTimeout(timerRef.current)
534
- if (!isBusy || stage === "idle") {
535
- return
536
- }
537
-
538
- /*
539
- console.log("progress function:", {
540
- stage,
541
- delay: progressDelayInMsPerStage[stage],
542
- progress,
543
- })
544
- */
545
- useStore.setState({
546
- // progress: Math.min(maxProgressPerStage[stage], progress + 1)
547
- progress: Math.min(100, progress + 1)
548
- })
549
-
550
- // timerRef.current = setTimeout(timerFn, progressDelayInMsPerStage[stage])
551
- timerRef.current = setTimeout(timerFn, 1200)
552
- }
553
-
554
- useEffect(() => {
555
- timerFn()
556
- clearTimeout(timerRef.current)
557
- if (!isBusy) { return }
558
- timerRef.current = setTimeout(timerFn, 0)
559
- }, [isBusy])
560
-
561
- // this is how we support query string parameters
562
- // ?prompt=hello <- set a default prompt
563
- // ?prompt=hello&autorun=true <- automatically run the app
564
- // ?orientation=landscape <- can be "landscape" or "portrait" (default)
565
- const searchParams = useSearchParams()
566
- const queryStringPrompt = (searchParams?.get('prompt') as string) || ""
567
- const queryStringAutorun = (searchParams?.get('autorun') as string) || ""
568
- const queryStringOrientation = (searchParams?.get('orientation') as string) || ""
569
- useEffect(() => {
570
- if (queryStringOrientation?.length > 1) {
571
- console.log(`orientation = "${queryStringOrientation}"`)
572
- const orientation =
573
- queryStringOrientation.trim().toLowerCase() === "landscape"
574
- ? ClapMediaOrientation.LANDSCAPE
575
- : ClapMediaOrientation.PORTRAIT
576
- setOrientation(orientation)
577
- }
578
- if (queryStringPrompt?.length > 1) {
579
- console.log(`prompt = "${queryStringPrompt}"`)
580
- if (queryStringPrompt !== promptDraftRef.current) {
581
- setStoryPromptDraft(queryStringPrompt)
582
- }
583
- const maybeAutorun = queryStringAutorun.trim().toLowerCase()
584
- console.log(`autorun = "${maybeAutorun}"`)
585
-
586
- // note: during development we will be called twice,
587
- // which is why we have a guard on busyRef.current
588
- if (maybeAutorun === "true" || maybeAutorun === "1" && !busyRef.current) {
589
- handleSubmit()
590
- }
591
- }
592
- }, [queryStringPrompt, queryStringAutorun, queryStringOrientation])
593
-
594
  return (
595
  <TooltipProvider>
596
  <div className={cn(
@@ -659,60 +95,7 @@ export function Main() {
659
 
660
  )}>
661
  <CardHeader>
662
- <div className="flex flex-col justify-start">
663
- <div className="
664
- flex flex-row
665
- items-center justify-center
666
- transition-all duration-200 ease-in-out
667
- px-3
668
-
669
- rounded-full">
670
- <div
671
- className="
672
- flex
673
- transition-all duration-200 ease-in-out
674
- items-center justify-center text-center
675
- w-10 h-10 md:w-12 md:h-12 lg:w-16 lg:h-16
676
- text-3xl md:text-4xl lg:text-5xl
677
- rounded-lg
678
- mr-2
679
- font-sans
680
- bg-amber-400 dark:bg-amber-400
681
-
682
- text-stone-950/80 dark:text-stone-950/80 font-bold
683
- "
684
- >AI</div>
685
- <div
686
- className="
687
- transition-all duration-200 ease-in-out
688
- text-amber-400 dark:text-amber-400
689
- text-3xl md:text-4xl lg:text-5xl
690
- "
691
-
692
- style={{ textShadow: "#00000035 0px 0px 2px" }}
693
-
694
- /*
695
- className="
696
- text-5xl
697
- bg-gradient-to-br from-yellow-300 to-yellow-500
698
- inline-block text-transparent bg-clip-text
699
- py-6
700
- "
701
- */
702
- >Stories Factory</div>
703
- </div>
704
-
705
- <p
706
- className="
707
- transition-all duration-200 ease-in-out
708
- text-stone-900/90 dark:text-stone-900/90
709
- text-lg md:text-xl lg:text-2xl
710
- text-center
711
- pt-2 md:pt-4
712
- "
713
- style={{ textShadow: "rgb(255 255 255 / 19%) 0px 0px 2px" }}
714
- >Make video stories using AI ✨</p>
715
- </div>
716
  </CardHeader>
717
  <CardContent
718
  className="flex flex-col space-y-3"
@@ -806,75 +189,24 @@ export function Main() {
806
  */}
807
 
808
  <div className="
809
-
810
- flex flex-row
811
- justify-between items-center
812
- space-x-3">
813
-
814
-
815
- {canSeeBetaFeatures &&
816
- <Tooltip>
817
- <TooltipTrigger asChild><Button
818
-
819
- onClick={openFilePicker}
820
- disabled={isBusy}
821
- // variant="ghost"
822
- className={cn(
823
- `text-xs md:text-sm lg:text-base`,
824
- `bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
825
- `font-bold`,
826
- `hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
827
- storyPromptDraft ? "opacity-100" : "opacity-80"
828
- )}
829
- >
830
- <span className="hidden xl:inline mr-1">Load .clap</span>
831
- <span className="inline xl:hidden mr-1">Load .clap</span>
832
- </Button></TooltipTrigger>
833
- <TooltipContent side="top">
834
- <p className="text-xs font-normal text-stone-100/90 text-center">
835
- Clap is a new AI format,<br/>check out the academy<br/>to learn more about it.
836
- </p>
837
- </TooltipContent>
838
- </Tooltip>}
839
-
840
-
841
-
842
- {canSeeBetaFeatures &&
843
- <Tooltip>
844
- <TooltipTrigger asChild><Button
845
- onClick={() => saveClap()}
846
- disabled={!currentClap || isBusy}
847
- // variant="ghost"
848
- className={cn(
849
- `text-xs md:text-sm lg:text-base`,
850
- `bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
851
- `font-bold`,
852
- `hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
853
- storyPromptDraft ? "opacity-100" : "opacity-80"
854
- )}
855
- >
856
- <span className="hidden xl:inline mr-1">Save .clap</span>
857
- <span className="inline xl:hidden mr-1">Save .clap</span>
858
- </Button></TooltipTrigger>
859
- <TooltipContent side="top">
860
- <p className="text-xs font-normal text-stone-100/90 text-center">
861
- Clap is a new AI format,<br/>check out the academy<br/>to learn more about it.
862
- </p>
863
- </TooltipContent>
864
- </Tooltip>
865
- }
866
  </div>
867
 
868
  <div className="
869
- flex flex-row
870
- justify-between items-center
871
- space-x-3
872
- select-none
873
- ">
874
 
875
-
876
- {/* RANDOMNESS SWITCH */}
877
- <div className="
878
  flex flex-row
879
  justify-between items-center
880
  cursor-pointer
@@ -901,10 +233,15 @@ export function Main() {
901
  >
902
  <GiRollingDices size={24} />
903
  </div>
904
- </div>
 
 
 
 
 
 
905
  {/* END OF RANDOMNESS SWITCH */}
906
 
907
-
908
  {/* ORIENTATION SWITCH */}
909
  <div className="
910
  flex flex-row
@@ -965,180 +302,10 @@ export function Main() {
965
 
966
  `-mt-[20px] -mb-[90px] md:-mt-0 md:-mb-0`,
967
  )}>
968
-
969
- <div className={cn(`
970
- -mt-8 md:mt-0
971
- transition-all duration-200 ease-in-out
972
- `,
973
- isLandscape
974
- ? `scale-[0.9] md:scale-[0.75] lg:scale-[0.9] xl:scale-[1.0] 2xl:scale-[1.1]`
975
- : `scale-[0.8] md:scale-[0.9] lg:scale-[1.1]`
976
- )}>
977
- <DeviceFrameset
978
- device="Nexus 5"
979
- // color="black"
980
-
981
- landscape={isLandscape}
982
-
983
- // note 1: videos are generated in 1024x576 or 576x1024
984
- // so we need to keep the same ratio here
985
-
986
- // note 2: width and height are fixed, if width always stays 512
987
- // that's because the landscape={} parameter will do the switch for us
988
-
989
- width={288}
990
- height={512}
991
- >
992
- <div className="
993
- flex flex-col items-center justify-center
994
- w-full h-full
995
- bg-black text-white
996
- ">
997
- {isBusy ? <div className="
998
- flex flex-col
999
- items-center justify-center
1000
- text-center space-y-1.5">
1001
- <p className="text-2xl font-bold">{progress}%</p>
1002
- <p className="text-base text-white/70">{isBusy
1003
- ? (
1004
- // note: some of those tasks are running in parallel,
1005
- // and some are super-slow (like music or video)
1006
- // by carefully selecting in which order we set the ternaries,
1007
- // we can create the illusion that we just have a succession of reasonably-sized tasks
1008
- storyGenerationStatus === "generating" ? "Writing story.."
1009
- : parseGenerationStatus === "generating" ? "Loading the project.."
1010
- : assetGenerationStatus === "generating" ? "Casting characters.."
1011
- : imageGenerationStatus === "generating" ? "Creating storyboards.."
1012
- : soundGenerationStatus === "generating" ? "Recording sounds.."
1013
- : videoGenerationStatus === "generating" ? "Filming shots.."
1014
- : musicGenerationStatus === "generating" ? "Producing music.."
1015
- : voiceGenerationStatus === "generating" ? "Recording dialogues.."
1016
- : finalGenerationStatus === "generating" ? "Editing final cut.."
1017
- : "Please wait.."
1018
- )
1019
- : status === "error"
1020
- ? <span>{error || ""}</span>
1021
- : <span>{error ? error : <span>&nbsp;</span>}</span> // to prevent layout changes
1022
- }</p>
1023
- </div>
1024
- : (currentVideo && currentVideo?.length > 128) ? <video
1025
- src={currentVideo}
1026
- controls
1027
- playsInline
1028
- // I think we can't autoplay with sound,
1029
- // so let's disable auto-play
1030
- // autoPlay
1031
- // muted
1032
- loop
1033
- className="object-cover"
1034
- style={{
1035
- }}
1036
- /> : <div className="
1037
- flex flex-col
1038
- items-center justify-center
1039
- text-lg text-center"></div>}
1040
- </div>
1041
-
1042
- <div className={cn(`
1043
- fixed
1044
- flex flex-row items-center justify-center
1045
- bg-transparent
1046
- font-sans
1047
- -mb-0
1048
- `,
1049
- isLandscape ? 'h-4' : 'h-14'
1050
- )}
1051
- style={{ width: isPortrait ? 288 : 512 }}>
1052
- <span className="text-stone-100/50 text-4xs"
1053
- style={{ textShadow: "rgb(0 0 0 / 80%) 0px 0px 2px" }}>
1054
- Powered by
1055
- </span>
1056
- <span className="ml-1 mr-0.5">
1057
- <Image src={HFLogo} alt="Hugging Face" width={14} height={13} />
1058
- </span>
1059
- <span className="text-stone-100/80 text-3xs font-semibold"
1060
- style={{ textShadow: "rgb(0 0 0 / 80%) 0px 0px 2px" }}>Hugging Face</span>
1061
-
1062
- </div>
1063
- </DeviceFrameset>
1064
-
1065
- {(currentVideo && currentVideo.length > 128) ? <div
1066
- className={cn(`
1067
- w-full
1068
- flex flex-row
1069
- items-center justify-center
1070
- transition-all duration-150 ease-in-out
1071
-
1072
- text-stone-800
1073
-
1074
- group
1075
- pt-2 md:pt-4
1076
- `,
1077
- isBusy ? 'opacity-50' : 'cursor-pointer opacity-100 hover:scale-110 active:scale-150 hover:text-stone-950 active:text-black'
1078
- )}
1079
- style={{ textShadow: "rgb(255 255 255 / 19%) 0px 0px 2px" }}
1080
- onClick={isBusy ? undefined : saveVideo}
1081
- >
1082
- <div className="
1083
- text-base md:text-lg lg:text-xl
1084
- transition-all duration-150 ease-out
1085
- group-hover:animate-swing
1086
- "><FaCloudDownloadAlt /></div>
1087
- <div className="text-xs md:text-sm lg:text-base">&nbsp;Download</div>
1088
- </div> : null}
1089
- </div>
1090
  </div>
1091
  </div>
1092
- <div
1093
- className="
1094
- fixed
1095
-
1096
- left-4 md:left-8
1097
- bottom-4
1098
- flex flex-col md:flex-row
1099
- md:items-center justify-center
1100
- space-y-4 md:space-x-4 md:space-y-0
1101
-
1102
- ">
1103
- <a
1104
- className="
1105
- flex
1106
- no-underline
1107
- animation-all duration-150 ease-in-out
1108
- group
1109
- text-stone-950/60 hover:text-stone-950/80 scale-95 hover:scale-100"
1110
- href="https://discord.gg/AEruz9B92B"
1111
- target="_blank">
1112
- <div className="
1113
- text-base md:text-lg lg:text-xl
1114
- transition-all duration-150 ease-out
1115
- group-hover:animate-swing
1116
- "><FaDiscord /></div>
1117
- <div className="text-xs md:text-sm lg:text-base ml-1.5">
1118
- <span className="hidden md:block">Chat on Discord</span>
1119
- <span className="block md:hidden">Discord</span>
1120
- </div>
1121
- </a>
1122
- <a
1123
- className="
1124
- flex
1125
- no-underline
1126
- animation-all duration-150 ease-in-out
1127
- group
1128
- text-stone-950/60 hover:text-stone-950/80 scale-95 hover:scale-100"
1129
- href="https://latent-store.notion.site/AI-Stories-Academy-8e3ce6ff2d5946ffadc94193619dd5cd"
1130
- target="_blank">
1131
- <div className="
1132
- text-base md:text-lg lg:text-xl
1133
- transition-all duration-150 ease-out
1134
- group-hover:animate-swing
1135
- "><GiSpellBook /></div>
1136
- <div className="text-xs md:text-sm lg:text-base ml-1.5">
1137
- <span className="hidden md:block">Prompt academy</span>
1138
- <span className="block md:hidden">Academy</span>
1139
- </div>
1140
- </a>
1141
- </div>
1142
  </div>
1143
  <Toaster />
1144
  </div>
 
1
  "use client"
2
 
3
+ import React from "react"
4
  import { IoMdPhonePortrait } from "react-icons/io"
5
  import { GiRollingDices } from "react-icons/gi"
6
+ import { ClapMediaOrientation } from "@aitube/clap"
 
 
 
 
 
 
 
 
7
 
8
  import { Card, CardContent, CardHeader } from "@/components/ui/card"
9
  import { Button } from "@/components/ui/button"
10
  import { Toaster } from "@/components/ui/sonner"
11
  import { TextareaField } from "@/components/form/textarea-field"
 
12
 
13
+ import { cn, generateRandomStory } from "@/lib/utils"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  import { defaultPrompt } from "./config"
15
  import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"
16
+ import { useOrientation, useProgressTimer, useQueryStringParams, useStoryPromptDraft } from "@/lib/hooks"
17
+ import { BottomBar, VideoPreview } from "@/components/interface"
18
+ import { MainTitle } from "@/components/interface/main-title"
19
+ import { LoadClapButton } from "@/components/interface/load-clap-button"
20
+ import { SaveClapButton } from "@/components/interface/save-clap-button"
21
+ import { useProcessors } from "@/lib/hooks/useProcessors"
22
+ import { Characters } from "@/components/interface/characters"
23
 
24
  export function Main() {
25
+ const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
26
+ const { isBusy } = useProgressTimer()
27
+ const { orientation, toggleOrientation } = useOrientation()
28
+ const { handleSubmit } = useProcessors()
29
+ useQueryStringParams()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  return (
31
  <TooltipProvider>
32
  <div className={cn(
 
95
 
96
  )}>
97
  <CardHeader>
98
+ <MainTitle />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
99
  </CardHeader>
100
  <CardContent
101
  className="flex flex-col space-y-3"
 
189
  */}
190
 
191
  <div className="
192
+ flex flex-row
193
+ justify-between items-center
194
+ space-x-3">
195
+ <LoadClapButton />
196
+ <SaveClapButton />
197
+ <Characters />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
198
  </div>
199
 
200
  <div className="
201
+ flex flex-row
202
+ justify-between items-center
203
+ space-x-3
204
+ select-none
205
+ ">
206
 
207
+ {/* RANDOMNESS SWITCH */}
208
+ <Tooltip>
209
+ <TooltipTrigger asChild><div className="
210
  flex flex-row
211
  justify-between items-center
212
  cursor-pointer
 
233
  >
234
  <GiRollingDices size={24} />
235
  </div>
236
+ </div></TooltipTrigger>
237
+ <TooltipContent side="top">
238
+ <p className="text-xs font-normal text-stone-100/90 text-center">
239
+ Generate a random prompt.
240
+ </p>
241
+ </TooltipContent>
242
+ </Tooltip>
243
  {/* END OF RANDOMNESS SWITCH */}
244
 
 
245
  {/* ORIENTATION SWITCH */}
246
  <div className="
247
  flex flex-row
 
302
 
303
  `-mt-[20px] -mb-[90px] md:-mt-0 md:-mb-0`,
304
  )}>
305
+ <VideoPreview />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
306
  </div>
307
  </div>
308
+ <BottomBar />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
309
  </div>
310
  <Toaster />
311
  </div>
src/components/interface/bottom-bar.tsx ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import React from "react"
4
+ import { FaDiscord } from "react-icons/fa"
5
+ import { GiSpellBook } from "react-icons/gi"
6
+
7
+ export function BottomBar() {
8
+
9
+ return (
10
+ <div
11
+ className="
12
+ fixed
13
+
14
+ left-4 md:left-8
15
+ bottom-4
16
+ flex flex-col md:flex-row
17
+ md:items-center justify-center
18
+ space-y-4 md:space-x-4 md:space-y-0
19
+
20
+ ">
21
+ <a
22
+ className="
23
+ flex
24
+ no-underline
25
+ animation-all duration-150 ease-in-out
26
+ group
27
+ text-stone-950/60 hover:text-stone-950/80 scale-95 hover:scale-100"
28
+ href="https://discord.gg/AEruz9B92B"
29
+ target="_blank">
30
+ <div className="
31
+ text-base md:text-lg lg:text-xl
32
+ transition-all duration-150 ease-out
33
+ group-hover:animate-swing
34
+ "><FaDiscord /></div>
35
+ <div className="text-xs md:text-sm lg:text-base ml-1.5">
36
+ <span className="hidden md:block">Chat on Discord</span>
37
+ <span className="block md:hidden">Discord</span>
38
+ </div>
39
+ </a>
40
+ <a
41
+ className="
42
+ flex
43
+ no-underline
44
+ animation-all duration-150 ease-in-out
45
+ group
46
+ text-stone-950/60 hover:text-stone-950/80 scale-95 hover:scale-100"
47
+ href="https://latent-store.notion.site/AI-Stories-Academy-8e3ce6ff2d5946ffadc94193619dd5cd"
48
+ target="_blank">
49
+ <div className="
50
+ text-base md:text-lg lg:text-xl
51
+ transition-all duration-150 ease-out
52
+ group-hover:animate-swing
53
+ "><GiSpellBook /></div>
54
+ <div className="text-xs md:text-sm lg:text-base ml-1.5">
55
+ <span className="hidden md:block">Prompt academy</span>
56
+ <span className="block md:hidden">Academy</span>
57
+ </div>
58
+ </a>
59
+ </div>
60
+ );
61
+ }
src/components/interface/character-button.tsx ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { ClapEntity } from "@aitube/clap"
2
+ import { HiCog6Tooth } from "react-icons/hi2"
3
+ import { MdFace2 } from "react-icons/md"
4
+ import { FaWrench } from "react-icons/fa6"
5
+ import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
6
+ import { useEntityPicture } from "@/lib/hooks/useEntityPicture"
7
+ import { cn } from "@/lib/utils"
8
+
9
+ export function CharacterButton({
10
+ entity
11
+ }: {
12
+ entity?: ClapEntity
13
+ }) {
14
+ const { picture, openFilePicker } = useEntityPicture(entity)
15
+
16
+ return (
17
+ <Tooltip>
18
+ <TooltipTrigger asChild><div className={cn(`
19
+ flex flex-row
20
+
21
+ justify-between items-center
22
+ cursor-pointer
23
+ transition-all duration-150 ease-in-out
24
+ hover:scale-110 active:scale-150
25
+ text-stone-800
26
+ hover:text-stone-950
27
+ active:text-black
28
+ group
29
+ `,
30
+ // picture ? ` opacity-100 ` : `opacity-60`
31
+ )}
32
+ onClick={openFilePicker}>
33
+
34
+ {picture
35
+ ? <div className={cn(`
36
+ flex
37
+ w-10 h-10 rounded-full
38
+ overflow-hidden
39
+ `,
40
+ picture
41
+ ? `
42
+ border-2 group-hover:border
43
+ border-stone-950/70 dark:border-stone-950/70
44
+ group-hover:shadow-md
45
+ `
46
+ : ``
47
+ )}>
48
+ <img
49
+ className="object-cover"
50
+ src={picture}
51
+ />
52
+ </div>: <MdFace2 size={24} />}
53
+
54
+ <div className="
55
+ -ml-2 mt-10
56
+ flex flex-row items-center justify-center
57
+ transition-all duration-150 ease-out
58
+ group-hover:animate-swing
59
+ opacity-0 group-hover:opacity-100
60
+ scale-0 group-hover:scale-100
61
+ "
62
+ >
63
+ {picture
64
+ ? <FaWrench size={16} />
65
+ : null
66
+ }
67
+ </div>
68
+ </div></TooltipTrigger>
69
+ <TooltipContent side="top">
70
+ <p className="text-xs font-normal text-stone-100/90 text-center">
71
+ Using this experimental<br/>feature may reduce<br/>
72
+ the quality of images.
73
+ </p>
74
+ </TooltipContent>
75
+ </Tooltip>
76
+ )
77
+ }
src/components/interface/characters.tsx ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useStore } from "@/app/store"
2
+ import { CharacterButton } from "./character-button"
3
+
4
+ export function Characters() {
5
+ const currentClap = useStore(s => s.currentClap)
6
+
7
+
8
+ return (
9
+ <div className="flex flex-row space-x-0">
10
+ {currentClap && currentClap.entities?.length > 0
11
+ // now: we only support displaying ONE entity for now
12
+ ? currentClap.entities.slice(0, 1).map(entity =>
13
+ <CharacterButton key={entity.id} entity={entity} />
14
+ )
15
+ : <CharacterButton />
16
+ }
17
+ </div>
18
+ )
19
+ }
src/components/interface/index.ts ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ export { VideoPreview } from "./video-preview"
2
+ export { BottomBar } from "./bottom-bar"
src/components/interface/load-clap-button.tsx ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import React from "react"
4
+
5
+ import { Button } from "@/components/ui/button"
6
+
7
+ import { cn } from "@/lib/utils"
8
+
9
+ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
10
+ import { useImportClap, useIsBusy, useStoryPromptDraft } from "@/lib/hooks"
11
+
12
+ export function LoadClapButton() {
13
+ const { isBusy } = useIsBusy()
14
+ const { openClapFilePicker } = useImportClap()
15
+ const { storyPromptDraft } = useStoryPromptDraft()
16
+
17
+ return (
18
+ <Tooltip>
19
+ <TooltipTrigger asChild><Button
20
+
21
+ onClick={openClapFilePicker}
22
+ disabled={isBusy}
23
+ // variant="ghost"
24
+ className={cn(
25
+ `text-xs md:text-sm lg:text-base`,
26
+ `bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
27
+ `font-bold`,
28
+ `hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
29
+ storyPromptDraft ? "opacity-100" : "opacity-80"
30
+ )}
31
+ >
32
+ <span className="hidden xl:inline mr-1">Load .clap</span>
33
+ <span className="inline xl:hidden mr-1">Load .clap</span>
34
+ </Button></TooltipTrigger>
35
+ <TooltipContent side="top">
36
+ <p className="text-xs font-normal text-stone-100/90 text-center">
37
+ Clap is a new AI format,<br/>check out the academy<br/>to learn more about it.
38
+ </p>
39
+ </TooltipContent>
40
+ </Tooltip>
41
+ )
42
+ }
src/components/interface/main-title.tsx ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ export function MainTitle() {
2
+ return (
3
+ <div className="flex flex-col justify-start">
4
+ <div className="
5
+ flex flex-row
6
+ items-center justify-center
7
+ transition-all duration-200 ease-in-out
8
+ px-3
9
+
10
+ rounded-full">
11
+ <div
12
+ className="
13
+ flex
14
+ transition-all duration-200 ease-in-out
15
+ items-center justify-center text-center
16
+ w-10 h-10 md:w-12 md:h-12 lg:w-16 lg:h-16
17
+ text-3xl md:text-4xl lg:text-5xl
18
+ rounded-lg
19
+ mr-2
20
+ font-sans
21
+ bg-amber-400 dark:bg-amber-400
22
+
23
+ text-stone-950/80 dark:text-stone-950/80 font-bold
24
+ "
25
+ >AI</div>
26
+ <div
27
+ className="
28
+ transition-all duration-200 ease-in-out
29
+ text-amber-400 dark:text-amber-400
30
+ text-3xl md:text-4xl lg:text-5xl
31
+ "
32
+ style={{ textShadow: "#00000035 0px 0px 2px" }}
33
+
34
+ /*
35
+ className="
36
+ text-5xl
37
+ bg-gradient-to-br from-yellow-300 to-yellow-500
38
+ inline-block text-transparent bg-clip-text
39
+ py-6
40
+ "
41
+ */
42
+ >Stories Factory</div>
43
+ </div>
44
+ <p
45
+ className="
46
+ transition-all duration-200 ease-in-out
47
+ text-stone-900/90 dark:text-stone-900/90
48
+ text-lg md:text-xl lg:text-2xl
49
+ text-center
50
+ pt-2 md:pt-4
51
+ "
52
+ style={{ textShadow: "rgb(255 255 255 / 19%) 0px 0px 2px" }}
53
+ >Make video stories using AI ✨</p>
54
+ </div>
55
+ )
56
+ }
src/components/interface/save-clap-button.tsx ADDED
@@ -0,0 +1,43 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import React from "react"
4
+
5
+ import { Button } from "@/components/ui/button"
6
+
7
+ import { useStore } from "../../app/store"
8
+ import { cn } from "@/lib/utils"
9
+
10
+ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
11
+ import { useIsBusy, useStoryPromptDraft } from "@/lib/hooks"
12
+
13
+ export function SaveClapButton() {
14
+ const { isBusy } = useIsBusy()
15
+ const currentClap = useStore(s => s.currentClap)
16
+ const saveClap = useStore(s => s.saveClap)
17
+ const { storyPromptDraft } = useStoryPromptDraft()
18
+
19
+ return (
20
+ <Tooltip>
21
+ <TooltipTrigger asChild><Button
22
+ onClick={() => saveClap()}
23
+ disabled={!currentClap || isBusy}
24
+ // variant="ghost"
25
+ className={cn(
26
+ `text-xs md:text-sm lg:text-base`,
27
+ `bg-stone-800/90 text-amber-400/100 dark:bg-stone-800/90 dark:text-amber-400/100`,
28
+ `font-bold`,
29
+ `hover:bg-stone-800/100 hover:text-amber-300/100 dark:hover:bg-stone-800/100 dark:hover:text-amber-300/100`,
30
+ storyPromptDraft ? "opacity-100" : "opacity-80"
31
+ )}
32
+ >
33
+ <span className="hidden xl:inline mr-1">Save .clap</span>
34
+ <span className="inline xl:hidden mr-1">Save .clap</span>
35
+ </Button></TooltipTrigger>
36
+ <TooltipContent side="top">
37
+ <p className="text-xs font-normal text-stone-100/90 text-center">
38
+ Clap is a new AI format,<br/>check out the academy<br/>to learn more about it.
39
+ </p>
40
+ </TooltipContent>
41
+ </Tooltip>
42
+ )
43
+ }
src/components/interface/video-preview.tsx ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import React from "react"
4
+ import { FaCloudDownloadAlt } from "react-icons/fa"
5
+ import Image from "next/image"
6
+ import { DeviceFrameset } from "react-device-frameset"
7
+ import "react-device-frameset/styles/marvel-devices.min.css"
8
+
9
+ import { useOrientation } from "@/lib/hooks/useOrientation"
10
+ import { useProgressTimer } from "@/lib/hooks/useProgressTimer"
11
+ import { cn } from "@/lib/utils/cn"
12
+
13
+ import { useStore } from "../../app/store"
14
+ import HFLogo from "../../app/hf-logo.svg"
15
+
16
+ export function VideoPreview() {
17
+ const status = useStore(s => s.status)
18
+ const parseGenerationStatus = useStore(s => s.parseGenerationStatus)
19
+ const storyGenerationStatus = useStore(s => s.storyGenerationStatus)
20
+ const assetGenerationStatus = useStore(s => s.assetGenerationStatus)
21
+ const soundGenerationStatus = useStore(s => s.soundGenerationStatus)
22
+ const musicGenerationStatus = useStore(s => s.musicGenerationStatus)
23
+ const voiceGenerationStatus = useStore(s => s.voiceGenerationStatus)
24
+ const imageGenerationStatus = useStore(s => s.imageGenerationStatus)
25
+ const videoGenerationStatus = useStore(s => s.videoGenerationStatus)
26
+ const finalGenerationStatus = useStore(s => s.finalGenerationStatus)
27
+ const currentVideo = useStore(s => s.currentVideo)
28
+
29
+ const error = useStore(s => s.error)
30
+
31
+ const saveVideo = useStore(s => s.saveVideo)
32
+
33
+
34
+ const { isBusy, progress } = useProgressTimer()
35
+
36
+ const {
37
+ isLandscape,
38
+ isPortrait,
39
+ } = useOrientation()
40
+
41
+ return (
42
+ <div className={cn(`
43
+ -mt-8 md:mt-0
44
+ transition-all duration-200 ease-in-out
45
+ `,
46
+ isLandscape
47
+ ? `scale-[0.9] md:scale-[0.75] lg:scale-[0.9] xl:scale-[1.0] 2xl:scale-[1.1]`
48
+ : `scale-[0.8] md:scale-[0.9] lg:scale-[1.1]`
49
+ )}>
50
+ <DeviceFrameset
51
+ device="Nexus 5"
52
+ // color="black"
53
+
54
+ landscape={isLandscape}
55
+
56
+ // note 1: videos are generated in 1024x576 or 576x1024
57
+ // so we need to keep the same ratio here
58
+
59
+ // note 2: width and height are fixed, if width always stays 512
60
+ // that's because the landscape={} parameter will do the switch for us
61
+
62
+ width={288}
63
+ height={512}
64
+ >
65
+ <div className="
66
+ flex flex-col items-center justify-center
67
+ w-full h-full
68
+ bg-black text-white
69
+ ">
70
+ {isBusy ? <div className="
71
+ flex flex-col
72
+ items-center justify-center
73
+ text-center space-y-1.5">
74
+ <p className="text-2xl font-bold">{progress}%</p>
75
+ <p className="text-base text-white/70">{isBusy
76
+ ? (
77
+ // note: some of those tasks are running in parallel,
78
+ // and some are super-slow (like music or video)
79
+ // by carefully selecting in which order we set the ternaries,
80
+ // we can create the illusion that we just have a succession of reasonably-sized tasks
81
+ storyGenerationStatus === "generating" ? "Writing story.."
82
+ : parseGenerationStatus === "generating" ? "Loading the project.."
83
+ : assetGenerationStatus === "generating" ? "Casting characters.."
84
+ : imageGenerationStatus === "generating" ? "Creating storyboards.."
85
+ : soundGenerationStatus === "generating" ? "Recording sounds.."
86
+ : videoGenerationStatus === "generating" ? "Filming shots.."
87
+ : musicGenerationStatus === "generating" ? "Producing music.."
88
+ : voiceGenerationStatus === "generating" ? "Recording dialogues.."
89
+ : finalGenerationStatus === "generating" ? "Editing final cut.."
90
+ : "Please wait.."
91
+ )
92
+ : status === "error"
93
+ ? <span>{error || ""}</span>
94
+ : <span>{error ? error : <span>&nbsp;</span>}</span> // to prevent layout changes
95
+ }</p>
96
+ </div>
97
+ : (currentVideo && currentVideo?.length > 128) ? <video
98
+ src={currentVideo}
99
+ controls
100
+ playsInline
101
+ // I think we can't autoplay with sound,
102
+ // so let's disable auto-play
103
+ // autoPlay
104
+ // muted
105
+ loop
106
+ className="object-cover"
107
+ style={{
108
+ }}
109
+ /> : <div className="
110
+ flex flex-col
111
+ items-center justify-center
112
+ text-lg text-center"></div>}
113
+ </div>
114
+
115
+ <div className={cn(`
116
+ fixed
117
+ flex flex-row items-center justify-center
118
+ bg-transparent
119
+ font-sans
120
+ -mb-0
121
+ `,
122
+ isLandscape ? 'h-4' : 'h-14'
123
+ )}
124
+ style={{ width: isPortrait ? 288 : 512 }}>
125
+ <span className="text-stone-100/50 text-4xs"
126
+ style={{ textShadow: "rgb(0 0 0 / 80%) 0px 0px 2px" }}>
127
+ Powered by
128
+ </span>
129
+ <span className="ml-1 mr-0.5">
130
+ <Image src={HFLogo} alt="Hugging Face" width={14} height={13} />
131
+ </span>
132
+ <span className="text-stone-100/80 text-3xs font-semibold"
133
+ style={{ textShadow: "rgb(0 0 0 / 80%) 0px 0px 2px" }}>Hugging Face</span>
134
+
135
+ </div>
136
+ </DeviceFrameset>
137
+
138
+ {(currentVideo && currentVideo.length > 128) ? <div
139
+ className={cn(`
140
+ w-full
141
+ flex flex-row
142
+ items-center justify-center
143
+ transition-all duration-150 ease-in-out
144
+
145
+ text-stone-800
146
+
147
+ group
148
+ pt-2 md:pt-4
149
+ `,
150
+ isBusy ? 'opacity-50' : 'cursor-pointer opacity-100 hover:scale-110 active:scale-150 hover:text-stone-950 active:text-black'
151
+ )}
152
+ style={{ textShadow: "rgb(255 255 255 / 19%) 0px 0px 2px" }}
153
+ onClick={isBusy ? undefined : saveVideo}
154
+ >
155
+ <div className="
156
+ text-base md:text-lg lg:text-xl
157
+ transition-all duration-150 ease-out
158
+ group-hover:animate-swing
159
+ "><FaCloudDownloadAlt /></div>
160
+ <div className="text-xs md:text-sm lg:text-base">&nbsp;Download</div>
161
+ </div> : null}
162
+ </div>
163
+ )
164
+ }
src/lib/hooks/index.tsx ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ export { useImportClap } from "./useImportClap"
2
+ export { useIsBusy } from "./useIsBusy"
3
+ export { useOrientation } from "./useOrientation"
4
+ export { useProgressTimer } from "./useProgressTimer"
5
+ export { useQueryStringParams } from "./useQueryStringParams"
6
+ export { useStoryPromptDraft } from "./useStoryPromptDraft"
src/lib/hooks/useEntityPicture.ts ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useEffect, useState } from "react"
2
+ import { useFilePicker } from "use-file-picker"
3
+
4
+ import { useStore } from "../../app/store"
5
+ import { ClapEntity, getClapAssetSourceType } from "@aitube/clap"
6
+ import { useOpenPictureFile } from "./useOpenPictureFile"
7
+
8
+ export function useEntityPicture(entity?: ClapEntity) {
9
+ const defaultPicture = entity?.imageId
10
+ const [currentPicture, setCurrentPicture] = useState(defaultPicture)
11
+ const { file, openFilePicker } = useOpenPictureFile()
12
+ const setMainCharacterImage = useStore(s => s.setMainCharacterImage)
13
+
14
+ const newPicture = file || currentPicture || defaultPicture
15
+
16
+ useEffect(() => {
17
+ setCurrentPicture(newPicture)
18
+ if (!newPicture) { return }
19
+
20
+ if (entity) {
21
+ entity.imageId = newPicture
22
+ } else {
23
+ setMainCharacterImage(newPicture)
24
+ }
25
+ }, [newPicture, JSON.stringify(entity)])
26
+
27
+ return { picture: newPicture, openFilePicker }
28
+ }
src/lib/hooks/useImportClap.ts ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useEffect, useRef } from "react"
2
+ import { useFilePicker } from "use-file-picker"
3
+ import { FileContent } from "use-file-picker/dist/interfaces"
4
+ import { ClapProject, updateClap } from "@aitube/clap"
5
+
6
+ import { useStore } from "../../app/store"
7
+ import { useProcessors } from "./useProcessors"
8
+
9
+ export function useImportClap() {
10
+
11
+ const setError = useStore(s => s.setError)
12
+ const setCurrentClap = useStore(s => s.setCurrentClap)
13
+ const loadClap = useStore(s => s.loadClap)
14
+
15
+ const {
16
+ generateMusic,
17
+ generateVideos,
18
+ generateFinalVideo,
19
+ } = useProcessors()
20
+
21
+ const importStory = async (fileData: FileContent<ArrayBuffer>): Promise<{
22
+ clap: ClapProject
23
+ regenerateVideo: boolean
24
+ }> => {
25
+ if (!fileData?.name) { throw new Error(`invalid file (missing file name)`) }
26
+
27
+ const {
28
+ setParseGenerationStatus,
29
+ } = useStore.getState()
30
+
31
+ setParseGenerationStatus("generating")
32
+
33
+ try {
34
+ const blob = new Blob([fileData.content])
35
+ const res = await loadClap(blob, fileData.name)
36
+
37
+ if (!res?.clap) { throw new Error(`failed to load the clap file`) }
38
+
39
+ setParseGenerationStatus("finished")
40
+ return res
41
+ } catch (err) {
42
+ console.error("failed to load the Clap file:", err)
43
+ setParseGenerationStatus("error")
44
+ throw err
45
+ }
46
+ }
47
+
48
+ const { openFilePicker: openClapFilePicker, filesContent } = useFilePicker({
49
+ accept: '.clap',
50
+ readAs: "ArrayBuffer"
51
+ })
52
+ const fileData = filesContent[0]
53
+
54
+ useEffect(() => {
55
+ const fn = async () => {
56
+ if (!fileData?.name) { return }
57
+
58
+ const { setStatus, setProgress } = useStore.getState()
59
+
60
+ setProgress(0)
61
+ setStatus("generating")
62
+
63
+ try {
64
+ let { clap, regenerateVideo } = await importStory(fileData)
65
+
66
+ // clap = await generateSounds(clap)
67
+
68
+ // setCurrentClap(clap)
69
+
70
+ console.log("loadClap(): clap = ", clap)
71
+
72
+ // it is important to skip regeneration if we already have a video
73
+ if (regenerateVideo) {
74
+ console.log(`regenerating music and videos..`)
75
+ const claps = await Promise.all([
76
+ generateMusic(clap),
77
+ generateVideos(clap)
78
+ ])
79
+
80
+ // console.log("finished processing the 2 tasks in parallel")
81
+
82
+ for (const newerClap of claps) {
83
+ clap = await updateClap(clap, newerClap, {
84
+ overwriteMeta: false,
85
+ inlineReplace: true,
86
+ })
87
+ }
88
+
89
+
90
+ setCurrentClap(clap)
91
+
92
+ await generateFinalVideo(clap)
93
+
94
+ } else {
95
+ console.log(`skipping music and video regeneration`)
96
+ }
97
+
98
+ setStatus("finished")
99
+ setProgress(100)
100
+ setError("")
101
+ } catch (err) {
102
+ console.error(`failed to import: `, err)
103
+ setStatus("error")
104
+ setError(`${err}`)
105
+ }
106
+
107
+ }
108
+ fn()
109
+ }, [fileData?.name])
110
+
111
+ return { openClapFilePicker }
112
+ }
src/lib/hooks/useIsBusy.ts ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useRef } from "react"
2
+
3
+ import { useStore } from "@/app/store"
4
+
5
+ export function useIsBusy() {
6
+ const isBusy = useStore(s => s.isBusy)
7
+ const busyRef = useRef(isBusy)
8
+ busyRef.current = isBusy
9
+
10
+ return {isBusy, busyRef }
11
+ }
src/lib/hooks/useOpenPictureFile.ts ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useEffect, useState } from "react"
2
+ import { useFilePicker } from "use-file-picker"
3
+
4
+ export function useOpenPictureFile() {
5
+ const [picture, setPicture] = useState("")
6
+
7
+ const { openFilePicker, filesContent } = useFilePicker({
8
+ readAs: 'DataURL',
9
+ accept: 'image/*',
10
+ })
11
+ const fileData = filesContent[0]
12
+
13
+ useEffect(() => {
14
+ if (!fileData?.name) { return }
15
+ setPicture(fileData.content)
16
+ }, [fileData?.name])
17
+
18
+ return { file: picture, openFilePicker }
19
+ }
src/lib/hooks/useOrientation.ts ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useStore } from "@/app/store"
2
+ import { ClapMediaOrientation } from "@aitube/clap"
3
+
4
+ export function useOrientation() {
5
+ const orientation = useStore(s => s.orientation)
6
+ const setOrientation = useStore(s => s.setOrientation)
7
+ const currentVideoOrientation = useStore(s => s.currentVideoOrientation)
8
+ const toggleOrientation = useStore(s => s.toggleOrientation)
9
+ // note: we are interested in the *current* video orientation,
10
+ // not the requested video orientation requested for the next video
11
+ const isLandscape = currentVideoOrientation === ClapMediaOrientation.LANDSCAPE
12
+ const isPortrait = currentVideoOrientation === ClapMediaOrientation.PORTRAIT
13
+ const isSquare = currentVideoOrientation === ClapMediaOrientation.SQUARE
14
+
15
+ return {
16
+ orientation,
17
+ setOrientation,
18
+ currentVideoOrientation,
19
+ toggleOrientation,
20
+ isLandscape,
21
+ isPortrait,
22
+ isSquare
23
+ }
24
+ }
src/lib/hooks/useProcessors.ts ADDED
@@ -0,0 +1,447 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import React, { useTransition } from "react"
4
+ import { ClapProject, ClapSegmentCategory, getClapAssetSourceType, newEntity, updateClap } from "@aitube/clap"
5
+
6
+ import { logImage } from "@/lib/utils"
7
+ import { useIsBusy, useStoryPromptDraft } from "@/lib/hooks"
8
+
9
+ import { createClap } from "@/app/server/aitube/createClap"
10
+ import { editClapEntities } from "@/app/server/aitube/editClapEntities"
11
+ import { editClapDialogues } from "@/app/server/aitube/editClapDialogues"
12
+ import { editClapStoryboards } from "@/app/server/aitube/editClapStoryboards"
13
+ import { editClapSounds } from "@/app/server/aitube/editClapSounds"
14
+ import { editClapMusic } from "@/app/server/aitube/editClapMusic"
15
+ import { editClapVideos } from "@/app/server/aitube/editClapVideos"
16
+ import { exportClapToVideo } from "@/app/server/aitube/exportClapToVideo"
17
+
18
+ import { useStore } from "../../app/store"
19
+
20
+ export function useProcessors() {
21
+ const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
22
+
23
+ const [_isPending, startTransition] = useTransition()
24
+
25
+ const mainCharacterImage = useStore(s => s.mainCharacterImage)
26
+ const mainCharacterVoice = useStore(s => s.mainCharacterVoice)
27
+
28
+ const currentClap = useStore(s => s.currentClap)
29
+ const setStoryPrompt = useStore(s => s.setStoryPrompt)
30
+ const setMainCharacterImage = useStore(s => s.setMainCharacterImage)
31
+ const setMainCharacterVoice = useStore(s => s.setMainCharacterVoice)
32
+ const setStatus = useStore(s => s.setStatus)
33
+
34
+ const error = useStore(s => s.error)
35
+ const setError = useStore(s => s.setError)
36
+ const setParseGenerationStatus = useStore(s => s.setParseGenerationStatus)
37
+ const setStoryGenerationStatus = useStore(s => s.setStoryGenerationStatus)
38
+ const setAssetGenerationStatus = useStore(s => s.setAssetGenerationStatus)
39
+ const setSoundGenerationStatus = useStore(s => s.setSoundGenerationStatus)
40
+ const setMusicGenerationStatus = useStore(s => s.setMusicGenerationStatus)
41
+ const setVoiceGenerationStatus = useStore(s => s.setVoiceGenerationStatus)
42
+ const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus)
43
+ const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus)
44
+ const setFinalGenerationStatus = useStore(s => s.setFinalGenerationStatus)
45
+ const setCurrentClap = useStore(s => s.setCurrentClap)
46
+ const setCurrentVideo = useStore(s => s.setCurrentVideo)
47
+ const setProgress = useStore(s => s.setProgress)
48
+
49
+ const { isBusy, busyRef } = useIsBusy()
50
+
51
+ const generateStory = async (): Promise<ClapProject> => {
52
+
53
+ let clap: ClapProject | undefined = undefined
54
+ try {
55
+ setProgress(0)
56
+
57
+ setStatus("generating")
58
+ setStoryGenerationStatus("generating")
59
+ setStoryPrompt(promptDraftRef.current)
60
+
61
+ clap = await createClap({
62
+ prompt: promptDraftRef.current,
63
+ orientation: useStore.getState().orientation,
64
+
65
+ turbo: false,
66
+ })
67
+
68
+ if (!clap) { throw new Error(`failed to create the clap`) }
69
+
70
+ if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) }
71
+
72
+ console.log(`handleSubmit(): received a clap = `, clap)
73
+
74
+ console.log(`handleSubmit(): copying over entities from the previous clap`)
75
+
76
+ console.log(`handleSubmit(): later we can add button(s) to clear the project and/or the character(s)`)
77
+ const { currentClap } = useStore.getState()
78
+
79
+ clap.entities = Array.isArray(currentClap?.entities) ? currentClap.entities : []
80
+
81
+ setCurrentClap(clap)
82
+ setStoryGenerationStatus("finished")
83
+
84
+ console.log("---------------- GENERATED STORY ----------------")
85
+ console.table(clap.segments, [
86
+ // 'startTimeInMs',
87
+ 'endTimeInMs',
88
+ // 'track',
89
+ 'category',
90
+ 'prompt'
91
+ ])
92
+ return clap
93
+ } catch (err) {
94
+ setStoryGenerationStatus("error")
95
+ throw err
96
+ }
97
+ }
98
+
99
+ const generateEntities = async (clap: ClapProject): Promise<ClapProject> => {
100
+ try {
101
+ // setProgress(20)
102
+ setAssetGenerationStatus("generating")
103
+ clap = await editClapEntities({
104
+ clap,
105
+
106
+ // generating entities requires a "smart" LLM
107
+ turbo: false,
108
+ // turbo: true,
109
+ }).then(r => r.promise)
110
+
111
+ if (!clap) { throw new Error(`failed to edit the entities`) }
112
+
113
+ console.log(`generateEntities(): received entities = `, clap)
114
+
115
+ if (mainCharacterImage) {
116
+ // we assume the main entity is the first one
117
+ let mainEntity = clap.entities.at(0)
118
+
119
+ if (mainEntity) {
120
+ console.log(`generateEntities(): replacing the main character's picture..`)
121
+ mainEntity.thumbnailUrl = mainCharacterImage
122
+ mainEntity.imagePrompt = ""
123
+ mainEntity.imageSourceType = getClapAssetSourceType(mainCharacterImage)
124
+ mainEntity.imageEngine = ""
125
+ mainEntity.imageId = mainCharacterImage
126
+ }
127
+ }
128
+
129
+ setAssetGenerationStatus("finished")
130
+ console.log("---------------- GENERATED ENTITIES ----------------")
131
+ console.table(clap.entities, [
132
+ 'category',
133
+ 'label',
134
+ 'imagePrompt',
135
+ 'appearance'
136
+ ])
137
+ return clap
138
+ } catch (err) {
139
+ setAssetGenerationStatus("error")
140
+ throw err
141
+ }
142
+ }
143
+
144
+ const generateSounds = async (clap: ClapProject): Promise<ClapProject> => {
145
+ try {
146
+ // setProgress(30)
147
+ setSoundGenerationStatus("generating")
148
+
149
+ clap = await editClapSounds({
150
+ clap,
151
+ turbo: false,
152
+ }).then(r => r.promise)
153
+
154
+ if (!clap) { throw new Error(`failed to edit the sound`) }
155
+
156
+ console.log(`handleSubmit(): received a clap with sound = `, clap)
157
+ setSoundGenerationStatus("finished")
158
+ console.log("---------------- GENERATED SOUND ----------------")
159
+ console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.SOUND), [
160
+ 'endTimeInMs',
161
+ 'prompt',
162
+ 'entityId',
163
+ ])
164
+ return clap
165
+ } catch (err) {
166
+ setSoundGenerationStatus("error")
167
+ throw err
168
+ }
169
+ }
170
+
171
+ const generateMusic = async (clap: ClapProject): Promise<ClapProject> => {
172
+ try {
173
+ // setProgress(30)
174
+ setMusicGenerationStatus("generating")
175
+
176
+ clap = await editClapMusic({
177
+ clap,
178
+ turbo: false,
179
+ }).then(r => r.promise)
180
+
181
+ if (!clap) { throw new Error(`failed to edit the music`) }
182
+
183
+ console.log(`handleSubmit(): received a clap with music = `, clap)
184
+ setMusicGenerationStatus("finished")
185
+ console.log("---------------- GENERATED MUSIC ----------------")
186
+ console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC), [
187
+ 'endTimeInMs',
188
+ 'prompt',
189
+ 'entityId',
190
+ ])
191
+ return clap
192
+ } catch (err) {
193
+ setMusicGenerationStatus("error")
194
+ throw err
195
+ }
196
+ }
197
+
198
+ const generateStoryboards = async (clap: ClapProject): Promise<ClapProject> => {
199
+ try {
200
+ // setProgress(40)
201
+ setImageGenerationStatus("generating")
202
+ clap = await editClapStoryboards({
203
+ clap,
204
+ // if we use entities, then we MUST use turbo
205
+ // that's because turbo uses PulID,
206
+ // but SDXL doesn't
207
+ turbo: false,
208
+ }).then(r => r.promise)
209
+
210
+ if (!clap) { throw new Error(`failed to edit the storyboards`) }
211
+
212
+ // const fusion =
213
+ console.log(`handleSubmit(): received storyboards = `, clap)
214
+
215
+ setImageGenerationStatus("finished")
216
+ console.log("---------------- GENERATED STORYBOARDS ----------------")
217
+ clap.segments
218
+ .filter(s => s.category === ClapSegmentCategory.STORYBOARD)
219
+ .forEach((s, i) => {
220
+ if (s.status === "completed" && s.assetUrl) {
221
+ // console.log(` [${i}] storyboard: ${s.prompt}`)
222
+ logImage(s.assetUrl, 0.35)
223
+ } else {
224
+ console.log(` [${i}] failed to generate storyboard`)
225
+ }
226
+ // console.log(`------------------`)
227
+ })
228
+ console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD), [
229
+ 'endTimeInMs',
230
+ 'prompt',
231
+ 'assetUrl'
232
+ ])
233
+ return clap
234
+ } catch (err) {
235
+ setImageGenerationStatus("error")
236
+ throw err
237
+ }
238
+ }
239
+
240
+ const generateVideos = async (clap: ClapProject): Promise<ClapProject> => {
241
+ try {
242
+ // setProgress(50)
243
+ setVideoGenerationStatus("generating")
244
+
245
+ clap = await editClapVideos({
246
+ clap,
247
+ turbo: false
248
+ }).then(r => r.promise)
249
+
250
+ if (!clap) { throw new Error(`failed to edit the videos`) }
251
+
252
+ console.log(`handleSubmit(): received individual video clips = `, clap)
253
+ setVideoGenerationStatus("finished")
254
+ console.log("---------------- GENERATED VIDEOS ----------------")
255
+ console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO), [
256
+ 'endTimeInMs',
257
+ 'prompt',
258
+ 'entityId',
259
+ ])
260
+ return clap
261
+ } catch (err) {
262
+ setVideoGenerationStatus("error")
263
+ throw err
264
+ }
265
+ }
266
+
267
+ const generateStoryboardsThenVideos = async (clap: ClapProject): Promise<ClapProject> => {
268
+ clap = await generateStoryboards(clap)
269
+ clap = await generateVideos(clap)
270
+ return clap
271
+ }
272
+
273
+
274
+ const generateDialogues = async (clap: ClapProject): Promise<ClapProject> => {
275
+ try {
276
+ // setProgress(70)
277
+ setVoiceGenerationStatus("generating")
278
+ clap = await editClapDialogues({
279
+ clap,
280
+ turbo: false,
281
+ }).then(r => r.promise)
282
+
283
+ if (!clap) { throw new Error(`failed to edit the dialogues`) }
284
+
285
+ console.log(`handleSubmit(): received dialogues = `, clap)
286
+ setVoiceGenerationStatus("finished")
287
+ console.log("---------------- GENERATED DIALOGUES ----------------")
288
+ console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.DIALOGUE), [
289
+ 'endTimeInMs',
290
+ 'prompt',
291
+ 'entityId',
292
+ ])
293
+ return clap
294
+ } catch (err) {
295
+ setVoiceGenerationStatus("error")
296
+ throw err
297
+ }
298
+ }
299
+
300
+ const generateFinalVideo = async (clap: ClapProject): Promise<string> => {
301
+
302
+ let assetUrl = ""
303
+ try {
304
+ // setProgress(85)
305
+ setFinalGenerationStatus("generating")
306
+ assetUrl = await exportClapToVideo({
307
+ clap,
308
+ turbo: false,
309
+ })
310
+
311
+ setCurrentVideo(assetUrl)
312
+
313
+ if (assetUrl.length < 128) { throw new Error(`generateFinalVideo(): the generated video is too small, so we failed`) }
314
+
315
+ console.log(`generateFinalVideo(): received a video: ${assetUrl.slice(0, 120)}...`)
316
+ setFinalGenerationStatus("finished")
317
+ return assetUrl
318
+ } catch (err) {
319
+ setFinalGenerationStatus("error")
320
+ throw err
321
+ }
322
+ }
323
+
324
+ const handleSubmit = async () => {
325
+ setStatus("generating")
326
+ busyRef.current = true
327
+
328
+ startTransition(async () => {
329
+ setStatus("generating")
330
+ busyRef.current = true
331
+
332
+ console.log(`handleSubmit(): generating a clap using prompt = "${promptDraftRef.current}" `)
333
+
334
+ try {
335
+ let clap = await generateStory()
336
+ setCurrentClap(clap)
337
+
338
+ const storyboards = clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD)
339
+
340
+ let mainCharacter = clap.entities.at(0)
341
+
342
+ // let's do something basic for now: we only support 1 entity (character)
343
+ // and we apply it to *all* the storyboards (we can always improve this later)
344
+ if (mainCharacter) {
345
+ console.log(`handleSubmit(): we use the clap's main character's face on all storyboards`)
346
+ storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
347
+ logImage(mainCharacter.imageId, 0.35)
348
+ } else if (mainCharacterImage) {
349
+ console.log(`handleSubmit(): declaring a new entity for our main character`)
350
+ const entityName = "person"
351
+ mainCharacter = newEntity({
352
+ category: ClapSegmentCategory.CHARACTER,
353
+ triggerName: entityName,
354
+ label: entityName,
355
+ description: entityName,
356
+ author: "auto",
357
+ thumbnailUrl: mainCharacterImage,
358
+
359
+ imagePrompt: "",
360
+ imageSourceType: getClapAssetSourceType(mainCharacterImage),
361
+ imageEngine: "",
362
+ imageId: mainCharacterImage,
363
+ audioPrompt: "",
364
+ })
365
+
366
+ clap.entities.push(mainCharacter!)
367
+ console.log(`handleSubmit(): we use the main character's face on all storyboards`)
368
+
369
+ storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
370
+ logImage(mainCharacterImage, 0.35)
371
+ }
372
+
373
+ const tasks = [
374
+ generateMusic(clap),
375
+ generateStoryboardsThenVideos(clap)
376
+ ]
377
+
378
+ const claps = await Promise.all(tasks)
379
+
380
+ console.log(`finished processing ${tasks.length} tasks in parallel`)
381
+
382
+ for (const newerClap of claps) {
383
+ clap = await updateClap(clap, newerClap, {
384
+ overwriteMeta: false,
385
+ inlineReplace: true,
386
+ })
387
+ setCurrentClap(clap)
388
+ }
389
+
390
+ /*
391
+ clap = await claps.reduce(async (existingClap, newerClap) =>
392
+ updateClap(existingClap, newerClap, {
393
+ overwriteMeta: false,
394
+ inlineReplace: true,
395
+ })
396
+ , Promise.resolve(clap)
397
+ */
398
+
399
+
400
+ // We can't have consistent characters with video (yet)
401
+ // clap = await generateEntities(clap)
402
+
403
+ /*
404
+ if (mainCharacterImage) {
405
+ console.log("handleSubmit(): User specified a main character image")
406
+ // various strategies here, for instance we can assume that the first character is the main character,
407
+ // or maybe a more reliable way is to count the number of occurrences.
408
+ // there is a risk of misgendering, so ideally we should add some kind of UI to do this,
409
+ // such as a list of characters.
410
+ }
411
+ */
412
+
413
+ // let's skip storyboards for now
414
+ // clap = await generateStoryboards(clap)
415
+
416
+ // clap = await generateVideos(clap)
417
+ // clap = await generateDialogues(clap)
418
+
419
+
420
+
421
+ console.log("final clap: ", clap)
422
+ setCurrentClap(clap)
423
+ await generateFinalVideo(clap)
424
+
425
+ setStatus("finished")
426
+ setError("")
427
+ } catch (err) {
428
+ console.error(`failed to generate: `, err)
429
+ setStatus("error")
430
+ setError(`Error, please contact an admin on Discord (${err})`)
431
+ }
432
+ })
433
+ }
434
+
435
+ return {
436
+ generateDialogues,
437
+ generateEntities,
438
+ generateFinalVideo,
439
+ generateMusic,
440
+ generateSounds,
441
+ generateStory,
442
+ generateStoryboards,
443
+ generateStoryboardsThenVideos,
444
+ generateVideos,
445
+ handleSubmit,
446
+ }
447
+ }
src/lib/hooks/useProgressTimer.ts ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useStore } from "@/app/store"
2
+ import { useEffect, useRef } from "react"
3
+ import { useIsBusy } from "./useIsBusy"
4
+
5
+ export function useProgressTimer() {
6
+ const runningRef = useRef(false)
7
+ const timerRef = useRef<NodeJS.Timeout>()
8
+
9
+ const progress = useStore(s => s.progress)
10
+ const stage = useStore(s => s.stage)
11
+ const { isBusy, busyRef } = useIsBusy()
12
+
13
+ const timerFn = async () => {
14
+ const { isBusy, progress, stage } = useStore.getState()
15
+
16
+ clearTimeout(timerRef.current)
17
+ if (!isBusy || stage === "idle") {
18
+ return
19
+ }
20
+
21
+ /*
22
+ console.log("progress function:", {
23
+ stage,
24
+ delay: progressDelayInMsPerStage[stage],
25
+ progress,
26
+ })
27
+ */
28
+ useStore.setState({
29
+ // progress: Math.min(maxProgressPerStage[stage], progress + 1)
30
+ progress: Math.min(100, progress + 1)
31
+ })
32
+
33
+ // timerRef.current = setTimeout(timerFn, progressDelayInMsPerStage[stage])
34
+ timerRef.current = setTimeout(timerFn, 1200)
35
+ }
36
+
37
+ useEffect(() => {
38
+ timerFn()
39
+ clearTimeout(timerRef.current)
40
+ if (!isBusy) { return }
41
+ timerRef.current = setTimeout(timerFn, 0)
42
+ }, [isBusy])
43
+
44
+ return { isBusy, busyRef, progress, stage }
45
+ }
src/lib/hooks/useQueryStringParams.ts ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { useEffect } from "react"
2
+ import { useSearchParams } from "next/navigation"
3
+ import { ClapMediaOrientation } from "@aitube/clap"
4
+
5
+ import { useStore } from "@/app/store"
6
+
7
+ import { useStoryPromptDraft } from "./useStoryPromptDraft"
8
+ import { useIsBusy } from "./useIsBusy"
9
+ import { useProcessors } from "./useProcessors"
10
+
11
+ export function useQueryStringParams() {
12
+ const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
13
+ const { busyRef } = useIsBusy()
14
+ const { handleSubmit } = useProcessors()
15
+
16
+ const setOrientation = useStore(s => s.setOrientation)
17
+ // this is how we support query string parameters
18
+ // ?prompt=hello <- set a default prompt
19
+ // ?prompt=hello&autorun=true <- automatically run the app
20
+ // ?orientation=landscape <- can be "landscape" or "portrait" (default)
21
+ const searchParams = useSearchParams()
22
+ const queryStringPrompt = (searchParams?.get('prompt') as string) || ""
23
+ const queryStringAutorun = (searchParams?.get('autorun') as string) || ""
24
+ const queryStringOrientation = (searchParams?.get('orientation') as string) || ""
25
+
26
+ useEffect(() => {
27
+ if (queryStringOrientation?.length > 1) {
28
+ console.log(`orientation = "${queryStringOrientation}"`)
29
+ const orientation =
30
+ queryStringOrientation.trim().toLowerCase() === "landscape"
31
+ ? ClapMediaOrientation.LANDSCAPE
32
+ : ClapMediaOrientation.PORTRAIT
33
+ setOrientation(orientation)
34
+ }
35
+ if (queryStringPrompt?.length > 1) {
36
+ console.log(`prompt = "${queryStringPrompt}"`)
37
+ if (queryStringPrompt !== promptDraftRef.current) {
38
+ setStoryPromptDraft(queryStringPrompt)
39
+ }
40
+ const maybeAutorun = queryStringAutorun.trim().toLowerCase()
41
+ console.log(`autorun = "${maybeAutorun}"`)
42
+
43
+ // note: during development we will be called twice,
44
+ // which is why we have a guard on busyRef.current
45
+ if (maybeAutorun === "true" || maybeAutorun === "1" && !busyRef.current) {
46
+ handleSubmit()
47
+ }
48
+ }
49
+ }, [queryStringPrompt, queryStringAutorun, queryStringOrientation])
50
+
51
+ }
src/lib/hooks/useStoryPromptDraft.ts ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import { useRef } from "react"
3
+ import { useLocalStorage } from "usehooks-ts"
4
+
5
+ import { defaultPrompt, localStorageStoryDraftKey } from "@/app/config"
6
+
7
+ export function useStoryPromptDraft() {
8
+ const [storyPromptDraft, setStoryPromptDraft] = useLocalStorage<string>(
9
+ localStorageStoryDraftKey,
10
+ defaultPrompt
11
+ )
12
+ const promptDraftRef = useRef("")
13
+ promptDraftRef.current = storyPromptDraft
14
+
15
+ return { storyPromptDraft, setStoryPromptDraft, promptDraftRef }
16
+ }
src/lib/utils/index.ts ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ export { cn } from "./cn"
2
+ export { generateRandomStory } from "./generateRandomStory"
3
+ export { logImage } from "./logImage"