jbilcke-hf HF staff commited on
Commit
83defd6
1 Parent(s): 0bf0c48

good progress on the AI Stories Factory

Browse files
package-lock.json CHANGED
@@ -50,6 +50,7 @@
50
  "@upstash/redis": "^1.28.3",
51
  "alchemy-sdk": "^3.2.1",
52
  "autoprefixer": "10.4.19",
 
53
  "class-variance-authority": "^0.7.0",
54
  "clsx": "^2.1.0",
55
  "cmdk": "^1.0.0",
 
50
  "@upstash/redis": "^1.28.3",
51
  "alchemy-sdk": "^3.2.1",
52
  "autoprefixer": "10.4.19",
53
+ "axios": "^1.6.8",
54
  "class-variance-authority": "^0.7.0",
55
  "clsx": "^2.1.0",
56
  "cmdk": "^1.0.0",
package.json CHANGED
@@ -51,6 +51,7 @@
51
  "@upstash/redis": "^1.28.3",
52
  "alchemy-sdk": "^3.2.1",
53
  "autoprefixer": "10.4.19",
 
54
  "class-variance-authority": "^0.7.0",
55
  "clsx": "^2.1.0",
56
  "cmdk": "^1.0.0",
 
51
  "@upstash/redis": "^1.28.3",
52
  "alchemy-sdk": "^3.2.1",
53
  "autoprefixer": "10.4.19",
54
+ "axios": "^1.6.8",
55
  "class-variance-authority": "^0.7.0",
56
  "clsx": "^2.1.0",
57
  "cmdk": "^1.0.0",
src/app/api/generate/storyboards/generateStoryboard.ts ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
3
+ import { generateSeed } from "@/lib/utils/generateSeed"
4
+ import { sleep } from "@/lib/utils/sleep"
5
+ import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
6
+ import { getValidNumber } from "@/lib/utils/getValidNumber"
7
+
8
+ export async function generateStoryboard({
9
+ prompt,
10
+ // negativePrompt,
11
+ width,
12
+ height,
13
+ seed,
14
+ }: {
15
+ prompt: string
16
+ // negativePrompt?: string
17
+ width?: number
18
+ height?: number
19
+ seed?: number
20
+ }) {
21
+
22
+ width = getValidNumber(width, 256, 8192, 512)
23
+ height = getValidNumber(height, 256, 8192, 288)
24
+
25
+ // console.log("calling await newRender")
26
+ prompt = getPositivePrompt(prompt)
27
+ const negativePrompt = getNegativePrompt()
28
+
29
+ let render = await newRender({
30
+ prompt,
31
+ negativePrompt,
32
+ nbFrames: 1,
33
+ nbFPS: 1,
34
+ nbSteps: 8,
35
+ width,
36
+ height,
37
+ turbo: true,
38
+ shouldRenewCache: true,
39
+ seed: seed || generateSeed()
40
+ })
41
+
42
+ let attempts = 10
43
+
44
+ while (attempts-- > 0) {
45
+ if (render.status === "completed") {
46
+ return render.assetUrl
47
+ }
48
+
49
+ if (render.status === "error") {
50
+ console.error(render.error)
51
+ throw new Error(`failed to generate the image ${render.error}`)
52
+ }
53
+
54
+ await sleep(2000) // minimum wait time
55
+
56
+ // console.log("asking getRender")
57
+ render = await getRender(render.renderId)
58
+ }
59
+
60
+ throw new Error(`failed to generate the image`)
61
+ }
src/app/api/generate/storyboards/route.ts CHANGED
@@ -5,8 +5,11 @@ import { parseClap } from "@/lib/clap/parseClap"
5
  import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
6
  import { getVideoPrompt } from "@/components/interface/latent-engine/core/prompts/getVideoPrompt"
7
  import { newSegment } from "@/lib/clap/newSegment"
8
- import { generateImage } from "@/components/interface/latent-engine/resolvers/image/generateImage"
9
  import { getToken } from "@/app/api/auth/getToken"
 
 
 
10
 
11
  // a helper to generate storyboards for a Clap
12
  // this is mostly used by external apps such as the Stories Factory
@@ -25,7 +28,7 @@ export async function POST(req: NextRequest) {
25
 
26
  if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) }
27
 
28
- console.log(`[api/generate/storyboards] detected ${clap.segments} segments`)
29
 
30
  const shotsSegments = clap.segments.filter(s => s.category === "camera")
31
  console.log(`[api/generate/storyboards] detected ${shotsSegments.length} shots`)
@@ -73,16 +76,18 @@ export async function POST(req: NextRequest) {
73
  // TASK 3: GENERATE MISSING STORYBOARD BITMAP
74
  if (!shotStoryboardSegment.assetUrl) {
75
  console.log(`[api/generate/storyboards] generating image..`)
76
- // note this will do a fetch to AiTube API
77
- // which is a bit weird since we are already inside the API, but it works
78
- //TODO Julian: maybe we could use an internal function call instead?
79
- shotStoryboardSegment.assetUrl = await generateImage({
80
- prompt: shotStoryboardSegment.prompt,
81
- width: clap.meta.width,
82
- height: clap.meta.height,
83
- token: jwtToken,
84
- })
85
 
 
 
 
 
 
 
 
 
 
 
 
86
  console.log(`[api/generate/storyboards] generated storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
87
  } else {
88
  console.log(`[api/generate/storyboards] there is already a storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
 
5
  import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
6
  import { getVideoPrompt } from "@/components/interface/latent-engine/core/prompts/getVideoPrompt"
7
  import { newSegment } from "@/lib/clap/newSegment"
8
+ import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
9
  import { getToken } from "@/app/api/auth/getToken"
10
+ import { generateSeed } from "@/lib/utils/generateSeed"
11
+ import { getPositivePrompt } from "../../utils/imagePrompts"
12
+ import { generateStoryboard } from "./generateStoryboard"
13
 
14
  // a helper to generate storyboards for a Clap
15
  // this is mostly used by external apps such as the Stories Factory
 
28
 
29
  if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) }
30
 
31
+ console.log(`[api/generate/storyboards] detected ${clap.segments.length} segments`)
32
 
33
  const shotsSegments = clap.segments.filter(s => s.category === "camera")
34
  console.log(`[api/generate/storyboards] detected ${shotsSegments.length} shots`)
 
76
  // TASK 3: GENERATE MISSING STORYBOARD BITMAP
77
  if (!shotStoryboardSegment.assetUrl) {
78
  console.log(`[api/generate/storyboards] generating image..`)
 
 
 
 
 
 
 
 
 
79
 
80
+ try {
81
+ shotStoryboardSegment.assetUrl = await generateStoryboard({
82
+ prompt: getPositivePrompt(shotStoryboardSegment.prompt),
83
+ width: clap.meta.width,
84
+ height: clap.meta.height,
85
+ })
86
+ } catch (err) {
87
+ console.log(`[api/generate/storyboards] failed to generate an image: ${err}`)
88
+ throw err
89
+ }
90
+
91
  console.log(`[api/generate/storyboards] generated storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
92
  } else {
93
  console.log(`[api/generate/storyboards] there is already a storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
src/components/interface/latent-engine/resolvers/image/generateImage.ts CHANGED
@@ -1,15 +1,23 @@
1
  import { aitubeApiUrl } from "../../core/config"
2
 
 
 
3
  export async function generateImage({
4
  prompt,
5
  width,
6
  height,
7
  token,
 
8
  }: {
9
  prompt: string
10
  width: number
11
  height: number
12
  token: string
 
 
 
 
 
13
  }): Promise<string> {
14
  const requestUri = `${aitubeApiUrl}/api/resolvers/image?t=${
15
  token
@@ -22,7 +30,28 @@ export async function generateImage({
22
  encodeURIComponent(prompt)
23
  }`
24
  const res = await fetch(requestUri)
25
- const blob = await res.blob()
26
- const url = URL.createObjectURL(blob)
27
- return url
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  }
 
1
  import { aitubeApiUrl } from "../../core/config"
2
 
3
+ // return an image
4
+ // note that this function can work either on the server-side or the client-side
5
  export async function generateImage({
6
  prompt,
7
  width,
8
  height,
9
  token,
10
+ mode = "data-uri"
11
  }: {
12
  prompt: string
13
  width: number
14
  height: number
15
  token: string
16
+
17
+ // data-uri are good for small files as they contain all the data, they can be sent over the network,
18
+ // object-uri are good for large files but can't be sent over the network (they are just a ID)
19
+ // this also means that object-uri cannot be used on the server-side
20
+ mode?: "data-uri" | "object-uri"
21
  }): Promise<string> {
22
  const requestUri = `${aitubeApiUrl}/api/resolvers/image?t=${
23
  token
 
30
  encodeURIComponent(prompt)
31
  }`
32
  const res = await fetch(requestUri)
33
+
34
+ // will only work on the server-side
35
+ if (mode === "object-uri") {
36
+ const blob = await res.blob()
37
+
38
+ return URL.createObjectURL(blob)
39
+ } else {
40
+ if (typeof window !== "undefined") {
41
+ const blob = await res.blob()
42
+
43
+ // on browser-side
44
+ const dataURL = await new Promise<string>((resolve, reject) => {
45
+ var a = new FileReader()
46
+ a.onload = function(e) { resolve(`${e.target?.result || ""}`) }
47
+ a.readAsDataURL(blob)
48
+ })
49
+ return dataURL
50
+ } else {
51
+ // NodeJS side
52
+ const buffer = await (res as any).buffer()
53
+ const contentType = res.headers.get("Content-Type")
54
+ return "data:" + contentType + ';base64,' + buffer.toString('base64')
55
+ }
56
+ }
57
  }
src/components/interface/latent-engine/resolvers/video/generateVideo.ts CHANGED
@@ -5,11 +5,17 @@ export async function generateVideo({
5
  width,
6
  height,
7
  token,
 
8
  }: {
9
  prompt: string
10
  width: number
11
  height: number
12
  token: string
 
 
 
 
 
13
  }): Promise<string> {
14
  const requestUri = `${aitubeApiUrl}/api/resolvers/video?t=${
15
  token
@@ -22,6 +28,24 @@ export async function generateVideo({
22
  }`
23
  const res = await fetch(requestUri)
24
  const blob = await res.blob()
25
- const url = URL.createObjectURL(blob)
26
- return url
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  }
 
5
  width,
6
  height,
7
  token,
8
+ mode = "data-uri",
9
  }: {
10
  prompt: string
11
  width: number
12
  height: number
13
  token: string
14
+
15
+ // data-uri are good for small files as they contain all the data, they can be sent over the network,
16
+ // object-uri are good for large files but can't be sent over the network (they are just a ID)
17
+ // this also means that object-uri cannot be used on the server-side
18
+ mode?: "data-uri" | "object-uri"
19
  }): Promise<string> {
20
  const requestUri = `${aitubeApiUrl}/api/resolvers/video?t=${
21
  token
 
28
  }`
29
  const res = await fetch(requestUri)
30
  const blob = await res.blob()
31
+
32
+ // will only work on the server-side
33
+ if (mode === "object-uri") {
34
+ return URL.createObjectURL(blob)
35
+ } else {
36
+ if (typeof window !== "undefined") {
37
+ // on browser-side
38
+ const dataURL = await new Promise<string>((resolve, reject) => {
39
+ var a = new FileReader()
40
+ a.onload = function(e) { resolve(`${e.target?.result || ""}`) }
41
+ a.readAsDataURL(blob)
42
+ })
43
+ return dataURL
44
+ } else {
45
+ // NodeJS side
46
+ const contentType = res.headers.get("Content-Type")
47
+ const buffer = await (res as any).buffer() as Buffer
48
+ return "data:" + contentType + ';base64,' + buffer.toString('base64')
49
+ }
50
+ }
51
  }
src/components/interface/latent-engine/resolvers/video/index.tsx CHANGED
@@ -20,6 +20,7 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
20
  width: clap.meta.width,
21
  height: clap.meta.height,
22
  token: useStore.getState().jwtToken,
 
23
  })
24
  // console.log(`resolveVideo: generated ${assetUrl}`)
25
 
 
20
  width: clap.meta.width,
21
  height: clap.meta.height,
22
  token: useStore.getState().jwtToken,
23
+ mode: "object-uri" // it's better for videos withing Chrome, apparently
24
  })
25
  // console.log(`resolveVideo: generated ${assetUrl}`)
26
 
src/components/interface/latent-engine/resolvers/video/index_legacy.tsx CHANGED
@@ -53,6 +53,7 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
53
  width: clap.meta.width,
54
  height: clap.meta.height,
55
  token: useStore.getState().jwtToken,
 
56
  }))
57
  }
58
 
 
53
  width: clap.meta.width,
54
  height: clap.meta.height,
55
  token: useStore.getState().jwtToken,
56
+ mode: "object-uri" // it's better for videos withing Chrome, apparently
57
  }))
58
  }
59
 
src/components/interface/latent-engine/resolvers/video/index_notSoGood.tsx CHANGED
@@ -53,6 +53,7 @@ export async function resolve(segment: ClapSegment, clap: ClapProject): Promise<
53
  width: clap.meta.width,
54
  height: clap.meta.height,
55
  token: useStore.getState().jwtToken,
 
56
  }))
57
  }
58
 
 
53
  width: clap.meta.width,
54
  height: clap.meta.height,
55
  token: useStore.getState().jwtToken,
56
+ mode: "object-uri" // it's better for videos withing Chrome, apparently
57
  }))
58
  }
59