jbilcke-hf HF staff commited on
Commit
a99df79
1 Parent(s): e336208
package-lock.json CHANGED
@@ -8,9 +8,9 @@
8
  "name": "@aitube/website",
9
  "version": "0.0.0",
10
  "dependencies": {
11
- "@aitube/clap": "0.0.20",
12
- "@aitube/client": "0.0.29",
13
- "@aitube/engine": "0.0.11",
14
  "@huggingface/hub": "0.12.3-oauth",
15
  "@huggingface/inference": "^2.7.0",
16
  "@jcoreio/async-throttle": "^1.6.0",
@@ -119,9 +119,9 @@
119
  }
120
  },
121
  "node_modules/@aitube/clap": {
122
- "version": "0.0.20",
123
- "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.20.tgz",
124
- "integrity": "sha512-QOEl3GTHRpaBtKGYoH+iabY9TfLYogOoIfc/YuwC01paO/woMASccjDloPGiPe9C0/P88AqB6s7bN3Dkz1v0TQ==",
125
  "dependencies": {
126
  "pure-uuid": "^1.8.1",
127
  "yaml": "^2.4.2"
@@ -131,22 +131,22 @@
131
  }
132
  },
133
  "node_modules/@aitube/client": {
134
- "version": "0.0.29",
135
- "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.29.tgz",
136
- "integrity": "sha512-7ASbsoKuk10RbF/d9uuGU2OBnMxrLd/O01rSH5GUYMTeWnJNjnEBy/klgCQxoA7qrO6VX7kJGI5MB2pfK9XJFA==",
137
  "dependencies": {
138
  "query-string": "^9.0.0"
139
  },
140
  "peerDependencies": {
141
- "@aitube/clap": "0.0.20"
142
  }
143
  },
144
  "node_modules/@aitube/engine": {
145
- "version": "0.0.11",
146
- "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.11.tgz",
147
- "integrity": "sha512-XMjhyakLvYvEmJFlJVWIdxymDhpAcW9RnZfGXNCsCPrjkrb6nhuVm3RDSRO2JwNPe/i7T4DaEDVeWavDiyQNMA==",
148
  "peerDependencies": {
149
- "@aitube/clap": "0.0.20"
150
  }
151
  },
152
  "node_modules/@alloc/quick-lru": {
@@ -3322,17 +3322,6 @@
3322
  "node": ">= 8"
3323
  }
3324
  },
3325
- "node_modules/anymatch/node_modules/picomatch": {
3326
- "version": "2.3.1",
3327
- "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
3328
- "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
3329
- "engines": {
3330
- "node": ">=8.6"
3331
- },
3332
- "funding": {
3333
- "url": "https://github.com/sponsors/jonschlinkert"
3334
- }
3335
- },
3336
  "node_modules/arg": {
3337
  "version": "5.0.2",
3338
  "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
@@ -3764,9 +3753,9 @@
3764
  }
3765
  },
3766
  "node_modules/caniuse-lite": {
3767
- "version": "1.0.30001620",
3768
- "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001620.tgz",
3769
- "integrity": "sha512-WJvYsOjd1/BYUY6SNGUosK9DUidBPDTnOARHp3fSmFO1ekdxaY6nKRttEVrfMmYi80ctS0kz1wiWmm14fVc3ew==",
3770
  "funding": [
3771
  {
3772
  "type": "opencollective",
@@ -6325,12 +6314,12 @@
6325
  }
6326
  },
6327
  "node_modules/micromatch": {
6328
- "version": "4.0.6",
6329
- "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.6.tgz",
6330
- "integrity": "sha512-Y4Ypn3oujJYxJcMacVgcs92wofTHxp9FzfDpQON4msDefoC0lb3ETvQLOdLcbhSwU1bz8HrL/1sygfBIHudrkQ==",
6331
  "dependencies": {
6332
  "braces": "^3.0.3",
6333
- "picomatch": "^4.0.2"
6334
  },
6335
  "engines": {
6336
  "node": ">=8.6"
@@ -6876,11 +6865,11 @@
6876
  "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
6877
  },
6878
  "node_modules/picomatch": {
6879
- "version": "4.0.2",
6880
- "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
6881
- "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==",
6882
  "engines": {
6883
- "node": ">=12"
6884
  },
6885
  "funding": {
6886
  "url": "https://github.com/sponsors/jonschlinkert"
@@ -7035,9 +7024,9 @@
7035
  }
7036
  },
7037
  "node_modules/postcss-selector-parser": {
7038
- "version": "6.0.16",
7039
- "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.16.tgz",
7040
- "integrity": "sha512-A0RVJrX+IUkVZbW3ClroRWurercFhieevHB38sr2+l9eUClMqome3LmEmnhlNy+5Mr2EYN6B2Kaw9wYdd+VHiw==",
7041
  "dependencies": {
7042
  "cssesc": "^3.0.0",
7043
  "util-deprecate": "^1.0.2"
@@ -7380,17 +7369,6 @@
7380
  "node": ">=8.10.0"
7381
  }
7382
  },
7383
- "node_modules/readdirp/node_modules/picomatch": {
7384
- "version": "2.3.1",
7385
- "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
7386
- "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
7387
- "engines": {
7388
- "node": ">=8.6"
7389
- },
7390
- "funding": {
7391
- "url": "https://github.com/sponsors/jonschlinkert"
7392
- }
7393
- },
7394
  "node_modules/reflect.getprototypeof": {
7395
  "version": "1.0.6",
7396
  "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz",
 
8
  "name": "@aitube/website",
9
  "version": "0.0.0",
10
  "dependencies": {
11
+ "@aitube/clap": "0.0.22",
12
+ "@aitube/client": "0.0.31",
13
+ "@aitube/engine": "0.0.12",
14
  "@huggingface/hub": "0.12.3-oauth",
15
  "@huggingface/inference": "^2.7.0",
16
  "@jcoreio/async-throttle": "^1.6.0",
 
119
  }
120
  },
121
  "node_modules/@aitube/clap": {
122
+ "version": "0.0.22",
123
+ "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.22.tgz",
124
+ "integrity": "sha512-oN+tfoy0fp95AiwNMVy6pbhkK9k/h/gZ3FTZPpF06bVLig4rZ/Bthyot+wy4E/hr9ERLeyGpfZVW8blRqHCAaw==",
125
  "dependencies": {
126
  "pure-uuid": "^1.8.1",
127
  "yaml": "^2.4.2"
 
131
  }
132
  },
133
  "node_modules/@aitube/client": {
134
+ "version": "0.0.31",
135
+ "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.31.tgz",
136
+ "integrity": "sha512-xjdPYRruuEWe8KdmWpMYB1ELylBRF2M6+BYr0K3VAf/fv4qcRqda+F9amitJ27h0t5rvagA7+MLLKFqlF7BT/A==",
137
  "dependencies": {
138
  "query-string": "^9.0.0"
139
  },
140
  "peerDependencies": {
141
+ "@aitube/clap": "0.0.22"
142
  }
143
  },
144
  "node_modules/@aitube/engine": {
145
+ "version": "0.0.12",
146
+ "resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.12.tgz",
147
+ "integrity": "sha512-MkV9FFk2JOjm6+ir5T4p65AzO95n1zjmXZevswer3ApXpMuMY0MuqC3krYhxj9cBF54S2C/mltUCk90cOHWP2Q==",
148
  "peerDependencies": {
149
+ "@aitube/clap": "0.0.22"
150
  }
151
  },
152
  "node_modules/@alloc/quick-lru": {
 
3322
  "node": ">= 8"
3323
  }
3324
  },
 
 
 
 
 
 
 
 
 
 
 
3325
  "node_modules/arg": {
3326
  "version": "5.0.2",
3327
  "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
 
3753
  }
3754
  },
3755
  "node_modules/caniuse-lite": {
3756
+ "version": "1.0.30001621",
3757
+ "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001621.tgz",
3758
+ "integrity": "sha512-+NLXZiviFFKX0fk8Piwv3PfLPGtRqJeq2TiNoUff/qB5KJgwecJTvCXDpmlyP/eCI/GUEmp/h/y5j0yckiiZrA==",
3759
  "funding": [
3760
  {
3761
  "type": "opencollective",
 
6314
  }
6315
  },
6316
  "node_modules/micromatch": {
6317
+ "version": "4.0.7",
6318
+ "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
6319
+ "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
6320
  "dependencies": {
6321
  "braces": "^3.0.3",
6322
+ "picomatch": "^2.3.1"
6323
  },
6324
  "engines": {
6325
  "node": ">=8.6"
 
6865
  "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
6866
  },
6867
  "node_modules/picomatch": {
6868
+ "version": "2.3.1",
6869
+ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
6870
+ "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
6871
  "engines": {
6872
+ "node": ">=8.6"
6873
  },
6874
  "funding": {
6875
  "url": "https://github.com/sponsors/jonschlinkert"
 
7024
  }
7025
  },
7026
  "node_modules/postcss-selector-parser": {
7027
+ "version": "6.1.0",
7028
+ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.0.tgz",
7029
+ "integrity": "sha512-UMz42UD0UY0EApS0ZL9o1XnLhSTtvvvLe5Dc2H2O56fvRZi+KulDyf5ctDhhtYJBGKStV2FL1fy6253cmLgqVQ==",
7030
  "dependencies": {
7031
  "cssesc": "^3.0.0",
7032
  "util-deprecate": "^1.0.2"
 
7369
  "node": ">=8.10.0"
7370
  }
7371
  },
 
 
 
 
 
 
 
 
 
 
 
7372
  "node_modules/reflect.getprototypeof": {
7373
  "version": "1.0.6",
7374
  "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz",
package.json CHANGED
@@ -10,9 +10,9 @@
10
  "lint": "next lint"
11
  },
12
  "dependencies": {
13
- "@aitube/clap": "0.0.20",
14
- "@aitube/client": "0.0.29",
15
- "@aitube/engine": "0.0.11",
16
  "@huggingface/hub": "0.12.3-oauth",
17
  "@huggingface/inference": "^2.7.0",
18
  "@jcoreio/async-throttle": "^1.6.0",
 
10
  "lint": "next lint"
11
  },
12
  "dependencies": {
13
+ "@aitube/clap": "0.0.22",
14
+ "@aitube/client": "0.0.31",
15
+ "@aitube/engine": "0.0.12",
16
  "@huggingface/hub": "0.12.3-oauth",
17
  "@huggingface/inference": "^2.7.0",
18
  "@jcoreio/async-throttle": "^1.6.0",
src/app/api/generators/clap/constants.ts CHANGED
@@ -1 +1,4 @@
1
  export const defaultSegmentDurationInMs = 2000
 
 
 
 
1
  export const defaultSegmentDurationInMs = 2000
2
+ // const defaultSegmentDurationInMs = 2916
3
+
4
+ export const MAX_PROMPT_LENGTH_IN_CHARS = 1024
src/app/api/v1/create/index.ts CHANGED
@@ -13,6 +13,7 @@ import { clapToLatentStory } from "../edit/entities/clapToLatentStory"
13
  import { generateRandomStory } from "./generateRandomStory"
14
  import { generateSoundPrompts } from "../edit/sounds/generateSoundPrompt"
15
  import { checkCaptions } from "./checkCaptions"
 
16
 
17
  // a helper to generate Clap stories from a few sentences
18
  // this is mostly used by external apps such as the Stories Factory
@@ -29,7 +30,8 @@ export async function create(request: {
29
  }): Promise<ClapProject> {
30
 
31
  // we limit to 512 characters
32
- let { prompt, hasCaptions } = checkCaptions(`${request?.prompt || ""}`.trim().slice(0, 512))
 
33
 
34
  // console.log("api/v1/create(): request:", request)
35
 
@@ -159,19 +161,19 @@ Output: `
159
  }))
160
 
161
  if (hasCaptions) {
162
- clap.segments.push(newSegment({
163
- track: 2,
164
- startTimeInMs: currentElapsedTimeInMs,
165
- endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
166
- assetDurationInMs: defaultSegmentDurationInMs,
167
- category: ClapSegmentCategory.INTERFACE,
168
- prompt: comment,
169
- // assetUrl: `data:text/plain;base64,${btoa(comment)}`,
170
- assetUrl: comment,
171
- outputType: ClapOutputType.TEXT,
172
- status: ClapSegmentStatus.TO_GENERATE,
173
- }))
174
- }
175
 
176
  clap.segments.push(newSegment({
177
  track: 3,
 
13
  import { generateRandomStory } from "./generateRandomStory"
14
  import { generateSoundPrompts } from "../edit/sounds/generateSoundPrompt"
15
  import { checkCaptions } from "./checkCaptions"
16
+ import { MAX_PROMPT_LENGTH_IN_CHARS } from "../../generators/clap/constants"
17
 
18
  // a helper to generate Clap stories from a few sentences
19
  // this is mostly used by external apps such as the Stories Factory
 
30
  }): Promise<ClapProject> {
31
 
32
  // we limit to 512 characters
33
+ let { prompt, hasCaptions } = checkCaptions(`${request?.prompt || ""}`.trim().slice(0, MAX_PROMPT_LENGTH_IN_CHARS))
34
+
35
 
36
  // console.log("api/v1/create(): request:", request)
37
 
 
161
  }))
162
 
163
  if (hasCaptions) {
164
+ clap.segments.push(newSegment({
165
+ track: 2,
166
+ startTimeInMs: currentElapsedTimeInMs,
167
+ endTimeInMs: currentElapsedTimeInMs + defaultSegmentDurationInMs,
168
+ assetDurationInMs: defaultSegmentDurationInMs,
169
+ category: ClapSegmentCategory.INTERFACE,
170
+ prompt: comment,
171
+ // assetUrl: `data:text/plain;base64,${btoa(comment)}`,
172
+ assetUrl: comment,
173
+ outputType: ClapOutputType.TEXT,
174
+ status: ClapSegmentStatus.TO_GENERATE,
175
+ }))
176
+ }
177
 
178
  clap.segments.push(newSegment({
179
  track: 3,
src/app/api/v1/edit/story/extendClapStory.ts CHANGED
@@ -11,6 +11,8 @@ import { ClapCompletionMode } from "@aitube/client"
11
  import { clapToLatentStory } from "../entities/clapToLatentStory"
12
  import { LatentStory } from "../../types"
13
  import { extendLatentStoryWithMoreShots } from "./extendLatentStoryWithMoreShots"
 
 
14
 
15
  export async function extendClapStory({
16
  prompt: maybePrompt,
@@ -30,9 +32,26 @@ export async function extendClapStory({
30
  turbo: boolean
31
  }): Promise<void> {
32
 
33
- const prompt = typeof maybePrompt === "string" && maybePrompt.length > 0
34
- ? maybePrompt
35
- : existingClap.meta.description
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
 
37
  const latentStory: LatentStory[] = await clapToLatentStory(existingClap)
38
 
@@ -43,16 +62,12 @@ export async function extendClapStory({
43
  turbo,
44
  })
45
 
46
- let hasCaptions = false
47
-
48
- let currentElapsedTimeInMs = 0
49
- for (const s of existingClap.segments) {
50
- if (s.category === ClapSegmentCategory.INTERFACE) { hasCaptions = true }
51
- if (s.endTimeInMs > currentElapsedTimeInMs) { currentElapsedTimeInMs = s.endTimeInMs }
52
- }
53
 
54
  // this is approximate - TTS generation will determine the final duration of each shot
 
55
  const defaultSegmentDurationInMs = 3000
 
56
 
57
  for (const { comment, image, voice } of shots) {
58
 
 
11
  import { clapToLatentStory } from "../entities/clapToLatentStory"
12
  import { LatentStory } from "../../types"
13
  import { extendLatentStoryWithMoreShots } from "./extendLatentStoryWithMoreShots"
14
+ import { MAX_PROMPT_LENGTH_IN_CHARS } from "@/app/api/generators/clap/constants"
15
+ import { checkCaptions } from "../../create/checkCaptions"
16
 
17
  export async function extendClapStory({
18
  prompt: maybePrompt,
 
32
  turbo: boolean
33
  }): Promise<void> {
34
 
35
+ let hasCaptions = false
36
+
37
+ let currentElapsedTimeInMs = 0
38
+ for (const s of existingClap.segments) {
39
+ if (s.category === ClapSegmentCategory.INTERFACE) { hasCaptions = true }
40
+ if (s.endTimeInMs > currentElapsedTimeInMs) { currentElapsedTimeInMs = s.endTimeInMs }
41
+ }
42
+
43
+ const { prompt, hasCaptions: hasCaptionInNewPrompt } = checkCaptions(
44
+ (
45
+ typeof maybePrompt === "string" && maybePrompt.length > 0
46
+ ? maybePrompt
47
+ : existingClap.meta.description
48
+ ).trim().slice(0, MAX_PROMPT_LENGTH_IN_CHARS)
49
+ )
50
+
51
+ // if the video already contains captions but the user asks for none for the subsequent scenes
52
+ if (!hasCaptionInNewPrompt && hasCaptions) {
53
+ hasCaptions = false
54
+ }
55
 
56
  const latentStory: LatentStory[] = await clapToLatentStory(existingClap)
57
 
 
62
  turbo,
63
  })
64
 
65
+ // console.log(`extendClapStory: going to extend, starting at ${currentElapsedTimeInMs}`)
 
 
 
 
 
 
66
 
67
  // this is approximate - TTS generation will determine the final duration of each shot
68
+
69
  const defaultSegmentDurationInMs = 3000
70
+ // const defaultSegmentDurationInMs = 2916
71
 
72
  for (const { comment, image, voice } of shots) {
73
 
src/app/api/v1/edit/story/route.ts CHANGED
@@ -21,7 +21,7 @@ export async function POST(req: NextRequest) {
21
  const qs = queryString.parseUrl(req.url || "")
22
  const query = (qs || {}).query
23
 
24
- const prompt = parsePrompt(query?.p, false)
25
  const mode = parseCompletionMode(query?.c)
26
  const turbo = parseTurbo(query?.t)
27
 
@@ -38,10 +38,10 @@ export async function POST(req: NextRequest) {
38
  meta: existingClap.meta
39
  })
40
 
41
- // console.log(`api/v1/edit/storyboards(): detected ${existingClap.segments.length} segments`)
42
 
43
  const allShotsSegments: ClapSegment[] = existingClap.segments.filter(s => s.category === ClapSegmentCategory.CAMERA)
44
- console.log(`api/v1/edit/storyboards(): detected ${allShotsSegments.length} shots`)
45
 
46
  await extendClapStory({
47
  prompt,
 
21
  const qs = queryString.parseUrl(req.url || "")
22
  const query = (qs || {}).query
23
 
24
+ const prompt = parsePrompt(query?.p, false) // <- false, to make it non-mandatory
25
  const mode = parseCompletionMode(query?.c)
26
  const turbo = parseTurbo(query?.t)
27
 
 
38
  meta: existingClap.meta
39
  })
40
 
41
+ // console.log(`api/v1/edit/story(): detected ${existingClap.segments.length} segments`)
42
 
43
  const allShotsSegments: ClapSegment[] = existingClap.segments.filter(s => s.category === ClapSegmentCategory.CAMERA)
44
+ console.log(`api/v1/edit/story(): detected ${allShotsSegments.length} shots`)
45
 
46
  await extendClapStory({
47
  prompt,