jbilcke-hf HF staff commited on
Commit
203cc67
1 Parent(s): b66c635
src/app/api/utils/extractFirstFrame.ts ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { promises as fs, existsSync } from 'node:fs';
2
+ import path from 'node:path';
3
+ import os from 'node:os';
4
+
5
+ import { v4 as uuidv4 } from 'uuid';
6
+ import ffmpeg from 'fluent-ffmpeg';
7
+
8
+ const validFormats = ['jpeg', 'png', 'webp'];
9
+
10
+ /**
11
+ * Extract the first frame from a video
12
+ *
13
+ * @param param0
14
+ * @returns
15
+ */
16
+ export async function extractFirstFrame({
17
+ inputVideo,
18
+ outputFormat = 'jpeg'
19
+ }: {
20
+ inputVideo?: string
21
+ outputFormat?: "jpeg" | "png" | "webp"
22
+
23
+ }): Promise<string> {
24
+
25
+ if (!inputVideo) {
26
+ throw new Error(`inputVideo must be a file path or a base64 data-uri`);
27
+ }
28
+
29
+ if (!validFormats.includes(outputFormat)) {
30
+ throw new Error(`Invalid output format. Choose one of: ${validFormats.join(', ')}`);
31
+ }
32
+
33
+ // Handle base64 input
34
+ let videoFilePath = inputVideo;
35
+ if (inputVideo.startsWith('data:')) {
36
+ const matches = inputVideo.match(/^data:video\/(\w+);base64,(.*)$/);
37
+ if (!matches) {
38
+ throw new Error('Invalid base64 input provided.');
39
+ }
40
+ const extension = matches[1];
41
+ const base64Content = matches[2];
42
+
43
+ videoFilePath = path.join(os.tmpdir(), `${uuidv4()}_inputVideo.${extension}`);
44
+ await fs.writeFile(videoFilePath, base64Content, 'base64');
45
+ } else if (!existsSync(videoFilePath)) {
46
+ throw new Error('Video file does not exist.');
47
+ }
48
+
49
+ // Create a temporary output file
50
+ const outputImagePath = path.join(os.tmpdir(), `${uuidv4()}.${outputFormat}`);
51
+
52
+ return new Promise((resolve, reject) => {
53
+ ffmpeg()
54
+ .input(videoFilePath)
55
+ .outputOptions([
56
+ '-vframes', '1', // Extract only one frame
57
+ '-f', 'image2', // Output format for the frame as image
58
+ '-an' // Disable audio
59
+ ])
60
+ .output(outputImagePath)
61
+ .on('error', (err) => {
62
+ reject(new Error(`FFmpeg error: ${err.message}`));
63
+ })
64
+ .on('end', async () => {
65
+ try {
66
+ const imageBuffer = await fs.readFile(outputImagePath);
67
+ const imageBase64 = `data:image/${outputFormat};base64,${imageBuffer.toString('base64')}`;
68
+ resolve(imageBase64);
69
+ } catch (error) {
70
+ reject(new Error(`Error reading the image file: ${error}`));
71
+ } finally {
72
+ // Clean up temporary files
73
+ if (inputVideo.startsWith('data:')) {
74
+ await fs.unlink(videoFilePath);
75
+ }
76
+ await fs.unlink(outputImagePath);
77
+ }
78
+ })
79
+ .run();
80
+ });
81
+ }
src/app/api/v1/edit/videos/processShot.ts CHANGED
@@ -16,6 +16,7 @@ import { getVideoPrompt } from "@aitube/engine"
16
  import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
17
 
18
  import { render } from "@/app/api/v1/render"
 
19
 
20
  export async function processShot({
21
  shotSegment,
@@ -42,7 +43,11 @@ export async function processShot({
42
 
43
  let shotVideoSegment: ClapSegment | undefined = shotVideoSegments.at(0)
44
 
45
- // console.log("bug here?", turbo)
 
 
 
 
46
 
47
  console.log(`[api/edit/videos] processShot: shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotVideoSegments.length} videos)`)
48
 
@@ -85,7 +90,7 @@ export async function processShot({
85
 
86
  // TASK 3: GENERATE MISSING VIDEO FILE
87
  if (!shotVideoSegment.assetUrl) {
88
- // console.log(`[api/edit/videos] processShot: generating video file..`)
89
 
90
  const debug = false
91
 
@@ -135,4 +140,56 @@ export async function processShot({
135
  } else {
136
  console.log(`[api/edit/videos] processShot: there is already a video file: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
137
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
138
  }
 
16
  import { getPositivePrompt } from "@/app/api/utils/imagePrompts"
17
 
18
  import { render } from "@/app/api/v1/render"
19
+ import { extractFirstFrame } from "@/app/api/utils/extractFirstFrame"
20
 
21
  export async function processShot({
22
  shotSegment,
 
43
 
44
  let shotVideoSegment: ClapSegment | undefined = shotVideoSegments.at(0)
45
 
46
+ const shotStoryboardSegments: ClapSegment[] = shotSegments.filter(s =>
47
+ s.category === ClapSegmentCategory.STORYBOARD
48
+ )
49
+
50
+ let shotStoryboardSegment: ClapSegment | undefined = shotStoryboardSegments.at(0)
51
 
52
  console.log(`[api/edit/videos] processShot: shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotVideoSegments.length} videos)`)
53
 
 
90
 
91
  // TASK 3: GENERATE MISSING VIDEO FILE
92
  if (!shotVideoSegment.assetUrl) {
93
+ console.log(`[api/edit/videos] processShot: generating video file..`)
94
 
95
  const debug = false
96
 
 
140
  } else {
141
  console.log(`[api/edit/videos] processShot: there is already a video file: ${shotVideoSegment?.assetUrl?.slice?.(0, 50)}...`)
142
  }
143
+
144
+ if (!shotVideoSegment.assetUrl) {
145
+ return
146
+ }
147
+
148
+ if (!shotStoryboardSegment) {
149
+ console.log(`[api/edit/videos] processShot: adding the missing storyboard segment`)
150
+
151
+ shotStoryboardSegment = newSegment({
152
+ track: 1,
153
+ startTimeInMs: shotSegment.startTimeInMs,
154
+ endTimeInMs: shotSegment.endTimeInMs,
155
+ assetDurationInMs: shotSegment.assetDurationInMs,
156
+ category: ClapSegmentCategory.STORYBOARD,
157
+ prompt: shotVideoSegment.prompt,
158
+ outputType: ClapOutputType.IMAGE,
159
+ status: "to_generate",
160
+ })
161
+
162
+ if (shotStoryboardSegment) {
163
+ existingClap.segments.push(shotStoryboardSegment)
164
+ }
165
+ }
166
+
167
+
168
+ //----------
169
+ if (
170
+ shotStoryboardSegment &&
171
+ (!shotStoryboardSegment.assetUrl || shotStoryboardSegment.status === "to_generate")
172
+ ) {
173
+ console.log(`[api/edit/videos] processShot: generating a missing storyboard asset`)
174
+
175
+ try {
176
+ shotStoryboardSegment.assetUrl = await extractFirstFrame({
177
+ inputVideo: shotVideoSegment.assetUrl,
178
+ outputFormat: "jpeg"
179
+ })
180
+ if (!shotStoryboardSegment.assetUrl) { throw new Error(`failed to extract the first frame`) }
181
+ console.warn(`[api/edit/videos] processShot: successfully fixed the missing storyboard`)
182
+
183
+ shotStoryboardSegment.status = "completed"
184
+ } catch (err) {
185
+ console.warn(`[api/edit/videos] processShot: couldn't generate the missing storyboard (probably an error with the ffmpeg config). Message:`, err)
186
+ shotStoryboardSegment.status = "to_generate"
187
+ }
188
+
189
+
190
+ if (shotStoryboardSegment && mode !== ClapCompletionMode.FULL) {
191
+ newerClap.segments.push(shotStoryboardSegment)
192
+ }
193
+ }
194
+
195
  }