Hugging Face's logo

Spaces:
jbilcke-hf
/
clapper
Running on CPU Upgrade

App Files Community
1
jbilcke-hf HF staff commited on
Commit
86259b4
•
1 Parent(s): eca0313

work on video import

Browse files
Files changed (28) hide show
  1. .gitattributes +1 -0
  2. .github/workflows/tests.yml +4 -2
  3. forge.config.js +51 -6
  4. main.js +17 -5
  5. package-lock.json +401 -181
  6. package.json +15 -6
  7. {src/app → public/images/logos}/CL.icns +0 -0
  8. {src/app → public/images/logos}/CL.iconset/icon_128x128.png +0 -0
  9. {src/app → public/images/logos}/CL.iconset/icon_128x128@2x.png +0 -0
  10. {src/app → public/images/logos}/CL.iconset/icon_16x16.png +0 -0
  11. {src/app → public/images/logos}/CL.iconset/icon_16x16@2x.png +0 -0
  12. {src/app → public/images/logos}/CL.iconset/icon_256x256.png +0 -0
  13. {src/app → public/images/logos}/CL.iconset/icon_256x256@2x.png +0 -0
  14. {src/app → public/images/logos}/CL.iconset/icon_32x32.png +0 -0
  15. {src/app → public/images/logos}/CL.iconset/icon_32x32@2x.png +0 -0
  16. {src/app → public/images/logos}/CL.iconset/icon_512x512.png +0 -0
  17. {src/app → public/images/logos}/CL.iconset/icon_512x512@2x.png +0 -0
  18. public/images/logos/CL.png +3 -0
  19. public/wasm/MediaInfoModule.wasm +3 -0
  20. src/lib/core/constants.ts +1 -1
  21. src/lib/utils/base64DataUriToFile.ts +12 -0
  22. src/services/io/createFullVideo.ts +11 -477
  23. src/services/io/{extractCaptionFromFrame.ts → extractCaptionFromFrameMoondream.ts} +3 -2
  24. src/services/io/extractCaptionsFromFrames.ts +95 -0
  25. src/services/io/extractFramesFromVideo.ts +97 -20
  26. src/services/io/ffmpegUtils.ts +478 -0
  27. src/services/io/parseFileIntoSegments.ts +95 -19
  28. src/services/io/useIO.ts +93 -5
.gitattributes CHANGED
@@ -5,3 +5,4 @@
5
  *.csv filter=lfs diff=lfs merge=lfs -text
6
  *.png filter=lfs diff=lfs merge=lfs -text
7
  *.excalidraw filter=lfs diff=lfs merge=lfs -text
 
 
5
  *.csv filter=lfs diff=lfs merge=lfs -text
6
  *.png filter=lfs diff=lfs merge=lfs -text
7
  *.excalidraw filter=lfs diff=lfs merge=lfs -text
8
+ *.wasm filter=lfs diff=lfs merge=lfs -text
.github/workflows/tests.yml CHANGED
@@ -46,10 +46,12 @@ jobs:
46
  run: npm install @playwright/test@${{ env.PLAYWRIGHT_VERSION }}
47
  - name: Install Playwright Browsers
48
  run: npx playwright install --with-deps ${{ inputs.browser }}
49
- if: steps.playwright-cache.outputs.cache-hit != 'true'
 
50
  - name: Install system dependencies for WebKit
51
  # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately
52
- if: ${{ inputs.browser == 'webkit' && steps.playwright-cache.outputs.cache-hit == 'true' }}
 
53
  run: npx playwright install-deps webkit
54
  - name: Run build
55
  run: npm run build:ci
 
46
  run: npm install @playwright/test@${{ env.PLAYWRIGHT_VERSION }}
47
  - name: Install Playwright Browsers
48
  run: npx playwright install --with-deps ${{ inputs.browser }}
49
+ # disabled for now as we have caching issues:
50
+ #_if: steps.playwright-cache.outputs.cache-hit != 'true'
51
  - name: Install system dependencies for WebKit
52
  # Some WebKit dependencies seem to lay outside the cache and will need to be installed separately
53
+ # disabled for now as we have caching issues:
54
+ #_if: ${{ inputs.browser == 'webkit' && steps.playwright-cache.outputs.cache-hit == 'true' }}
55
  run: npx playwright install-deps webkit
56
  - name: Run build
57
  run: npm run build:ci
forge.config.js CHANGED
@@ -1,24 +1,68 @@
 
 
 
 
 
 
 
 
 
 
1
  module.exports = {
2
  packagerConfig: {
3
- asar: false, // true,
4
- icon: "./public/icon",
5
- osxSign: {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  },
7
  rebuildConfig: {},
8
  makers: [
9
  {
10
  name: '@electron-forge/maker-squirrel',
11
- config: {},
 
 
12
  },
13
  {
14
  name: '@electron-forge/maker-zip',
15
  platforms: ['darwin'],
 
16
  },
17
  {
18
  name: '@electron-forge/maker-deb',
19
  config: {
20
  options: {
21
- icon: './public/icon.png'
22
  }
23
  },
24
  },
@@ -26,7 +70,7 @@ module.exports = {
26
  name: '@electron-forge/maker-dmg',
27
  config: {
28
  options: {
29
- icon: './public/icon.icns'
30
  }
31
  },
32
  },
@@ -45,3 +89,4 @@ module.exports = {
45
  */
46
  ],
47
  };
 
 
1
+ // for some reason using forge.config.ts doesn't work,
2
+ //
3
+ // it says:
4
+ // Failed to load: /Users/jbilcke/Projects/clapper/forge.config.ts
5
+ // An unhandled rejection has occurred inside Forge:
6
+ // SyntaxError: Unexpected token 'export'
7
+ //
8
+ // so we cannot use this:
9
+ // import type { ForgeConfig } from '@electron-forge/shared-types';
10
+ // export const config: ForgeConfig = {
11
  module.exports = {
12
  packagerConfig: {
13
+ name: "Clapper",
14
+ asar: true,
15
+ icon: "./public/images/logos/CL.png",
16
+ osxSign: {},
17
+
18
+ // One or more files to be copied directly into the app's
19
+ // Contents/Resources directory for macOS target platforms
20
+ // and the resources directory for other target platforms.
21
+ // The resources directory can be referenced in the packaged
22
+ // app via the process.resourcesPath value.
23
+ extraResource: [
24
+ ".next/standalone"
25
+ ],
26
+ // ignore: ['^\\/public$', '^\\/node_modules$', '^\\/src$', '^\\/[.].+'],
27
+
28
+ // Walks the node_modules dependency tree to remove all of
29
+ // the packages specified in the devDependencies section of
30
+ // package.json from the outputted Electron app.
31
+ prune: true,
32
+
33
+ ignore: [
34
+ '^\\/.next$',
35
+ '^\\/src$',
36
+ '^\\/documentation$',
37
+ '^\\/test-results$',
38
+ '^\\/playwright-report$',
39
+ '^\\/.github$',
40
+ '^\\/public$',
41
+ '^\\/out$',
42
+ '^\\/tests$',
43
+ '^\\/Dockerfile$',
44
+ '^\\/package-lock.json$',
45
+ '^\\/.git$',
46
+ ],
47
  },
48
  rebuildConfig: {},
49
  makers: [
50
  {
51
  name: '@electron-forge/maker-squirrel',
52
+ config: {
53
+ authors: "Clapper contributors"
54
+ }
55
  },
56
  {
57
  name: '@electron-forge/maker-zip',
58
  platforms: ['darwin'],
59
+ config: {},
60
  },
61
  {
62
  name: '@electron-forge/maker-deb',
63
  config: {
64
  options: {
65
+ icon: './public/images/logos/CL.png'
66
  }
67
  },
68
  },
 
70
  name: '@electron-forge/maker-dmg',
71
  config: {
72
  options: {
73
+ icon: './public/images/logos/CL.icns'
74
  }
75
  },
76
  },
 
89
  */
90
  ],
91
  };
92
+
main.js CHANGED
@@ -1,6 +1,16 @@
1
  const fs = require('fs')
2
  const path = require('path')
 
 
 
 
 
 
 
 
3
  const dotenv = require('dotenv')
 
 
4
 
5
  dotenv.config()
6
 
@@ -20,12 +30,14 @@ try {
20
 
21
  const { app, BrowserWindow, screen } = require('electron')
22
 
23
- const currentDir = process.cwd()
24
-
25
- const mainServerPath = path.join(currentDir, '.next/standalone/server.js')
26
 
27
- // now we load the main server
28
- require(mainServerPath)
 
 
 
 
 
29
 
30
  // TODO: load the proxy server (for AI providers that refuse browser-side clients)
31
  // const proxyServerPath = path.join(currentDir, '.next/standalone/proxy-server.js')
 
1
  const fs = require('fs')
2
  const path = require('path')
3
+
4
+ // -----------------------------------------------------------
5
+ //
6
+ // attention: if you add dependencies here, you might have to edit
7
+ // forge.config.js, , the part about:
8
+ // '^\\/node_modules/(?!dotenv)$',
9
+ //
10
+ // if you have an idea to do that automatically, let me know
11
  const dotenv = require('dotenv')
12
+ //
13
+ // -----------------------------------------------------------
14
 
15
  dotenv.config()
16
 
 
30
 
31
  const { app, BrowserWindow, screen } = require('electron')
32
 
 
 
 
33
 
34
+ try {
35
+ // used when the app is built with `npm run electron:make`
36
+ require(path.join(process.resourcesPath, 'standalone/server.js'))
37
+ } catch (err) {
38
+ // used when the app is started with `npm run electron:start`
39
+ require(path.join(process.cwd(), '.next/standalone/server.js'))
40
+ }
41
 
42
  // TODO: load the proxy server (for AI providers that refuse browser-side clients)
43
  // const proxyServerPath = path.join(currentDir, '.next/standalone/proxy-server.js')
package-lock.json CHANGED
@@ -1,19 +1,19 @@
1
  {
2
- "name": "@aitube/clapper",
3
  "version": "0.0.5",
4
  "lockfileVersion": 3,
5
  "requires": true,
6
  "packages": {
7
  "": {
8
- "name": "@aitube/clapper",
9
  "version": "0.0.5",
10
  "license": "GPL-3.0-only",
11
  "dependencies": {
12
  "@aitube/broadway": "0.0.22",
13
  "@aitube/clap": "0.0.30",
14
- "@aitube/clapper-services": "0.0.28",
15
  "@aitube/engine": "0.0.26",
16
- "@aitube/timeline": "0.0.42",
17
  "@fal-ai/serverless-client": "^0.13.0",
18
  "@ffmpeg/ffmpeg": "^0.12.10",
19
  "@ffmpeg/util": "^0.12.1",
@@ -110,6 +110,7 @@
110
  "@electron-forge/maker-squirrel": "^7.4.0",
111
  "@electron-forge/maker-zip": "^7.4.0",
112
  "@electron-forge/plugin-auto-unpack-natives": "^7.4.0",
 
113
  "@playwright/test": "^1.45.1",
114
  "@testing-library/react": "^16.0.0",
115
  "@types/fluent-ffmpeg": "^2.1.24",
@@ -160,12 +161,12 @@
160
  }
161
  },
162
  "node_modules/@aitube/clapper-services": {
163
- "version": "0.0.28",
164
- "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.28.tgz",
165
- "integrity": "sha512-YmiPGAGtZcgqqnXmgtWvFyd9TL5eCpF343+ndvDa+aorQ43PkIKo7f3df550njOICM5KZhSBRFEIh1c5Dsi7Yg==",
166
  "peerDependencies": {
167
  "@aitube/clap": "0.0.30",
168
- "@aitube/timeline": "0.0.42",
169
  "@monaco-editor/react": "4.6.0",
170
  "monaco-editor": "0.50.0",
171
  "react": "*",
@@ -191,9 +192,9 @@
191
  }
192
  },
193
  "node_modules/@aitube/timeline": {
194
- "version": "0.0.42",
195
- "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.42.tgz",
196
- "integrity": "sha512-H9vvHrrOBsTpU7AeM1+PJ7TbGr0Jb63aRGBXXL8CUVawX2L92BIkoEBXhGSVePP6yeS6oUv/u86Z2rHPLW9ChQ==",
197
  "dependencies": {
198
  "date-fns": "^3.6.0",
199
  "react-virtualized-auto-sizer": "^1.0.24"
@@ -2037,6 +2038,43 @@
2037
  "node": ">= 16.4.0"
2038
  }
2039
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2040
  "node_modules/@electron-forge/shared-types": {
2041
  "version": "7.4.0",
2042
  "resolved": "https://registry.npmjs.org/@electron-forge/shared-types/-/shared-types-7.4.0.tgz",
@@ -2513,6 +2551,18 @@
2513
  "node": ">=12"
2514
  }
2515
  },
 
 
 
 
 
 
 
 
 
 
 
 
2516
  "node_modules/@electron/rebuild/node_modules/http-proxy-agent": {
2517
  "version": "5.0.0",
2518
  "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
@@ -2575,6 +2625,18 @@
2575
  "node": ">=10"
2576
  }
2577
  },
 
 
 
 
 
 
 
 
 
 
 
 
2578
  "node_modules/@electron/rebuild/node_modules/minipass-collect": {
2579
  "version": "1.0.2",
2580
  "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz",
@@ -2694,6 +2756,12 @@
2694
  "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
2695
  }
2696
  },
 
 
 
 
 
 
2697
  "node_modules/@electron/universal": {
2698
  "version": "2.0.1",
2699
  "resolved": "https://registry.npmjs.org/@electron/universal/-/universal-2.0.1.tgz",
@@ -3846,24 +3914,24 @@
3846
  }
3847
  },
3848
  "node_modules/@inquirer/confirm": {
3849
- "version": "3.1.16",
3850
- "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.16.tgz",
3851
- "integrity": "sha512-DXgLZim+YVTk05zRywvFRfJt2Jje7sZ4DO6Ss9RpGtgXEd/T0IiTqubHWst0IazCwdPI9g/06Rtm/nm4IBFJBA==",
3852
  "dependencies": {
3853
- "@inquirer/core": "^9.0.4",
3854
- "@inquirer/type": "^1.5.0"
3855
  },
3856
  "engines": {
3857
  "node": ">=18"
3858
  }
3859
  },
3860
  "node_modules/@inquirer/core": {
3861
- "version": "9.0.4",
3862
- "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.0.4.tgz",
3863
- "integrity": "sha512-46LaWACIctSfVKTu71ziFlqO8SVLhWGSxvaHpf0frfDTphSSpIfeNo5ZH/kJPHYJw4VgPGf/9c3zJN/FnCdaIQ==",
3864
  "dependencies": {
3865
- "@inquirer/figures": "^1.0.4",
3866
- "@inquirer/type": "^1.5.0",
3867
  "@types/mute-stream": "^0.0.4",
3868
  "@types/node": "^20.14.11",
3869
  "@types/wrap-ansi": "^3.0.0",
@@ -3981,17 +4049,17 @@
3981
  }
3982
  },
3983
  "node_modules/@inquirer/figures": {
3984
- "version": "1.0.4",
3985
- "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.4.tgz",
3986
- "integrity": "sha512-R7Gsg6elpuqdn55fBH2y9oYzrU/yKrSmIsDX4ROT51vohrECFzTf2zw9BfUbOW8xjfmM2QbVoVYdTwhrtEKWSQ==",
3987
  "engines": {
3988
  "node": ">=18"
3989
  }
3990
  },
3991
  "node_modules/@inquirer/type": {
3992
- "version": "1.5.0",
3993
- "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.0.tgz",
3994
- "integrity": "sha512-L/UdayX9Z1lLN+itoTKqJ/X4DX5DaWu2Sruwt4XgZzMNv32x4qllbzMX4MbJlz0yxAQtU19UvABGOjmdq1u3qA==",
3995
  "dependencies": {
3996
  "mute-stream": "^1.0.0"
3997
  },
@@ -4052,15 +4120,6 @@
4052
  "node": ">=18.0.0"
4053
  }
4054
  },
4055
- "node_modules/@isaacs/fs-minipass/node_modules/minipass": {
4056
- "version": "7.1.2",
4057
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
4058
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
4059
- "optional": true,
4060
- "engines": {
4061
- "node": ">=16 || 14 >=14.17"
4062
- }
4063
- },
4064
  "node_modules/@jridgewell/gen-mapping": {
4065
  "version": "0.3.5",
4066
  "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz",
@@ -4449,15 +4508,6 @@
4449
  "url": "https://github.com/sponsors/isaacs"
4450
  }
4451
  },
4452
- "node_modules/@next/eslint-plugin-next/node_modules/minipass": {
4453
- "version": "7.1.2",
4454
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
4455
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
4456
- "dev": true,
4457
- "engines": {
4458
- "node": ">=16 || 14 >=14.17"
4459
- }
4460
- },
4461
  "node_modules/@next/swc-darwin-arm64": {
4462
  "version": "14.2.5",
4463
  "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.5.tgz",
@@ -4698,6 +4748,148 @@
4698
  "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
4699
  }
4700
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4701
  "node_modules/@open-draft/deferred-promise": {
4702
  "version": "2.2.0",
4703
  "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz",
@@ -5882,25 +6074,25 @@
5882
  "integrity": "sha512-/RVXdLvJxLg4QKvMoM5WlwNR9ViO9z8B/qPcc+C0Sa/teJY7QG7kJ441DwzOjMYEY7GmU4dj5EcGHIkKZiQZCA=="
5883
  },
5884
  "node_modules/@react-spring/animated": {
5885
- "version": "9.7.3",
5886
- "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.3.tgz",
5887
- "integrity": "sha512-5CWeNJt9pNgyvuSzQH+uy2pvTg8Y4/OisoscZIR8/ZNLIOI+CatFBhGZpDGTF/OzdNFsAoGk3wiUYTwoJ0YIvw==",
5888
  "dependencies": {
5889
- "@react-spring/shared": "~9.7.3",
5890
- "@react-spring/types": "~9.7.3"
5891
  },
5892
  "peerDependencies": {
5893
  "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
5894
  }
5895
  },
5896
  "node_modules/@react-spring/core": {
5897
- "version": "9.7.3",
5898
- "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.3.tgz",
5899
- "integrity": "sha512-IqFdPVf3ZOC1Cx7+M0cXf4odNLxDC+n7IN3MDcVCTIOSBfqEcBebSv+vlY5AhM0zw05PDbjKrNmBpzv/AqpjnQ==",
5900
  "dependencies": {
5901
- "@react-spring/animated": "~9.7.3",
5902
- "@react-spring/shared": "~9.7.3",
5903
- "@react-spring/types": "~9.7.3"
5904
  },
5905
  "funding": {
5906
  "type": "opencollective",
@@ -5911,30 +6103,31 @@
5911
  }
5912
  },
5913
  "node_modules/@react-spring/rafz": {
5914
- "version": "9.6.1",
5915
- "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.6.1.tgz",
5916
- "integrity": "sha512-v6qbgNRpztJFFfSE3e2W1Uz+g8KnIBs6SmzCzcVVF61GdGfGOuBrbjIcp+nUz301awVmREKi4eMQb2Ab2gGgyQ=="
5917
  },
5918
  "node_modules/@react-spring/shared": {
5919
- "version": "9.7.3",
5920
- "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.3.tgz",
5921
- "integrity": "sha512-NEopD+9S5xYyQ0pGtioacLhL2luflh6HACSSDUZOwLHoxA5eku1UPuqcJqjwSD6luKjjLfiLOspxo43FUHKKSA==",
5922
  "dependencies": {
5923
- "@react-spring/types": "~9.7.3"
 
5924
  },
5925
  "peerDependencies": {
5926
  "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
5927
  }
5928
  },
5929
  "node_modules/@react-spring/three": {
5930
- "version": "9.7.3",
5931
- "resolved": "https://registry.npmjs.org/@react-spring/three/-/three-9.7.3.tgz",
5932
- "integrity": "sha512-Q1p512CqUlmMK8UMBF/Rj79qndhOWq4XUTayxMP9S892jiXzWQuj+xC3Xvm59DP/D4JXusXpxxqfgoH+hmOktA==",
5933
  "dependencies": {
5934
- "@react-spring/animated": "~9.7.3",
5935
- "@react-spring/core": "~9.7.3",
5936
- "@react-spring/shared": "~9.7.3",
5937
- "@react-spring/types": "~9.7.3"
5938
  },
5939
  "peerDependencies": {
5940
  "@react-three/fiber": ">=6.0",
@@ -5943,9 +6136,9 @@
5943
  }
5944
  },
5945
  "node_modules/@react-spring/types": {
5946
- "version": "9.7.3",
5947
- "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.3.tgz",
5948
- "integrity": "sha512-Kpx/fQ/ZFX31OtlqVEFfgaD1ACzul4NksrvIgYfIFq9JpDHFwQkMVZ10tbo0FU/grje4rcL4EIrjekl3kYwgWw=="
5949
  },
5950
  "node_modules/@react-three/drei": {
5951
  "version": "9.108.4",
@@ -6018,6 +6211,11 @@
6018
  "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
6019
  }
6020
  },
 
 
 
 
 
6021
  "node_modules/@react-three/drei/node_modules/@react-spring/shared": {
6022
  "version": "9.6.1",
6023
  "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.6.1.tgz",
@@ -8375,6 +8573,12 @@
8375
  }
8376
  ]
8377
  },
 
 
 
 
 
 
8378
  "node_modules/bidi-js": {
8379
  "version": "1.0.3",
8380
  "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
@@ -8474,6 +8678,12 @@
8474
  "dev": true,
8475
  "optional": true
8476
  },
 
 
 
 
 
 
8477
  "node_modules/bowser": {
8478
  "version": "2.11.0",
8479
  "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz",
@@ -8648,18 +8858,6 @@
8648
  "balanced-match": "^1.0.0"
8649
  }
8650
  },
8651
- "node_modules/cacache/node_modules/fs-minipass": {
8652
- "version": "3.0.3",
8653
- "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz",
8654
- "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==",
8655
- "dev": true,
8656
- "dependencies": {
8657
- "minipass": "^7.0.3"
8658
- },
8659
- "engines": {
8660
- "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
8661
- }
8662
- },
8663
  "node_modules/cacache/node_modules/glob": {
8664
  "version": "10.4.5",
8665
  "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
@@ -8716,15 +8914,6 @@
8716
  "url": "https://github.com/sponsors/isaacs"
8717
  }
8718
  },
8719
- "node_modules/cacache/node_modules/minipass": {
8720
- "version": "7.1.2",
8721
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
8722
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
8723
- "dev": true,
8724
- "engines": {
8725
- "node": ">=16 || 14 >=14.17"
8726
- }
8727
- },
8728
  "node_modules/cacheable-lookup": {
8729
  "version": "5.0.4",
8730
  "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
@@ -8830,9 +9019,9 @@
8830
  }
8831
  },
8832
  "node_modules/caniuse-lite": {
8833
- "version": "1.0.30001642",
8834
- "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001642.tgz",
8835
- "integrity": "sha512-3XQ0DoRgLijXJErLSl+bLnJ+Et4KqV1PY6JJBGAFlsNsz31zeAIncyeZfLCabHK/jtSh+671RM9YMldxjUPZtA==",
8836
  "funding": [
8837
  {
8838
  "type": "opencollective",
@@ -9996,6 +10185,12 @@
9996
  "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
9997
  "dev": true
9998
  },
 
 
 
 
 
 
9999
  "node_modules/dequal": {
10000
  "version": "2.0.3",
10001
  "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
@@ -12179,15 +12374,15 @@
12179
  }
12180
  },
12181
  "node_modules/fs-minipass": {
12182
- "version": "2.1.0",
12183
- "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
12184
- "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
12185
  "dev": true,
12186
  "dependencies": {
12187
- "minipass": "^3.0.0"
12188
  },
12189
  "engines": {
12190
- "node": ">= 8"
12191
  }
12192
  },
12193
  "node_modules/fs-temp": {
@@ -13585,6 +13780,15 @@
13585
  "node": ">=8"
13586
  }
13587
  },
 
 
 
 
 
 
 
 
 
13588
  "node_modules/is-potential-custom-element-name": {
13589
  "version": "1.0.1",
13590
  "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
@@ -14431,15 +14635,6 @@
14431
  "node": "^16.14.0 || >=18.0.0"
14432
  }
14433
  },
14434
- "node_modules/make-fetch-happen/node_modules/minipass": {
14435
- "version": "7.1.2",
14436
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
14437
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
14438
- "dev": true,
14439
- "engines": {
14440
- "node": ">=16 || 14 >=14.17"
14441
- }
14442
- },
14443
  "node_modules/map-age-cleaner": {
14444
  "version": "0.1.3",
14445
  "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
@@ -14600,15 +14795,11 @@
14600
  }
14601
  },
14602
  "node_modules/minipass": {
14603
- "version": "3.3.6",
14604
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
14605
- "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
14606
- "dev": true,
14607
- "dependencies": {
14608
- "yallist": "^4.0.0"
14609
- },
14610
  "engines": {
14611
- "node": ">=8"
14612
  }
14613
  },
14614
  "node_modules/minipass-collect": {
@@ -14623,15 +14814,6 @@
14623
  "node": ">=16 || 14 >=14.17"
14624
  }
14625
  },
14626
- "node_modules/minipass-collect/node_modules/minipass": {
14627
- "version": "7.1.2",
14628
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
14629
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
14630
- "dev": true,
14631
- "engines": {
14632
- "node": ">=16 || 14 >=14.17"
14633
- }
14634
- },
14635
  "node_modules/minipass-fetch": {
14636
  "version": "3.0.5",
14637
  "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.5.tgz",
@@ -14649,15 +14831,6 @@
14649
  "encoding": "^0.1.13"
14650
  }
14651
  },
14652
- "node_modules/minipass-fetch/node_modules/minipass": {
14653
- "version": "7.1.2",
14654
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
14655
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
14656
- "dev": true,
14657
- "engines": {
14658
- "node": ">=16 || 14 >=14.17"
14659
- }
14660
- },
14661
  "node_modules/minipass-flush": {
14662
  "version": "1.0.5",
14663
  "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz",
@@ -14670,6 +14843,24 @@
14670
  "node": ">= 8"
14671
  }
14672
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14673
  "node_modules/minipass-pipeline": {
14674
  "version": "1.2.4",
14675
  "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz",
@@ -14682,6 +14873,24 @@
14682
  "node": ">=8"
14683
  }
14684
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14685
  "node_modules/minipass-sized": {
14686
  "version": "1.0.3",
14687
  "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz",
@@ -14694,7 +14903,19 @@
14694
  "node": ">=8"
14695
  }
14696
  },
14697
- "node_modules/minipass/node_modules/yallist": {
 
 
 
 
 
 
 
 
 
 
 
 
14698
  "version": "4.0.0",
14699
  "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
14700
  "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
@@ -14713,6 +14934,18 @@
14713
  "node": ">= 8"
14714
  }
14715
  },
 
 
 
 
 
 
 
 
 
 
 
 
14716
  "node_modules/minizlib/node_modules/yallist": {
14717
  "version": "4.0.0",
14718
  "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
@@ -15194,15 +15427,6 @@
15194
  "url": "https://github.com/sponsors/isaacs"
15195
  }
15196
  },
15197
- "node_modules/node-gyp/node_modules/minipass": {
15198
- "version": "7.1.2",
15199
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
15200
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
15201
- "dev": true,
15202
- "engines": {
15203
- "node": ">=16 || 14 >=14.17"
15204
- }
15205
- },
15206
  "node_modules/node-gyp/node_modules/which": {
15207
  "version": "4.0.0",
15208
  "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz",
@@ -15606,15 +15830,6 @@
15606
  "url": "https://github.com/sponsors/isaacs"
15607
  }
15608
  },
15609
- "node_modules/onnxruntime-node/node_modules/minipass": {
15610
- "version": "7.1.2",
15611
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
15612
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
15613
- "optional": true,
15614
- "engines": {
15615
- "node": ">=16 || 14 >=14.17"
15616
- }
15617
- },
15618
  "node_modules/onnxruntime-node/node_modules/minizlib": {
15619
  "version": "3.0.1",
15620
  "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz",
@@ -16051,14 +16266,6 @@
16051
  "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
16052
  "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="
16053
  },
16054
- "node_modules/path-scurry/node_modules/minipass": {
16055
- "version": "7.1.2",
16056
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
16057
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
16058
- "engines": {
16059
- "node": ">=16 || 14 >=14.17"
16060
- }
16061
- },
16062
  "node_modules/path-to-regexp": {
16063
  "version": "6.2.2",
16064
  "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz",
@@ -18060,15 +18267,6 @@
18060
  "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
18061
  }
18062
  },
18063
- "node_modules/ssri/node_modules/minipass": {
18064
- "version": "7.1.2",
18065
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
18066
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
18067
- "dev": true,
18068
- "engines": {
18069
- "node": ">=16 || 14 >=14.17"
18070
- }
18071
- },
18072
  "node_modules/stackback": {
18073
  "version": "0.0.2",
18074
  "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz",
@@ -18583,14 +18781,6 @@
18583
  "url": "https://github.com/sponsors/isaacs"
18584
  }
18585
  },
18586
- "node_modules/sucrase/node_modules/minipass": {
18587
- "version": "7.1.2",
18588
- "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
18589
- "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
18590
- "engines": {
18591
- "node": ">=16 || 14 >=14.17"
18592
- }
18593
- },
18594
  "node_modules/sudo-prompt": {
18595
  "version": "9.2.1",
18596
  "resolved": "https://registry.npmjs.org/sudo-prompt/-/sudo-prompt-9.2.1.tgz",
@@ -18736,6 +18926,30 @@
18736
  "node": ">=10"
18737
  }
18738
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18739
  "node_modules/tar/node_modules/minipass": {
18740
  "version": "5.0.0",
18741
  "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
@@ -19370,6 +19584,12 @@
19370
  "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
19371
  }
19372
  },
 
 
 
 
 
 
19373
  "node_modules/universalify": {
19374
  "version": "2.0.1",
19375
  "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
 
1
  {
2
+ "name": "clapper",
3
  "version": "0.0.5",
4
  "lockfileVersion": 3,
5
  "requires": true,
6
  "packages": {
7
  "": {
8
+ "name": "clapper",
9
  "version": "0.0.5",
10
  "license": "GPL-3.0-only",
11
  "dependencies": {
12
  "@aitube/broadway": "0.0.22",
13
  "@aitube/clap": "0.0.30",
14
+ "@aitube/clapper-services": "0.0.29",
15
  "@aitube/engine": "0.0.26",
16
+ "@aitube/timeline": "0.0.43",
17
  "@fal-ai/serverless-client": "^0.13.0",
18
  "@ffmpeg/ffmpeg": "^0.12.10",
19
  "@ffmpeg/util": "^0.12.1",
 
110
  "@electron-forge/maker-squirrel": "^7.4.0",
111
  "@electron-forge/maker-zip": "^7.4.0",
112
  "@electron-forge/plugin-auto-unpack-natives": "^7.4.0",
113
+ "@electron-forge/publisher-github": "^7.4.0",
114
  "@playwright/test": "^1.45.1",
115
  "@testing-library/react": "^16.0.0",
116
  "@types/fluent-ffmpeg": "^2.1.24",
 
161
  }
162
  },
163
  "node_modules/@aitube/clapper-services": {
164
+ "version": "0.0.29",
165
+ "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.29.tgz",
166
+ "integrity": "sha512-61UH/TQwPcvXArEkPnGNm+IQulaW3zNh73pzihdU2kkqufGzUYCNSd/jHJh9dLqQm3lZtm6QMN2RReFrzGuLNQ==",
167
  "peerDependencies": {
168
  "@aitube/clap": "0.0.30",
169
+ "@aitube/timeline": "0.0.43",
170
  "@monaco-editor/react": "4.6.0",
171
  "monaco-editor": "0.50.0",
172
  "react": "*",
 
192
  }
193
  },
194
  "node_modules/@aitube/timeline": {
195
+ "version": "0.0.43",
196
+ "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.43.tgz",
197
+ "integrity": "sha512-TnzKrB955YeDKOMWsnniGbQ+qulCmGptMfhNjDLEqA6jRcsnPVUFCR2dQBqWGNn6KfFPXmDvSi0Sihy7Oj98Aw==",
198
  "dependencies": {
199
  "date-fns": "^3.6.0",
200
  "react-virtualized-auto-sizer": "^1.0.24"
 
2038
  "node": ">= 16.4.0"
2039
  }
2040
  },
2041
+ "node_modules/@electron-forge/publisher-github": {
2042
+ "version": "7.4.0",
2043
+ "resolved": "https://registry.npmjs.org/@electron-forge/publisher-github/-/publisher-github-7.4.0.tgz",
2044
+ "integrity": "sha512-hrxKNssJyU8Yuz0qv384y5RKojMG0nWeG7/kidjp8PX/RnqjGRU/JJ0Worl28g8LGiLt5R5JIfNLngLaFMn8tg==",
2045
+ "dev": true,
2046
+ "dependencies": {
2047
+ "@electron-forge/publisher-base": "7.4.0",
2048
+ "@electron-forge/shared-types": "7.4.0",
2049
+ "@octokit/core": "^3.2.4",
2050
+ "@octokit/plugin-retry": "^3.0.9",
2051
+ "@octokit/request-error": "^2.0.5",
2052
+ "@octokit/rest": "^18.0.11",
2053
+ "@octokit/types": "^6.1.2",
2054
+ "chalk": "^4.0.0",
2055
+ "debug": "^4.3.1",
2056
+ "fs-extra": "^10.0.0",
2057
+ "log-symbols": "^4.0.0",
2058
+ "mime-types": "^2.1.25"
2059
+ },
2060
+ "engines": {
2061
+ "node": ">= 16.4.0"
2062
+ }
2063
+ },
2064
+ "node_modules/@electron-forge/publisher-github/node_modules/fs-extra": {
2065
+ "version": "10.1.0",
2066
+ "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
2067
+ "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
2068
+ "dev": true,
2069
+ "dependencies": {
2070
+ "graceful-fs": "^4.2.0",
2071
+ "jsonfile": "^6.0.1",
2072
+ "universalify": "^2.0.0"
2073
+ },
2074
+ "engines": {
2075
+ "node": ">=12"
2076
+ }
2077
+ },
2078
  "node_modules/@electron-forge/shared-types": {
2079
  "version": "7.4.0",
2080
  "resolved": "https://registry.npmjs.org/@electron-forge/shared-types/-/shared-types-7.4.0.tgz",
 
2551
  "node": ">=12"
2552
  }
2553
  },
2554
+ "node_modules/@electron/rebuild/node_modules/fs-minipass": {
2555
+ "version": "2.1.0",
2556
+ "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
2557
+ "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
2558
+ "dev": true,
2559
+ "dependencies": {
2560
+ "minipass": "^3.0.0"
2561
+ },
2562
+ "engines": {
2563
+ "node": ">= 8"
2564
+ }
2565
+ },
2566
  "node_modules/@electron/rebuild/node_modules/http-proxy-agent": {
2567
  "version": "5.0.0",
2568
  "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
 
2625
  "node": ">=10"
2626
  }
2627
  },
2628
+ "node_modules/@electron/rebuild/node_modules/minipass": {
2629
+ "version": "3.3.6",
2630
+ "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
2631
+ "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
2632
+ "dev": true,
2633
+ "dependencies": {
2634
+ "yallist": "^4.0.0"
2635
+ },
2636
+ "engines": {
2637
+ "node": ">=8"
2638
+ }
2639
+ },
2640
  "node_modules/@electron/rebuild/node_modules/minipass-collect": {
2641
  "version": "1.0.2",
2642
  "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz",
 
2756
  "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
2757
  }
2758
  },
2759
+ "node_modules/@electron/rebuild/node_modules/yallist": {
2760
+ "version": "4.0.0",
2761
+ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
2762
+ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
2763
+ "dev": true
2764
+ },
2765
  "node_modules/@electron/universal": {
2766
  "version": "2.0.1",
2767
  "resolved": "https://registry.npmjs.org/@electron/universal/-/universal-2.0.1.tgz",
 
3914
  }
3915
  },
3916
  "node_modules/@inquirer/confirm": {
3917
+ "version": "3.1.17",
3918
+ "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.17.tgz",
3919
+ "integrity": "sha512-qCpt/AABzPynz8tr69VDvhcjwmzAryipWXtW8Vi6m651da4H/d0Bdn55LkxXD7Rp2gfgxvxzTdb66AhIA8gzBA==",
3920
  "dependencies": {
3921
+ "@inquirer/core": "^9.0.5",
3922
+ "@inquirer/type": "^1.5.1"
3923
  },
3924
  "engines": {
3925
  "node": ">=18"
3926
  }
3927
  },
3928
  "node_modules/@inquirer/core": {
3929
+ "version": "9.0.5",
3930
+ "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.0.5.tgz",
3931
+ "integrity": "sha512-QWG41I7vn62O9stYKg/juKXt1PEbr/4ZZCPb4KgXDQGwgA9M5NBTQ7FnOvT1ridbxkm/wTxLCNraUs7y47pIRQ==",
3932
  "dependencies": {
3933
+ "@inquirer/figures": "^1.0.5",
3934
+ "@inquirer/type": "^1.5.1",
3935
  "@types/mute-stream": "^0.0.4",
3936
  "@types/node": "^20.14.11",
3937
  "@types/wrap-ansi": "^3.0.0",
 
4049
  }
4050
  },
4051
  "node_modules/@inquirer/figures": {
4052
+ "version": "1.0.5",
4053
+ "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.5.tgz",
4054
+ "integrity": "sha512-79hP/VWdZ2UVc9bFGJnoQ/lQMpL74mGgzSYX1xUqCVk7/v73vJCMw1VuyWN1jGkZ9B3z7THAbySqGbCNefcjfA==",
4055
  "engines": {
4056
  "node": ">=18"
4057
  }
4058
  },
4059
  "node_modules/@inquirer/type": {
4060
+ "version": "1.5.1",
4061
+ "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.1.tgz",
4062
+ "integrity": "sha512-m3YgGQlKNS0BM+8AFiJkCsTqHEFCWn6s/Rqye3mYwvqY6LdfUv12eSwbsgNzrYyrLXiy7IrrjDLPysaSBwEfhw==",
4063
  "dependencies": {
4064
  "mute-stream": "^1.0.0"
4065
  },
 
4120
  "node": ">=18.0.0"
4121
  }
4122
  },
 
 
 
 
 
 
 
 
 
4123
  "node_modules/@jridgewell/gen-mapping": {
4124
  "version": "0.3.5",
4125
  "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz",
 
4508
  "url": "https://github.com/sponsors/isaacs"
4509
  }
4510
  },
 
 
 
 
 
 
 
 
 
4511
  "node_modules/@next/swc-darwin-arm64": {
4512
  "version": "14.2.5",
4513
  "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.5.tgz",
 
4748
  "node": "^12.13.0 || ^14.15.0 || >=16.0.0"
4749
  }
4750
  },
4751
+ "node_modules/@octokit/auth-token": {
4752
+ "version": "2.5.0",
4753
+ "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.5.0.tgz",
4754
+ "integrity": "sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==",
4755
+ "dev": true,
4756
+ "dependencies": {
4757
+ "@octokit/types": "^6.0.3"
4758
+ }
4759
+ },
4760
+ "node_modules/@octokit/core": {
4761
+ "version": "3.6.0",
4762
+ "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.6.0.tgz",
4763
+ "integrity": "sha512-7RKRKuA4xTjMhY+eG3jthb3hlZCsOwg3rztWh75Xc+ShDWOfDDATWbeZpAHBNRpm4Tv9WgBMOy1zEJYXG6NJ7Q==",
4764
+ "dev": true,
4765
+ "dependencies": {
4766
+ "@octokit/auth-token": "^2.4.4",
4767
+ "@octokit/graphql": "^4.5.8",
4768
+ "@octokit/request": "^5.6.3",
4769
+ "@octokit/request-error": "^2.0.5",
4770
+ "@octokit/types": "^6.0.3",
4771
+ "before-after-hook": "^2.2.0",
4772
+ "universal-user-agent": "^6.0.0"
4773
+ }
4774
+ },
4775
+ "node_modules/@octokit/endpoint": {
4776
+ "version": "6.0.12",
4777
+ "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz",
4778
+ "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==",
4779
+ "dev": true,
4780
+ "dependencies": {
4781
+ "@octokit/types": "^6.0.3",
4782
+ "is-plain-object": "^5.0.0",
4783
+ "universal-user-agent": "^6.0.0"
4784
+ }
4785
+ },
4786
+ "node_modules/@octokit/graphql": {
4787
+ "version": "4.8.0",
4788
+ "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.8.0.tgz",
4789
+ "integrity": "sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==",
4790
+ "dev": true,
4791
+ "dependencies": {
4792
+ "@octokit/request": "^5.6.0",
4793
+ "@octokit/types": "^6.0.3",
4794
+ "universal-user-agent": "^6.0.0"
4795
+ }
4796
+ },
4797
+ "node_modules/@octokit/openapi-types": {
4798
+ "version": "12.11.0",
4799
+ "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-12.11.0.tgz",
4800
+ "integrity": "sha512-VsXyi8peyRq9PqIz/tpqiL2w3w80OgVMwBHltTml3LmVvXiphgeqmY9mvBw9Wu7e0QWk/fqD37ux8yP5uVekyQ==",
4801
+ "dev": true
4802
+ },
4803
+ "node_modules/@octokit/plugin-paginate-rest": {
4804
+ "version": "2.21.3",
4805
+ "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.21.3.tgz",
4806
+ "integrity": "sha512-aCZTEf0y2h3OLbrgKkrfFdjRL6eSOo8komneVQJnYecAxIej7Bafor2xhuDJOIFau4pk0i/P28/XgtbyPF0ZHw==",
4807
+ "dev": true,
4808
+ "dependencies": {
4809
+ "@octokit/types": "^6.40.0"
4810
+ },
4811
+ "peerDependencies": {
4812
+ "@octokit/core": ">=2"
4813
+ }
4814
+ },
4815
+ "node_modules/@octokit/plugin-request-log": {
4816
+ "version": "1.0.4",
4817
+ "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz",
4818
+ "integrity": "sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA==",
4819
+ "dev": true,
4820
+ "peerDependencies": {
4821
+ "@octokit/core": ">=3"
4822
+ }
4823
+ },
4824
+ "node_modules/@octokit/plugin-rest-endpoint-methods": {
4825
+ "version": "5.16.2",
4826
+ "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.16.2.tgz",
4827
+ "integrity": "sha512-8QFz29Fg5jDuTPXVtey05BLm7OB+M8fnvE64RNegzX7U+5NUXcOcnpTIK0YfSHBg8gYd0oxIq3IZTe9SfPZiRw==",
4828
+ "dev": true,
4829
+ "dependencies": {
4830
+ "@octokit/types": "^6.39.0",
4831
+ "deprecation": "^2.3.1"
4832
+ },
4833
+ "peerDependencies": {
4834
+ "@octokit/core": ">=3"
4835
+ }
4836
+ },
4837
+ "node_modules/@octokit/plugin-retry": {
4838
+ "version": "3.0.9",
4839
+ "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-3.0.9.tgz",
4840
+ "integrity": "sha512-r+fArdP5+TG6l1Rv/C9hVoty6tldw6cE2pRHNGmFPdyfrc696R6JjrQ3d7HdVqGwuzfyrcaLAKD7K8TX8aehUQ==",
4841
+ "dev": true,
4842
+ "dependencies": {
4843
+ "@octokit/types": "^6.0.3",
4844
+ "bottleneck": "^2.15.3"
4845
+ }
4846
+ },
4847
+ "node_modules/@octokit/request": {
4848
+ "version": "5.6.3",
4849
+ "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.3.tgz",
4850
+ "integrity": "sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==",
4851
+ "dev": true,
4852
+ "dependencies": {
4853
+ "@octokit/endpoint": "^6.0.1",
4854
+ "@octokit/request-error": "^2.1.0",
4855
+ "@octokit/types": "^6.16.1",
4856
+ "is-plain-object": "^5.0.0",
4857
+ "node-fetch": "^2.6.7",
4858
+ "universal-user-agent": "^6.0.0"
4859
+ }
4860
+ },
4861
+ "node_modules/@octokit/request-error": {
4862
+ "version": "2.1.0",
4863
+ "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz",
4864
+ "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==",
4865
+ "dev": true,
4866
+ "dependencies": {
4867
+ "@octokit/types": "^6.0.3",
4868
+ "deprecation": "^2.0.0",
4869
+ "once": "^1.4.0"
4870
+ }
4871
+ },
4872
+ "node_modules/@octokit/rest": {
4873
+ "version": "18.12.0",
4874
+ "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-18.12.0.tgz",
4875
+ "integrity": "sha512-gDPiOHlyGavxr72y0guQEhLsemgVjwRePayJ+FcKc2SJqKUbxbkvf5kAZEWA/MKvsfYlQAMVzNJE3ezQcxMJ2Q==",
4876
+ "dev": true,
4877
+ "dependencies": {
4878
+ "@octokit/core": "^3.5.1",
4879
+ "@octokit/plugin-paginate-rest": "^2.16.8",
4880
+ "@octokit/plugin-request-log": "^1.0.4",
4881
+ "@octokit/plugin-rest-endpoint-methods": "^5.12.0"
4882
+ }
4883
+ },
4884
+ "node_modules/@octokit/types": {
4885
+ "version": "6.41.0",
4886
+ "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.41.0.tgz",
4887
+ "integrity": "sha512-eJ2jbzjdijiL3B4PrSQaSjuF2sPEQPVCPzBvTHJD9Nz+9dw2SGH4K4xeQJ77YfTq5bRQ+bD8wT11JbeDPmxmGg==",
4888
+ "dev": true,
4889
+ "dependencies": {
4890
+ "@octokit/openapi-types": "^12.11.0"
4891
+ }
4892
+ },
4893
  "node_modules/@open-draft/deferred-promise": {
4894
  "version": "2.2.0",
4895
  "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz",
 
6074
  "integrity": "sha512-/RVXdLvJxLg4QKvMoM5WlwNR9ViO9z8B/qPcc+C0Sa/teJY7QG7kJ441DwzOjMYEY7GmU4dj5EcGHIkKZiQZCA=="
6075
  },
6076
  "node_modules/@react-spring/animated": {
6077
+ "version": "9.7.4",
6078
+ "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.7.4.tgz",
6079
+ "integrity": "sha512-7As+8Pty2QlemJ9O5ecsuPKjmO0NKvmVkRR1n6mEotFgWar8FKuQt2xgxz3RTgxcccghpx1YdS1FCdElQNexmQ==",
6080
  "dependencies": {
6081
+ "@react-spring/shared": "~9.7.4",
6082
+ "@react-spring/types": "~9.7.4"
6083
  },
6084
  "peerDependencies": {
6085
  "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
6086
  }
6087
  },
6088
  "node_modules/@react-spring/core": {
6089
+ "version": "9.7.4",
6090
+ "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.7.4.tgz",
6091
+ "integrity": "sha512-GzjA44niEJBFUe9jN3zubRDDDP2E4tBlhNlSIkTChiNf9p4ZQlgXBg50qbXfSXHQPHak/ExYxwhipKVsQ/sUTw==",
6092
  "dependencies": {
6093
+ "@react-spring/animated": "~9.7.4",
6094
+ "@react-spring/shared": "~9.7.4",
6095
+ "@react-spring/types": "~9.7.4"
6096
  },
6097
  "funding": {
6098
  "type": "opencollective",
 
6103
  }
6104
  },
6105
  "node_modules/@react-spring/rafz": {
6106
+ "version": "9.7.4",
6107
+ "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.7.4.tgz",
6108
+ "integrity": "sha512-mqDI6rW0Ca8IdryOMiXRhMtVGiEGLIO89vIOyFQXRIwwIMX30HLya24g9z4olDvFyeDW3+kibiKwtZnA4xhldA=="
6109
  },
6110
  "node_modules/@react-spring/shared": {
6111
+ "version": "9.7.4",
6112
+ "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.7.4.tgz",
6113
+ "integrity": "sha512-bEPI7cQp94dOtCFSEYpxvLxj0+xQfB5r9Ru1h8OMycsIq7zFZon1G0sHrBLaLQIWeMCllc4tVDYRTLIRv70C8w==",
6114
  "dependencies": {
6115
+ "@react-spring/rafz": "~9.7.4",
6116
+ "@react-spring/types": "~9.7.4"
6117
  },
6118
  "peerDependencies": {
6119
  "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
6120
  }
6121
  },
6122
  "node_modules/@react-spring/three": {
6123
+ "version": "9.7.4",
6124
+ "resolved": "https://registry.npmjs.org/@react-spring/three/-/three-9.7.4.tgz",
6125
+ "integrity": "sha512-HKUhrrvWW7F/MAroObOloqcYyFqsUHp1ANIDvPVxk9cSh7veW7gQbJm2Sc7Ka+L4gVJEwSkS+MRfr8kk+sRZBw==",
6126
  "dependencies": {
6127
+ "@react-spring/animated": "~9.7.4",
6128
+ "@react-spring/core": "~9.7.4",
6129
+ "@react-spring/shared": "~9.7.4",
6130
+ "@react-spring/types": "~9.7.4"
6131
  },
6132
  "peerDependencies": {
6133
  "@react-three/fiber": ">=6.0",
 
6136
  }
6137
  },
6138
  "node_modules/@react-spring/types": {
6139
+ "version": "9.7.4",
6140
+ "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.7.4.tgz",
6141
+ "integrity": "sha512-iQVztO09ZVfsletMiY+DpT/JRiBntdsdJ4uqk3UJFhrhS8mIC9ZOZbmfGSRs/kdbNPQkVyzucceDicQ/3Mlj9g=="
6142
  },
6143
  "node_modules/@react-three/drei": {
6144
  "version": "9.108.4",
 
6211
  "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
6212
  }
6213
  },
6214
+ "node_modules/@react-three/drei/node_modules/@react-spring/rafz": {
6215
+ "version": "9.6.1",
6216
+ "resolved": "https://registry.npmjs.org/@react-spring/rafz/-/rafz-9.6.1.tgz",
6217
+ "integrity": "sha512-v6qbgNRpztJFFfSE3e2W1Uz+g8KnIBs6SmzCzcVVF61GdGfGOuBrbjIcp+nUz301awVmREKi4eMQb2Ab2gGgyQ=="
6218
+ },
6219
  "node_modules/@react-three/drei/node_modules/@react-spring/shared": {
6220
  "version": "9.6.1",
6221
  "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.6.1.tgz",
 
8573
  }
8574
  ]
8575
  },
8576
+ "node_modules/before-after-hook": {
8577
+ "version": "2.2.3",
8578
+ "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz",
8579
+ "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==",
8580
+ "dev": true
8581
+ },
8582
  "node_modules/bidi-js": {
8583
  "version": "1.0.3",
8584
  "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
 
8678
  "dev": true,
8679
  "optional": true
8680
  },
8681
+ "node_modules/bottleneck": {
8682
+ "version": "2.19.5",
8683
+ "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
8684
+ "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==",
8685
+ "dev": true
8686
+ },
8687
  "node_modules/bowser": {
8688
  "version": "2.11.0",
8689
  "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz",
 
8858
  "balanced-match": "^1.0.0"
8859
  }
8860
  },
 
 
 
 
 
 
 
 
 
 
 
 
8861
  "node_modules/cacache/node_modules/glob": {
8862
  "version": "10.4.5",
8863
  "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
 
8914
  "url": "https://github.com/sponsors/isaacs"
8915
  }
8916
  },
 
 
 
 
 
 
 
 
 
8917
  "node_modules/cacheable-lookup": {
8918
  "version": "5.0.4",
8919
  "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
 
9019
  }
9020
  },
9021
  "node_modules/caniuse-lite": {
9022
+ "version": "1.0.30001643",
9023
+ "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001643.tgz",
9024
+ "integrity": "sha512-ERgWGNleEilSrHM6iUz/zJNSQTP8Mr21wDWpdgvRwcTXGAq6jMtOUPP4dqFPTdKqZ2wKTdtB+uucZ3MRpAUSmg==",
9025
  "funding": [
9026
  {
9027
  "type": "opencollective",
 
10185
  "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
10186
  "dev": true
10187
  },
10188
+ "node_modules/deprecation": {
10189
+ "version": "2.3.1",
10190
+ "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
10191
+ "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==",
10192
+ "dev": true
10193
+ },
10194
  "node_modules/dequal": {
10195
  "version": "2.0.3",
10196
  "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
 
12374
  }
12375
  },
12376
  "node_modules/fs-minipass": {
12377
+ "version": "3.0.3",
12378
+ "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz",
12379
+ "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==",
12380
  "dev": true,
12381
  "dependencies": {
12382
+ "minipass": "^7.0.3"
12383
  },
12384
  "engines": {
12385
+ "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
12386
  }
12387
  },
12388
  "node_modules/fs-temp": {
 
13780
  "node": ">=8"
13781
  }
13782
  },
13783
+ "node_modules/is-plain-object": {
13784
+ "version": "5.0.0",
13785
+ "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
13786
+ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==",
13787
+ "dev": true,
13788
+ "engines": {
13789
+ "node": ">=0.10.0"
13790
+ }
13791
+ },
13792
  "node_modules/is-potential-custom-element-name": {
13793
  "version": "1.0.1",
13794
  "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
 
14635
  "node": "^16.14.0 || >=18.0.0"
14636
  }
14637
  },
 
 
 
 
 
 
 
 
 
14638
  "node_modules/map-age-cleaner": {
14639
  "version": "0.1.3",
14640
  "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz",
 
14795
  }
14796
  },
14797
  "node_modules/minipass": {
14798
+ "version": "7.1.2",
14799
+ "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
14800
+ "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
 
 
 
 
14801
  "engines": {
14802
+ "node": ">=16 || 14 >=14.17"
14803
  }
14804
  },
14805
  "node_modules/minipass-collect": {
 
14814
  "node": ">=16 || 14 >=14.17"
14815
  }
14816
  },
 
 
 
 
 
 
 
 
 
14817
  "node_modules/minipass-fetch": {
14818
  "version": "3.0.5",
14819
  "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.5.tgz",
 
14831
  "encoding": "^0.1.13"
14832
  }
14833
  },
 
 
 
 
 
 
 
 
 
14834
  "node_modules/minipass-flush": {
14835
  "version": "1.0.5",
14836
  "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz",
 
14843
  "node": ">= 8"
14844
  }
14845
  },
14846
+ "node_modules/minipass-flush/node_modules/minipass": {
14847
+ "version": "3.3.6",
14848
+ "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
14849
+ "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
14850
+ "dev": true,
14851
+ "dependencies": {
14852
+ "yallist": "^4.0.0"
14853
+ },
14854
+ "engines": {
14855
+ "node": ">=8"
14856
+ }
14857
+ },
14858
+ "node_modules/minipass-flush/node_modules/yallist": {
14859
+ "version": "4.0.0",
14860
+ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
14861
+ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
14862
+ "dev": true
14863
+ },
14864
  "node_modules/minipass-pipeline": {
14865
  "version": "1.2.4",
14866
  "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz",
 
14873
  "node": ">=8"
14874
  }
14875
  },
14876
+ "node_modules/minipass-pipeline/node_modules/minipass": {
14877
+ "version": "3.3.6",
14878
+ "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
14879
+ "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
14880
+ "dev": true,
14881
+ "dependencies": {
14882
+ "yallist": "^4.0.0"
14883
+ },
14884
+ "engines": {
14885
+ "node": ">=8"
14886
+ }
14887
+ },
14888
+ "node_modules/minipass-pipeline/node_modules/yallist": {
14889
+ "version": "4.0.0",
14890
+ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
14891
+ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
14892
+ "dev": true
14893
+ },
14894
  "node_modules/minipass-sized": {
14895
  "version": "1.0.3",
14896
  "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz",
 
14903
  "node": ">=8"
14904
  }
14905
  },
14906
+ "node_modules/minipass-sized/node_modules/minipass": {
14907
+ "version": "3.3.6",
14908
+ "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
14909
+ "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
14910
+ "dev": true,
14911
+ "dependencies": {
14912
+ "yallist": "^4.0.0"
14913
+ },
14914
+ "engines": {
14915
+ "node": ">=8"
14916
+ }
14917
+ },
14918
+ "node_modules/minipass-sized/node_modules/yallist": {
14919
  "version": "4.0.0",
14920
  "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
14921
  "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
 
14934
  "node": ">= 8"
14935
  }
14936
  },
14937
+ "node_modules/minizlib/node_modules/minipass": {
14938
+ "version": "3.3.6",
14939
+ "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
14940
+ "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
14941
+ "dev": true,
14942
+ "dependencies": {
14943
+ "yallist": "^4.0.0"
14944
+ },
14945
+ "engines": {
14946
+ "node": ">=8"
14947
+ }
14948
+ },
14949
  "node_modules/minizlib/node_modules/yallist": {
14950
  "version": "4.0.0",
14951
  "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
 
15427
  "url": "https://github.com/sponsors/isaacs"
15428
  }
15429
  },
 
 
 
 
 
 
 
 
 
15430
  "node_modules/node-gyp/node_modules/which": {
15431
  "version": "4.0.0",
15432
  "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz",
 
15830
  "url": "https://github.com/sponsors/isaacs"
15831
  }
15832
  },
 
 
 
 
 
 
 
 
 
15833
  "node_modules/onnxruntime-node/node_modules/minizlib": {
15834
  "version": "3.0.1",
15835
  "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz",
 
16266
  "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
16267
  "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="
16268
  },
 
 
 
 
 
 
 
 
16269
  "node_modules/path-to-regexp": {
16270
  "version": "6.2.2",
16271
  "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz",
 
18267
  "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
18268
  }
18269
  },
 
 
 
 
 
 
 
 
 
18270
  "node_modules/stackback": {
18271
  "version": "0.0.2",
18272
  "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz",
 
18781
  "url": "https://github.com/sponsors/isaacs"
18782
  }
18783
  },
 
 
 
 
 
 
 
 
18784
  "node_modules/sudo-prompt": {
18785
  "version": "9.2.1",
18786
  "resolved": "https://registry.npmjs.org/sudo-prompt/-/sudo-prompt-9.2.1.tgz",
 
18926
  "node": ">=10"
18927
  }
18928
  },
18929
+ "node_modules/tar/node_modules/fs-minipass": {
18930
+ "version": "2.1.0",
18931
+ "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
18932
+ "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
18933
+ "dev": true,
18934
+ "dependencies": {
18935
+ "minipass": "^3.0.0"
18936
+ },
18937
+ "engines": {
18938
+ "node": ">= 8"
18939
+ }
18940
+ },
18941
+ "node_modules/tar/node_modules/fs-minipass/node_modules/minipass": {
18942
+ "version": "3.3.6",
18943
+ "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
18944
+ "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
18945
+ "dev": true,
18946
+ "dependencies": {
18947
+ "yallist": "^4.0.0"
18948
+ },
18949
+ "engines": {
18950
+ "node": ">=8"
18951
+ }
18952
+ },
18953
  "node_modules/tar/node_modules/minipass": {
18954
  "version": "5.0.0",
18955
  "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
 
19584
  "node": "^14.17.0 || ^16.13.0 || >=18.0.0"
19585
  }
19586
  },
19587
+ "node_modules/universal-user-agent": {
19588
+ "version": "6.0.1",
19589
+ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz",
19590
+ "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==",
19591
+ "dev": true
19592
+ },
19593
  "node_modules/universalify": {
19594
  "version": "2.0.1",
19595
  "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
package.json CHANGED
@@ -5,11 +5,19 @@
5
  "description": "🎬 Clapper",
6
  "license": "GPL-3.0-only",
7
  "main": "main.js",
 
 
 
 
 
 
 
8
  "scripts": {
9
  "dev": "npm i && npm run checks && next dev",
10
- "build": "npm i && npm run checks && next build && npm run build:copyassets",
 
11
  "build:copyassets": "cp -R public .next/standalone/public && cp -R .next/static .next/standalone/.next/static",
12
- "build:ci": "next build && npm run build:copyassets",
13
  "start": "next start",
14
  "start:prod": "node .next/standalone/server.js",
15
  "checks": "npm run format:fix && npm run lint",
@@ -23,15 +31,15 @@
23
  "test:e2e": "npx playwright test",
24
  "electron": "npm run build && electron .",
25
  "electron:start": "npm run build && electron-forge start",
26
- "electron:package": "electron-forge package",
27
- "electron:make": "electron-forge make"
28
  },
29
  "dependencies": {
30
  "@aitube/broadway": "0.0.22",
31
  "@aitube/clap": "0.0.30",
32
- "@aitube/clapper-services": "0.0.28",
33
  "@aitube/engine": "0.0.26",
34
- "@aitube/timeline": "0.0.42",
35
  "@fal-ai/serverless-client": "^0.13.0",
36
  "@ffmpeg/ffmpeg": "^0.12.10",
37
  "@ffmpeg/util": "^0.12.1",
@@ -128,6 +136,7 @@
128
  "@electron-forge/maker-squirrel": "^7.4.0",
129
  "@electron-forge/maker-zip": "^7.4.0",
130
  "@electron-forge/plugin-auto-unpack-natives": "^7.4.0",
 
131
  "@playwright/test": "^1.45.1",
132
  "@testing-library/react": "^16.0.0",
133
  "@types/fluent-ffmpeg": "^2.1.24",
 
5
  "description": "🎬 Clapper",
6
  "license": "GPL-3.0-only",
7
  "main": "main.js",
8
+ "files": [
9
+ "./main.js"
10
+ ],
11
+ "directories": {
12
+ "src:": "./src",
13
+ "public:": "./public"
14
+ },
15
  "scripts": {
16
  "dev": "npm i && npm run checks && next dev",
17
+ "build": "npm i && npm run prepare && npm run checks && rm -Rf out && next build && npm run build:copyassets",
18
+ "build:ci": "rm -Rf out && npm run prepare && next build && npm run build:copyassets",
19
  "build:copyassets": "cp -R public .next/standalone/public && cp -R .next/static .next/standalone/.next/static",
20
+ "prepare": "cp -R node_modules/mediainfo.js/dist/MediaInfoModule.wasm public/wasm/",
21
  "start": "next start",
22
  "start:prod": "node .next/standalone/server.js",
23
  "checks": "npm run format:fix && npm run lint",
 
31
  "test:e2e": "npx playwright test",
32
  "electron": "npm run build && electron .",
33
  "electron:start": "npm run build && electron-forge start",
34
+ "electron:package": "npm run build && electron-forge package",
35
+ "electron:make": "npm run build && electron-forge make"
36
  },
37
  "dependencies": {
38
  "@aitube/broadway": "0.0.22",
39
  "@aitube/clap": "0.0.30",
40
+ "@aitube/clapper-services": "0.0.29",
41
  "@aitube/engine": "0.0.26",
42
+ "@aitube/timeline": "0.0.43",
43
  "@fal-ai/serverless-client": "^0.13.0",
44
  "@ffmpeg/ffmpeg": "^0.12.10",
45
  "@ffmpeg/util": "^0.12.1",
 
136
  "@electron-forge/maker-squirrel": "^7.4.0",
137
  "@electron-forge/maker-zip": "^7.4.0",
138
  "@electron-forge/plugin-auto-unpack-natives": "^7.4.0",
139
+ "@electron-forge/publisher-github": "^7.4.0",
140
  "@playwright/test": "^1.45.1",
141
  "@testing-library/react": "^16.0.0",
142
  "@types/fluent-ffmpeg": "^2.1.24",
{src/app → public/images/logos}/CL.icns RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_128x128.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_128x128@2x.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_16x16.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_16x16@2x.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_256x256.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_256x256@2x.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_32x32.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_32x32@2x.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_512x512.png RENAMED
File without changes
{src/app → public/images/logos}/CL.iconset/icon_512x512@2x.png RENAMED
File without changes
public/images/logos/CL.png ADDED

Git LFS Details

  • SHA256: 1ad55d8b1b92c5d31c1dc66452af1907790dfa5d9d778386f3f12842434e7f6f
  • Pointer size: 130 Bytes
  • Size of remote file: 60.4 kB
public/wasm/MediaInfoModule.wasm ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0a209b7ad5152420f6c739f5bf10e9c02df79798f8f75fbe413c6d1be422bbf
3
+ size 2355621
src/lib/core/constants.ts CHANGED
@@ -3,7 +3,7 @@
3
  export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
4
 
5
  export const APP_NAME = 'Clapper.app'
6
- export const APP_REVISION = 'r20240721-0835'
7
 
8
  export const APP_DOMAIN = 'Clapper.app'
9
  export const APP_LINK = 'https://clapper.app'
 
3
  export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
4
 
5
  export const APP_NAME = 'Clapper.app'
6
+ export const APP_REVISION = 'r20240722-0205'
7
 
8
  export const APP_DOMAIN = 'Clapper.app'
9
  export const APP_LINK = 'https://clapper.app'
src/lib/utils/base64DataUriToFile.ts ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ export function base64DataUriToFile(dataUrl: string, fileName: string) {
2
+ var arr = dataUrl.split(',')
3
+ const st = `${arr[0] || ''}`
4
+ const mime = `${st.match(/:(.*?);/)?.[1] || ''}`
5
+ const bstr = atob(arr[arr.length - 1])
6
+ let n = bstr.length
7
+ const u8arr = new Uint8Array(n)
8
+ while (n--) {
9
+ u8arr[n] = bstr.charCodeAt(n)
10
+ }
11
+ return new File([u8arr], fileName, { type: mime })
12
+ }
src/services/io/createFullVideo.ts CHANGED
@@ -1,481 +1,15 @@
1
  import { UUID } from '@aitube/clap'
2
- import { FFmpeg } from '@ffmpeg/ffmpeg'
3
- import { toBlobURL } from '@ffmpeg/util'
4
-
5
- const TAG = 'io/createFullVideo'
6
-
7
- export type FFMPegVideoInput = {
8
- data: Uint8Array | null
9
- startTimeInMs: number
10
- endTimeInMs: number
11
- durationInSecs: number
12
- }
13
-
14
- export type FFMPegAudioInput = FFMPegVideoInput
15
-
16
- /**
17
- * Download and load single and multi-threading FFMPeg.
18
- * MT for video
19
- * ST for audio (as MT has issues with it)
20
- * toBlobURL is used to bypass CORS issues, urls with the same domain can be used directly.
21
- */
22
- async function initializeFFmpeg() {
23
- const [ffmpegSt, ffmpegMt] = [new FFmpeg(), new FFmpeg()]
24
- const baseStURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd'
25
- const baseMtURL = 'https://unpkg.com/@ffmpeg/core-mt@0.12.6/dist/umd'
26
-
27
- ffmpegSt.on('log', ({ message }) => {
28
- console.log(TAG, 'FFmpeg Single-Thread:', message)
29
- })
30
-
31
- ffmpegMt.on('log', ({ message }) => {
32
- console.log(TAG, 'FFmpeg Multi-Thread:', message)
33
- })
34
-
35
- await ffmpegSt.load({
36
- coreURL: await toBlobURL(`${baseStURL}/ffmpeg-core.js`, 'text/javascript'),
37
- wasmURL: await toBlobURL(
38
- `${baseStURL}/ffmpeg-core.wasm`,
39
- 'application/wasm'
40
- ),
41
- })
42
-
43
- await ffmpegMt.load({
44
- coreURL: await toBlobURL(`${baseMtURL}/ffmpeg-core.js`, 'text/javascript'),
45
- wasmURL: await toBlobURL(
46
- `${baseMtURL}/ffmpeg-core.wasm`,
47
- 'application/wasm'
48
- ),
49
- workerURL: await toBlobURL(
50
- `${baseMtURL}/ffmpeg-core.worker.js`,
51
- 'text/javascript'
52
- ),
53
- })
54
-
55
- return [ffmpegSt, ffmpegMt] as [FFmpeg, FFmpeg]
56
- }
57
-
58
- /**
59
- * Get loaded FFmpeg.
60
- */
61
- let ffmpegInstance: [FFmpeg, FFmpeg]
62
- export async function loadFFmpegSt() {
63
- if (!ffmpegInstance) ffmpegInstance = await initializeFFmpeg()
64
- return ffmpegInstance[0]
65
- }
66
-
67
- export async function loadFFmpegMt() {
68
- if (!ffmpegInstance) ffmpegInstance = await initializeFFmpeg()
69
- return ffmpegInstance[1]
70
- }
71
-
72
- /**
73
- * Creates an exclusive logger for the FFmpeg calls inside the provided method,
74
- * it calculates the progress based on raw FFmpeg logs and the provided `totalTimeInMs`.
75
- *
76
- * @param totalTimeInMs
77
- * @param method
78
- * @param callback
79
- * @param {number} callback.progress - The progress of the FFmpeg process from 0 to 100.
80
- * @returns
81
- */
82
- async function captureFFmpegProgress(
83
- ffmpeg: FFmpeg,
84
- totalTimeInMs: number,
85
- method: () => any,
86
- callback: (progress: number) => void
87
- ): Promise<any> {
88
- const extractProgressTimeMsFromLogs = (log: string): number | null => {
89
- // `frame` for videos, `size` for audios
90
- if (!log.startsWith('frame') && !log.startsWith('size')) return null
91
- const timeRegex = /time=(\d{2}):(\d{2}):(\d{2})\.(\d{2})/
92
- const match = log.match(timeRegex)
93
- if (match) {
94
- const hours = parseInt(match[1])
95
- const minutes = parseInt(match[2])
96
- const seconds = parseInt(match[3])
97
- const centiseconds = parseInt(match[4])
98
- const totalMilliseconds =
99
- hours * 3600000 + minutes * 60000 + seconds * 1000 + centiseconds * 10
100
- return totalMilliseconds
101
- }
102
- return null
103
- }
104
- let ffmpegLog = true
105
- ffmpeg.on('log', ({ message }) => {
106
- if (!ffmpegLog) return
107
- const timeInMs = extractProgressTimeMsFromLogs(message)
108
- if (timeInMs) callback((timeInMs / totalTimeInMs) * 100)
109
- })
110
- const result = await method()
111
- ffmpegLog = false
112
- return result
113
- }
114
-
115
- /**
116
- * It will calculate a proportional progress between a targetProgress and a startProgress
117
- *
118
- * @param startProgress e.g. 50
119
- * @param progress e.g. 50
120
- * @param targetProgress e.g. 70
121
- * @returns e.g. 60, because 50% of progress between 70% and 50%, would result on 60%
122
- */
123
- function calculateProgress(
124
- startProgress: number,
125
- progress: number,
126
- targetProgress: number
127
- ): number {
128
- return startProgress + (progress * (targetProgress - startProgress)) / 100
129
- }
130
-
131
- /**
132
- * Creates an empty black video and appends it to the
133
- * provided `fileListContentArray`.
134
- *
135
- * @param duration time in milliseconds
136
- * @param width
137
- * @param height
138
- * @param filename
139
- * @param fileListContentArray fileList.txt where to append the file name
140
- * @param onProgress callback to capture the progress of this method
141
- */
142
- export async function addEmptyVideo(
143
- durationInSecs: number,
144
- width: number,
145
- height: number,
146
- filename: string,
147
- fileListContentArray: string[],
148
- onProgress?: (progress: number, message?: string) => void
149
- ) {
150
- const ffmpeg = await loadFFmpegMt()
151
- let targetPartialProgress = 0
152
-
153
- // For some reason, creating empty video with silent audio
154
- // in one exec doesn't work, we need to split it.
155
-
156
- console.log(
157
- TAG,
158
- 'Creating empty video',
159
- filename,
160
- width,
161
- height,
162
- durationInSecs
163
- )
164
- let currentProgress = 0
165
- targetPartialProgress = 50
166
-
167
- await captureFFmpegProgress(
168
- ffmpeg,
169
- durationInSecs * 1000,
170
- async () => {
171
- await ffmpeg.exec([
172
- '-f',
173
- 'lavfi',
174
- '-i',
175
- `color=c=black:s=${width}x${height}:d=${durationInSecs}`,
176
- '-c:v',
177
- 'libx264',
178
- '-t',
179
- `${durationInSecs}`,
180
- '-loglevel',
181
- 'verbose',
182
- `base_${filename}`,
183
- ])
184
- },
185
- (progress) => {
186
- onProgress?.((progress / 100) * targetPartialProgress)
187
- }
188
- )
189
-
190
- console.log(
191
- TAG,
192
- 'Adding silent audio to empty video',
193
- filename,
194
- width,
195
- height,
196
- durationInSecs
197
- )
198
- currentProgress = 50
199
- targetPartialProgress = 100
200
-
201
- const exitCode = await ffmpeg.exec([
202
- '-i',
203
- `base_${filename}`,
204
- '-f',
205
- 'lavfi',
206
- '-i',
207
- 'anullsrc',
208
- '-c:v',
209
- 'copy',
210
- '-c:a',
211
- 'aac',
212
- '-t',
213
- `${durationInSecs}`,
214
- '-loglevel',
215
- 'verbose',
216
- filename,
217
- ])
218
-
219
- if (exitCode) {
220
- throw new Error(`${TAG}: Unexpect error while creating empty video`)
221
- }
222
-
223
- console.log(TAG, 'Empty video created', filename)
224
- fileListContentArray.push(`file ${filename}`)
225
- }
226
-
227
- /**
228
- * Creates the full mixed audio including silence
229
- * segments and loads it into ffmpeg with the given `filename`.
230
- * @param onProgress callback to capture the progress of this method
231
- * @throws Error if ffmpeg returns exit code 1
232
- */
233
- export async function createFullAudio(
234
- audios: FFMPegAudioInput[],
235
- filename: string,
236
- totalVideoDurationInMs: number,
237
- onProgress?: (progress: number, message: string) => void
238
- ): Promise<void> {
239
- console.log(TAG, 'Creating full audio', filename)
240
-
241
- const ffmpeg = await loadFFmpegSt()
242
- const filterComplexParts = []
243
- const baseFilename = `base_${filename}`
244
- let currentProgress = 0
245
- let targetProgress = 25
246
-
247
- // To mix audios at given times, we need a first empty base audio track
248
-
249
- await captureFFmpegProgress(
250
- ffmpeg,
251
- totalVideoDurationInMs,
252
- async () => {
253
- await ffmpeg.exec([
254
- '-f',
255
- 'lavfi',
256
- '-i',
257
- 'anullsrc',
258
- '-t',
259
- `${totalVideoDurationInMs / 1000}`,
260
- '-loglevel',
261
- 'verbose',
262
- !audios.length ? filename : baseFilename,
263
- ])
264
- },
265
- (progress) => {
266
- onProgress?.(
267
- calculateProgress(currentProgress, progress, targetProgress),
268
- 'Creating base audio...'
269
- )
270
- }
271
- )
272
-
273
- // If there is no audios, the base audio is the final one
274
- if (!audios.length) return onProgress?.(100, 'Prepared audios...')
275
-
276
- currentProgress = targetProgress
277
- targetProgress = 50
278
-
279
- // Mix audios based on their start times
280
-
281
- const audioInputFiles = ['-i', baseFilename]
282
- for (let index = 0; index < audios.length; index++) {
283
- onProgress?.(currentProgress, 'Creating base audio...')
284
- console.log(TAG, `Processing audio #${index}`)
285
- const audio = audios[index]
286
- const expectedProgressForItem = ((1 / audios.length) * targetProgress) / 100
287
- if (!audio.data) continue
288
- const audioFilename = `audio_${UUID()}.mp3`
289
- await ffmpeg.writeFile(audioFilename, audio.data)
290
- audioInputFiles.push('-i', audioFilename)
291
- const delay = audio.startTimeInMs
292
- const durationInSecs = audio.endTimeInMs - audio.startTimeInMs / 1000
293
- filterComplexParts.push(
294
- `[${index + 1}:a]atrim=0:${durationInSecs},adelay=${delay}|${delay}[delayed${index}]`
295
- )
296
- currentProgress += expectedProgressForItem * 100
297
- }
298
-
299
- const amixInputs = `[0:a]${audios.map((_, index) => `[delayed${index}]`).join('')}amix=inputs=${audios.length + 1}:duration=longest`
300
- filterComplexParts.push(`${amixInputs}[a]`)
301
- const filterComplex = filterComplexParts.join('; ')
302
-
303
- currentProgress = targetProgress
304
- targetProgress = 100
305
-
306
- const createFullAudioExitCode = await captureFFmpegProgress(
307
- ffmpeg,
308
- totalVideoDurationInMs,
309
- async () => {
310
- await ffmpeg.exec([
311
- ...audioInputFiles,
312
- '-filter_complex',
313
- filterComplex,
314
- '-map',
315
- '[a]',
316
- '-t',
317
- `${totalVideoDurationInMs / 1000}`,
318
- '-loglevel',
319
- 'verbose',
320
- filename,
321
- ])
322
- },
323
- (progress) => {
324
- onProgress?.(
325
- calculateProgress(currentProgress, progress, targetProgress),
326
- 'Mixing audios...'
327
- )
328
- }
329
- )
330
-
331
- if (createFullAudioExitCode) {
332
- throw new Error(`${TAG}: Error while creating full audio!`)
333
- }
334
- onProgress?.(targetProgress, 'Prepared audios...')
335
- }
336
-
337
- /**
338
- * Creates the full silent video including empty black
339
- * segments and loads it into ffmpeg with the given `filename`.
340
- * @param onProgress callback to capture the progress of this method
341
- * @throws Error if ffmpeg returns exit code 1
342
- */
343
- export async function createFullSilentVideo(
344
- videos: FFMPegVideoInput[],
345
- filename: string,
346
- totalVideoDurationInMs: number,
347
- width: number,
348
- height: number,
349
- excludeEmptyContent = false,
350
- onProgress?: (progress: number, message: string) => void
351
- ) {
352
- const ffmpeg = await loadFFmpegMt()
353
- const fileList = 'fileList.txt'
354
- const fileListContentArray = []
355
-
356
- // Complete array of videos including concatenated empty segments
357
- // This is helpful for cleaner progress log
358
- let lastStartTimeVideoInMs = 0
359
- let videosWithGaps: FFMPegVideoInput[]
360
-
361
- if (!videos.length) {
362
- videosWithGaps = [
363
- {
364
- startTimeInMs: 0,
365
- endTimeInMs: totalVideoDurationInMs,
366
- data: null,
367
- durationInSecs: totalVideoDurationInMs / 1000,
368
- },
369
- ]
370
- } else {
371
- videosWithGaps = videos.reduce((arr: FFMPegVideoInput[], video, index) => {
372
- const emptyVideoDurationInMs =
373
- video.startTimeInMs - lastStartTimeVideoInMs
374
- if (emptyVideoDurationInMs) {
375
- arr.push({
376
- startTimeInMs: lastStartTimeVideoInMs,
377
- endTimeInMs: lastStartTimeVideoInMs + emptyVideoDurationInMs,
378
- data: null,
379
- durationInSecs: emptyVideoDurationInMs / 1000,
380
- })
381
- }
382
- arr.push(video)
383
- lastStartTimeVideoInMs = video.endTimeInMs
384
- if (
385
- index == videos.length - 1 &&
386
- lastStartTimeVideoInMs < totalVideoDurationInMs
387
- ) {
388
- arr.push({
389
- startTimeInMs: lastStartTimeVideoInMs,
390
- endTimeInMs: totalVideoDurationInMs,
391
- data: null,
392
- durationInSecs:
393
- (totalVideoDurationInMs - lastStartTimeVideoInMs) / 1000,
394
- })
395
- }
396
- return arr
397
- }, [])
398
- }
399
-
400
- onProgress?.(0, 'Preparing videos...')
401
-
402
- // Arbitrary percentage, as `concat` is fast,
403
- // then estimate the generation of gap videos
404
- // as the 70% of the work
405
- let currentProgress = 0
406
- let targetProgress = 70
407
-
408
- for (const video of videosWithGaps) {
409
- const expectedProgressForItem =
410
- (((video.durationInSecs * 1000) / totalVideoDurationInMs) *
411
- targetProgress) /
412
- 100
413
- if (!video.data) {
414
- if (excludeEmptyContent) continue
415
- let collectedProgress = 0
416
- await addEmptyVideo(
417
- video.durationInSecs,
418
- width,
419
- height,
420
- `empty_video_${UUID()}.mp4`,
421
- fileListContentArray,
422
- (progress) => {
423
- const subProgress = progress / 100
424
- currentProgress +=
425
- (expectedProgressForItem * subProgress - collectedProgress) * 100
426
- console.log(TAG, 'Current progress', currentProgress)
427
- onProgress?.(currentProgress, 'Preparing videos...')
428
- collectedProgress = expectedProgressForItem * subProgress
429
- }
430
- )
431
- } else {
432
- const videoFilename = `video_${UUID()}.mp4`
433
- await ffmpeg.writeFile(videoFilename, video.data)
434
- fileListContentArray.push(`file ${videoFilename}`)
435
- currentProgress += expectedProgressForItem * 100
436
- console.log(TAG, 'Current progress', currentProgress)
437
- onProgress?.(currentProgress, 'Preparing videos...')
438
- }
439
- }
440
-
441
- onProgress?.(targetProgress, 'Concatenating videos...')
442
- currentProgress = 70
443
- targetProgress = 100
444
-
445
- const fileListContent = fileListContentArray.join('\n')
446
- await ffmpeg.writeFile(fileList, fileListContent)
447
-
448
- const creatBaseFullVideoExitCode = await captureFFmpegProgress(
449
- ffmpeg,
450
- totalVideoDurationInMs,
451
- async () => {
452
- await ffmpeg.exec([
453
- '-f',
454
- 'concat',
455
- '-safe',
456
- '0',
457
- '-i',
458
- fileList,
459
- '-loglevel',
460
- 'verbose',
461
- '-c',
462
- 'copy',
463
- filename,
464
- ])
465
- },
466
- (progress: number) => {
467
- onProgress?.(
468
- calculateProgress(currentProgress, progress, targetProgress),
469
- 'Merging audio and video...'
470
- )
471
- }
472
- )
473
-
474
- if (creatBaseFullVideoExitCode) {
475
- throw new Error(`${TAG}: Error while creating base full video!`)
476
- }
477
- onProgress?.(targetProgress, 'Concatenating videos...')
478
- }
479
 
480
  /**
481
  * Creates full video with audio using `@ffmpeg/ffmpeg` multi-core,
 
1
  import { UUID } from '@aitube/clap'
2
+ import {
3
+ calculateProgress,
4
+ captureFFmpegProgress,
5
+ createFullAudio,
6
+ createFullSilentVideo,
7
+ FFMPegAudioInput,
8
+ FFMPegVideoInput,
9
+ loadFFmpegMt,
10
+ loadFFmpegSt,
11
+ TAG,
12
+ } from './ffmpegUtils'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
 
14
  /**
15
  * Creates full video with audio using `@ffmpeg/ffmpeg` multi-core,
src/services/io/{extractCaptionFromFrame.ts → extractCaptionFromFrameMoondream.ts} RENAMED
@@ -5,7 +5,7 @@ import {
5
  RawImage,
6
  } from '@xenova/transformers'
7
 
8
- export async function extractCaptionFromFrame(
9
  imageInBase64DataUri: string
10
  ): Promise<string> {
11
  if (!(navigator as any).gpu) {
@@ -16,7 +16,8 @@ export async function extractCaptionFromFrame(
16
  2. You need to enable WebGPU (depends on your browser, see below)
17
 
18
  2.1 For Chrome: Perform the following operations in the Chrome / Microsoft Edge address bar
19
- The chrome://flags/#enable-unsafe-webgpu flag must be enabled (not enable-webgpu-developer-features). Linux experimental support also requires launching the browser with --enable-features=Vulkan.
 
20
 
21
  2.2 For Safari 18 (macOS 15): WebGPU is enabled by default
22
 
 
5
  RawImage,
6
  } from '@xenova/transformers'
7
 
8
+ export async function extractCaptionFromFrameMoondream(
9
  imageInBase64DataUri: string
10
  ): Promise<string> {
11
  if (!(navigator as any).gpu) {
 
16
  2. You need to enable WebGPU (depends on your browser, see below)
17
 
18
  2.1 For Chrome: Perform the following operations in the Chrome / Microsoft Edge address bar
19
+ The chrome://flags/#enable-unsafe-webgpu flag must be enabled (not enable-webgpu-developer-features).
20
+ Linux experimental support also requires launching the browser with --enable-features=Vulkan.
21
 
22
  2.2 For Safari 18 (macOS 15): WebGPU is enabled by default
23
 
src/services/io/extractCaptionsFromFrames.ts ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import {
2
+ AutoProcessor,
3
+ AutoTokenizer,
4
+ Florence2ForConditionalGeneration,
5
+ RawImage,
6
+ } from '@xenova/transformers'
7
+
8
+ export async function extractCaptionsFromFrames(
9
+ images: string[] = [],
10
+ onProgress: (
11
+ progress: number,
12
+ storyboardIndex: number,
13
+ nbStoryboards: number
14
+ ) => void
15
+ ): Promise<string[]> {
16
+ if (!(navigator as any).gpu) {
17
+ throw new Error(`Please enable WebGPU to analyze video frames:
18
+
19
+ 1. You need a modern browser such as Google Chrome 113+, Microsoft Edge 113+, Safari 18 (macOS 15), Firefox Nightly
20
+
21
+ 2. You need to enable WebGPU (depends on your browser, see below)
22
+
23
+ 2.1 For Chrome: Perform the following operations in the Chrome / Microsoft Edge address bar
24
+ The chrome://flags/#enable-unsafe-webgpu flag must be enabled (not enable-webgpu-developer-features).
25
+ Linux experimental support also requires launching the browser with --enable-features=Vulkan.
26
+
27
+ 2.2 For Safari 18 (macOS 15): WebGPU is enabled by default
28
+
29
+ 2.3 For Firefox Nightly: Type about:config in the address bar and set 'dom.webgpu.enabled" to true
30
+ `)
31
+ }
32
+
33
+ let progress = 0
34
+ onProgress(progress, 0, images.length)
35
+ // for code example, see:
36
+ // https://github.com/xenova/transformers.js/pull/545#issuecomment-2183625876
37
+
38
+ // Load model, processor, and tokenizer
39
+ const model_id = 'onnx-community/Florence-2-base-ft'
40
+ const model = await Florence2ForConditionalGeneration.from_pretrained(
41
+ model_id,
42
+ {
43
+ dtype: 'fp32',
44
+ }
45
+ )
46
+
47
+ onProgress((progress = 5), 0, images.length)
48
+
49
+ const processor = await AutoProcessor.from_pretrained(model_id)
50
+
51
+ onProgress((progress = 10), 0, images.length)
52
+
53
+ const tokenizer = await AutoTokenizer.from_pretrained(model_id)
54
+
55
+ onProgress((progress = 15), 0, images.length)
56
+
57
+ // not all prompts will work properly, see the official examples:
58
+ // https://huggingface.co/microsoft/Florence-2-base-ft/blob/e7a5acc73559546de6e12ec0319cd7cc1fa2437c/processing_florence2.py#L115-L117
59
+
60
+ // Prepare text inputs
61
+ const prompts = 'Describe with a paragraph what is shown in the image.'
62
+ const text_inputs = tokenizer(prompts)
63
+
64
+ let i = 1
65
+ const captions: string[] = []
66
+ for (const imageInBase64DataUri of images) {
67
+ console.log('analyzing image:', imageInBase64DataUri.slice(0, 64))
68
+ // Prepare vision inputs
69
+ const image = await RawImage.fromURL(imageInBase64DataUri)
70
+ const vision_inputs = await processor(image)
71
+
72
+ console.log(' - generating caption..')
73
+ // Generate text
74
+ const generated_ids = await model.generate({
75
+ ...text_inputs,
76
+ ...vision_inputs,
77
+ max_new_tokens: 100,
78
+ })
79
+
80
+ // Decode generated text
81
+ const generated_text = tokenizer.batch_decode(generated_ids, {
82
+ skip_special_tokens: true,
83
+ })
84
+
85
+ const caption = `${generated_text[0] || ''}`
86
+ console.log(' - caption:', caption)
87
+
88
+ const relativeProgress = i / images.length
89
+
90
+ progress += relativeProgress * 75
91
+ onProgress(progress, i, images.length)
92
+ captions.push(caption)
93
+ }
94
+ return captions
95
+ }
src/services/io/extractFramesFromVideo.ts CHANGED
@@ -1,3 +1,5 @@
 
 
1
  import { FFmpeg } from '@ffmpeg/ffmpeg'
2
  import { toBlobURL } from '@ffmpeg/util'
3
  import mediaInfoFactory, {
@@ -17,14 +19,20 @@ interface FrameExtractorOptions {
17
  maxHeight: number
18
  sceneSamplingRate: number // Percentage of additional frames between scene changes (0-100)
19
  onProgress?: (progress: number) => void // Callback function for progress updates
 
20
  }
21
 
22
- async function extractFramesFromVideo(
23
  videoBlob: Blob,
24
  options: FrameExtractorOptions
25
  ): Promise<string[]> {
26
  // Initialize MediaInfo
27
- const mediaInfo = await mediaInfoFactory({ format: 'object' })
 
 
 
 
 
28
 
29
  // Get video duration using MediaInfo
30
  const getSize = () => videoBlob.size
@@ -42,19 +50,33 @@ async function extractFramesFromVideo(
42
  reader.readAsArrayBuffer(videoBlob.slice(offset, offset + chunkSize))
43
  })
44
 
 
 
 
 
45
  const result = await mediaInfo.analyzeData(getSize, readChunk)
 
 
 
46
 
47
  let duration: number = 0
48
 
49
  for (const track of result.media?.track || []) {
50
- /// '@type': "General" | "Video" | "Audio" | "Text" | "Image" | "Menu" | "Other"
 
 
 
51
  let maybeDuration: number = 0
52
  if (track['@type'] === 'Audio') {
53
  const audioTrack = track as AudioTrack
54
- maybeDuration = audioTrack.Duration || 0
 
 
55
  } else if (track['@type'] === 'Video') {
56
  const videoTrack = track as VideoTrack
57
- maybeDuration = videoTrack.Duration || 0
 
 
58
  }
59
  if (
60
  typeof maybeDuration === 'number' &&
@@ -69,6 +91,10 @@ async function extractFramesFromVideo(
69
  throw new Error('Could not determine video duration (or it is length 0)')
70
  }
71
 
 
 
 
 
72
  // Initialize FFmpeg
73
  const ffmpeg = new FFmpeg()
74
  const baseURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd'
@@ -78,17 +104,26 @@ async function extractFramesFromVideo(
78
  wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, 'application/wasm'),
79
  })
80
 
 
 
 
 
81
  // Write video file to FFmpeg's file system
82
  const videoUint8Array = new Uint8Array(await videoBlob.arrayBuffer())
83
  await ffmpeg.writeFile('input.mp4', videoUint8Array)
84
-
 
 
85
  // Prepare FFmpeg command
86
  const sceneFilter = `select='gt(scene,0.4)'`
87
  const additionalFramesFilter = `select='not(mod(n,${Math.floor(100 / options.sceneSamplingRate)}))'`
88
- const scaleFilter = `scale=iw*min(${options.maxWidth}/iw\,${options.maxHeight}/ih):ih*min(${options.maxWidth}/iw\,${options.maxHeight}/ih)`
89
 
90
  let lastProgress = 0
91
  ffmpeg.on('log', ({ message }) => {
 
 
 
92
  const timeMatch = message.match(/time=(\d{2}):(\d{2}):(\d{2}\.\d{2})/)
93
  if (timeMatch) {
94
  const [, hours, minutes, seconds] = timeMatch
@@ -102,40 +137,82 @@ async function extractFramesFromVideo(
102
  }
103
  })
104
 
105
- await ffmpeg.exec([
106
  '-i',
107
  'input.mp4',
 
 
108
  '-vf',
109
  `${sceneFilter},${additionalFramesFilter},${scaleFilter}`,
110
  '-vsync',
111
- '0',
112
  '-q:v',
113
  '2',
 
 
 
 
114
  `frames_%03d.${options.format}`,
115
- ])
 
 
 
 
 
 
 
 
 
 
 
116
 
117
  // Read generated frames
118
  const files = await ffmpeg.listDir('/')
 
 
 
119
  const frameFiles = files.filter(
120
  (file) =>
121
  file.name.startsWith('frames_') &&
122
  file.name.endsWith(`.${options.format}`)
123
  )
 
 
 
124
 
125
  const frames: string[] = []
 
 
126
  for (let i = 0; i < frameFiles.length; i++) {
127
  const file = frameFiles[i]
128
- const frameData = await ffmpeg.readFile(file.name)
129
- const base64Frame = btoa(
130
- String.fromCharCode.apply(null, frameData as unknown as number[])
131
- )
132
- frames.push(`data:image/${options.format};base64,${base64Frame}`)
133
-
134
- // Update progress for frame processing (from 90% to 100%)
135
- options.onProgress?.(90 + Math.round(((i + 1) / frameFiles.length) * 10))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
  }
137
 
 
 
 
138
  return frames
139
  }
140
-
141
- export default extractFramesFromVideo
 
1
+ 'use client'
2
+
3
  import { FFmpeg } from '@ffmpeg/ffmpeg'
4
  import { toBlobURL } from '@ffmpeg/util'
5
  import mediaInfoFactory, {
 
19
  maxHeight: number
20
  sceneSamplingRate: number // Percentage of additional frames between scene changes (0-100)
21
  onProgress?: (progress: number) => void // Callback function for progress updates
22
+ debug?: boolean
23
  }
24
 
25
+ export async function extractFramesFromVideo(
26
  videoBlob: Blob,
27
  options: FrameExtractorOptions
28
  ): Promise<string[]> {
29
  // Initialize MediaInfo
30
+ const mediaInfo = await mediaInfoFactory({
31
+ format: 'object',
32
+ locateFile: () => {
33
+ return '/wasm/MediaInfoModule.wasm'
34
+ },
35
+ })
36
 
37
  // Get video duration using MediaInfo
38
  const getSize = () => videoBlob.size
 
50
  reader.readAsArrayBuffer(videoBlob.slice(offset, offset + chunkSize))
51
  })
52
 
53
+ if (options.debug) {
54
+ console.log('calling await mediaInfo.analyzeData(getSize, readChunk)')
55
+ }
56
+
57
  const result = await mediaInfo.analyzeData(getSize, readChunk)
58
+ if (options.debug) {
59
+ console.log('result = ', result)
60
+ }
61
 
62
  let duration: number = 0
63
 
64
  for (const track of result.media?.track || []) {
65
+ if (options.debug) {
66
+ console.log('track = ', track)
67
+ }
68
+
69
  let maybeDuration: number = 0
70
  if (track['@type'] === 'Audio') {
71
  const audioTrack = track as AudioTrack
72
+ maybeDuration = audioTrack.Duration
73
+ ? parseFloat(`${audioTrack.Duration || 0}`)
74
+ : 0
75
  } else if (track['@type'] === 'Video') {
76
  const videoTrack = track as VideoTrack
77
+ maybeDuration = videoTrack.Duration
78
+ ? parseFloat(`${videoTrack.Duration || 0}`)
79
+ : 0
80
  }
81
  if (
82
  typeof maybeDuration === 'number' &&
 
91
  throw new Error('Could not determine video duration (or it is length 0)')
92
  }
93
 
94
+ if (options.debug) {
95
+ console.log('duration in seconds:', duration)
96
+ }
97
+
98
  // Initialize FFmpeg
99
  const ffmpeg = new FFmpeg()
100
  const baseURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd'
 
104
  wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, 'application/wasm'),
105
  })
106
 
107
+ if (options.debug) {
108
+ console.log('FFmpeg loaded!')
109
+ }
110
+
111
  // Write video file to FFmpeg's file system
112
  const videoUint8Array = new Uint8Array(await videoBlob.arrayBuffer())
113
  await ffmpeg.writeFile('input.mp4', videoUint8Array)
114
+ if (options.debug) {
115
+ console.log('input.mp4 written!')
116
+ }
117
  // Prepare FFmpeg command
118
  const sceneFilter = `select='gt(scene,0.4)'`
119
  const additionalFramesFilter = `select='not(mod(n,${Math.floor(100 / options.sceneSamplingRate)}))'`
120
+ const scaleFilter = `scale='min(${options.maxWidth},iw)':min'(${options.maxHeight},ih)':force_original_aspect_ratio=decrease`
121
 
122
  let lastProgress = 0
123
  ffmpeg.on('log', ({ message }) => {
124
+ if (options.debug) {
125
+ console.log('FFmpeg log:', message)
126
+ }
127
  const timeMatch = message.match(/time=(\d{2}):(\d{2}):(\d{2}\.\d{2})/)
128
  if (timeMatch) {
129
  const [, hours, minutes, seconds] = timeMatch
 
137
  }
138
  })
139
 
140
+ const ffmpegCommand = [
141
  '-i',
142
  'input.mp4',
143
+ '-loglevel',
144
+ 'verbose',
145
  '-vf',
146
  `${sceneFilter},${additionalFramesFilter},${scaleFilter}`,
147
  '-vsync',
148
+ '2',
149
  '-q:v',
150
  '2',
151
+ '-f',
152
+ 'image2',
153
+ '-frames:v',
154
+ '1000', // Limit the number of frames to extract
155
  `frames_%03d.${options.format}`,
156
+ ]
157
+
158
+ if (options.debug) {
159
+ console.log('Executing FFmpeg command:', ffmpegCommand.join(' '))
160
+ }
161
+
162
+ try {
163
+ await ffmpeg.exec(ffmpegCommand)
164
+ } catch (error) {
165
+ console.error('FFmpeg execution error:', error)
166
+ throw error
167
+ }
168
 
169
  // Read generated frames
170
  const files = await ffmpeg.listDir('/')
171
+ if (options.debug) {
172
+ console.log('All files in FFmpeg filesystem:', files)
173
+ }
174
  const frameFiles = files.filter(
175
  (file) =>
176
  file.name.startsWith('frames_') &&
177
  file.name.endsWith(`.${options.format}`)
178
  )
179
+ if (options.debug) {
180
+ console.log('Frame files found:', frameFiles.length)
181
+ }
182
 
183
  const frames: string[] = []
184
+ const encoder = new TextEncoder()
185
+
186
  for (let i = 0; i < frameFiles.length; i++) {
187
  const file = frameFiles[i]
188
+ if (options.debug) {
189
+ console.log(`Processing frame file: ${file.name}`)
190
+ }
191
+ try {
192
+ const frameData = await ffmpeg.readFile(file.name)
193
+
194
+ // Convert Uint8Array to Base64 string without using btoa
195
+ let binary = ''
196
+ const bytes = new Uint8Array(frameData as any)
197
+ const len = bytes.byteLength
198
+ for (let i = 0; i < len; i++) {
199
+ binary += String.fromCharCode(bytes[i])
200
+ }
201
+ const base64Frame = window.btoa(binary)
202
+
203
+ frames.push(`data:image/${options.format};base64,${base64Frame}`)
204
+
205
+ // Update progress for frame processing (from 90% to 100%)
206
+ options.onProgress?.(90 + Math.round(((i + 1) / frameFiles.length) * 10))
207
+ } catch (error) {
208
+ console.error(`Error processing frame ${file.name}:`, error)
209
+ // You can choose to either skip this frame or throw an error
210
+ // throw error; // Uncomment this line if you want to stop processing on any error
211
+ }
212
  }
213
 
214
+ if (options.debug) {
215
+ console.log(`Total frames processed: ${frames.length}`)
216
+ }
217
  return frames
218
  }
 
 
src/services/io/ffmpegUtils.ts ADDED
@@ -0,0 +1,478 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { UUID } from '@aitube/clap'
2
+ import { FFmpeg } from '@ffmpeg/ffmpeg'
3
+ import { toBlobURL } from '@ffmpeg/util'
4
+
5
+ export const TAG = 'io/createFullVideo'
6
+
7
+ export type FFMPegVideoInput = {
8
+ data: Uint8Array | null
9
+ startTimeInMs: number
10
+ endTimeInMs: number
11
+ durationInSecs: number
12
+ }
13
+
14
+ export type FFMPegAudioInput = FFMPegVideoInput
15
+
16
+ /**
17
+ * Download and load single and multi-threading FFMPeg.
18
+ * MT for video
19
+ * ST for audio (as MT has issues with it)
20
+ * toBlobURL is used to bypass CORS issues, urls with the same domain can be used directly.
21
+ */
22
+ export async function initializeFFmpeg() {
23
+ const [ffmpegSt, ffmpegMt] = [new FFmpeg(), new FFmpeg()]
24
+ const baseStURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd'
25
+ const baseMtURL = 'https://unpkg.com/@ffmpeg/core-mt@0.12.6/dist/umd'
26
+
27
+ ffmpegSt.on('log', ({ message }) => {
28
+ console.log(TAG, 'FFmpeg Single-Thread:', message)
29
+ })
30
+
31
+ ffmpegMt.on('log', ({ message }) => {
32
+ console.log(TAG, 'FFmpeg Multi-Thread:', message)
33
+ })
34
+
35
+ await ffmpegSt.load({
36
+ coreURL: await toBlobURL(`${baseStURL}/ffmpeg-core.js`, 'text/javascript'),
37
+ wasmURL: await toBlobURL(
38
+ `${baseStURL}/ffmpeg-core.wasm`,
39
+ 'application/wasm'
40
+ ),
41
+ })
42
+
43
+ await ffmpegMt.load({
44
+ coreURL: await toBlobURL(`${baseMtURL}/ffmpeg-core.js`, 'text/javascript'),
45
+ wasmURL: await toBlobURL(
46
+ `${baseMtURL}/ffmpeg-core.wasm`,
47
+ 'application/wasm'
48
+ ),
49
+ workerURL: await toBlobURL(
50
+ `${baseMtURL}/ffmpeg-core.worker.js`,
51
+ 'text/javascript'
52
+ ),
53
+ })
54
+
55
+ return [ffmpegSt, ffmpegMt] as [FFmpeg, FFmpeg]
56
+ }
57
+
58
+ /**
59
+ * Get loaded FFmpeg.
60
+ */
61
+ let ffmpegInstance: [FFmpeg, FFmpeg]
62
+ export async function loadFFmpegSt() {
63
+ if (!ffmpegInstance) ffmpegInstance = await initializeFFmpeg()
64
+ return ffmpegInstance[0]
65
+ }
66
+
67
+ export async function loadFFmpegMt() {
68
+ if (!ffmpegInstance) ffmpegInstance = await initializeFFmpeg()
69
+ return ffmpegInstance[1]
70
+ }
71
+
72
+ /**
73
+ * Creates an exclusive logger for the FFmpeg calls inside the provided method,
74
+ * it calculates the progress based on raw FFmpeg logs and the provided `totalTimeInMs`.
75
+ *
76
+ * @param totalTimeInMs
77
+ * @param method
78
+ * @param callback
79
+ * @param {number} callback.progress - The progress of the FFmpeg process from 0 to 100.
80
+ * @returns
81
+ */
82
+ export async function captureFFmpegProgress(
83
+ ffmpeg: FFmpeg,
84
+ totalTimeInMs: number,
85
+ method: () => any,
86
+ callback: (progress: number) => void
87
+ ): Promise<any> {
88
+ const extractProgressTimeMsFromLogs = (log: string): number | null => {
89
+ // `frame` for videos, `size` for audios
90
+ if (!log.startsWith('frame') && !log.startsWith('size')) return null
91
+ const timeRegex = /time=(\d{2}):(\d{2}):(\d{2})\.(\d{2})/
92
+ const match = log.match(timeRegex)
93
+ if (match) {
94
+ const hours = parseInt(match[1])
95
+ const minutes = parseInt(match[2])
96
+ const seconds = parseInt(match[3])
97
+ const centiseconds = parseInt(match[4])
98
+ const totalMilliseconds =
99
+ hours * 3600000 + minutes * 60000 + seconds * 1000 + centiseconds * 10
100
+ return totalMilliseconds
101
+ }
102
+ return null
103
+ }
104
+ let ffmpegLog = true
105
+ ffmpeg.on('log', ({ message }) => {
106
+ if (!ffmpegLog) return
107
+ const timeInMs = extractProgressTimeMsFromLogs(message)
108
+ if (timeInMs) callback((timeInMs / totalTimeInMs) * 100)
109
+ })
110
+ const result = await method()
111
+ ffmpegLog = false
112
+ return result
113
+ }
114
+
115
+ /**
116
+ * It will calculate a proportional progress between a targetProgress and a startProgress
117
+ *
118
+ * @param startProgress e.g. 50
119
+ * @param progress e.g. 50
120
+ * @param targetProgress e.g. 70
121
+ * @returns e.g. 60, because 50% of progress between 70% and 50%, would result on 60%
122
+ */
123
+ export function calculateProgress(
124
+ startProgress: number,
125
+ progress: number,
126
+ targetProgress: number
127
+ ): number {
128
+ return startProgress + (progress * (targetProgress - startProgress)) / 100
129
+ }
130
+
131
+ /**
132
+ * Creates an empty black video and appends it to the
133
+ * provided `fileListContentArray`.
134
+ *
135
+ * @param duration time in milliseconds
136
+ * @param width
137
+ * @param height
138
+ * @param filename
139
+ * @param fileListContentArray fileList.txt where to append the file name
140
+ * @param onProgress callback to capture the progress of this method
141
+ */
142
+ export async function addEmptyVideo(
143
+ durationInSecs: number,
144
+ width: number,
145
+ height: number,
146
+ filename: string,
147
+ fileListContentArray: string[],
148
+ onProgress?: (progress: number, message?: string) => void
149
+ ) {
150
+ const ffmpeg = await loadFFmpegMt()
151
+ let targetPartialProgress = 0
152
+
153
+ // For some reason, creating empty video with silent audio
154
+ // in one exec doesn't work, we need to split it.
155
+
156
+ console.log(
157
+ TAG,
158
+ 'Creating empty video',
159
+ filename,
160
+ width,
161
+ height,
162
+ durationInSecs
163
+ )
164
+ let currentProgress = 0
165
+ targetPartialProgress = 50
166
+
167
+ await captureFFmpegProgress(
168
+ ffmpeg,
169
+ durationInSecs * 1000,
170
+ async () => {
171
+ await ffmpeg.exec([
172
+ '-f',
173
+ 'lavfi',
174
+ '-i',
175
+ `color=c=black:s=${width}x${height}:d=${durationInSecs}`,
176
+ '-c:v',
177
+ 'libx264',
178
+ '-t',
179
+ `${durationInSecs}`,
180
+ '-loglevel',
181
+ 'verbose',
182
+ `base_${filename}`,
183
+ ])
184
+ },
185
+ (progress) => {
186
+ onProgress?.((progress / 100) * targetPartialProgress)
187
+ }
188
+ )
189
+
190
+ console.log(
191
+ TAG,
192
+ 'Adding silent audio to empty video',
193
+ filename,
194
+ width,
195
+ height,
196
+ durationInSecs
197
+ )
198
+ currentProgress = 50
199
+ targetPartialProgress = 100
200
+
201
+ const exitCode = await ffmpeg.exec([
202
+ '-i',
203
+ `base_${filename}`,
204
+ '-f',
205
+ 'lavfi',
206
+ '-i',
207
+ 'anullsrc',
208
+ '-c:v',
209
+ 'copy',
210
+ '-c:a',
211
+ 'aac',
212
+ '-t',
213
+ `${durationInSecs}`,
214
+ '-loglevel',
215
+ 'verbose',
216
+ filename,
217
+ ])
218
+
219
+ if (exitCode) {
220
+ throw new Error(`${TAG}: Unexpect error while creating empty video`)
221
+ }
222
+
223
+ console.log(TAG, 'Empty video created', filename)
224
+ fileListContentArray.push(`file ${filename}`)
225
+ }
226
+
227
+ /**
228
+ * Creates the full mixed audio including silence
229
+ * segments and loads it into ffmpeg with the given `filename`.
230
+ * @param onProgress callback to capture the progress of this method
231
+ * @throws Error if ffmpeg returns exit code 1
232
+ */
233
+ export async function createFullAudio(
234
+ audios: FFMPegAudioInput[],
235
+ filename: string,
236
+ totalVideoDurationInMs: number,
237
+ onProgress?: (progress: number, message: string) => void
238
+ ): Promise<void> {
239
+ console.log(TAG, 'Creating full audio', filename)
240
+
241
+ const ffmpeg = await loadFFmpegSt()
242
+ const filterComplexParts = []
243
+ const baseFilename = `base_${filename}`
244
+ let currentProgress = 0
245
+ let targetProgress = 25
246
+
247
+ // To mix audios at given times, we need a first empty base audio track
248
+
249
+ await captureFFmpegProgress(
250
+ ffmpeg,
251
+ totalVideoDurationInMs,
252
+ async () => {
253
+ await ffmpeg.exec([
254
+ '-f',
255
+ 'lavfi',
256
+ '-i',
257
+ 'anullsrc',
258
+ '-t',
259
+ `${totalVideoDurationInMs / 1000}`,
260
+ '-loglevel',
261
+ 'verbose',
262
+ !audios.length ? filename : baseFilename,
263
+ ])
264
+ },
265
+ (progress) => {
266
+ onProgress?.(
267
+ calculateProgress(currentProgress, progress, targetProgress),
268
+ 'Creating base audio...'
269
+ )
270
+ }
271
+ )
272
+
273
+ // If there is no audios, the base audio is the final one
274
+ if (!audios.length) return onProgress?.(100, 'Prepared audios...')
275
+
276
+ currentProgress = targetProgress
277
+ targetProgress = 50
278
+
279
+ // Mix audios based on their start times
280
+
281
+ const audioInputFiles = ['-i', baseFilename]
282
+ for (let index = 0; index < audios.length; index++) {
283
+ onProgress?.(currentProgress, 'Creating base audio...')
284
+ console.log(TAG, `Processing audio #${index}`)
285
+ const audio = audios[index]
286
+ const expectedProgressForItem = ((1 / audios.length) * targetProgress) / 100
287
+ if (!audio.data) continue
288
+ const audioFilename = `audio_${UUID()}.mp3`
289
+ await ffmpeg.writeFile(audioFilename, audio.data)
290
+ audioInputFiles.push('-i', audioFilename)
291
+ const delay = audio.startTimeInMs
292
+ const durationInSecs = audio.endTimeInMs - audio.startTimeInMs / 1000
293
+ filterComplexParts.push(
294
+ `[${index + 1}:a]atrim=0:${durationInSecs},adelay=${delay}|${delay}[delayed${index}]`
295
+ )
296
+ currentProgress += expectedProgressForItem * 100
297
+ }
298
+
299
+ const amixInputs = `[0:a]${audios.map((_, index) => `[delayed${index}]`).join('')}amix=inputs=${audios.length + 1}:duration=longest`
300
+ filterComplexParts.push(`${amixInputs}[a]`)
301
+ const filterComplex = filterComplexParts.join('; ')
302
+
303
+ currentProgress = targetProgress
304
+ targetProgress = 100
305
+
306
+ const createFullAudioExitCode = await captureFFmpegProgress(
307
+ ffmpeg,
308
+ totalVideoDurationInMs,
309
+ async () => {
310
+ await ffmpeg.exec([
311
+ ...audioInputFiles,
312
+ '-filter_complex',
313
+ filterComplex,
314
+ '-map',
315
+ '[a]',
316
+ '-t',
317
+ `${totalVideoDurationInMs / 1000}`,
318
+ '-loglevel',
319
+ 'verbose',
320
+ filename,
321
+ ])
322
+ },
323
+ (progress) => {
324
+ onProgress?.(
325
+ calculateProgress(currentProgress, progress, targetProgress),
326
+ 'Mixing audios...'
327
+ )
328
+ }
329
+ )
330
+
331
+ if (createFullAudioExitCode) {
332
+ throw new Error(`${TAG}: Error while creating full audio!`)
333
+ }
334
+ onProgress?.(targetProgress, 'Prepared audios...')
335
+ }
336
+
337
+ /**
338
+ * Creates the full silent video including empty black
339
+ * segments and loads it into ffmpeg with the given `filename`.
340
+ * @param onProgress callback to capture the progress of this method
341
+ * @throws Error if ffmpeg returns exit code 1
342
+ */
343
+ export async function createFullSilentVideo(
344
+ videos: FFMPegVideoInput[],
345
+ filename: string,
346
+ totalVideoDurationInMs: number,
347
+ width: number,
348
+ height: number,
349
+ excludeEmptyContent = false,
350
+ onProgress?: (progress: number, message: string) => void
351
+ ) {
352
+ const ffmpeg = await loadFFmpegMt()
353
+ const fileList = 'fileList.txt'
354
+ const fileListContentArray = []
355
+
356
+ // Complete array of videos including concatenated empty segments
357
+ // This is helpful for cleaner progress log
358
+ let lastStartTimeVideoInMs = 0
359
+ let videosWithGaps: FFMPegVideoInput[]
360
+
361
+ if (!videos.length) {
362
+ videosWithGaps = [
363
+ {
364
+ startTimeInMs: 0,
365
+ endTimeInMs: totalVideoDurationInMs,
366
+ data: null,
367
+ durationInSecs: totalVideoDurationInMs / 1000,
368
+ },
369
+ ]
370
+ } else {
371
+ videosWithGaps = videos.reduce((arr: FFMPegVideoInput[], video, index) => {
372
+ const emptyVideoDurationInMs =
373
+ video.startTimeInMs - lastStartTimeVideoInMs
374
+ if (emptyVideoDurationInMs) {
375
+ arr.push({
376
+ startTimeInMs: lastStartTimeVideoInMs,
377
+ endTimeInMs: lastStartTimeVideoInMs + emptyVideoDurationInMs,
378
+ data: null,
379
+ durationInSecs: emptyVideoDurationInMs / 1000,
380
+ })
381
+ }
382
+ arr.push(video)
383
+ lastStartTimeVideoInMs = video.endTimeInMs
384
+ if (
385
+ index == videos.length - 1 &&
386
+ lastStartTimeVideoInMs < totalVideoDurationInMs
387
+ ) {
388
+ arr.push({
389
+ startTimeInMs: lastStartTimeVideoInMs,
390
+ endTimeInMs: totalVideoDurationInMs,
391
+ data: null,
392
+ durationInSecs:
393
+ (totalVideoDurationInMs - lastStartTimeVideoInMs) / 1000,
394
+ })
395
+ }
396
+ return arr
397
+ }, [])
398
+ }
399
+
400
+ onProgress?.(0, 'Preparing videos...')
401
+
402
+ // Arbitrary percentage, as `concat` is fast,
403
+ // then estimate the generation of gap videos
404
+ // as the 70% of the work
405
+ let currentProgress = 0
406
+ let targetProgress = 70
407
+
408
+ for (const video of videosWithGaps) {
409
+ const expectedProgressForItem =
410
+ (((video.durationInSecs * 1000) / totalVideoDurationInMs) *
411
+ targetProgress) /
412
+ 100
413
+ if (!video.data) {
414
+ if (excludeEmptyContent) continue
415
+ let collectedProgress = 0
416
+ await addEmptyVideo(
417
+ video.durationInSecs,
418
+ width,
419
+ height,
420
+ `empty_video_${UUID()}.mp4`,
421
+ fileListContentArray,
422
+ (progress) => {
423
+ const subProgress = progress / 100
424
+ currentProgress +=
425
+ (expectedProgressForItem * subProgress - collectedProgress) * 100
426
+ console.log(TAG, 'Current progress', currentProgress)
427
+ onProgress?.(currentProgress, 'Preparing videos...')
428
+ collectedProgress = expectedProgressForItem * subProgress
429
+ }
430
+ )
431
+ } else {
432
+ const videoFilename = `video_${UUID()}.mp4`
433
+ await ffmpeg.writeFile(videoFilename, video.data)
434
+ fileListContentArray.push(`file ${videoFilename}`)
435
+ currentProgress += expectedProgressForItem * 100
436
+ console.log(TAG, 'Current progress', currentProgress)
437
+ onProgress?.(currentProgress, 'Preparing videos...')
438
+ }
439
+ }
440
+
441
+ onProgress?.(targetProgress, 'Concatenating videos...')
442
+ currentProgress = 70
443
+ targetProgress = 100
444
+
445
+ const fileListContent = fileListContentArray.join('\n')
446
+ await ffmpeg.writeFile(fileList, fileListContent)
447
+
448
+ const creatBaseFullVideoExitCode = await captureFFmpegProgress(
449
+ ffmpeg,
450
+ totalVideoDurationInMs,
451
+ async () => {
452
+ await ffmpeg.exec([
453
+ '-f',
454
+ 'concat',
455
+ '-safe',
456
+ '0',
457
+ '-i',
458
+ fileList,
459
+ '-loglevel',
460
+ 'verbose',
461
+ '-c',
462
+ 'copy',
463
+ filename,
464
+ ])
465
+ },
466
+ (progress: number) => {
467
+ onProgress?.(
468
+ calculateProgress(currentProgress, progress, targetProgress),
469
+ 'Merging audio and video...'
470
+ )
471
+ }
472
+ )
473
+
474
+ if (creatBaseFullVideoExitCode) {
475
+ throw new Error(`${TAG}: Error while creating base full video!`)
476
+ }
477
+ onProgress?.(targetProgress, 'Concatenating videos...')
478
+ }
src/services/io/parseFileIntoSegments.ts CHANGED
@@ -5,6 +5,7 @@ import {
5
  ClapOutputType,
6
  ClapSegmentCategory,
7
  ClapSegmentStatus,
 
8
  newSegment,
9
  UUID,
10
  } from '@aitube/clap'
@@ -13,6 +14,9 @@ import {
13
  SegmentEditionStatus,
14
  SegmentVisibility,
15
  TimelineSegment,
 
 
 
16
  } from '@aitube/timeline'
17
 
18
  import { blobToBase64DataUri } from '@/lib/utils/blobToBase64DataUri'
@@ -22,12 +26,21 @@ import { ResourceCategory, ResourceType } from '@aitube/clapper-services'
22
 
23
  export async function parseFileIntoSegments({
24
  file,
 
 
 
25
  }: {
26
  /**
27
  * The file to import
28
  */
29
  file: File
 
 
 
 
30
  }): Promise<TimelineSegment[]> {
 
 
31
  // console.log(`parseFileIntoSegments(): filename = ${file.name}`)
32
  // console.log(`parseFileIntoSegments(): file size = ${file.size} bytes`)
33
  // console.log(`parseFileIntoSegments(): file type = ${file.type}`)
@@ -38,16 +51,69 @@ export async function parseFileIntoSegments({
38
  'TODO: open a popup to ask if this is a voice character sample, dialogue, music etc'
39
  )
40
 
41
- let type: ResourceType = 'misc'
42
- let resourceCategory: ResourceCategory = 'misc'
43
-
44
  const newSegments: TimelineSegment[] = []
45
 
46
  switch (file.type) {
47
- case 'image/webp':
48
- type = 'image'
49
- resourceCategory = 'control_image'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  break
 
51
 
52
  case 'audio/mpeg': // this is the "official" one
53
  case 'audio/mp3': // this is just an alias
@@ -56,10 +122,10 @@ export async function parseFileIntoSegments({
56
  case 'audio/x-mp4': // should be rare, normally is is audio/mp4
57
  case 'audio/m4a': // shouldn't exist
58
  case 'audio/x-m4a': // should be rare, normally is is audio/mp4
59
- case 'audio/webm':
60
  // for background track, or as an inspiration track, or a voice etc
61
- type = 'audio'
62
- resourceCategory = 'background_music'
63
 
64
  // TODO: add caption analysis
65
  const { durationInMs, durationInSteps, bpm, audioBuffer } =
@@ -71,11 +137,12 @@ export async function parseFileIntoSegments({
71
  })
72
 
73
  // TODO: use the correct drop time
74
- const startTimeInMs = 0
75
- const startTimeInSteps = 1
76
-
77
- const endTimeInSteps = durationInSteps
78
- const endTimeInMs = startTimeInMs + durationInMs
 
79
 
80
  // ok let's stop for a minute there:
81
  // if someone drops a .mp3, and assuming we don't yet have the UI to select the category,
@@ -92,6 +159,7 @@ export async function parseFileIntoSegments({
92
  startTimeInMs, // start time of the segment
93
  endTimeInMs, // end time of the segment (startTimeInMs + durationInMs)
94
  status: ClapSegmentStatus.COMPLETED,
 
95
  // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index
96
  label: `${file.name} (${Math.round(durationInMs / 1000)}s @ ${Math.round(bpm * 100) / 100} BPM)`, // a short label to name the segment (optional, can be human or LLM-defined)
97
  category,
@@ -104,6 +172,11 @@ export async function parseFileIntoSegments({
104
  const timelineSegment = await clapSegmentToTimelineSegment(
105
  newSegment(newSegmentData)
106
  )
 
 
 
 
 
107
  timelineSegment.outputType = ClapOutputType.AUDIO
108
  timelineSegment.outputGain = 1.0
109
  timelineSegment.audioBuffer = audioBuffer
@@ -116,28 +189,31 @@ export async function parseFileIntoSegments({
116
  // poof! type disappears.. it's magic
117
  newSegments.push(timelineSegment)
118
  break
 
119
 
120
- case 'text/plain':
121
  // for dialogue, prompts..
122
- type = 'text'
123
- resourceCategory = 'text_prompt'
124
  break
 
125
 
126
- default:
127
  console.log(`unrecognized file type "${file.type}"`)
128
  break
 
129
  }
130
 
131
  // note: we always upload the files, because even if it is an unhandled format (eg. a PDF)
132
  // this can still be part of the project as a resource for humans (inspiration, guidelines etc)
133
 
 
134
  const id = UUID()
135
  const fileName = `${id}.${extension}`
136
 
137
  const storage = `resources`
138
  const filePath = `${type}/${fileName}`
139
 
140
- /*
141
  const { data, error } = await supabase
142
  .storage
143
  .from('avatars')
 
5
  ClapOutputType,
6
  ClapSegmentCategory,
7
  ClapSegmentStatus,
8
+ isValidNumber,
9
  newSegment,
10
  UUID,
11
  } from '@aitube/clap'
 
14
  SegmentEditionStatus,
15
  SegmentVisibility,
16
  TimelineSegment,
17
+ useTimeline,
18
+ TimelineStore,
19
+ DEFAULT_DURATION_IN_MS_PER_STEP,
20
  } from '@aitube/timeline'
21
 
22
  import { blobToBase64DataUri } from '@/lib/utils/blobToBase64DataUri'
 
26
 
27
  export async function parseFileIntoSegments({
28
  file,
29
+ track,
30
+ startTimeInMs: maybeStartTimeInMs,
31
+ endTimeInMs: maybeEndTimeInMs,
32
  }: {
33
  /**
34
  * The file to import
35
  */
36
  file: File
37
+
38
+ track?: number
39
+ startTimeInMs?: number
40
+ endTimeInMs?: number
41
  }): Promise<TimelineSegment[]> {
42
+ const timeline: TimelineStore = useTimeline.getState()
43
+ const { cursorTimestampAtInMs } = timeline
44
  // console.log(`parseFileIntoSegments(): filename = ${file.name}`)
45
  // console.log(`parseFileIntoSegments(): file size = ${file.size} bytes`)
46
  // console.log(`parseFileIntoSegments(): file type = ${file.type}`)
 
51
  'TODO: open a popup to ask if this is a voice character sample, dialogue, music etc'
52
  )
53
 
 
 
 
54
  const newSegments: TimelineSegment[] = []
55
 
56
  switch (file.type) {
57
+ case 'image/jpeg':
58
+ case 'image/png':
59
+ case 'image/avif':
60
+ case 'image/heic':
61
+ case 'image/webp': {
62
+ const type: ResourceType = 'image'
63
+ const resourceCategory: ResourceCategory = 'control_image'
64
+
65
+ // ok let's stop for a minute there:
66
+ // if someone drops a .mp3, and assuming we don't yet have the UI to select the category,
67
+ // do you think it should be a SOUND, a VOICE or a MUSIC by default?
68
+ // I expect people will use AI service providers for sound and voice,
69
+ // maybe in some case music too, but there are also many people
70
+ // who will want to use their own track eg. to create a music video
71
+ const category = ClapSegmentCategory.STORYBOARD
72
+
73
+ const assetUrl = await blobToBase64DataUri(file)
74
+
75
+ const startTimeInMs = isValidNumber(maybeStartTimeInMs)
76
+ ? maybeStartTimeInMs!
77
+ : cursorTimestampAtInMs
78
+ const durationInSteps = 4
79
+ const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP
80
+ const endTimeInMs = isValidNumber(maybeEndTimeInMs)
81
+ ? maybeEndTimeInMs!
82
+ : startTimeInMs + durationInMs
83
+
84
+ const newSegmentData: Partial<TimelineSegment> = {
85
+ prompt: 'Storyboard', // note: this can be set later with an automatic captioning worker
86
+ startTimeInMs, // start time of the segment
87
+ endTimeInMs, // end time of the segment (startTimeInMs + durationInMs)
88
+ status: ClapSegmentStatus.COMPLETED,
89
+ // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index
90
+ label: `${file.name}`, // a short label to name the segment (optional, can be human or LLM-defined)
91
+ category,
92
+ assetUrl,
93
+ assetDurationInMs: endTimeInMs,
94
+ assetSourceType: ClapAssetSource.DATA,
95
+ assetFileFormat: `${file.type}`,
96
+ }
97
+
98
+ const timelineSegment = await clapSegmentToTimelineSegment(
99
+ newSegment(newSegmentData)
100
+ )
101
+
102
+ if (isValidNumber(track)) {
103
+ timelineSegment.track = track
104
+ }
105
+
106
+ timelineSegment.outputType = ClapOutputType.IMAGE
107
+
108
+ // we assume we want it to be immediately visible
109
+ timelineSegment.visibility = SegmentVisibility.VISIBLE
110
+
111
+ // console.log("newSegment:", audioSegment)
112
+
113
+ // poof! type disappears.. it's magic
114
+ newSegments.push(timelineSegment)
115
  break
116
+ }
117
 
118
  case 'audio/mpeg': // this is the "official" one
119
  case 'audio/mp3': // this is just an alias
 
122
  case 'audio/x-mp4': // should be rare, normally is is audio/mp4
123
  case 'audio/m4a': // shouldn't exist
124
  case 'audio/x-m4a': // should be rare, normally is is audio/mp4
125
+ case 'audio/webm': {
126
  // for background track, or as an inspiration track, or a voice etc
127
+ const type: ResourceType = 'audio'
128
+ const resourceCategory: ResourceCategory = 'background_music'
129
 
130
  // TODO: add caption analysis
131
  const { durationInMs, durationInSteps, bpm, audioBuffer } =
 
137
  })
138
 
139
  // TODO: use the correct drop time
140
+ const startTimeInMs = isValidNumber(maybeStartTimeInMs)
141
+ ? maybeStartTimeInMs!
142
+ : 0
143
+ const endTimeInMs = isValidNumber(maybeEndTimeInMs)
144
+ ? maybeEndTimeInMs!
145
+ : startTimeInMs + durationInMs
146
 
147
  // ok let's stop for a minute there:
148
  // if someone drops a .mp3, and assuming we don't yet have the UI to select the category,
 
159
  startTimeInMs, // start time of the segment
160
  endTimeInMs, // end time of the segment (startTimeInMs + durationInMs)
161
  status: ClapSegmentStatus.COMPLETED,
162
+ track,
163
  // track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index
164
  label: `${file.name} (${Math.round(durationInMs / 1000)}s @ ${Math.round(bpm * 100) / 100} BPM)`, // a short label to name the segment (optional, can be human or LLM-defined)
165
  category,
 
172
  const timelineSegment = await clapSegmentToTimelineSegment(
173
  newSegment(newSegmentData)
174
  )
175
+
176
+ if (isValidNumber(track)) {
177
+ timelineSegment.track = track
178
+ }
179
+
180
  timelineSegment.outputType = ClapOutputType.AUDIO
181
  timelineSegment.outputGain = 1.0
182
  timelineSegment.audioBuffer = audioBuffer
 
189
  // poof! type disappears.. it's magic
190
  newSegments.push(timelineSegment)
191
  break
192
+ }
193
 
194
+ case 'text/plain': {
195
  // for dialogue, prompts..
196
+ const type: ResourceType = 'text'
197
+ const resourceCategory: ResourceCategory = 'text_prompt'
198
  break
199
+ }
200
 
201
+ default: {
202
  console.log(`unrecognized file type "${file.type}"`)
203
  break
204
+ }
205
  }
206
 
207
  // note: we always upload the files, because even if it is an unhandled format (eg. a PDF)
208
  // this can still be part of the project as a resource for humans (inspiration, guidelines etc)
209
 
210
+ /*
211
  const id = UUID()
212
  const fileName = `${id}.${extension}`
213
 
214
  const storage = `resources`
215
  const filePath = `${type}/${fileName}`
216
 
 
217
  const { data, error } = await supabase
218
  .storage
219
  .from('avatars')
src/services/io/useIO.ts CHANGED
@@ -18,6 +18,7 @@ import {
18
  TimelineSegment,
19
  removeFinalVideosAndConvertToTimelineSegments,
20
  getFinalVideo,
 
21
  } from '@aitube/timeline'
22
  import { ParseScriptProgressUpdate, parseScriptToClap } from '@aitube/broadway'
23
  import { IOStore, TaskCategory, TaskVisibility } from '@aitube/clapper-services'
@@ -40,11 +41,11 @@ import {
40
  formatSegmentForExport,
41
  } from '@/lib/utils/formatSegmentForExport'
42
  import { sleep } from '@/lib/utils/sleep'
43
- import {
44
- FFMPegAudioInput,
45
- FFMPegVideoInput,
46
- createFullVideo,
47
- } from './createFullVideo'
48
 
49
  export const useIO = create<IOStore>((set, get) => ({
50
  ...getDefaultIOState(),
@@ -107,6 +108,93 @@ export const useIO = create<IOStore>((set, get) => ({
107
  }
108
 
109
  const isVideoFile = fileType.startsWith('video/')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110
  }
111
  },
112
  openScreenplay: async (
 
18
  TimelineSegment,
19
  removeFinalVideosAndConvertToTimelineSegments,
20
  getFinalVideo,
21
+ DEFAULT_DURATION_IN_MS_PER_STEP,
22
  } from '@aitube/timeline'
23
  import { ParseScriptProgressUpdate, parseScriptToClap } from '@aitube/broadway'
24
  import { IOStore, TaskCategory, TaskVisibility } from '@aitube/clapper-services'
 
41
  formatSegmentForExport,
42
  } from '@/lib/utils/formatSegmentForExport'
43
  import { sleep } from '@/lib/utils/sleep'
44
+ import { FFMPegAudioInput, FFMPegVideoInput } from './ffmpegUtils'
45
+ import { createFullVideo } from './createFullVideo'
46
+ import { extractFramesFromVideo } from './extractFramesFromVideo'
47
+ import { extractCaptionsFromFrames } from './extractCaptionsFromFrames'
48
+ import { base64DataUriToFile } from '@/lib/utils/base64DataUriToFile'
49
 
50
  export const useIO = create<IOStore>((set, get) => ({
51
  ...getDefaultIOState(),
 
108
  }
109
 
110
  const isVideoFile = fileType.startsWith('video/')
111
+ if (isVideoFile) {
112
+ const storyboardExtractionTask = useTasks.getState().add({
113
+ category: TaskCategory.IMPORT,
114
+ visibility: TaskVisibility.BLOCKER,
115
+ initialMessage: `Extracting storyboards..`,
116
+ successMessage: `Extracting storyboards.. 100% done`,
117
+ value: 0,
118
+ })
119
+
120
+ const frames = await extractFramesFromVideo(file, {
121
+ format: 'png', // in theory we could also use 'jpg', but this freezes FFmpeg
122
+ maxWidth: 1024,
123
+ maxHeight: 576,
124
+ sceneSamplingRate: 100,
125
+ onProgress: (progress: number) => {
126
+ storyboardExtractionTask.setProgress({
127
+ message: `Extracting storyboards.. ${progress}% done`,
128
+ value: progress,
129
+ })
130
+ },
131
+ })
132
+
133
+ // optional: reset the project
134
+ await timeline.setClap(newClap())
135
+
136
+ const track = 1
137
+ let i = 0
138
+ let startTimeInMs = 0
139
+ const durationInSteps = 4
140
+ const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP
141
+ let endTimeInMs = startTimeInMs + durationInMs
142
+
143
+ for (const frame of frames) {
144
+ const frameFile = base64DataUriToFile(frame, `storyboard_${i++}.png`)
145
+ const newSegments = await parseFileIntoSegments({
146
+ file: frameFile,
147
+ startTimeInMs,
148
+ endTimeInMs,
149
+ track,
150
+ })
151
+
152
+ startTimeInMs += durationInMs
153
+ endTimeInMs += durationInMs
154
+
155
+ console.log('calling timeline.addSegments with:', newSegments)
156
+ await timeline.addSegments({
157
+ segments: newSegments,
158
+ track,
159
+ })
160
+ }
161
+
162
+ storyboardExtractionTask.success()
163
+
164
+ const enableCaptioning = false
165
+
166
+ if (enableCaptioning) {
167
+ const captioningTask = useTasks.getState().add({
168
+ category: TaskCategory.IMPORT,
169
+ // visibility: TaskVisibility.BLOCKER,
170
+
171
+ // since this is very long task, we can run it in the background
172
+ visibility: TaskVisibility.BACKGROUND,
173
+ initialMessage: `Analyzing storyboards..`,
174
+ successMessage: `Analyzing storyboards.. 100% done`,
175
+ value: 0,
176
+ })
177
+
178
+ console.log('calling extractCaptionsFromFrames() with:', frames)
179
+ const captions = await extractCaptionsFromFrames(
180
+ frames,
181
+ (
182
+ progress: number,
183
+ storyboardIndex: number,
184
+ nbStoryboards: number
185
+ ) => {
186
+ captioningTask.setProgress({
187
+ message: `Analyzing storyboards (${progress}%)`,
188
+ value: progress,
189
+ })
190
+ }
191
+ )
192
+ console.log('captions:', captions)
193
+ // TODO: add
194
+
195
+ captioningTask.success()
196
+ }
197
+ }
198
  }
199
  },
200
  openScreenplay: async (