first version of the video analyzer

package-lock.json

@@ -11,9 +11,9 @@
       "dependencies": {
         "@aitube/broadway": "0.0.22",
         "@aitube/clap": "0.0.30",
-        "@aitube/clapper-services": "0.0.29",
+        "@aitube/clapper-services": "0.0.34",
         "@aitube/engine": "0.0.26",
-        "@aitube/timeline": "0.0.43",
+        "@aitube/timeline": "0.0.44",
         "@fal-ai/serverless-client": "^0.13.0",
         "@ffmpeg/ffmpeg": "^0.12.10",
         "@ffmpeg/util": "^0.12.1",
@@ -161,12 +161,12 @@
       }
     },
     "node_modules/@aitube/clapper-services": {
-      "version": "0.0.29",
-      "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.29.tgz",
-      "integrity": "sha512-61UH/TQwPcvXArEkPnGNm+IQulaW3zNh73pzihdU2kkqufGzUYCNSd/jHJh9dLqQm3lZtm6QMN2RReFrzGuLNQ==",
+      "version": "0.0.34",
+      "resolved": "https://registry.npmjs.org/@aitube/clapper-services/-/clapper-services-0.0.34.tgz",
+      "integrity": "sha512-d0HruUyWRIXozO67W+2iEUTuBdbojGPn9BnIf6cvxkVbywLwy4hKaN+SD+yQwOi/jqoqu+TTQYUoWSF93JDVEQ==",
       "peerDependencies": {
         "@aitube/clap": "0.0.30",
-        "@aitube/timeline": "0.0.43",
+        "@aitube/timeline": "0.0.44",
         "@monaco-editor/react": "4.6.0",
         "monaco-editor": "0.50.0",
         "react": "*",
@@ -192,9 +192,9 @@
       }
     },
     "node_modules/@aitube/timeline": {
-      "version": "0.0.43",
-      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.43.tgz",
-      "integrity": "sha512-TnzKrB955YeDKOMWsnniGbQ+qulCmGptMfhNjDLEqA6jRcsnPVUFCR2dQBqWGNn6KfFPXmDvSi0Sihy7Oj98Aw==",
+      "version": "0.0.44",
+      "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.44.tgz",
+      "integrity": "sha512-iELTtmLONWR7zuGLLr9cJRlMuNoBXWxZzgGerDeXa5VyQhDmjj4shLOlZLP78PiIVHMdRwZr16IN6ob899VmMw==",
       "dependencies": {
         "date-fns": "^3.6.0",
         "react-virtualized-auto-sizer": "^1.0.24"
@@ -3348,20 +3348,20 @@
       }
     },
     "node_modules/@floating-ui/core": {
-      "version": "1.6.4",
-      "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.4.tgz",
-      "integrity": "sha512-a4IowK4QkXl4SCWTGUR0INAfEOX3wtsYw3rKK5InQEHMGObkR8Xk44qYQD9P4r6HHw0iIfK6GUKECmY8sTkqRA==",
+      "version": "1.6.5",
+      "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.5.tgz",
+      "integrity": "sha512-8GrTWmoFhm5BsMZOTHeGD2/0FLKLQQHvO/ZmQga4tKempYRLz8aqJGqXVuQgisnMObq2YZ2SgkwctN1LOOxcqA==",
       "dependencies": {
-        "@floating-ui/utils": "^0.2.4"
+        "@floating-ui/utils": "^0.2.5"
       }
     },
     "node_modules/@floating-ui/dom": {
-      "version": "1.6.7",
-      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.7.tgz",
-      "integrity": "sha512-wmVfPG5o2xnKDU4jx/m4w5qva9FWHcnZ8BvzEe90D/RpwsJaTAVYPEPdQ8sbr/N8zZTAHlZUTQdqg8ZUbzHmng==",
+      "version": "1.6.8",
+      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.8.tgz",
+      "integrity": "sha512-kx62rP19VZ767Q653wsP1XZCGIirkE09E0QUGNYTM/ttbbQHqcGPdSfWFxUyyNLc/W6aoJRBajOSXhP6GXjC0Q==",
       "dependencies": {
         "@floating-ui/core": "^1.6.0",
-        "@floating-ui/utils": "^0.2.4"
+        "@floating-ui/utils": "^0.2.5"
       }
     },
     "node_modules/@floating-ui/react-dom": {
@@ -3377,9 +3377,9 @@
       }
     },
     "node_modules/@floating-ui/utils": {
-      "version": "0.2.4",
-      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.4.tgz",
-      "integrity": "sha512-dWO2pw8hhi+WrXq1YJy2yCuWoL20PddgGaqTgVe4cOS9Q6qklXCiA1tJEqX6BEwRNSCP84/afac9hd4MS+zEUA=="
+      "version": "0.2.5",
+      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.5.tgz",
+      "integrity": "sha512-sTcG+QZ6fdEUObICavU+aB3Mp8HY4n14wYHdxK4fXjPmv3PXZZeY5RaguJmGyeH/CJQhX3fqKUtS4qc1LoHwhQ=="
     },
     "node_modules/@gar/promisify": {
       "version": "1.1.3",
@@ -3430,9 +3430,9 @@
       }
     },
     "node_modules/@huggingface/inference/node_modules/@huggingface/tasks": {
-      "version": "0.11.2",
-      "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.11.2.tgz",
-      "integrity": "sha512-vlwUJsj/QJcR/oLXvV+JBKheaVk9pqfAPYiS136cjHEDTeTW5/+ePpM6uKOc56oxqwrUjh5T0JylHJU8vyqr1A=="
+      "version": "0.11.3",
+      "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.11.3.tgz",
+      "integrity": "sha512-IYq4OdlySdscjkFwm6iIqP1ZgKl4OGhvQFJWI7Yxpq2V8RmXcgIjiqk/65S6Ap7i+eyCdlOC4qweVy/ICNE0JA=="
     },
     "node_modules/@huggingface/jinja": {
       "version": "0.2.2",
@@ -4206,15 +4206,15 @@
       }
     },
     "node_modules/@langchain/core": {
-      "version": "0.2.17",
-      "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.17.tgz",
-      "integrity": "sha512-WnFiZ7R/ZUVeHO2IgcSL7Tu+CjApa26Iy99THJP5fax/NF8UQCc/ZRcw2Sb/RUuRPVm6ALDass0fSQE1L9YNJg==",
+      "version": "0.2.18",
+      "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.18.tgz",
+      "integrity": "sha512-ru542BwNcsnDfjTeDbIkFIchwa54ctHZR+kVrC8U9NPS9/36iM8p8ruprOV7Zccj/oxtLE5UpEhV+9MZhVcFlA==",
       "dependencies": {
         "ansi-styles": "^5.0.0",
         "camelcase": "6",
         "decamelize": "1.2.0",
         "js-tiktoken": "^1.0.12",
-        "langsmith": "~0.1.30",
+        "langsmith": "~0.1.39",
         "ml-distance": "^4.0.0",
         "mustache": "^4.2.0",
         "p-queue": "^6.6.2",
@@ -6141,9 +6141,9 @@
       "integrity": "sha512-iQVztO09ZVfsletMiY+DpT/JRiBntdsdJ4uqk3UJFhrhS8mIC9ZOZbmfGSRs/kdbNPQkVyzucceDicQ/3Mlj9g=="
     },
     "node_modules/@react-three/drei": {
-      "version": "9.108.4",
-      "resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-9.108.4.tgz",
-      "integrity": "sha512-YyPVG7+np6G8CJRVVdEfgK+bou7cvp8v9R7k4NSHsoi5EokFPG03tkCjniRiz5SzQyN+E8kCiMogI9oZaop5+g==",
+      "version": "9.109.0",
+      "resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-9.109.0.tgz",
+      "integrity": "sha512-LlJ1k0DO5UvBdjuv6WuSP5jXb1mXsQY3VeQTfzivCsHJH9pUsbxutLL7mk84w9MI7cZytv2Qcx2nU2HBm0eNpQ==",
       "dependencies": {
         "@babel/runtime": "^7.11.2",
         "@mediapipe/tasks-vision": "0.10.8",
@@ -7313,9 +7313,9 @@
       }
     },
     "node_modules/@testing-library/dom": {
-      "version": "10.3.2",
-      "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.3.2.tgz",
-      "integrity": "sha512-0bxIdP9mmPiOJ6wHLj8bdJRq+51oddObeCGdEf6PNEhYd93ZYAN+lPRnEOVFtheVwDM7+p+tza3LAQgp0PTudg==",
+      "version": "10.4.0",
+      "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz",
+      "integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==",
       "dev": true,
       "peer": true,
       "dependencies": {
@@ -7844,13 +7844,13 @@
       }
     },
     "node_modules/@vitest/expect": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.0.3.tgz",
-      "integrity": "sha512-X6AepoOYePM0lDNUPsGXTxgXZAl3EXd0GYe/MZyVE4HzkUqyUVC6S3PrY5mClDJ6/7/7vALLMV3+xD/Ko60Hqg==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.0.4.tgz",
+      "integrity": "sha512-39jr5EguIoanChvBqe34I8m1hJFI4+jxvdOpD7gslZrVQBKhh8H9eD7J/LJX4zakrw23W+dITQTDqdt43xVcJw==",
       "dev": true,
       "dependencies": {
-        "@vitest/spy": "2.0.3",
-        "@vitest/utils": "2.0.3",
+        "@vitest/spy": "2.0.4",
+        "@vitest/utils": "2.0.4",
         "chai": "^5.1.1",
         "tinyrainbow": "^1.2.0"
       },
@@ -7859,9 +7859,9 @@
       }
     },
     "node_modules/@vitest/pretty-format": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.0.3.tgz",
-      "integrity": "sha512-URM4GLsB2xD37nnTyvf6kfObFafxmycCL8un3OC9gaCs5cti2u+5rJdIflZ2fUJUen4NbvF6jCufwViAFLvz1g==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.0.4.tgz",
+      "integrity": "sha512-RYZl31STbNGqf4l2eQM1nvKPXE0NhC6Eq0suTTePc4mtMQ1Fn8qZmjV4emZdEdG2NOWGKSCrHZjmTqDCDoeFBw==",
       "dev": true,
       "dependencies": {
         "tinyrainbow": "^1.2.0"
@@ -7871,12 +7871,12 @@
       }
     },
     "node_modules/@vitest/runner": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.0.3.tgz",
-      "integrity": "sha512-EmSP4mcjYhAcuBWwqgpjR3FYVeiA4ROzRunqKltWjBfLNs1tnMLtF+qtgd5ClTwkDP6/DGlKJTNa6WxNK0bNYQ==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.0.4.tgz",
+      "integrity": "sha512-Gk+9Su/2H2zNfNdeJR124gZckd5st4YoSuhF1Rebi37qTXKnqYyFCd9KP4vl2cQHbtuVKjfEKrNJxHHCW8thbQ==",
       "dev": true,
       "dependencies": {
-        "@vitest/utils": "2.0.3",
+        "@vitest/utils": "2.0.4",
         "pathe": "^1.1.2"
       },
       "funding": {
@@ -7884,12 +7884,12 @@
       }
     },
     "node_modules/@vitest/snapshot": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.0.3.tgz",
-      "integrity": "sha512-6OyA6v65Oe3tTzoSuRPcU6kh9m+mPL1vQ2jDlPdn9IQoUxl8rXhBnfICNOC+vwxWY684Vt5UPgtcA2aPFBb6wg==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.0.4.tgz",
+      "integrity": "sha512-or6Mzoz/pD7xTvuJMFYEtso1vJo1S5u6zBTinfl+7smGUhqybn6VjzCDMhmTyVOFWwkCMuNjmNNxnyXPgKDoPw==",
       "dev": true,
       "dependencies": {
-        "@vitest/pretty-format": "2.0.3",
+        "@vitest/pretty-format": "2.0.4",
         "magic-string": "^0.30.10",
         "pathe": "^1.1.2"
       },
@@ -7898,9 +7898,9 @@
       }
     },
     "node_modules/@vitest/spy": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.0.3.tgz",
-      "integrity": "sha512-sfqyAw/ypOXlaj4S+w8689qKM1OyPOqnonqOc9T91DsoHbfN5mU7FdifWWv3MtQFf0lEUstEwR9L/q/M390C+A==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.0.4.tgz",
+      "integrity": "sha512-uTXU56TNoYrTohb+6CseP8IqNwlNdtPwEO0AWl+5j7NelS6x0xZZtP0bDWaLvOfUbaYwhhWp1guzXUxkC7mW7Q==",
       "dev": true,
       "dependencies": {
         "tinyspy": "^3.0.0"
@@ -7910,12 +7910,12 @@
       }
     },
     "node_modules/@vitest/utils": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.0.3.tgz",
-      "integrity": "sha512-c/UdELMuHitQbbc/EVctlBaxoYAwQPQdSNwv7z/vHyBKy2edYZaFgptE27BRueZB7eW8po+cllotMNTDpL3HWg==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.0.4.tgz",
+      "integrity": "sha512-Zc75QuuoJhOBnlo99ZVUkJIuq4Oj0zAkrQ2VzCqNCx6wAwViHEh5Fnp4fiJTE9rA+sAoXRf00Z9xGgfEzV6fzQ==",
       "dev": true,
       "dependencies": {
-        "@vitest/pretty-format": "2.0.3",
+        "@vitest/pretty-format": "2.0.4",
         "estree-walker": "^3.0.3",
         "loupe": "^3.1.1",
         "tinyrainbow": "^1.2.0"
@@ -7926,7 +7926,7 @@
     },
     "node_modules/@xenova/transformers": {
       "version": "3.0.0-alpha.0",
-      "resolved": "git+ssh://git@github.com/xenova/transformers.js.git#96f19b062429ee4569ceeb4695aea90f0f456e63",
+      "resolved": "git+ssh://git@github.com/xenova/transformers.js.git#c6aeb4be1bc1cdfa72e9d050f77b97dc9c8af362",
       "dependencies": {
         "@huggingface/jinja": "^0.2.2",
         "onnxruntime-web": "^1.18.0",
@@ -15062,9 +15062,9 @@
       }
     },
     "node_modules/msw/node_modules/type-fest": {
-      "version": "4.22.1",
-      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.22.1.tgz",
-      "integrity": "sha512-9tHNEa0Ov81YOopiVkcCJVz5TM6AEQ+CHHjFIktqPnE3NV0AHIkx+gh9tiCl58m/66wWxkOC9eltpa75J4lQPA==",
+      "version": "4.23.0",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.23.0.tgz",
+      "integrity": "sha512-ZiBujro2ohr5+Z/hZWHESLz3g08BBdrdLMieYFULJO+tWc437sn8kQsWLJoZErY8alNhxre9K4p3GURAG11n+w==",
       "engines": {
         "node": ">=16"
       },
@@ -15452,9 +15452,9 @@
       }
     },
     "node_modules/node-releases": {
-      "version": "2.0.17",
-      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.17.tgz",
-      "integrity": "sha512-Ww6ZlOiEQfPfXM45v17oabk77Z7mg5bOt7AjDyzy7RjK9OrLrLC8dyZQoAPEOtFX9SaNf1Tdvr5gRJWdTJj7GA=="
+      "version": "2.0.18",
+      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz",
+      "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g=="
     },
     "node_modules/nopt": {
       "version": "7.2.1",
@@ -15877,9 +15877,9 @@
       }
     },
     "node_modules/onnxruntime-node/node_modules/tar": {
-      "version": "7.4.0",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.0.tgz",
-      "integrity": "sha512-XQs0S8fuAkQWuqhDeCdMlJXDX80D7EOVLDPVFkna9yQfzS+PHKgfxcei0jf6/+QAWcjqrnC8uM3fSAnrQl+XYg==",
+      "version": "7.4.1",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.1.tgz",
+      "integrity": "sha512-dDJzpQf7Nud96mCs3wtw+XUiWGpi9WHxytSusrg0lYlj/Kr11DnB5hfw5bNDQNzx52JJ2Vy+7l8AFivp6H7ETA==",
       "optional": true,
       "dependencies": {
         "@isaacs/fs-minipass": "^4.0.0",
@@ -15916,9 +15916,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.52.7",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.52.7.tgz",
-      "integrity": "sha512-dgxA6UZHary6NXUHEDj5TWt8ogv0+ibH+b4pT5RrWMjiRZVylNwLcw/2ubDrX5n0oUmHX/ZgudMJeemxzOvz7A==",
+      "version": "4.53.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.53.0.tgz",
+      "integrity": "sha512-XoMaJsSLuedW5eoMEMmZbdNoXgML3ujcU5KfwRnC6rnbmZkHE2Q4J/SArwhqCxQRqJwHnQUj1LpiROmKPExZJA==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -16939,9 +16939,9 @@
       }
     },
     "node_modules/query-string": {
-      "version": "9.0.0",
-      "resolved": "https://registry.npmjs.org/query-string/-/query-string-9.0.0.tgz",
-      "integrity": "sha512-4EWwcRGsO2H+yzq6ddHcVqkCQ2EFUSfDMEjF8ryp8ReymyZhIuaFRGLomeOQLkrzacMHoyky2HW0Qe30UbzkKw==",
+      "version": "9.1.0",
+      "resolved": "https://registry.npmjs.org/query-string/-/query-string-9.1.0.tgz",
+      "integrity": "sha512-t6dqMECpCkqfyv2FfwVS1xcB6lgXW/0XZSaKdsCNGYkqMO76AFiJEg4vINzoDKcZa6MS7JX+OHIjwh06K5vczw==",
       "dependencies": {
         "decode-uri-component": "^0.4.1",
         "filter-obj": "^5.1.0",
@@ -19852,9 +19852,9 @@
       }
     },
     "node_modules/vite-node": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.0.3.tgz",
-      "integrity": "sha512-14jzwMx7XTcMB+9BhGQyoEAmSl0eOr3nrnn+Z12WNERtOvLN+d2scbRUvyni05rT3997Bg+rZb47NyP4IQPKXg==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.0.4.tgz",
+      "integrity": "sha512-ZpJVkxcakYtig5iakNeL7N3trufe3M6vGuzYAr4GsbCTwobDeyPJpE4cjDhhPluv8OvQCFzu2LWp6GkoKRITXA==",
       "dev": true,
       "dependencies": {
         "cac": "^6.7.14",
@@ -19888,18 +19888,18 @@
       }
     },
     "node_modules/vitest": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.0.3.tgz",
-      "integrity": "sha512-o3HRvU93q6qZK4rI2JrhKyZMMuxg/JRt30E6qeQs6ueaiz5hr1cPj+Sk2kATgQzMMqsa2DiNI0TIK++1ULx8Jw==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.0.4.tgz",
+      "integrity": "sha512-luNLDpfsnxw5QSW4bISPe6tkxVvv5wn2BBs/PuDRkhXZ319doZyLOBr1sjfB5yCEpTiU7xCAdViM8TNVGPwoog==",
       "dev": true,
       "dependencies": {
         "@ampproject/remapping": "^2.3.0",
-        "@vitest/expect": "2.0.3",
-        "@vitest/pretty-format": "^2.0.3",
-        "@vitest/runner": "2.0.3",
-        "@vitest/snapshot": "2.0.3",
-        "@vitest/spy": "2.0.3",
-        "@vitest/utils": "2.0.3",
+        "@vitest/expect": "2.0.4",
+        "@vitest/pretty-format": "^2.0.4",
+        "@vitest/runner": "2.0.4",
+        "@vitest/snapshot": "2.0.4",
+        "@vitest/spy": "2.0.4",
+        "@vitest/utils": "2.0.4",
         "chai": "^5.1.1",
         "debug": "^4.3.5",
         "execa": "^8.0.1",
@@ -19910,8 +19910,8 @@
         "tinypool": "^1.0.0",
         "tinyrainbow": "^1.2.0",
         "vite": "^5.0.0",
-        "vite-node": "2.0.3",
-        "why-is-node-running": "^2.2.2"
+        "vite-node": "2.0.4",
+        "why-is-node-running": "^2.3.0"
       },
       "bin": {
         "vitest": "vitest.mjs"
@@ -19925,8 +19925,8 @@
       "peerDependencies": {
         "@edge-runtime/vm": "*",
         "@types/node": "^18.0.0 || >=20.0.0",
-        "@vitest/browser": "2.0.3",
-        "@vitest/ui": "2.0.3",
+        "@vitest/browser": "2.0.4",
+        "@vitest/ui": "2.0.4",
         "happy-dom": "*",
         "jsdom": "*"
       },

package.json

@@ -37,9 +37,9 @@
   "dependencies": {
     "@aitube/broadway": "0.0.22",
     "@aitube/clap": "0.0.30",
-    "@aitube/clapper-services": "0.0.29",
+    "@aitube/clapper-services": "0.0.34",
     "@aitube/engine": "0.0.26",
-    "@aitube/timeline": "0.0.43",
+    "@aitube/timeline": "0.0.44",
     "@fal-ai/serverless-client": "^0.13.0",
     "@ffmpeg/ffmpeg": "^0.12.10",
     "@ffmpeg/util": "^0.12.1",

public/images/onboarding/get-started.xcf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:864be2d20925acf9ff343ac45c3d877c656eb2b474f335316526181a204a82be
+size 70110

public/images/onboarding/pick-an-example.xcf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29d46c5ed0ae1b1a4d9d11cf1103baf52b50ce204221d3667b7f481394ed75ba
+size 107646

src/app/main.tsx

@@ -5,7 +5,6 @@ import { ReflexContainer, ReflexSplitter, ReflexElement } from 'react-reflex'
 import { useSearchParams } from 'next/navigation'
 import { DndProvider, useDrop } from 'react-dnd'
 import { HTML5Backend, NativeTypes } from 'react-dnd-html5-backend'
-import { useTimeline } from '@aitube/timeline'
 
 import { Toaster } from '@/components/ui/sonner'
 import { cn } from '@/lib/utils'
@@ -14,22 +13,23 @@ import { Monitor } from '@/components/monitor'
 
 import { SettingsDialog } from '@/components/settings'
 import { LoadingDialog } from '@/components/dialogs/loader/LoadingDialog'
-import { useUI } from '@/services/ui'
+import { useUI, useIO } from '@/services'
 import { TopBar } from '@/components/toolbars/top-bar'
 import { Timeline } from '@/components/core/timeline'
-import { useIO } from '@/services/io/useIO'
 import { ChatView } from '@/components/assistant/ChatView'
 import { Editors } from '@/components/editors/Editors'
+import { useTheme } from '@/services/ui/useTheme'
 
 type DroppableThing = { files: File[] }
 
 function MainContent() {
   const ref = useRef<HTMLDivElement>(null)
-  const isEmpty = useTimeline((s) => s.isEmpty)
+  const showWelcomeScreen = useUI((s) => s.showWelcomeScreen)
   const showTimeline = useUI((s) => s.showTimeline)
   const showAssistant = useUI((s) => s.showAssistant)
-
+  const theme = useTheme()
   const openFiles = useIO((s) => s.openFiles)
+  const isTopMenuOpen = useUI((s) => s.isTopMenuOpen)
 
   const [{ isOver, canDrop }, connectFileDrop] = useDrop({
     accept: [NativeTypes.FILE],
@@ -67,8 +67,7 @@ function MainContent() {
       <TopBar />
       <div
         className={cn(
-          `flex h-[calc(100vh-40px)] w-screen flex-row overflow-hidden`,
-          isEmpty ? 'opacity-0' : 'opacity-100'
+          `flex h-[calc(100vh-40px)] w-screen flex-row overflow-hidden`
         )}
       >
         <ReflexContainer orientation="vertical">
@@ -109,6 +108,60 @@ function MainContent() {
         </ReflexContainer>
       </div>
 
+      <div
+        className={cn(
+          showWelcomeScreen
+            ? 'pointer-events-auto z-[101] flex'
+            : 'pointer-events-none hidden',
+          `fixed top-[40px] h-[calc(100vh-40px)] w-screen flex-row overflow-hidden`,
+          `items-center justify-center`,
+          `bg-stone-950`
+        )}
+      >
+        <div
+          className="flex h-full w-full items-center justify-center"
+          style={{
+            backgroundImage:
+              'repeating-radial-gradient( circle at 0 0, transparent 0, #000000 7px ), repeating-linear-gradient( #37353455, #373534 )',
+          }}
+        >
+          <div
+            className={cn(
+              `pointer-events-none absolute left-[100px] top-[20px]`,
+              `opacity-90`
+            )}
+          >
+            <img
+              src="/images/onboarding/get-started.png"
+              width="180"
+              className=""
+            ></img>
+          </div>
+          <div
+            className={cn(
+              `pointer-events-none absolute left-[305px] top-[140px]`,
+              `transition-all duration-200 ease-out`,
+              isTopMenuOpen ? 'scale-100 opacity-90' : 'scale-90 opacity-0'
+            )}
+          >
+            <img src="/images/onboarding/pick-an-example.png" width="140"></img>
+          </div>
+          <div className="flex flex-col items-center justify-center space-y-6">
+            <h1 className="text-6xl font-bold">
+              Welcome to{' '}
+              <span className="" style={{ color: theme.defaultPrimaryColor }}>
+                Clapper
+              </span>
+              .
+            </h1>
+            <div className="flex flex-col items-center justify-center space-y-2 text-center text-2xl font-semibold">
+              <p>A free and open-source AI video editor,</p>
+              <p>designed for the age of generative filmmaking.</p>
+            </div>
+          </div>
+        </div>
+      </div>
+
       <SettingsDialog />
       <LoadingDialog />
       <Toaster />

src/components/toolbars/top-menu/file/index.tsx

@@ -17,6 +17,7 @@ import {
 import { useOpenFilePicker, useQueryStringParams } from '@/lib/hooks'
 import { IframeWarning } from '@/components/dialogs/iframe-warning'
 import { useIO, useUI } from '@/services'
+import { newClap } from '@aitube/clap'
 
 export function TopMenuFile() {
   const { clapUrl } = useQueryStringParams({
@@ -44,6 +45,9 @@ export function TopMenuFile() {
 
   const hasBetaAccess = useUI((s) => s.hasBetaAccess)
 
+  const showWelcomeScreen = useUI((s) => s.showWelcomeScreen)
+  const setShowWelcomeScreen = useUI((s) => s.setShowWelcomeScreen)
+
   useEffect(() => {
     ;(async () => {
       if (!clapUrl) {
@@ -67,21 +71,21 @@ export function TopMenuFile() {
       <MenubarMenu>
         <MenubarTrigger>File</MenubarTrigger>
         <MenubarContent>
-          {hasBetaAccess && (
-            <MenubarItem
-              onClick={() => {
-                openClapUrl('/samples/claps/empty_project.clap')
-              }}
-            >
-              New Project<MenubarShortcut>⌘N</MenubarShortcut>
-            </MenubarItem>
-          )}
+          <MenubarItem
+            onClick={() => {
+              setClap(newClap())
+              setShowWelcomeScreen(false)
+            }}
+          >
+            New Project<MenubarShortcut>⌘N</MenubarShortcut>
+          </MenubarItem>
+          <MenubarSeparator />
           <MenubarItem
             onClick={() => {
               openFilePicker()
             }}
           >
-            Open file (.clap, .txt)<MenubarShortcut>⌘O</MenubarShortcut>
+            Open project (.clap)<MenubarShortcut>⌘O</MenubarShortcut>
           </MenubarItem>
           <MenubarItem
             onClick={() => {
@@ -92,7 +96,7 @@ export function TopMenuFile() {
           </MenubarItem>
           <MenubarSeparator />
           <MenubarSub>
-            <MenubarSubTrigger>Examples</MenubarSubTrigger>
+            <MenubarSubTrigger>Import an example</MenubarSubTrigger>
             <MenubarSubContent>
               <MenubarItem
                 onClick={() => {
@@ -157,20 +161,46 @@ export function TopMenuFile() {
           <MenubarSeparator />
           <MenubarItem
             onClick={() => {
-              saveVideoFile()
+              openFilePicker()
+            }}
+          >
+            Import screenplay (.txt)
+          </MenubarItem>
+          <MenubarItem
+            onClick={() => {
+              openFilePicker()
+            }}
+          >
+            Import video (.mp4)
+          </MenubarItem>
+          {/*
+          In case we want to show a video import wizard UI:
+
+          <MenubarItem
+            onClick={() => {
+              openFilePicker()
             }}
           >
-            Export project to MP4
+            Import video (.mp4)
           </MenubarItem>
           <MenubarSeparator />
+          */}
+
+          <MenubarSeparator />
+          <MenubarItem
+            onClick={() => {
+              saveVideoFile()
+            }}
+          >
+            Export full video (.mp4)
+          </MenubarItem>
           <MenubarItem
             onClick={() => {
               saveZipFile()
             }}
           >
-            Export project to .zip
+            Export all assets (.zip)
           </MenubarItem>
-          <MenubarSeparator />
           {/*
         <MenubarItem onClick={() => {
           saveKdenline()

src/components/toolbars/top-menu/index.tsx

@@ -19,10 +19,17 @@ import { TopMenuPlugins } from './plugins'
 export function TopMenu() {
   const isBusyResolving = useResolver((s) => s.isBusyResolving)
 
+  const setIsTopMenuOpen = useUI((s) => s.setIsTopMenuOpen)
+
   const hasBetaAccess = useUI((s) => s.hasBetaAccess)
 
   return (
-    <Menubar className="ml-1 w-full">
+    <Menubar
+      className="ml-1 w-full"
+      onValueChange={(value) => {
+        setIsTopMenuOpen(!!value)
+      }}
+    >
       <TopMenuLogo />
       <TopMenuFile />
       {hasBetaAccess && <TopMenuEdit />}

src/lib/core/constants.ts

@@ -3,7 +3,7 @@
 export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
 
 export const APP_NAME = 'Clapper.app'
-export const APP_REVISION = 'r20240722-0205'
+export const APP_REVISION = 'r20240722-2258'
 
 export const APP_DOMAIN = 'Clapper.app'
 export const APP_LINK = 'https://clapper.app'

src/lib/hooks/useOpenFilePicker.ts

@@ -4,15 +4,24 @@ import { useFilePicker } from 'use-file-picker'
 import { parseFileName } from '@/services/io/parseFileName'
 import { useIO } from '@/services/io/useIO'
 
-const supportedExtensions = ['clap', 'txt']
+const defaultSupportedExtensions = ['clap', 'txt', 'mp4', 'mp3']
 
-export function useOpenFilePicker() {
+export function useOpenFilePicker(
+  {
+    supportedExtensions = defaultSupportedExtensions,
+  }: {
+    supportedExtensions: string[]
+  } = {
+    supportedExtensions: defaultSupportedExtensions,
+  }
+) {
   const [isLoading, setIsLoading] = useState(false)
   const openClapBlob = useIO((s) => s.openClapBlob)
   const openScreenplay = useIO((s) => s.openScreenplay)
+  const openVideo = useIO((s) => s.openVideo)
 
   const { openFilePicker, filesContent, loading } = useFilePicker({
-    accept: ['clap', 'txt'].map((ext) => `.${ext}`),
+    accept: supportedExtensions.map((ext) => `.${ext}`),
     readAs: 'ArrayBuffer',
   })
 
@@ -27,7 +36,7 @@ export function useOpenFilePicker() {
 
       const { fileName, projectName, extension } = parseFileName(input)
 
-      if (!supportedExtensions.includes(extension)) {
+      if (!defaultSupportedExtensions.includes(extension)) {
         console.error(`unsupported extension "${extension}"`)
         return
       }
@@ -52,10 +61,27 @@ export function useOpenFilePicker() {
         } finally {
           setIsLoading(false)
         }
+      } else if (extension === 'mp4') {
+        try {
+          setIsLoading(true)
+          await openVideo(projectName, fileName, blob)
+        } catch (err) {
+          console.error('failed to load the Clap file:', err)
+        } finally {
+          setIsLoading(false)
+        }
+      } else if (extension === 'mp3') {
+        alert('Initializing a project from a mp3 is not supported yet')
       }
     }
     fn()
-  }, [fileData?.name, fileData?.content, openClapBlob, openScreenplay])
+  }, [
+    fileData?.name,
+    fileData?.content,
+    openClapBlob,
+    openScreenplay,
+    openVideo,
+  ])
 
   return {
     openFilePicker,

src/lib/utils/base64DataUriToFile.ts

@@ -1,5 +1,5 @@
 export function base64DataUriToFile(dataUrl: string, fileName: string) {
-  var arr = dataUrl.split(',')
+  var arr = `${dataUrl || ''}`.split(',')
   const st = `${arr[0] || ''}`
   const mime = `${st.match(/:(.*?);/)?.[1] || ''}`
   const bstr = atob(arr[arr.length - 1])

src/services/io/extractFramesFromVideo.ts

@@ -1,6 +1,7 @@
 'use client'
 
 import { FFmpeg } from '@ffmpeg/ffmpeg'
+import { FileData } from '@ffmpeg/ffmpeg/dist/esm/types'
 import { toBlobURL } from '@ffmpeg/util'
 import mediaInfoFactory, {
   Track,
@@ -12,6 +13,7 @@ import mediaInfoFactory, {
   MenuTrack,
   OtherTrack,
 } from 'mediainfo.js'
+import { fileDataToBase64 } from './fileDataToBase64'
 
 interface FrameExtractorOptions {
   format: 'png' | 'jpg'
@@ -20,6 +22,7 @@ interface FrameExtractorOptions {
   sceneSamplingRate: number // Percentage of additional frames between scene changes (0-100)
   onProgress?: (progress: number) => void // Callback function for progress updates
   debug?: boolean
+  autoCrop?: boolean // New option to enable automatic cropping
 }
 
 export async function extractFramesFromVideo(
@@ -114,28 +117,57 @@ export async function extractFramesFromVideo(
   if (options.debug) {
     console.log('input.mp4 written!')
   }
-  // Prepare FFmpeg command
-  const sceneFilter = `select='gt(scene,0.4)'`
-  const additionalFramesFilter = `select='not(mod(n,${Math.floor(100 / options.sceneSamplingRate)}))'`
-  const scaleFilter = `scale='min(${options.maxWidth},iw)':min'(${options.maxHeight},ih)':force_original_aspect_ratio=decrease`
 
-  let lastProgress = 0
-  ffmpeg.on('log', ({ message }) => {
+  let cropParams = ''
+
+  if (options.autoCrop) {
+    // First pass: Detect crop parameters
+    const cropDetectCommand = [
+      '-i',
+      'input.mp4',
+      '-vf',
+      'cropdetect=limit=0.1:round=2:reset=0',
+      '-f',
+      'null',
+      '-t',
+      '10', // Analyze first 10 seconds
+      '-',
+    ]
+
     if (options.debug) {
-      console.log('FFmpeg log:', message)
+      console.log(
+        'Executing crop detection command:',
+        cropDetectCommand.join(' ')
+      )
     }
-    const timeMatch = message.match(/time=(\d{2}):(\d{2}):(\d{2}\.\d{2})/)
-    if (timeMatch) {
-      const [, hours, minutes, seconds] = timeMatch
-      const currentTime =
-        parseInt(hours) * 3600 + parseInt(minutes) * 60 + parseFloat(seconds)
-      const progress = Math.min(100, Math.round((currentTime / duration) * 100))
-      if (progress > lastProgress) {
-        lastProgress = progress
-        options.onProgress?.(progress)
+
+    ffmpeg.on('log', ({ message }) => {
+      const cropMatch = message.match(/crop=(\d+:\d+:\d+:\d+)/)
+      if (cropMatch) {
+        cropParams = cropMatch[1]
       }
+    })
+
+    await ffmpeg.exec(cropDetectCommand)
+
+    if (options.debug) {
+      console.log('Detected crop parameters:', cropParams)
     }
-  })
+
+    if (!cropParams) {
+      console.warn('No crop parameters detected. Proceeding without cropping.')
+    }
+  }
+
+  // Main processing command
+  const sceneFilter = `select='gt(scene,0.2)'`
+  const additionalFramesFilter = `select='not(mod(n,${Math.floor(100 / options.sceneSamplingRate)}))'`
+  const scaleFilter = `scale='min(${options.maxWidth},iw)':min'(${options.maxHeight},ih)':force_original_aspect_ratio=decrease`
+
+  let filterChain = `${sceneFilter},${additionalFramesFilter},${scaleFilter}`
+  if (options.autoCrop && cropParams) {
+    filterChain = `crop=${cropParams},${filterChain}`
+  }
 
   const ffmpegCommand = [
     '-i',
@@ -143,7 +175,7 @@ export async function extractFramesFromVideo(
     '-loglevel',
     'verbose',
     '-vf',
-    `${sceneFilter},${additionalFramesFilter},${scaleFilter}`,
+    filterChain,
     '-vsync',
     '2',
     '-q:v',
@@ -156,9 +188,30 @@ export async function extractFramesFromVideo(
   ]
 
   if (options.debug) {
-    console.log('Executing FFmpeg command:', ffmpegCommand.join(' '))
+    console.log('Executing main FFmpeg command:', ffmpegCommand.join(' '))
   }
 
+  let lastProgress = 0
+  ffmpeg.on('log', ({ message }) => {
+    if (options.debug) {
+      console.log('FFmpeg log:', message)
+    }
+    const timeMatch = message.match(/time=(\d{2}):(\d{2}):(\d{2}\.\d{2})/)
+    if (timeMatch) {
+      const [, hours, minutes, seconds] = timeMatch
+      const currentTime =
+        parseInt(hours) * 3600 + parseInt(minutes) * 60 + parseFloat(seconds)
+      const progress = Math.min(100, Math.round((currentTime / duration) * 100))
+      if (progress > lastProgress) {
+        lastProgress = progress
+        options.onProgress?.(progress)
+      }
+    }
+  })
+
+  if (options.debug) {
+    console.log('Executing FFmpeg command:', ffmpegCommand.join(' '))
+  }
   try {
     await ffmpeg.exec(ffmpegCommand)
   } catch (error) {
@@ -189,16 +242,9 @@ export async function extractFramesFromVideo(
       console.log(`Processing frame file: ${file.name}`)
     }
     try {
-      const frameData = await ffmpeg.readFile(file.name)
-
-      // Convert Uint8Array to Base64 string without using btoa
-      let binary = ''
-      const bytes = new Uint8Array(frameData as any)
-      const len = bytes.byteLength
-      for (let i = 0; i < len; i++) {
-        binary += String.fromCharCode(bytes[i])
-      }
-      const base64Frame = window.btoa(binary)
+      const frameData: FileData = await ffmpeg.readFile(file.name)
+
+      const base64Frame = fileDataToBase64(frameData)
 
       frames.push(`data:image/${options.format};base64,${base64Frame}`)
 

src/services/io/extractScenesFromVideo.ts

@@ -0,0 +1,369 @@
+'use client'
+
+import { FFmpeg } from '@ffmpeg/ffmpeg'
+import { toBlobURL } from '@ffmpeg/util'
+import mediaInfoFactory, { VideoTrack, AudioTrack } from 'mediainfo.js'
+import { fileDataToBase64 } from './fileDataToBase64'
+
+interface ExtractorOptions {
+  frameFormat: 'png' | 'jpg'
+  maxWidth: number
+  maxHeight: number
+  framesPerScene: number
+  onProgress?: (progress: number) => void
+  debug?: boolean
+  autoCrop?: boolean
+  sceneThreshold?: number
+  minSceneDuration?: number
+}
+
+interface SceneData {
+  sceneIndex: number
+  startTimeInMs: number
+  endTimeInMs: number
+  video: string
+  frames: string[]
+}
+
+export async function extractScenesFromVideo(
+  videoBlob: Blob,
+  options: ExtractorOptions
+): Promise<SceneData[]> {
+  const ffmpeg = new FFmpeg()
+  const baseURL = 'https://unpkg.com/@ffmpeg/core@0.12.6/dist/umd'
+
+  try {
+    console.log(`getting duration..`)
+
+    const duration = await getVideoDuration(videoBlob)
+    if (!duration) {
+      throw new Error(`couldn't get the video duration`)
+    }
+    if (options.debug) {
+      console.log('Video duration in seconds:', duration)
+    }
+
+    console.log(`loading FFmpeg..`)
+
+    await ffmpeg.load({
+      coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, 'text/javascript'),
+      wasmURL: await toBlobURL(
+        `${baseURL}/ffmpeg-core.wasm`,
+        'application/wasm'
+      ),
+    })
+
+    if (options.debug) {
+      console.log('FFmpeg loaded')
+    }
+
+    const videoUint8Array = new Uint8Array(await videoBlob.arrayBuffer())
+    await ffmpeg.writeFile('input.mp4', videoUint8Array)
+
+    console.log(`detecting crop parameters..`)
+
+    let cropParams = ''
+    if (options.autoCrop) {
+      cropParams = await detectCropParameters(ffmpeg, options)
+    }
+
+    const sceneThreshold = options.sceneThreshold || 0.2
+    const minSceneDuration = options.minSceneDuration || 1
+
+    const sceneDetectionFilter = `select='gt(scene,${sceneThreshold})'`
+    const scaleFilter = `scale='min(${options.maxWidth},iw)':min'(${options.maxHeight},ih)':force_original_aspect_ratio=decrease`
+
+    let filterChain = `${sceneDetectionFilter},${scaleFilter}`
+    if (cropParams) {
+      filterChain = `crop=${cropParams},${filterChain}`
+    }
+    console.log(`detecting scenes..`)
+
+    const sceneTimestamps = await detectScenes(
+      ffmpeg,
+      filterChain,
+      options,
+      duration
+    )
+
+    console.log(`detected ${sceneTimestamps.length} scenes`)
+
+    const scenes: SceneData[] = []
+
+    for (let i = 0; i < sceneTimestamps.length; i++) {
+      const startTime = sceneTimestamps[i]
+      const endTime =
+        i < sceneTimestamps.length - 1
+          ? sceneTimestamps[i + 1]
+          : duration * 1000
+      const sceneDuration = endTime - startTime
+      console.log(`processing scene ${i}`)
+
+      try {
+        const sceneData = await processScene(
+          ffmpeg,
+          i,
+          startTime,
+          endTime,
+          sceneDuration,
+          options
+        )
+        scenes.push(sceneData)
+      } catch (error) {
+        console.error(`Error processing scene ${i}:`, error)
+      }
+
+      options.onProgress?.(Math.round(((i + 1) / sceneTimestamps.length) * 100))
+    }
+
+    if (options.debug) {
+      console.log(`Total scenes processed: ${scenes.length}`)
+    }
+
+    return scenes
+  } catch (error) {
+    console.error('Error in extractFramesAndScenesFromVideo:', error)
+    throw error
+  } finally {
+    try {
+      await ffmpeg.terminate()
+    } catch (error) {
+      console.error('Error terminating FFmpeg:', error)
+    }
+  }
+}
+
+async function getVideoDuration(
+  videoBlob: Blob,
+  debug: boolean = false
+): Promise<number> {
+  // Initialize MediaInfo
+  const mediaInfo = await mediaInfoFactory({
+    format: 'object',
+    locateFile: () => {
+      return '/wasm/MediaInfoModule.wasm'
+    },
+  })
+
+  // Get video duration using MediaInfo
+  const getSize = () => videoBlob.size
+  const readChunk = (chunkSize: number, offset: number) =>
+    new Promise<Uint8Array>((resolve, reject) => {
+      const reader = new FileReader()
+      reader.onload = (event) => {
+        if (event.target?.result instanceof ArrayBuffer) {
+          resolve(new Uint8Array(event.target.result))
+        } else {
+          reject(new Error('Failed to read chunk'))
+        }
+      }
+      reader.onerror = (error) => reject(error)
+      reader.readAsArrayBuffer(videoBlob.slice(offset, offset + chunkSize))
+    })
+
+  if (debug) {
+    console.log('calling await mediaInfo.analyzeData(getSize, readChunk)')
+  }
+
+  const result = await mediaInfo.analyzeData(getSize, readChunk)
+  if (debug) {
+    console.log('result = ', result)
+  }
+
+  let duration: number = 0
+
+  for (const track of result.media?.track || []) {
+    if (debug) {
+      console.log('track = ', track)
+    }
+
+    let maybeDuration: number = 0
+    if (track['@type'] === 'Audio') {
+      const audioTrack = track as AudioTrack
+      maybeDuration = audioTrack.Duration
+        ? parseFloat(`${audioTrack.Duration || 0}`)
+        : 0
+    } else if (track['@type'] === 'Video') {
+      const videoTrack = track as VideoTrack
+      maybeDuration = videoTrack.Duration
+        ? parseFloat(`${videoTrack.Duration || 0}`)
+        : 0
+    }
+    if (
+      typeof maybeDuration === 'number' &&
+      isFinite(maybeDuration) &&
+      !isNaN(maybeDuration)
+    ) {
+      duration = maybeDuration
+    }
+  }
+  return duration
+}
+
+async function detectCropParameters(
+  ffmpeg: FFmpeg,
+  options: ExtractorOptions
+): Promise<string> {
+  const cropDetectCommand = [
+    '-i',
+    'input.mp4',
+    '-vf',
+    'cropdetect=limit=0.1:round=2:reset=0',
+    '-f',
+    'null',
+    '-t',
+    '10',
+    '-',
+  ]
+
+  if (options.debug) {
+    console.log(
+      'Executing crop detection command:',
+      cropDetectCommand.join(' ')
+    )
+  }
+
+  let cropParams = ''
+  ffmpeg.on('log', ({ message }) => {
+    const cropMatch = message.match(/crop=(\d+:\d+:\d+:\d+)/)
+    if (cropMatch) {
+      cropParams = cropMatch[1]
+    }
+  })
+
+  await ffmpeg.exec(cropDetectCommand)
+
+  if (options.debug) {
+    console.log('Detected crop parameters:', cropParams)
+  }
+
+  return cropParams
+}
+
+async function detectScenes(
+  ffmpeg: FFmpeg,
+  filterChain: string,
+  options: ExtractorOptions,
+  duration: number
+): Promise<number[]> {
+  const extractScenesCommand = [
+    '-i',
+    'input.mp4',
+    '-filter_complex',
+    `${filterChain},metadata=print:file=scenes.txt`,
+    '-f',
+    'null',
+    '-',
+  ]
+
+  if (options.debug) {
+    console.log(
+      'Executing scene detection command:',
+      extractScenesCommand.join(' ')
+    )
+  }
+
+  await ffmpeg.exec(extractScenesCommand)
+
+  const scenesMetadata = await ffmpeg.readFile('scenes.txt')
+  const decodedMetadata = new TextDecoder().decode(scenesMetadata as Uint8Array)
+
+  if (options.debug) {
+    console.log('Scenes metadata:', decodedMetadata)
+  }
+
+  const sceneTimestamps = decodedMetadata
+    .split('\n')
+    .filter((line) => line.includes('pts_time'))
+    .map((line) => parseFloat(line.split('pts_time:')[1]) * 1000) // Convert to milliseconds
+
+  // Add start and end timestamps
+  sceneTimestamps.unshift(0)
+  sceneTimestamps.push(duration * 1000)
+
+  // Filter out scenes that are too short
+  const filteredScenes = sceneTimestamps.filter((timestamp, index, array) => {
+    if (index === 0) return true
+    const sceneDuration = timestamp - array[index - 1]
+    return sceneDuration >= (options.minSceneDuration || 1) * 1000
+  })
+
+  return filteredScenes
+}
+
+async function processScene(
+  ffmpeg: FFmpeg,
+  index: number,
+  startTime: number,
+  endTime: number,
+  duration: number,
+  options: ExtractorOptions
+): Promise<SceneData> {
+  const extractSceneCommand = [
+    '-ss',
+    (startTime / 1000).toString(),
+    '-i',
+    'input.mp4',
+    '-t',
+    (duration / 1000).toString(),
+    '-c:v',
+    'libx264',
+    '-preset',
+    'ultrafast',
+    '-crf',
+    '23',
+    '-c:a',
+    'aac',
+    `scene_${index}.mp4`,
+  ]
+  // console.log(`calling ffmpeg.exec(extractSceneCommand)`, extractSceneCommand)
+  await ffmpeg.exec(extractSceneCommand)
+
+  // Calculate frame interval to get the desired number of frames
+  const frameInterval = Math.max(
+    1,
+    Math.floor(duration / (1000 * options.framesPerScene))
+  )
+
+  const extractFramesCommand = [
+    '-i',
+    `scene_${index}.mp4`,
+    '-vf',
+    `select='not(mod(n,${frameInterval}))',setpts=N/FRAME_RATE/TB`,
+    '-frames:v',
+    options.framesPerScene.toString(),
+    '-vsync',
+    '0',
+    '-q:v',
+    '2',
+    '-f',
+    'image2',
+    `scene_${index}_frame_%03d.${options.frameFormat}`,
+  ]
+  // console.log(`calling ffmpeg.exec(extractFramesCommand)`, extractFramesCommand)
+  await ffmpeg.exec(extractFramesCommand)
+
+  const sceneVideo = await ffmpeg.readFile(`scene_${index}.mp4`)
+  const frameFiles = (await ffmpeg.listDir('/')).filter(
+    (file) =>
+      file.name.startsWith(`scene_${index}_frame_`) &&
+      file.name.endsWith(`.${options.frameFormat}`)
+  )
+
+  const frames: string[] = []
+  for (const frameFile of frameFiles) {
+    const frameData = await ffmpeg.readFile(frameFile.name)
+    const base64Frame = fileDataToBase64(frameData)
+    frames.push(`data:image/${options.frameFormat};base64,${base64Frame}`)
+  }
+
+  const base64Video = fileDataToBase64(sceneVideo)
+
+  return {
+    sceneIndex: index,
+    startTimeInMs: Math.round(startTime),
+    endTimeInMs: Math.round(endTime),
+    video: `data:video/mp4;base64,${base64Video}`,
+    frames,
+  }
+}

src/services/io/fileDataToBase64.ts

@@ -0,0 +1,13 @@
+import { FileData } from '@ffmpeg/ffmpeg/dist/esm/types'
+
+export function fileDataToBase64(fileData: FileData): string {
+  // Convert Uint8Array to Base64 string without using btoa
+  let binary = ''
+  const bytes = new Uint8Array(fileData as any)
+  const len = bytes.byteLength
+  for (let i = 0; i < len; i++) {
+    binary += String.fromCharCode(bytes[i])
+  }
+
+  return window.btoa(binary)
+}

src/services/io/parseFileIntoSegments.ts

@@ -81,7 +81,41 @@ export async function parseFileIntoSegments({
         ? maybeEndTimeInMs!
         : startTimeInMs + durationInMs
 
-      const newSegmentData: Partial<TimelineSegment> = {
+      const partialVideo: Partial<TimelineSegment> = {
+        category: ClapSegmentCategory.VIDEO,
+        startTimeInMs,
+        endTimeInMs,
+
+        prompt: 'movie',
+        label: 'movie', // `${file.name.split(".")[0] || "Untitled"}`, // a short label to name the segment (optional, can be human or LLM-defined)
+
+        outputType: ClapOutputType.VIDEO,
+        status: ClapSegmentStatus.TO_GENERATE,
+
+        assetUrl: '',
+        assetDurationInMs: durationInMs,
+        assetSourceType: ClapAssetSource.EMPTY,
+        assetFileFormat: undefined,
+        track: track ? track : undefined,
+      }
+
+      const video = await clapSegmentToTimelineSegment(newSegment(partialVideo))
+
+      if (isValidNumber(track)) {
+        video.track = track
+      }
+
+      video.outputType = ClapOutputType.VIDEO
+
+      // we assume we want it to be immediately visible
+      video.visibility = SegmentVisibility.VISIBLE
+
+      // console.log("newSegment:", audioSegment)
+
+      // poof! type disappears.. it's magic
+      newSegments.push(video)
+
+      const partialStoryboard: Partial<TimelineSegment> = {
         prompt: 'Storyboard', // note: this can be set later with an automatic captioning worker
         startTimeInMs, // start time of the segment
         endTimeInMs, // end time of the segment (startTimeInMs + durationInMs)
@@ -90,28 +124,31 @@ export async function parseFileIntoSegments({
         label: `${file.name}`, // a short label to name the segment (optional, can be human or LLM-defined)
         category,
         assetUrl,
-        assetDurationInMs: endTimeInMs,
+        assetDurationInMs: durationInMs,
         assetSourceType: ClapAssetSource.DATA,
         assetFileFormat: `${file.type}`,
+
+        // important: we try to go below
+        track: track ? track + 1 : undefined,
       }
 
-      const timelineSegment = await clapSegmentToTimelineSegment(
-        newSegment(newSegmentData)
+      const storyboard = await clapSegmentToTimelineSegment(
+        newSegment(partialStoryboard)
       )
 
       if (isValidNumber(track)) {
-        timelineSegment.track = track
+        storyboard.track = track
       }
 
-      timelineSegment.outputType = ClapOutputType.IMAGE
+      storyboard.outputType = ClapOutputType.IMAGE
 
       // we assume we want it to be immediately visible
-      timelineSegment.visibility = SegmentVisibility.VISIBLE
+      storyboard.visibility = SegmentVisibility.VISIBLE
 
       // console.log("newSegment:", audioSegment)
 
       // poof! type disappears.. it's magic
-      newSegments.push(timelineSegment)
+      newSegments.push(storyboard)
       break
     }
 

src/services/io/useIO.ts

@@ -3,7 +3,10 @@
 import {
   ClapAssetSource,
   ClapEntity,
+  ClapMediaOrientation,
+  ClapOutputType,
   ClapProject,
+  ClapSegment,
   ClapSegmentCategory,
   ClapSegmentStatus,
   getClapAssetSourceType,
@@ -11,6 +14,7 @@ import {
   newSegment,
   parseClap,
   serializeClap,
+  UUID,
 } from '@aitube/clap'
 import {
   TimelineStore,
@@ -19,6 +23,7 @@ import {
   removeFinalVideosAndConvertToTimelineSegments,
   getFinalVideo,
   DEFAULT_DURATION_IN_MS_PER_STEP,
+  clapSegmentToTimelineSegment,
 } from '@aitube/timeline'
 import { ParseScriptProgressUpdate, parseScriptToClap } from '@aitube/broadway'
 import { IOStore, TaskCategory, TaskVisibility } from '@aitube/clapper-services'
@@ -43,9 +48,11 @@ import {
 import { sleep } from '@/lib/utils/sleep'
 import { FFMPegAudioInput, FFMPegVideoInput } from './ffmpegUtils'
 import { createFullVideo } from './createFullVideo'
-import { extractFramesFromVideo } from './extractFramesFromVideo'
+import { extractScenesFromVideo } from './extractScenesFromVideo'
 import { extractCaptionsFromFrames } from './extractCaptionsFromFrames'
 import { base64DataUriToFile } from '@/lib/utils/base64DataUriToFile'
+import { useUI } from '../ui'
+import { getTypeAndExtension } from '@/lib/utils/getTypeAndExtension'
 
 export const useIO = create<IOStore>((set, get) => ({
   ...getDefaultIOState(),
@@ -59,7 +66,7 @@ export const useIO = create<IOStore>((set, get) => ({
     timeline.clear()
   },
   openFiles: async (files: File[]) => {
-    const { openClapBlob, openScreenplay } = get()
+    const { openClapBlob, openScreenplay, openVideo } = get()
     const timeline: TimelineStore = useTimeline.getState()
     const { segments, addSegments } = timeline
 
@@ -101,101 +108,166 @@ export const useIO = create<IOStore>((set, get) => ({
         const newSegments = await parseFileIntoSegments({ file })
 
         console.log('calling  timeline.addSegments with:', newSegments)
-        await timeline.addSegments({
-          segments: newSegments,
-        })
+        await timeline.addSegments({ segments: newSegments })
+
         return
       }
 
       const isVideoFile = fileType.startsWith('video/')
       if (isVideoFile) {
-        const storyboardExtractionTask = useTasks.getState().add({
-          category: TaskCategory.IMPORT,
-          visibility: TaskVisibility.BLOCKER,
-          initialMessage: `Extracting storyboards..`,
-          successMessage: `Extracting storyboards.. 100% done`,
-          value: 0,
-        })
+        await openVideo(projectName, fileName, file)
+        return
+      }
+    }
+    useUI.getState().setShowWelcomeScreen(false)
+  },
+  openVideo: async (
+    projectName: string,
+    fileName: string,
+    fileContent: string | Blob
+  ): Promise<void> => {
+    const timeline: TimelineStore = useTimeline.getState()
 
-        const frames = await extractFramesFromVideo(file, {
-          format: 'png', // in theory we could also use 'jpg', but this freezes FFmpeg
-          maxWidth: 1024,
-          maxHeight: 576,
-          sceneSamplingRate: 100,
-          onProgress: (progress: number) => {
-            storyboardExtractionTask.setProgress({
-              message: `Extracting storyboards.. ${progress}% done`,
-              value: progress,
-            })
-          },
+    const sceneExtractionTask = useTasks.getState().add({
+      category: TaskCategory.IMPORT,
+      visibility: TaskVisibility.BLOCKER,
+      initialMessage: `Starting up, can take a few minutes..`,
+      successMessage: `Extracting scenes.. 100%`,
+      value: 0,
+    })
+
+    const file =
+      typeof fileContent === 'string'
+        ? base64DataUriToFile(fileContent, fileName)
+        : fileContent
+
+    const scenes = await extractScenesFromVideo(file, {
+      frameFormat: 'png', // in theory we could also use 'jpg', but this freezes FFmpeg
+      maxWidth: 1024,
+      maxHeight: 576,
+      framesPerScene: 1,
+      autoCrop: true,
+      sceneThreshold: 0.1,
+      minSceneDuration: 1,
+      debug: true,
+      onProgress: (progress: number) => {
+        sceneExtractionTask.setProgress({
+          message: `Extracting scenes.. ${progress}%`,
+          value: progress,
         })
+      },
+    })
 
-        // optional: reset the project
-        // await timeline.setClap(newClap())
-
-        const track = 1
-        let i = 0
-        let startTimeInMs = 0
-        const durationInSteps = 4
-        const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP
-        let endTimeInMs = startTimeInMs + durationInMs
-
-        for (const frame of frames) {
-          const frameFile = base64DataUriToFile(frame, `storyboard_${i++}.png`)
-          const newSegments = await parseFileIntoSegments({
-            file: frameFile,
-            startTimeInMs,
-            endTimeInMs,
-            track,
-          })
+    // optional: reset the project
+    // await timeline.setClap(newClap())
+
+    let currentStoryboardIndex = 0
+    let startTimeInMs = 0
+    const durationInSteps = 4
+    const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP
+    let endTimeInMs = startTimeInMs + durationInMs
+
+    // TODO: extract info from the original video to determine things like
+    // the orientation, duration..
+    timeline.setClap(
+      newClap({
+        meta: {
+          id: UUID(),
+          title: projectName,
+          description: `${projectName} (${fileName})`,
+          synopsis: '',
+          licence:
+            "This OpenClap file is just a conversion from the original screenplay and doesn't claim any copyright or intellectual property. All rights reserved to the original intellectual property and copyright holders. Using OpenClap isn't piracy.",
+
+          orientation: ClapMediaOrientation.LANDSCAPE,
+          durationInMs: frames.length * durationInMs,
+
+          // TODO: those should come from the Clapper user settings
+
+          width: 1024,
+          height: 576,
+
+          defaultVideoModel: '', // <-- we should deprecate this no?
+          extraPositivePrompt: '',
+          screenplay: '',
+          isLoop: false,
+          isInteractive: false,
+        },
+      })
+    )
 
-          startTimeInMs += durationInMs
-          endTimeInMs += durationInMs
+    for (const scene of scenes) {
+      console.log('parsing scene:', scene)
+      try {
+        const frameFile = base64DataUriToFile(
+          scene.frames[0],
+          `storyboard_${++currentStoryboardIndex}.png`
+        )
 
-          console.log('calling timeline.addSegments with:', newSegments)
-          await timeline.addSegments({
-            segments: newSegments,
-            track,
-          })
-        }
+        const assetDurationInMs = scene.endTimeInMs - scene.startTimeInMs
+
+        // this returns multiple segments (video, image..)
+        const newSegments = await parseFileIntoSegments({
+          file: frameFile,
+          startTimeInMs: scene.startTimeInMs,
+          endTimeInMs: scene.endTimeInMs,
+        })
 
-        storyboardExtractionTask.success()
+        for (const newSegment of newSegments) {
+          newSegment.assetDurationInMs = assetDurationInMs
+          if (newSegment.category === ClapSegmentCategory.VIDEO) {
+            const { assetFileFormat, outputType } = getTypeAndExtension(
+              scene.video
+            )
+            newSegment.assetFileFormat = assetFileFormat
+            newSegment.assetUrl = scene.video
+            newSegment.status = ClapSegmentStatus.COMPLETED
+            newSegment.outputType = outputType
+          }
+        }
+        await timeline.addSegments({ segments: newSegments })
+      } catch (err) {
+        console.error(`failed to process scene:`, scene)
+        console.error(err)
+      }
+    }
 
-        const enableCaptioning = false
+    sceneExtractionTask.success()
 
-        if (enableCaptioning) {
-          const captioningTask = useTasks.getState().add({
-            category: TaskCategory.IMPORT,
-            // visibility: TaskVisibility.BLOCKER,
+    const enableCaptioning = false
 
-            // since this is very long task, we can run it in the background
-            visibility: TaskVisibility.BACKGROUND,
-            initialMessage: `Analyzing storyboards..`,
-            successMessage: `Analyzing storyboards.. 100% done`,
-            value: 0,
-          })
+    if (enableCaptioning) {
+      const captioningTask = useTasks.getState().add({
+        category: TaskCategory.IMPORT,
+        // visibility: TaskVisibility.BLOCKER,
 
-          console.log('calling extractCaptionsFromFrames() with:', frames)
-          const captions = await extractCaptionsFromFrames(
-            frames,
-            (
-              progress: number,
-              storyboardIndex: number,
-              nbStoryboards: number
-            ) => {
-              captioningTask.setProgress({
-                message: `Analyzing storyboards (${progress}%)`,
-                value: progress,
-              })
-            }
-          )
-          console.log('captions:', captions)
-          // TODO: add
+        // since this is very long task, we can run it in the background
+        visibility: TaskVisibility.BACKGROUND,
+        initialMessage: `Analyzing storyboards..`,
+        successMessage: `Analyzing storyboards.. 100% done`,
+        value: 0,
+      })
 
-          captioningTask.success()
+      console.log('calling extractCaptionsFromFrames() with:', frames)
+      /*
+      const captions = await extractCaptionsFromFrames(
+        frames,
+        (progress: number, storyboardIndex: number, nbStoryboards: number) => {
+          captioningTask.setProgress({
+            message: `Analyzing storyboards (${progress}%)`,
+            value: progress,
+          })
         }
-      }
+      )
+  
+      console.log('captions:', captions)
+      */
+      // TODO: add
+
+      captioningTask.success()
     }
+
+    useUI.getState().setShowWelcomeScreen(false)
   },
   openScreenplay: async (
     projectName: string,
@@ -270,6 +342,7 @@ export const useIO = create<IOStore>((set, get) => ({
       task.fail(`${err || 'unknown screenplay import error'}`)
     } finally {
     }
+    useUI.getState().setShowWelcomeScreen(false)
   },
   openScreenplayUrl: async (url: string) => {
     const timeline: TimelineStore = useTimeline.getState()
@@ -326,6 +399,7 @@ export const useIO = create<IOStore>((set, get) => ({
     } catch (err) {
       task.fail(`${err || 'unknown error'}`)
     }
+    useUI.getState().setShowWelcomeScreen(false)
   },
   saveAnyFile: (blob: Blob, fileName: string) => {
     // Create an object URL for the compressed clap blob
@@ -391,6 +465,7 @@ export const useIO = create<IOStore>((set, get) => ({
     } catch (err) {
       task.fail(`${err || 'unknown error'}`)
     }
+    useUI.getState().setShowWelcomeScreen(false)
   },
   openClapBlob: async (projectName: string, fileName: string, blob: Blob) => {
     const timeline: TimelineStore = useTimeline.getState()
@@ -423,6 +498,7 @@ export const useIO = create<IOStore>((set, get) => ({
     } catch (err) {
       task.fail(`${err || 'unknown error'}`)
     }
+    useUI.getState().setShowWelcomeScreen(false)
   },
   saveClap: async () => {
     const { saveAnyFile } = get()
@@ -726,7 +802,9 @@ export const useIO = create<IOStore>((set, get) => ({
     }
   },
 
-  openMLT: async (file: File) => {},
+  openMLT: async (file: File) => {
+    useUI.getState().setShowWelcomeScreen(false)
+  },
   saveMLT: async () => {},
   generateMLT: async (): Promise<string> => {
     const timeline: TimelineStore = useTimeline.getState()
@@ -1001,7 +1079,9 @@ export const useIO = create<IOStore>((set, get) => ({
 </mlt>`
   },
 
-  openKdenline: async (file: File) => {},
+  openKdenline: async (file: File) => {
+    useUI.getState().setShowWelcomeScreen(false)
+  },
 
   saveKdenline: async () => {
     const { saveAnyFile } = get()
@@ -1062,7 +1142,9 @@ export const useIO = create<IOStore>((set, get) => ({
     */
   },
 
-  openOpenTimelineIO: async (file: File) => {},
+  openOpenTimelineIO: async (file: File) => {
+    useUI.getState().setShowWelcomeScreen(false)
+  },
 
   saveOpenTimelineIO: async () => {},
 
@@ -1086,6 +1168,9 @@ export const useIO = create<IOStore>((set, get) => ({
     }
 
     const { entities } = await parseClap(file)
+
+    useUI.getState().setShowWelcomeScreen(false)
+
     return entities
   },
 }))

src/services/resolver/useResolver.ts

@@ -597,10 +597,12 @@ export const useResolver = create<ResolverStore>((set, get) => ({
       ) as TimelineSegment
 
       if (newSegment.outputType === ClapOutputType.AUDIO) {
-        try {
-          newSegment.audioBuffer = await getAudioBuffer(newSegment.assetUrl)
-        } catch (err) {
-          console.error(`failed to load the audio file: ${err}`)
+        if (newSegment.assetUrl) {
+          try {
+            newSegment.audioBuffer = await getAudioBuffer(newSegment.assetUrl)
+          } catch (err) {
+            console.error(`failed to load the audio file: ${err}`)
+          }
         }
       }
 

src/services/ui/getDefaultUIState.ts

@@ -6,6 +6,8 @@ import {
 
 export function getDefaultUIState(): UIState {
   const state: UIState = {
+    isTopMenuOpen: false,
+    showWelcomeScreen: true,
     hasBetaAccess: false,
     themeName: 'backstage',
     showApiKeys: false,

src/services/ui/useUI.ts

@@ -21,6 +21,13 @@ export const useUI = create<UIStore>()(
   persist(
     (set, get) => ({
       ...getDefaultUIState(),
+      setIsTopMenuOpen: (isTopMenuOpen: boolean) => {
+        set({ isTopMenuOpen })
+      },
+      setShowWelcomeScreen: (showWelcomeScreen: boolean) => {
+        console.log('setShowWelcomeScreen called with:', showWelcomeScreen)
+        set({ showWelcomeScreen: showWelcomeScreen })
+      },
       setHasBetaAccess: (hasBetaAccess: boolean) => {
         set({ hasBetaAccess })
       },