diff --git a/package-lock.json b/package-lock.json index 044f2114fa5bc40895f033fb7e74b4961d4125b9..4517fc0889fae8c477b2dddd72d14a1ffa9e2670 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,8 +10,10 @@ "dependencies": { "@aitube/clap": "0.0.24", "@aitube/engine": "0.0.15", - "@aitube/timeline": "0.0.8", + "@aitube/timeline": "0.0.11", + "@fal-ai/serverless-client": "^0.10.3", "@huggingface/hub": "^0.15.0", + "@huggingface/inference": "^2.7.0", "@radix-ui/react-accordion": "^1.1.2", "@radix-ui/react-avatar": "^1.0.4", "@radix-ui/react-checkbox": "^1.0.4", @@ -100,9 +102,9 @@ } }, "node_modules/@aitube/timeline": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.8.tgz", - "integrity": "sha512-+Eqhdr2tGCU46r9mJ+H/so+QB6BObqerI5j3oG2iSaMSalFIgP1K8zuigMhN/FzKkESF3ORY77ciZnZupcmBDA==", + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.11.tgz", + "integrity": "sha512-/ndCCCdh0kY4gSObxS+cSdN+G5B2fdDKHgcAT5lrtifsenERFuWza4nUd7JbGQCS1/8AKs7OFrE6lvP243KBvw==", "dependencies": { "date-fns": "^3.6.0", "react-virtualized-auto-sizer": "^1.0.24" @@ -149,11 +151,11 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.6.tgz", - "integrity": "sha512-ZJhac6FkEd1yhG2AHOmfcXG4ceoLltoCVJjN5XsWN9BifBQr+cHJbWi0h68HZuSORq+3WtJ2z0hwF2NG1b5kcA==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz", + "integrity": "sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==", "dependencies": { - "@babel/highlight": "^7.24.6", + "@babel/highlight": "^7.24.7", "picocolors": "^1.0.0" }, "engines": { @@ -161,30 +163,30 @@ } }, "node_modules/@babel/compat-data": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.6.tgz", - "integrity": "sha512-aC2DGhBq5eEdyXWqrDInSqQjO0k8xtPRf5YylULqx8MCd6jBtzqfta/3ETMRpuKIc5hyswfO80ObyA1MvkCcUQ==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.7.tgz", + "integrity": "sha512-qJzAIcv03PyaWqxRgO4mSU3lihncDT296vnyuE2O8uA4w3UHWI4S3hgeZd1L8W1Bft40w9JxJ2b412iDUFFRhw==", "peer": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/core": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.6.tgz", - "integrity": "sha512-qAHSfAdVyFmIvl0VHELib8xar7ONuSHrE2hLnsaWkYNTI68dmi1x8GYDhJjMI/e7XWal9QBlZkwbOnkcw7Z8gQ==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.7.tgz", + "integrity": "sha512-nykK+LEK86ahTkX/3TgauT0ikKoNCfKHEaZYTUVupJdTLzGNvrblu4u6fa7DhZONAltdf8e662t/abY8idrd/g==", "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.24.6", - "@babel/generator": "^7.24.6", - "@babel/helper-compilation-targets": "^7.24.6", - "@babel/helper-module-transforms": "^7.24.6", - "@babel/helpers": "^7.24.6", - "@babel/parser": "^7.24.6", - "@babel/template": "^7.24.6", - "@babel/traverse": "^7.24.6", - "@babel/types": "^7.24.6", + "@babel/code-frame": "^7.24.7", + "@babel/generator": "^7.24.7", + "@babel/helper-compilation-targets": "^7.24.7", + "@babel/helper-module-transforms": "^7.24.7", + "@babel/helpers": "^7.24.7", + "@babel/parser": "^7.24.7", + "@babel/template": "^7.24.7", + "@babel/traverse": "^7.24.7", + "@babel/types": "^7.24.7", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", @@ -209,11 +211,11 @@ } }, "node_modules/@babel/generator": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.6.tgz", - "integrity": "sha512-S7m4eNa6YAPJRHmKsLHIDJhNAGNKoWNiWefz1MBbpnt8g9lvMDl1hir4P9bo/57bQEmuwEhnRU/AMWsD0G/Fbg==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.7.tgz", + "integrity": "sha512-oipXieGC3i45Y1A41t4tAqpnEZWgB/lC6Ehh6+rOviR5XWpTtMmLN+fGjz9vOiNRt0p6RtO6DtD0pdU3vpqdSA==", "dependencies": { - "@babel/types": "^7.24.6", + "@babel/types": "^7.24.7", "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.25", "jsesc": "^2.5.1" @@ -223,24 +225,24 @@ } }, "node_modules/@babel/helper-annotate-as-pure": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.24.6.tgz", - "integrity": "sha512-DitEzDfOMnd13kZnDqns1ccmftwJTS9DMkyn9pYTxulS7bZxUxpMly3Nf23QQ6NwA4UB8lAqjbqWtyvElEMAkg==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.24.7.tgz", + "integrity": "sha512-BaDeOonYvhdKw+JoMVkAixAAJzG2jVPIwWoKBPdYuY9b452e2rPuI9QPYh3KpofZ3pW2akOmwZLOiOsHMiqRAg==", "dependencies": { - "@babel/types": "^7.24.6" + "@babel/types": "^7.24.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.24.6.tgz", - "integrity": "sha512-VZQ57UsDGlX/5fFA7GkVPplZhHsVc+vuErWgdOiysI9Ksnw0Pbbd6pnPiR/mmJyKHgyIW0c7KT32gmhiF+cirg==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.24.7.tgz", + "integrity": "sha512-ctSdRHBi20qWOfy27RUb4Fhp07KSJ3sXcuSvTrXrc4aG8NSYDo1ici3Vhg9bg69y5bj0Mr1lh0aeEgTvc12rMg==", "peer": true, "dependencies": { - "@babel/compat-data": "^7.24.6", - "@babel/helper-validator-option": "^7.24.6", + "@babel/compat-data": "^7.24.7", + "@babel/helper-validator-option": "^7.24.7", "browserslist": "^4.22.2", "lru-cache": "^5.1.1", "semver": "^6.3.1" @@ -268,58 +270,62 @@ } }, "node_modules/@babel/helper-environment-visitor": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.24.6.tgz", - "integrity": "sha512-Y50Cg3k0LKLMjxdPjIl40SdJgMB85iXn27Vk/qbHZCFx/o5XO3PSnpi675h1KEmmDb6OFArfd5SCQEQ5Q4H88g==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.24.7.tgz", + "integrity": "sha512-DoiN84+4Gnd0ncbBOM9AZENV4a5ZiL39HYMyZJGZ/AZEykHYdJw0wW3kdcsh9/Kn+BRXHLkkklZ51ecPKmI1CQ==", + "dependencies": { + "@babel/types": "^7.24.7" + }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-function-name": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.24.6.tgz", - "integrity": "sha512-xpeLqeeRkbxhnYimfr2PC+iA0Q7ljX/d1eZ9/inYbmfG2jpl8Lu3DyXvpOAnrS5kxkfOWJjioIMQsaMBXFI05w==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.24.7.tgz", + "integrity": "sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA==", "dependencies": { - "@babel/template": "^7.24.6", - "@babel/types": "^7.24.6" + "@babel/template": "^7.24.7", + "@babel/types": "^7.24.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-hoist-variables": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.24.6.tgz", - "integrity": "sha512-SF/EMrC3OD7dSta1bLJIlrsVxwtd0UpjRJqLno6125epQMJ/kyFmpTT4pbvPbdQHzCHg+biQ7Syo8lnDtbR+uA==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.24.7.tgz", + "integrity": "sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ==", "dependencies": { - "@babel/types": "^7.24.6" + "@babel/types": "^7.24.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-imports": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.6.tgz", - "integrity": "sha512-a26dmxFJBF62rRO9mmpgrfTLsAuyHk4e1hKTUkD/fcMfynt8gvEKwQPQDVxWhca8dHoDck+55DFt42zV0QMw5g==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz", + "integrity": "sha512-8AyH3C+74cgCVVXow/myrynrAGv+nTVg5vKu2nZph9x7RcRwzmh0VFallJuFTZ9mx6u4eSdXZfcOzSqTUm0HCA==", "dependencies": { - "@babel/types": "^7.24.6" + "@babel/traverse": "^7.24.7", + "@babel/types": "^7.24.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.24.6.tgz", - "integrity": "sha512-Y/YMPm83mV2HJTbX1Qh2sjgjqcacvOlhbzdCCsSlblOKjSYmQqEbO6rUniWQyRo9ncyfjT8hnUjlG06RXDEmcA==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.24.7.tgz", + "integrity": "sha512-1fuJEwIrp+97rM4RWdO+qrRsZlAeL1lQJoPqtCYWv0NL115XM93hIH4CSRln2w52SqvmY5hqdtauB6QFCDiZNQ==", "peer": true, "dependencies": { - "@babel/helper-environment-visitor": "^7.24.6", - "@babel/helper-module-imports": "^7.24.6", - "@babel/helper-simple-access": "^7.24.6", - "@babel/helper-split-export-declaration": "^7.24.6", - "@babel/helper-validator-identifier": "^7.24.6" + "@babel/helper-environment-visitor": "^7.24.7", + "@babel/helper-module-imports": "^7.24.7", + "@babel/helper-simple-access": "^7.24.7", + "@babel/helper-split-export-declaration": "^7.24.7", + "@babel/helper-validator-identifier": "^7.24.7" }, "engines": { "node": ">=6.9.0" @@ -329,80 +335,81 @@ } }, "node_modules/@babel/helper-plugin-utils": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.6.tgz", - "integrity": "sha512-MZG/JcWfxybKwsA9N9PmtF2lOSFSEMVCpIRrbxccZFLJPrJciJdG/UhSh5W96GEteJI2ARqm5UAHxISwRDLSNg==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.7.tgz", + "integrity": "sha512-Rq76wjt7yz9AAc1KnlRKNAi/dMSVWgDRx43FHoJEbcYU6xOWaE2dVPwcdTukJrjxS65GITyfbvEYHvkirZ6uEg==", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-simple-access": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.24.6.tgz", - "integrity": "sha512-nZzcMMD4ZhmB35MOOzQuiGO5RzL6tJbsT37Zx8M5L/i9KSrukGXWTjLe1knIbb/RmxoJE9GON9soq0c0VEMM5g==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.24.7.tgz", + "integrity": "sha512-zBAIvbCMh5Ts+b86r/CjU+4XGYIs+R1j951gxI3KmmxBMhCg4oQMsv6ZXQ64XOm/cvzfU1FmoCyt6+owc5QMYg==", "peer": true, "dependencies": { - "@babel/types": "^7.24.6" + "@babel/traverse": "^7.24.7", + "@babel/types": "^7.24.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-split-export-declaration": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.24.6.tgz", - "integrity": "sha512-CvLSkwXGWnYlF9+J3iZUvwgAxKiYzK3BWuo+mLzD/MDGOZDj7Gq8+hqaOkMxmJwmlv0iu86uH5fdADd9Hxkymw==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.24.7.tgz", + "integrity": "sha512-oy5V7pD+UvfkEATUKvIjvIAH/xCzfsFVw7ygW2SI6NClZzquT+mwdTfgfdbUiceh6iQO0CHtCPsyze/MZ2YbAA==", "dependencies": { - "@babel/types": "^7.24.6" + "@babel/types": "^7.24.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-string-parser": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.6.tgz", - "integrity": "sha512-WdJjwMEkmBicq5T9fm/cHND3+UlFa2Yj8ALLgmoSQAJZysYbBjw+azChSGPN4DSPLXOcooGRvDwZWMcF/mLO2Q==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.7.tgz", + "integrity": "sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg==", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.6.tgz", - "integrity": "sha512-4yA7s865JHaqUdRbnaxarZREuPTHrjpDT+pXoAZ1yhyo6uFnIEpS8VMu16siFOHDpZNKYv5BObhsB//ycbICyw==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz", + "integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-option": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.24.6.tgz", - "integrity": "sha512-Jktc8KkF3zIkePb48QO+IapbXlSapOW9S+ogZZkcO6bABgYAxtZcjZ/O005111YLf+j4M84uEgwYoidDkXbCkQ==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.24.7.tgz", + "integrity": "sha512-yy1/KvjhV/ZCL+SM7hBrvnZJ3ZuT9OuZgIJAGpPEToANvc3iM6iDvBnRjtElWibHU6n8/LPR/EjX9EtIEYO3pw==", "peer": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helpers": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.24.6.tgz", - "integrity": "sha512-V2PI+NqnyFu1i0GyTd/O/cTpxzQCYioSkUIRmgo7gFEHKKCg5w46+r/A6WeUR1+P3TeQ49dspGPNd/E3n9AnnA==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.24.7.tgz", + "integrity": "sha512-NlmJJtvcw72yRJRcnCmGvSi+3jDEg8qFu3z0AFoymmzLx5ERVWyzd9kVXr7Th9/8yIJi2Zc6av4Tqz3wFs8QWg==", "peer": true, "dependencies": { - "@babel/template": "^7.24.6", - "@babel/types": "^7.24.6" + "@babel/template": "^7.24.7", + "@babel/types": "^7.24.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/highlight": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.6.tgz", - "integrity": "sha512-2YnuOp4HAk2BsBrJJvYCbItHx0zWscI1C3zgWkz+wDyD9I7GIVrfnLyrR4Y1VR+7p+chAEcrgRQYZAGIKMV7vQ==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.7.tgz", + "integrity": "sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==", "dependencies": { - "@babel/helper-validator-identifier": "^7.24.6", + "@babel/helper-validator-identifier": "^7.24.7", "chalk": "^2.4.2", "js-tokens": "^4.0.0", "picocolors": "^1.0.0" @@ -457,9 +464,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.6.tgz", - "integrity": "sha512-eNZXdfU35nJC2h24RznROuOpO94h6x8sg9ju0tT9biNtLZ2vuP8SduLqqV+/8+cebSLV9SJEAN5Z3zQbJG/M+Q==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.7.tgz", + "integrity": "sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw==", "bin": { "parser": "bin/babel-parser.js" }, @@ -468,11 +475,11 @@ } }, "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.24.6.tgz", - "integrity": "sha512-lWfvAIFNWMlCsU0DRUun2GpFwZdGTukLaHJqRh1JRb80NdAP5Sb1HDHB5X9P9OtgZHQl089UzQkpYlBq2VTPRw==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.24.7.tgz", + "integrity": "sha512-6ddciUPe/mpMnOKv/U+RSd2vvVy+Yw/JfBB0ZHYjEZt9NLHmCUylNYlsbqCCS1Bffjlb0fCwC9Vqz+sBz6PsiQ==", "dependencies": { - "@babel/helper-plugin-utils": "^7.24.6" + "@babel/helper-plugin-utils": "^7.24.7" }, "engines": { "node": ">=6.9.0" @@ -482,9 +489,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.6.tgz", - "integrity": "sha512-Ja18XcETdEl5mzzACGd+DKgaGJzPTCow7EglgwTmHdwokzDFYh/MHua6lU6DV/hjF2IaOJ4oX2nqnjG7RElKOw==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.7.tgz", + "integrity": "sha512-UwgBRMjJP+xv857DCngvqXI3Iq6J4v0wXmwc6sapg+zyhbwmQX67LUEFrkK5tbyJ30jGuG3ZvWpBiB9LCy1kWw==", "dependencies": { "regenerator-runtime": "^0.14.0" }, @@ -493,31 +500,31 @@ } }, "node_modules/@babel/template": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.6.tgz", - "integrity": "sha512-3vgazJlLwNXi9jhrR1ef8qiB65L1RK90+lEQwv4OxveHnqC3BfmnHdgySwRLzf6akhlOYenT+b7AfWq+a//AHw==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.7.tgz", + "integrity": "sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig==", "dependencies": { - "@babel/code-frame": "^7.24.6", - "@babel/parser": "^7.24.6", - "@babel/types": "^7.24.6" + "@babel/code-frame": "^7.24.7", + "@babel/parser": "^7.24.7", + "@babel/types": "^7.24.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.24.6.tgz", - "integrity": "sha512-OsNjaJwT9Zn8ozxcfoBc+RaHdj3gFmCmYoQLUII1o6ZrUwku0BMg80FoOTPx+Gi6XhcQxAYE4xyjPTo4SxEQqw==", - "dependencies": { - "@babel/code-frame": "^7.24.6", - "@babel/generator": "^7.24.6", - "@babel/helper-environment-visitor": "^7.24.6", - "@babel/helper-function-name": "^7.24.6", - "@babel/helper-hoist-variables": "^7.24.6", - "@babel/helper-split-export-declaration": "^7.24.6", - "@babel/parser": "^7.24.6", - "@babel/types": "^7.24.6", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.24.7.tgz", + "integrity": "sha512-yb65Ed5S/QAcewNPh0nZczy9JdYXkkAbIsEo+P7BE7yO3txAY30Y/oPa3QkQ5It3xVG2kpKMg9MsdxZaO31uKA==", + "dependencies": { + "@babel/code-frame": "^7.24.7", + "@babel/generator": "^7.24.7", + "@babel/helper-environment-visitor": "^7.24.7", + "@babel/helper-function-name": "^7.24.7", + "@babel/helper-hoist-variables": "^7.24.7", + "@babel/helper-split-export-declaration": "^7.24.7", + "@babel/parser": "^7.24.7", + "@babel/types": "^7.24.7", "debug": "^4.3.1", "globals": "^11.1.0" }, @@ -534,12 +541,12 @@ } }, "node_modules/@babel/types": { - "version": "7.24.6", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.6.tgz", - "integrity": "sha512-WaMsgi6Q8zMgMth93GvWPXkhAIEobfsIkLTacoVZoK1J0CevIPGYY2Vo5YvJGqyHqXM6P4ppOYGsIRU8MM9pFQ==", + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.7.tgz", + "integrity": "sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q==", "dependencies": { - "@babel/helper-string-parser": "^7.24.6", - "@babel/helper-validator-identifier": "^7.24.6", + "@babel/helper-string-parser": "^7.24.7", + "@babel/helper-validator-identifier": "^7.24.7", "to-fast-properties": "^2.0.0" }, "engines": { @@ -566,6 +573,15 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/runtime": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.2.0.tgz", + "integrity": "sha512-bV21/9LQmcQeCPEg3BDFtvwL6cwiTMksYNWQQ4KOxCZikEGalWtenoZ0wCiukJINlGCIi2KXx01g4FoH/LxpzQ==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@emotion/is-prop-valid": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.2.2.tgz", @@ -641,6 +657,20 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@fal-ai/serverless-client": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@fal-ai/serverless-client/-/serverless-client-0.10.3.tgz", + "integrity": "sha512-Mi1TU9zjrDNuRf462fVBu8PTL4dSuIwOJYn56HQvOHx+ABhbsKD0azCALSANglRx0L6KAXSeQKXZpIAJdvowCQ==", + "dependencies": { + "@msgpack/msgpack": "^3.0.0-beta2", + "eventsource-parser": "^1.1.2", + "robot3": "^0.4.1", + "uuid-random": "^1.3.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@floating-ui/core": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.2.tgz", @@ -687,10 +717,21 @@ "node": ">=18" } }, + "node_modules/@huggingface/inference": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.7.0.tgz", + "integrity": "sha512-u7Fn637Q3f7nUB1tajM4CgzhvoFQkOQr5W5Fm+2wT9ETgGoLBh25BLlYPTJRjAd2WY01s71v0lqAwNvHHCc3mg==", + "dependencies": { + "@huggingface/tasks": "^0.10.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@huggingface/tasks": { - "version": "0.10.9", - "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.10.9.tgz", - "integrity": "sha512-KdGnpw7sTLgr9nUwH6NrMHi/6/yBe5HdeXeKWauajseKZXwyrTYwNYIzMnh8/hE7Qa2UnseAdFMpCqe54vGd4A==" + "version": "0.10.11", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.10.11.tgz", + "integrity": "sha512-talNbXgxT8fsqt8DhhF7mNZxfeCc8fAJWe1yquBzd4pA4PH/J504Z41uijo36HNoI8q3KGZ89A884sIcG3ak6Q==" }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", @@ -747,6 +788,31 @@ "@img/sharp-libvips-darwin-arm64": "1.0.2" } }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.4.tgz", + "integrity": "sha512-0l7yRObwtTi82Z6ebVI2PnHT8EB2NxBgpK2MiKJZJ7cz32R4lxd001ecMhzzsZig3Yv9oclvqqdV93jo9hy+Dw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.2" + } + }, "node_modules/@img/sharp-libvips-darwin-arm64": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.2.tgz", @@ -768,6 +834,366 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.2.tgz", + "integrity": "sha512-Ofw+7oaWa0HiiMiKWqqaZbaYV3/UGL2wAPeLuJTx+9cXpCRdvQhCLG0IH8YGwM0yGWGLpsF4Su9vM1o6aer+Fw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "macos": ">=10.13", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.2.tgz", + "integrity": "sha512-iLWCvrKgeFoglQxdEwzu1eQV04o8YeYGFXtfWU26Zr2wWT3q3MTzC+QTCO3ZQfWd3doKHT4Pm2kRmLbupT+sZw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.2.tgz", + "integrity": "sha512-x7kCt3N00ofFmmkkdshwj3vGPCnmiDh7Gwnd4nUwZln2YjqPxV1NlTyZOvoDWdKQVDL911487HOueBvrpflagw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.2.tgz", + "integrity": "sha512-cmhQ1J4qVhfmS6szYW7RT+gLJq9dH2i4maq+qyXayUSn9/3iY2ZeWpbAgSpSVbV2E1JUL2Gg7pwnYQ1h8rQIog==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.2.tgz", + "integrity": "sha512-E441q4Qdb+7yuyiADVi5J+44x8ctlrqn8XgkDTwr4qPJzWkaHwD489iZ4nGDgcuya4iMN3ULV6NwbhRZJ9Z7SQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.2.tgz", + "integrity": "sha512-3CAkndNpYUrlDqkCM5qhksfE+qSIREVpyoeHIU6jd48SJZViAmznoQQLAv4hVXF7xyUB9zf+G++e2v1ABjCbEQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.2.tgz", + "integrity": "sha512-VI94Q6khIHqHWNOh6LLdm9s2Ry4zdjWJwH56WoiJU7NTeDwyApdZZ8c+SADC8OH98KWNQXnE01UdJ9CSfZvwZw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.4.tgz", + "integrity": "sha512-RUgBD1c0+gCYZGCCe6mMdTiOFS0Zc/XrN0fYd6hISIKcDUbAW5NtSQW9g/powkrXYm6Vzwd6y+fqmExDuCdHNQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.4.tgz", + "integrity": "sha512-2800clwVg1ZQtxwSoTlHvtm9ObgAax7V6MTAB/hDT945Tfyy3hVkmiHpeLPCKYqYR1Gcmv1uDZ3a4OFwkdBL7Q==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.4.tgz", + "integrity": "sha512-h3RAL3siQoyzSoH36tUeS0PDmb5wINKGYzcLB5C6DIiAn2F3udeFAum+gj8IbA/82+8RGCTn7XW8WTFnqag4tQ==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.31", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.4.tgz", + "integrity": "sha512-GoR++s0XW9DGVi8SUGQ/U4AeIzLdNjHka6jidVwapQ/JebGVQIpi52OdyxCNVRE++n1FCLzjDovJNozif7w/Aw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.2" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.4.tgz", + "integrity": "sha512-nhr1yC3BlVrKDTl6cO12gTpXMl4ITBUZieehFvMntlCXFzH2bvKG76tBL2Y/OqhupZt81pR7R+Q5YhJxW0rGgQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.2" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.4.tgz", + "integrity": "sha512-uCPTku0zwqDmZEOi4ILyGdmW76tH7dm8kKlOIV1XC5cLyJ71ENAAqarOHQh0RLfpIpbV5KOpXzdU6XkJtS0daw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.2" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.4.tgz", + "integrity": "sha512-Bmmauh4sXUsUqkleQahpdNXKvo+wa1V9KhT2pDA4VJGKwnKMJXiSTGphn0gnJrlooda0QxCtXc6RX1XAU6hMnQ==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.1.1" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.4.tgz", + "integrity": "sha512-99SJ91XzUhYHbx7uhK3+9Lf7+LjwMGQZMDlO/E/YVJ7Nc3lyDFZPGhjwiYdctoH2BOzW9+TnfqcaMKt0jHLdqw==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.4.tgz", + "integrity": "sha512-3QLocdTRVIrFNye5YocZl+KKpYKP+fksi1QhmOArgx7GyhIbQp/WrJRu176jm8IxromS7RIkzMiMINVdBtC8Aw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -868,6 +1294,14 @@ "three": ">= 0.159.0" } }, + "node_modules/@msgpack/msgpack": { + "version": "3.0.0-beta2", + "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.0.0-beta2.tgz", + "integrity": "sha512-y+l1PNV0XDyY8sM3YtuMLK5vE3/hkfId+Do8pLo/OPxfxuFAUwcGz3oiiUuV46/aBpwTzZ+mRWVMtlSKbradhw==", + "engines": { + "node": ">= 14" + } + }, "node_modules/@next/env": { "version": "14.2.3", "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.3.tgz", @@ -1058,9 +1492,9 @@ } }, "node_modules/@pmndrs/uikit": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/@pmndrs/uikit/-/uikit-0.3.4.tgz", - "integrity": "sha512-2TlnuS+aYq62Sz3U+l/D+e8DDxt2v0Q8qMrILWXJWLh+42TtKUSpjugnDKgJXC9fjxrgtWFC7/Y4oREVsHALdA==", + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@pmndrs/uikit/-/uikit-0.3.6.tgz", + "integrity": "sha512-TGcOV2cJTdNFUyEWfTlxuiZmb+jZtBe+CS5ZdR7Weco3BE0m830x0M2UEkPGdw+6Ch8IzPKf5w2pEPtVPCA6QQ==", "dependencies": { "@preact/signals-core": "^1.5.1", "inline-style-parser": "^0.2.3", @@ -2443,18 +2877,19 @@ } }, "node_modules/@react-three/uikit": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/@react-three/uikit/-/uikit-0.3.4.tgz", - "integrity": "sha512-AYI4HIg1nJ8Z+fgcqqUWZog/TgUiemdJBr6fF2/3KP4UNb+6KsWi2PH02EZDqJ/aL0x/zZDgT5TynB9rwCEefg==", + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@react-three/uikit/-/uikit-0.3.6.tgz", + "integrity": "sha512-kDKcFR+XbfUIfS7tnViMzJetA01TysZTVXJm2x3Ve+/viYJuX9ISCKFABNQ76iVWNYMhobS2VAb0/8l+shUJLA==", "dependencies": { - "@pmndrs/uikit": "^0.3.4", + "@pmndrs/uikit": "^0.3.6", "@preact/signals-core": "^1.5.1", "chalk": "^5.3.0", "commander": "^12.0.0", "ora": "^8.0.1", "prettier": "^3.2.5", "prompts": "^2.4.2", - "zod": "^3.22.4" + "zod": "^3.22.4", + "zustand": "^4.5.2" }, "bin": { "uikit": "dist/cli/index.js" @@ -2465,11 +2900,11 @@ } }, "node_modules/@react-three/uikit-lucide": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/@react-three/uikit-lucide/-/uikit-lucide-0.3.4.tgz", - "integrity": "sha512-f8TlmmZsJW/o4Lvan2B6gl9YA4qy9cUmjqJv5s+BC4Pad2GHyE4Y/YgVnGUKwFXtEA0XZDmRrICK0xZv/dlF4A==", + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@react-three/uikit-lucide/-/uikit-lucide-0.3.6.tgz", + "integrity": "sha512-cWzzVX7EC2in5A+NOj1i7yzAkvgYjSIM7/ZxFmMCmsB2Hw55m/Cj/IPJwx/0uzJB03EBwWc5mV+C1UTmX4ZuVA==", "dependencies": { - "@react-three/uikit": "^0.3.4" + "@react-three/uikit": "^0.3.6" } }, "node_modules/@rushstack/eslint-patch": { @@ -3315,9 +3750,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001627", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001627.tgz", - "integrity": "sha512-4zgNiB8nTyV/tHhwZrFs88ryjls/lHiqFhrxCW4qSTeuRByBVnPYpDInchOIySWknznucaf31Z4KYqjfbrecVw==", + "version": "1.0.30001628", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001628.tgz", + "integrity": "sha512-S3BnR4Kh26TBxbi5t5kpbcUlLJb9lhtDXISDPwOfI+JoC+ik0QksvkZtUVyikw3hjnkgkMPSJ8oIM9yMm9vflA==", "funding": [ { "type": "opencollective", @@ -4100,9 +4535,9 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, "node_modules/electron-to-chromium": { - "version": "1.4.789", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.789.tgz", - "integrity": "sha512-0VbyiaXoT++Fi2vHGo2ThOeS6X3vgRCWrjPeO2FeIAWL6ItiSJ9BqlH8LfCXe3X1IdcG+S0iLoNaxQWhfZoGzQ==" + "version": "1.4.790", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.790.tgz", + "integrity": "sha512-eVGeQxpaBYbomDBa/Mehrs28MdvCXfJmEFzaMFsv8jH/MJDLIylJN81eTJ5kvx7B7p18OiPK0BkC06lydEy63A==" }, "node_modules/emoji-regex": { "version": "9.2.2", @@ -4110,9 +4545,9 @@ "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, "node_modules/enhanced-resolve": { - "version": "5.16.1", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.16.1.tgz", - "integrity": "sha512-4U5pNsuDl0EhuZpq46M5xPslstkviJuhrdobaRDBk2Jy2KO37FDAJl4lb2KlNabxT0m4MTK2UHNrsAcphE8nyw==", + "version": "5.17.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.0.tgz", + "integrity": "sha512-dwDPwZL0dmye8Txp2gzFmA6sxALaSvdRDjPH0viLcKrtlOL3tw62nWWweVD1SdILDTJrbrL6tdWVN58Wo6U3eA==", "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.2.0" @@ -4755,6 +5190,14 @@ "node": ">=0.8.x" } }, + "node_modules/eventsource-parser": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.1.2.tgz", + "integrity": "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==", + "engines": { + "node": ">=14.18" + } + }, "node_modules/extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -5809,9 +6252,9 @@ } }, "node_modules/jiti": { - "version": "1.21.0", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.0.tgz", - "integrity": "sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==", + "version": "1.21.3", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.3.tgz", + "integrity": "sha512-uy2bNX5zQ+tESe+TiC7ilGRz8AtRGmnJH55NC5S0nSUjvvvM2hJHmefHErugGXN4pNv4Qx7vLsnNw9qJ9mtIsw==", "bin": { "jiti": "bin/jiti.js" } @@ -6819,9 +7262,9 @@ } }, "node_modules/prettier": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.0.tgz", - "integrity": "sha512-J9odKxERhCQ10OC2yb93583f6UnYutOeiV5i0zEDS7UGTdUt0u+y8erxl3lBKvwo/JHyyoEdXjwp4dke9oyZ/g==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.1.tgz", + "integrity": "sha512-7CAwy5dRsxs8PHXT3twixW9/OEll8MLE0VRPCJyl7CkS6VHGPSlsVaWTiASPTyGyYRyApxlaWTzwUxVNrhcwDg==", "bin": { "prettier": "bin/prettier.cjs" }, @@ -7380,6 +7823,11 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/robot3": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/robot3/-/robot3-0.4.1.tgz", + "integrity": "sha512-hzjy826lrxzx8eRgv80idkf8ua1JAepRc9Efdtj03N3KNJuznQCPlyCJ7gnUmDFwZCLQjxy567mQVKmdv2BsXQ==" + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -8234,9 +8682,9 @@ } }, "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" }, "node_modules/tunnel-rat": { "version": "0.1.2", @@ -8584,6 +9032,11 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/uuid-random": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/uuid-random/-/uuid-random-1.3.2.tgz", + "integrity": "sha512-UOzej0Le/UgkbWEO8flm+0y+G+ljUon1QWTEZOq1rnMAsxo2+SckbiZdKzAHHlVh6gJqI1TjC/xwgR50MuCrBQ==" + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index 52e525977899659f4c95371bcf025dcfc79c9ab6..6f222b1735db6d19d92c20b12991c112d73e2587 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,10 @@ "dependencies": { "@aitube/clap": "0.0.24", "@aitube/engine": "0.0.15", - "@aitube/timeline": "0.0.8", + "@aitube/timeline": "0.0.11", + "@fal-ai/serverless-client": "^0.10.3", "@huggingface/hub": "^0.15.0", + "@huggingface/inference": "^2.7.0", "@radix-ui/react-accordion": "^1.1.2", "@radix-ui/react-avatar": "^1.0.4", "@radix-ui/react-checkbox": "^1.0.4", diff --git a/src/app/api/render/providers/comfy-comfyicu/index.ts b/src/app/api/render/providers/comfy-comfyicu/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..10af95fb7ba809d97bea954caad51458f76e1e8f --- /dev/null +++ b/src/app/api/render/providers/comfy-comfyicu/index.ts @@ -0,0 +1,32 @@ +import Replicate from 'replicate' + +import { RenderRequest } from "@/types" +import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap" +import { getVideoPrompt } from "@aitube/engine" + +export async function renderSegment(request: RenderRequest): Promise { + + if (request.segment.category !== ClapSegmentCategory.STORYBOARD) { + throw new Error(`Clapper doesn't support ${request.segment.category} generation for provider "Comfy.icu". Please open a pull request with (working code) to solve this!`) + } + + const segment: ClapSegment = { ...request.segment } + + const visualPrompt = getVideoPrompt( + request.segments, + request.entities + ) + + try { + + throw new Error(`Not Implemented!`) + + } catch (err) { + console.error(`failed to call Comfy.icu: `, err) + segment.assetUrl = '' + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.status = ClapSegmentStatus.TO_GENERATE + } + + return segment +} \ No newline at end of file diff --git a/src/app/api/render/providers/comfy-huggingface/index.ts b/src/app/api/render/providers/comfy-huggingface/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..fa42295f4c4b5f5c372b60694e3eada84f0036df --- /dev/null +++ b/src/app/api/render/providers/comfy-huggingface/index.ts @@ -0,0 +1,30 @@ +import { RenderRequest } from "@/types" +import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap" +import { getVideoPrompt } from "@aitube/engine" + +export async function renderSegment(request: RenderRequest): Promise { + + if (request.segment.category !== ClapSegmentCategory.STORYBOARD) { + throw new Error(`Clapper doesn't support ${request.segment.category} generation for provider "Comfy Hugging Face". Please open a pull request with (working code) to solve this!`) + } + + const segment: ClapSegment = { ...request.segment } + + const visualPrompt = getVideoPrompt( + request.segments, + request.entities + ) + + try { + + throw new Error(`Not Implemented!`) + + } catch (err) { + console.error(`failed to call Hugging Face: `, err) + segment.assetUrl = '' + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.status = ClapSegmentStatus.TO_GENERATE + } + + return segment +} \ No newline at end of file diff --git a/src/app/api/render/providers/comfy-replicate/index.ts b/src/app/api/render/providers/comfy-replicate/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..c304c744e1305af08de17652f1c5f2afba6a9e05 --- /dev/null +++ b/src/app/api/render/providers/comfy-replicate/index.ts @@ -0,0 +1,30 @@ +import { ClapSegment, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap" + +import { RenderRequest } from "@/types" +import { getComfyWorkflow } from "../comfy/getComfyWorkflow" +import { runWorkflow } from "./runWorkflow" + +export async function renderSegment(request: RenderRequest): Promise { + + const workflow = getComfyWorkflow(request) + + // TODO support Hugging Face as well + // const await runWithHuggingFace({ + const segment: ClapSegment = { ...request.segment } + + try { + segment.assetUrl = await runWorkflow({ + apiKey: request.settings.replicateApiKey, + workflow, + }) + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + } catch (err) { + console.error(`failed to call Replicate: `, err) + segment.assetUrl = '' + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.status = ClapSegmentStatus.TO_GENERATE + // request.segment.status = ClapSegmentStatus.ERROR + } + + return segment +} \ No newline at end of file diff --git a/src/server/comfy/replicate.ts b/src/app/api/render/providers/comfy-replicate/runWorkflow.ts similarity index 71% rename from src/server/comfy/replicate.ts rename to src/app/api/render/providers/comfy-replicate/runWorkflow.ts index 5fb485140d6c82954a26f3566e20c39ccc4582bb..f78e73977b5cbfc43674c4803cac05e805c6c34e 100644 --- a/src/server/comfy/replicate.ts +++ b/src/app/api/render/providers/comfy-replicate/runWorkflow.ts @@ -1,8 +1,7 @@ -"use server" import Replicate from 'replicate' -export async function run({ +export async function runWorkflow({ apiKey, workflow }: { @@ -13,6 +12,8 @@ export async function run({ const replicate = new Replicate({ auth: apiKey }) // https://replicate.com/fofr/any-comfyui-workflow + // https://replicate.com/guides/comfyui/run-comfyui-on-replicate + const cogId = "fofr/any-comfyui-workflow:74f12621dc9f9b7cdca50d03941b8ddb3a368d7f5a1bb16fb7e1b87f05d96bf5" const output = await replicate.run(cogId, { @@ -21,7 +22,7 @@ export async function run({ } }) - console.log(`response:`, output) + console.log(`response from Replicate:`, output) - return "" + return output as any } \ No newline at end of file diff --git a/src/server/comfy/getComfyWorkflow.ts b/src/app/api/render/providers/comfy/getComfyWorkflow.ts similarity index 71% rename from src/server/comfy/getComfyWorkflow.ts rename to src/app/api/render/providers/comfy/getComfyWorkflow.ts index 578440c17373741017ec2a328a120b7a9cabf5e4..8ff3b448f10398139ad53e7ceac04924e2890978 100644 --- a/src/server/comfy/getComfyWorkflow.ts +++ b/src/app/api/render/providers/comfy/getComfyWorkflow.ts @@ -9,9 +9,9 @@ export function getComfyWorkflow(request: RenderRequest) { let comfyWorkflow = "{}" if (request.segment.category === ClapSegmentCategory.STORYBOARD) { - comfyWorkflow = request.storyboardWorkflow + comfyWorkflow = request.settings.comfyWorkflowForStoryboard } else if (request.segment.category === ClapSegmentCategory.VIDEO) { - comfyWorkflow = request.videoWorkflow + comfyWorkflow = request.settings.comfyWorkflowForVideo } // parse the node array from the ComfyUI workflow @@ -22,13 +22,21 @@ export function getComfyWorkflow(request: RenderRequest) { request.entities ) - for (const node of nodes) { + const output: Record = {} + + nodes.forEach((node, i) => { if (typeof node.inputs.text === "string") { if (node._meta.title.includes("Prompt")) { node.inputs.text = visualPrompt } } - } - - return JSON.stringify(nodes) + output[`${i}`] = node + }) + + console.log("DEBUG:", { + nodes, + output + }) + + return JSON.stringify(output) } \ No newline at end of file diff --git a/src/app/api/render/providers/falai/index.ts b/src/app/api/render/providers/falai/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..a2754948430722b0c8fa8ccdee719c79ece24017 --- /dev/null +++ b/src/app/api/render/providers/falai/index.ts @@ -0,0 +1,67 @@ +import * as fal from '@fal-ai/serverless-client' + +import { RenderRequest } from "@/types" +import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap" +import { getVideoPrompt } from "@aitube/engine" +import { fetchContentToBase64 } from '@/lib/utils/fetchContentToBase64' + +export async function renderSegment(request: RenderRequest): Promise { + + if (request.segment.category !== ClapSegmentCategory.STORYBOARD) { + throw new Error(`Clapper doesn't support ${request.segment.category} generation for provider "Fal.ai". Please open a pull request with (working code) to solve this!`) + } + + fal.config({ + credentials: request.settings.falAiApiKey + }) + + const segment: ClapSegment = { ...request.segment } + + const visualPrompt = getVideoPrompt( + request.segments, + request.entities + ) + + try { + + const result = await fal.subscribe(request.settings.falAiModelForImage, { + input: { + prompt: visualPrompt + }, + logs: true, + onQueueUpdate: (update) => { + if (update.status === "IN_PROGRESS") { + update.logs?.map((log) => log.message).forEach(console.log); + } + }, + }) as { + prompt: string; + images: { url: string; content_type: string }[] + } + + // { + // "images": [ + // { + // "url": "", + // "content_type": "image/jpeg" + // } + // ], + // "prompt": "" + // } + + console.log(`response from Fal.ai:`, result) + + // TODO: this might be also in videos, since we are gonna generate videos as well + const content = result.images[0]?.url || "" + + segment.assetUrl = await fetchContentToBase64(content) + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + } catch (err) { + console.error(`failed to call Fal.ai: `, err) + segment.assetUrl = '' + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.status = ClapSegmentStatus.TO_GENERATE + } + + return segment +} \ No newline at end of file diff --git a/src/app/api/render/providers/huggingface/index.ts b/src/app/api/render/providers/huggingface/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..054112134dd5ff6089eb7674f0eb26315d5b585d --- /dev/null +++ b/src/app/api/render/providers/huggingface/index.ts @@ -0,0 +1,37 @@ +import { HfInference, HfInferenceEndpoint } from "@huggingface/inference" + +import { RenderRequest } from "@/types" +import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap" +import { getVideoPrompt } from "@aitube/engine" +import { blobToBase64DataUri } from "@/lib/utils/blobToBase64DataUri" + +export async function renderSegment(request: RenderRequest): Promise { + + const hf: HfInferenceEndpoint = new HfInference(request.settings.huggingFaceApiKey) + + if (request.segment.category !== ClapSegmentCategory.STORYBOARD) { + throw new Error(`Clapper doesn't support ${request.segment.category} generation for provider "Hugging Face". Please open a pull request with (working code) to solve this!`) + } + + const segment: ClapSegment = { ...request.segment } + + try { + const blob: Blob = await hf.textToImage({ + model: request.settings.huggingFaceModelForImage, + inputs: getVideoPrompt( + request.segments, + request.entities + ) + }) + + segment.assetUrl = await blobToBase64DataUri(blob) + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + } catch (err) { + console.error(`failed to call Replicate: `, err) + segment.assetUrl = '' + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.status = ClapSegmentStatus.TO_GENERATE + } + + return segment +} \ No newline at end of file diff --git a/src/app/api/render/providers/modelslab/index.ts b/src/app/api/render/providers/modelslab/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..0d6e8994a4de69fad7c88f692e5fa61d1484b8bd --- /dev/null +++ b/src/app/api/render/providers/modelslab/index.ts @@ -0,0 +1,30 @@ +import { RenderRequest } from "@/types" +import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap" +import { getVideoPrompt } from "@aitube/engine" + +export async function renderSegment(request: RenderRequest): Promise { + + if (request.segment.category !== ClapSegmentCategory.STORYBOARD) { + throw new Error(`Clapper doesn't support ${request.segment.category} generation for provider "ModelsLab". Please open a pull request with (working code) to solve this!`) + } + + const segment: ClapSegment = { ...request.segment } + + const visualPrompt = getVideoPrompt( + request.segments, + request.entities + ) + + try { + + throw new Error(`Not Implemented!`) + + } catch (err) { + console.error(`failed to call ModelsLab: `, err) + segment.assetUrl = '' + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.status = ClapSegmentStatus.TO_GENERATE + } + + return segment +} \ No newline at end of file diff --git a/src/app/api/render/providers/replicate/index.ts b/src/app/api/render/providers/replicate/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..bf11bf700c314f3878a0f25a31856a6be120219b --- /dev/null +++ b/src/app/api/render/providers/replicate/index.ts @@ -0,0 +1,42 @@ +import Replicate from 'replicate' + +import { RenderRequest } from "@/types" +import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap" +import { getVideoPrompt } from "@aitube/engine" +import { fetchContentToBase64 } from '@/lib/utils/fetchContentToBase64' + +export async function renderSegment(request: RenderRequest): Promise { + + const replicate = new Replicate({ auth: request.settings.huggingFaceApiKey }) + + if (request.segment.category !== ClapSegmentCategory.STORYBOARD) { + throw new Error(`Clapper doesn't support ${request.segment.category} generation for provider "Replicate". Please open a pull request with (working code) to solve this!`) + } + + const segment: ClapSegment = { ...request.segment } + + try { + + const output = await replicate.run( + request.settings.replicateModelForImage as any, { + input: { + prompt: getVideoPrompt( + request.segments, + request.entities + ) + } + }) + + console.log(`Replicate replied: `, output) + + segment.assetUrl = await fetchContentToBase64(`${output}`) + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + } catch (err) { + console.error(`failed to call Replicate: `, err) + segment.assetUrl = '' + segment.assetSourceType = getClapAssetSourceType(segment.assetUrl) + segment.status = ClapSegmentStatus.TO_GENERATE + } + + return segment +} \ No newline at end of file diff --git a/src/app/api/render/route.ts b/src/app/api/render/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..efd8855fd573d6e9aceb979af31c30ab4e0b5d51 --- /dev/null +++ b/src/app/api/render/route.ts @@ -0,0 +1,56 @@ +import { NextResponse, NextRequest } from "next/server" + +import { renderSegment as renderSegmentUsingHuggingFace } from "./providers/huggingface" +import { renderSegment as renderSegmentUsingComfyReplicate } from "./providers/comfy-replicate" +import { renderSegment as renderSegmentUsingReplicate } from "./providers/replicate" +import { renderSegment as renderSegmentUsingComfyComfyIcu } from "./providers/comfy-comfyicu" +import { renderSegment as renderSegmentUsingFalAi } from "./providers/falai" +import { renderSegment as renderSegmentUsingModelsLab } from "./providers/modelslab" + +import { ComputeProvider, RenderRequest } from "@/types" +import { ClapSegmentCategory } from "@aitube/clap" + +export async function POST(req: NextRequest) { + // do we really need to secure it? + // I mean.. in the end, the user is using their own credentials, + // so they cannot siphon free OpenAI, HF, Replicate tokens + console.log(`TODO Julian: secure the endpoint`) + // await throwIfInvalidToken(req.headers.get("Authorization")) + const request = (await req.json()) as RenderRequest + + const provider = + request.segment.category === ClapSegmentCategory.STORYBOARD + ? request.settings.storyboardProvider + : request.segment.category === ClapSegmentCategory.VIDEO + ? request.settings.videoProvider + : request.segment.category === ClapSegmentCategory.DIALOGUE + ? request.settings.speechProvider + : request.segment.category === ClapSegmentCategory.SOUND + ? request.settings.soundProvider + : request.segment.category === ClapSegmentCategory.MUSIC + ? request.settings.musicProvider + : null + + if (!provider) { throw new Error(`Segments of category ${request.segment.category} are not supported yet`)} + + const renderSegment = + provider === ComputeProvider.HUGGINGFACE + ? renderSegmentUsingHuggingFace + : provider === ComputeProvider.COMFY_HUGGINGFACE + ? renderSegmentUsingComfyReplicate + : provider === ComputeProvider.REPLICATE + ? renderSegmentUsingReplicate + : provider === ComputeProvider.COMFY_COMFYICU + ? renderSegmentUsingComfyComfyIcu + : provider === ComputeProvider.FALAI + ? renderSegmentUsingFalAi + : provider === ComputeProvider.MODELSLAB + ? renderSegmentUsingModelsLab + : null + + if (!renderSegment) { throw new Error(`Provider ${provider} is not supported yet`)} + + const segment = await renderSegment(request) + + return NextResponse.json(segment) +} diff --git a/src/app/main.tsx b/src/app/main.tsx index 61b50565b21207260de1dc75a98c4707ebff04d1..295da489ac68e28b081949eaef4516a22a02fb88 100644 --- a/src/app/main.tsx +++ b/src/app/main.tsx @@ -16,9 +16,12 @@ import { TopBar } from "@/components/interface/top-bar" import { useTimelineState } from "@aitube/timeline" import { SettingsDialog } from "@/components/settings" import { LoadingDialog } from "@/components/interface/loader/LoadingDialog" +import { useSettingsView } from "@/settings/view" export function Main() { const isEmpty = useTimelineState(s => s.isEmpty) + const showTimeline = useSettingsView((s) => s.showTimeline) + return (
- + - + diff --git a/src/components/form/form-file.tsx b/src/components/form/form-file.tsx index a4c88880a26c15466d5ad15354cb03f263f24fd9..f22ed1c6c290bbbc0545f36b239f2f165cb790eb 100644 --- a/src/components/form/form-file.tsx +++ b/src/components/form/form-file.tsx @@ -47,7 +47,7 @@ export function FormFile({ ({ ({ onSelect(selectedItem?.value) }} defaultValue={selectedItemId}> - + diff --git a/src/components/interface/loader/index.tsx b/src/components/interface/loader/index.tsx index ea27f875da1479c7ce7950d16908bdda3966054e..64b743c7d4531c2f8412c198947aab4edd84913b 100644 --- a/src/components/interface/loader/index.tsx +++ b/src/components/interface/loader/index.tsx @@ -12,7 +12,7 @@ export function Loader({ return (
{ + + console.log(`custom segmentRender() called with:`, { segment, segments, entities }) + const request: RenderRequest = { + settings: useSettingsRendering.getState().getSettings(), + segment, + segments, + entities, + } + + const res = await fetch("/api/render", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(request) + }) + + const newSegment = (await res.json()) as ClapSegment + + return newSegment +} + export function Timeline() { + const [_isPending, startTransition] = useTransition() + + const isReady = useTimelineState(s => s.isReady) + + const storyboardRenderingStrategy = useSettingsRendering(s => s.storyboardRenderingStrategy) + const setStoryboardRenderingStrategy = useTimelineState(s => s.setStoryboardRenderingStrategy) + useEffect(() => { + if (isReady) setStoryboardRenderingStrategy(storyboardRenderingStrategy) + }, [isReady, setStoryboardRenderingStrategy, storyboardRenderingStrategy]) + + const videoRenderingStrategy = useSettingsRendering(s => s.videoRenderingStrategy) + const setVideoRenderingStrategy = useTimelineState(s => s.setVideoRenderingStrategy) + useEffect(() => { + if (isReady) setVideoRenderingStrategy(videoRenderingStrategy) + }, [isReady, setVideoRenderingStrategy, videoRenderingStrategy]) + + const getSettings = useSettingsRendering(s => s.getSettings) + const setSegmentRenderer = useTimelineState(s => s.setSegmentRenderer) + + useEffect(() => { + setSegmentRenderer(segmentRenderer) + }, [isReady]) + return ( - {/* - - - - */} - - {/**/} + + ) } diff --git a/src/components/interface/top-menu/rendering/index.tsx b/src/components/interface/top-menu/settings/index.tsx similarity index 97% rename from src/components/interface/top-menu/rendering/index.tsx rename to src/components/interface/top-menu/settings/index.tsx index 018ef187ff7c8e6966ca556a13a39385727f914b..f7f8cc195bf8766b4cdf68c8f1d855c04785e06c 100644 --- a/src/components/interface/top-menu/rendering/index.tsx +++ b/src/components/interface/top-menu/settings/index.tsx @@ -14,10 +14,10 @@ import { MenubarTrigger } from "@/components/ui/menubar" import { useSettingsRendering } from "@/settings/rendering" -import { RenderingStrategy } from "@/types" +import { RenderingStrategy } from "@aitube/timeline" import { useSettingsView } from "@/settings/view" -export function TopMenuRendering() { +export function TopMenuSettings() { const setShowSettings = useSettingsView(s => s.setShowSettings) const storyboardRenderingStrategy = useSettingsRendering((s) => s.storyboardRenderingStrategy) @@ -35,7 +35,7 @@ export function TopMenuRendering() { return ( - Rendering + Settings diff --git a/src/components/interface/top-menu/view/index.tsx b/src/components/interface/top-menu/view/index.tsx index 080941ceedcebc13560392085b1fdb8ebfb06525..76ea7f2ea89b155136543573a103211ee009cd06 100644 --- a/src/components/interface/top-menu/view/index.tsx +++ b/src/components/interface/top-menu/view/index.tsx @@ -55,6 +55,8 @@ export function TopMenuView() { }}> Toggle fullscreen + + {/* Show timeline + { @@ -92,6 +95,7 @@ export function TopMenuView() { return false }} >Show video player + */} ) diff --git a/src/components/settings/SettingsSectionRendering.tsx b/src/components/settings/SettingsSectionRendering.tsx deleted file mode 100644 index 525da7d3df733862c4fb84b2e72fc4614190ece7..0000000000000000000000000000000000000000 --- a/src/components/settings/SettingsSectionRendering.tsx +++ /dev/null @@ -1,68 +0,0 @@ -import { FormSection } from "@/components/form/form-section" -import { useSettingsRendering } from "@/settings/rendering" -import { FormSelect } from "../form/form-select" -import { ComfyVendor } from "@/types" -import { FormInput } from "../form/form-input" -import { APP_NAME } from "@/lib/core/constants" - -export function SettingsSectionRendering() { - - const comfyUiApiVendor = useSettingsRendering(s => s.comfyUiApiVendor) - const setComfyUiApiVendor = useSettingsRendering(s => s.setComfyUiApiVendor) - - const comfyUiApiKey = useSettingsRendering(s => s.comfyUiApiKey) - const setComfyUiApiKey = useSettingsRendering(s => s.setComfyUiApiKey) - - const storyboardRenderingStrategy = useSettingsRendering(s => s.storyboardRenderingStrategy) - const setStoryboardRenderingStrategy = useSettingsRendering(s => s.setStoryboardRenderingStrategy) - - const videoRenderingStrategy = useSettingsRendering(s => s.videoRenderingStrategy) - const setVideoRenderingStrategy = useSettingsRendering(s => s.setVideoRenderingStrategy) - - const maxNbAssetsToGenerateInParallel = useSettingsRendering(s => s.maxNbAssetsToGenerateInParallel) - const setMaxNbAssetsToGenerateInParallel = useSettingsRendering(s => s.setMaxNbAssetsToGenerateInParallel) - - const availableComfyUiApiVendors = { - [ComfyVendor.NONE]: "No provider", - // [ComfyVendor.REPLICATE]: "Replicate", - // [ComfyVendor.HUGGINGFACE]: "HuggingFace", - // [ComfyVendor.CUSTOM]: "Custom server", - } - - return ( -
- - - label="Service provider for ComfyUI" - selectedItemId={comfyUiApiVendor} - selectedItemLabel={ - (availableComfyUiApiVendors as any)[comfyUiApiVendor] - || ComfyVendor.NONE - } - items={Object.entries(availableComfyUiApiVendors).map(([vendor, label]) => ({ - id: vendor, - label, - disabled: (vendor as ComfyVendor) === ComfyVendor.HUGGINGFACE || (vendor as ComfyVendor) === ComfyVendor.CUSTOM, - value: vendor as ComfyVendor, - }))} - onSelect={setComfyUiApiVendor} - horizontal - /> - - {comfyUiApiVendor === ComfyVendor.REPLICATE && ( - <> - -

- Note: preferences and credentials are stored inside your browser local storage. {APP_NAME} uses them to perform API calls on your behalf, then forget them immediately. -

- - )} - -
- ) -} \ No newline at end of file diff --git a/src/components/settings/assistant.tsx b/src/components/settings/assistant.tsx new file mode 100644 index 0000000000000000000000000000000000000000..b14ee5cdb39ca6592ab6ff8bb72e4e51bc2b8b3d --- /dev/null +++ b/src/components/settings/assistant.tsx @@ -0,0 +1,35 @@ +import { FormSection } from "@/components/form/form-section" +import { useSettingsRendering } from "@/settings/rendering" +import { ComputeProvider } from "@/types" + +import { FormSelect } from "../form/form-select" +import { availableComputeProvidersForAssistant } from "./constants" + +export function SettingsSectionAssistant() { + const assistantProvider = useSettingsRendering(s => s.assistantProvider) + const setAssistantProvider = useSettingsRendering(s => s.setAssistantProvider) + + return ( +
+ + + + label="Assistant provider" + selectedItemId={assistantProvider} + selectedItemLabel={ + (availableComputeProvidersForAssistant as any)[assistantProvider] + || ComputeProvider.NONE + } + items={Object.entries(availableComputeProvidersForAssistant).map(([provider, label]) => ({ + id: provider, + label, + disabled: false, + value: provider as ComputeProvider, + }))} + onSelect={setAssistantProvider} + horizontal + /> + +
+ ) +} \ No newline at end of file diff --git a/src/components/settings/constants.ts b/src/components/settings/constants.ts new file mode 100644 index 0000000000000000000000000000000000000000..6ee5a32b4aa9eb6808ff97d43ecf76be7dd38c13 --- /dev/null +++ b/src/components/settings/constants.ts @@ -0,0 +1,78 @@ +import { ComfyIcuAccelerator, ComputeProvider } from "@/types" + +export const availableComputeProviders = { + [ComputeProvider.NONE]: "No default provider", + [ComputeProvider.HUGGINGFACE]: "HuggingFace", + [ComputeProvider.REPLICATE]: "Replicate (Cog & ComfyUI)", + [ComputeProvider.COMFY_COMFYICU]: "Comfy.icu (ComfyUI workflow)", + [ComputeProvider.FALAI]: "Fal.ai", + [ComputeProvider.MODELSLAB]: "ModelsLab", + // [ComputeProvider.CUSTOM]: "Custom server", +} + +export const availableComputeProvidersForAssistant = { + [ComputeProvider.NONE]: "No assistant provider", + [ComputeProvider.HUGGINGFACE]: "Hugging Face (Inference API)", + [ComputeProvider.GROQ]: "Groq", + [ComputeProvider.OPENAI]: "OpenAI (ChatGPT)", +} + +export const availableComputeProvidersForStoryboards = { + [ComputeProvider.NONE]: "No storyboard provider", + [ComputeProvider.HUGGINGFACE]: "HuggingFace", + [ComputeProvider.REPLICATE]: "Replicate (Cog API)", + [ComputeProvider.COMFY_REPLICATE]: "Replicate (Comfy API)", + [ComputeProvider.COMFY_COMFYICU]: "Comfy.icu (Comfy API)", + [ComputeProvider.FALAI]: "Fal.ai", + [ComputeProvider.MODELSLAB]: "ModelsLab.com", +} + +export const availableComputeProvidersForVideos = { + [ComputeProvider.NONE]: "No video provider", + [ComputeProvider.HUGGINGFACE]: "HuggingFace", + [ComputeProvider.REPLICATE]: "Replicate (Cog API)", + [ComputeProvider.COMFY_REPLICATE]: "Replicate (Comfy API)", + [ComputeProvider.COMFY_COMFYICU]: "Comfy.icu (Comfy API)", + [ComputeProvider.FALAI]: "Fal.ai", + [ComputeProvider.MODELSLAB]: "ModelsLab.com", +} + +export const availableComputeProvidersForMusic = { + [ComputeProvider.NONE]: "No music provider", + [ComputeProvider.HUGGINGFACE]: "HuggingFace", + [ComputeProvider.COMFY_REPLICATE]: "Replicate (Comfy API)", + [ComputeProvider.COMFY_COMFYICU]: "Comfy.icu (Comfy API)", + [ComputeProvider.STABILITYAI]: "StabilityAI (Stable Audio API)", + [ComputeProvider.FALAI]: "Fal.ai", + [ComputeProvider.MODELSLAB]: "ModelsLab.com", +} + +export const availableComputeProvidersForSound = { + [ComputeProvider.NONE]: "No sound provider", + [ComputeProvider.HUGGINGFACE]: "HuggingFace", + [ComputeProvider.COMFY_REPLICATE]: "Replicate (Comfy API)", + [ComputeProvider.COMFY_COMFYICU]: "Comfy.icu (Comfy API)", + [ComputeProvider.STABILITYAI]: "StabilityAI (Stable Audio API)", + [ComputeProvider.ELEVENLABS]: "Eleven Labs (Sound Effects API)", + +} + +export const availableComputeProvidersForSpeech = { + [ComputeProvider.NONE]: "No speech provider", + [ComputeProvider.ELEVENLABS]: "Eleven Labs (TTS API)", + [ComputeProvider.STABILITYAI]: "StabilityAI (Stable Audio API)", + [ComputeProvider.HUGGINGFACE]: "HuggingFace", + [ComputeProvider.COMFY_REPLICATE]: "Replicate (Comfy API)", + [ComputeProvider.COMFY_COMFYICU]: "Comfy.icu (Comfy API)", + [ComputeProvider.FALAI]: "Fal.ai", + [ComputeProvider.MODELSLAB]: "ModelsLab.com", +} + +export const availableComfyIcuAccelerators = { + [ComfyIcuAccelerator.L4]: "L4", + [ComfyIcuAccelerator.T4]: "T4", + [ComfyIcuAccelerator.A10]: "A10", + [ComfyIcuAccelerator.A100_40GB]: "A100 (40 GB)", + [ComfyIcuAccelerator.A100_80GB]: "A100 (80 GB)", + [ComfyIcuAccelerator.H100]: "H100", +} \ No newline at end of file diff --git a/src/components/settings/index.tsx b/src/components/settings/index.tsx index 885ddcb1aa451fa7d5f4d9f773c7b72d451a6fe3..0439480ab0ff968c364fc2a15ba13591ecb5038b 100644 --- a/src/components/settings/index.tsx +++ b/src/components/settings/index.tsx @@ -6,7 +6,13 @@ import { ScrollArea } from "@/components/ui/scroll-area" import { cn } from "@/lib/utils" import { useSettingsView } from "@/settings/view" -import { SettingsSectionRendering } from "./SettingsSectionRendering" +import { SettingsSectionProvider } from "./provider" +import { SettingsSectionAssistant } from "./assistant" +import { SettingsSectionStoryboard } from "./storyboard" +import { SettingsSectionVideo } from "./video" +import { SettingsSectionSound } from "./sound" +import { SettingsSectionMusic } from "./music" +import { SettingsSectionSpeech } from "./speech" export function SettingsDialog() { @@ -15,21 +21,33 @@ export function SettingsDialog() { const [_isPending, startTransition] = useTransition() const panels = { - rendering: , + provider: , + assistant: , + storyboard: , + video: , + speech: , + music: , + sound: , } const panelLabels = { - rendering: "Rendering", + provider: "Providers", + assistant: "Assistant", + storyboard: "Image", + video: "Video", + speech: "Speech", + music: "Music", + sound: "Sound", } as any - const [configPanel, setConfigPanel] = useState("rendering") + const [configPanel, setConfigPanel] = useState("provider") return ( @@ -46,7 +64,7 @@ export function SettingsDialog() {
diff --git a/src/components/settings/music.tsx b/src/components/settings/music.tsx new file mode 100644 index 0000000000000000000000000000000000000000..c08667c99f66cb490455e79c233967764360ee2b --- /dev/null +++ b/src/components/settings/music.tsx @@ -0,0 +1,87 @@ +import { FormSection } from "@/components/form/form-section" +import { getDefaultSettingsRendering, useSettingsRendering } from "@/settings/rendering" +import { ComputeProvider } from "@/types" + +import { FormSelect } from "../form/form-select" +import { availableComputeProvidersForMusic } from "./constants" +import { FormInput } from "../form/form-input" + +export function SettingsSectionMusic() { + const defaultSettings = getDefaultSettingsRendering() + + const musicProvider = useSettingsRendering(s => s.musicProvider) + const setMusicProvider = useSettingsRendering(s => s.setMusicProvider) + + const huggingFaceModelForMusic = useSettingsRendering(s => s.huggingFaceModelForMusic) + const setHuggingFaceModelForMusic = useSettingsRendering(s => s.setHuggingFaceModelForMusic) + + const replicateModelForMusic = useSettingsRendering(s => s.replicateModelForMusic) + const setReplicateModelForMusic = useSettingsRendering(s => s.setReplicateModelForMusic) + + const falAiModelForMusic = useSettingsRendering(s => s.falAiModelForMusic) + const setFalAiModelForMusic = useSettingsRendering(s => s.setFalAiModelForMusic) + + const modelsLabModelForMusic = useSettingsRendering(s => s.modelsLabModelForMusic) + const setModelsLabModelForMusic = useSettingsRendering(s => s.setModelsLabModelForMusic) + + const comfyWorkflowForMusic = useSettingsRendering(s => s.comfyWorkflowForMusic) + const setComfyWorkflowForMusic = useSettingsRendering(s => s.setComfyWorkflowForMusic) + + return ( +
+ + + + label="Music provider" + selectedItemId={musicProvider} + selectedItemLabel={ + (availableComputeProvidersForMusic as any)[musicProvider] + || ComputeProvider.NONE + } + items={Object.entries(availableComputeProvidersForMusic).map(([provider, label]) => ({ + id: provider, + label, + disabled: false, + value: provider as ComputeProvider, + }))} + onSelect={setMusicProvider} + horizontal + /> + {musicProvider.startsWith("COMFY_") + ? + : // "proprietary" parameters + <> + {musicProvider === ComputeProvider.HUGGINGFACE && } + {musicProvider === ComputeProvider.REPLICATE && } + {musicProvider === ComputeProvider.FALAI && } + {musicProvider === ComputeProvider.MODELSLAB && } + } + +
+ ) +} \ No newline at end of file diff --git a/src/components/settings/provider.tsx b/src/components/settings/provider.tsx new file mode 100644 index 0000000000000000000000000000000000000000..3f3004d27b4a21e43312573a9e3df6b78ab837df --- /dev/null +++ b/src/components/settings/provider.tsx @@ -0,0 +1,97 @@ +import { FormSection } from "@/components/form/form-section" +import { getDefaultSettingsRendering, useSettingsRendering } from "@/settings/rendering" +import { FormSelect } from "../form/form-select" +import { ComfyIcuAccelerator } from "@/types" +import { FormInput } from "../form/form-input" +import { APP_NAME } from "@/lib/core/constants" +import { availableComfyIcuAccelerators } from "./constants" + +export function SettingsSectionProvider() { + const defaultSettings = getDefaultSettingsRendering() + + const replicateApiKey = useSettingsRendering(s => s.replicateApiKey) + const setReplicateApiKey = useSettingsRendering(s => s.setReplicateApiKey) + + const comfyIcuApiKey = useSettingsRendering(s => s.comfyIcuApiKey) + const setComfyIcuApiKey = useSettingsRendering(s => s.setComfyIcuApiKey) + + const comfyIcuAccelerator = useSettingsRendering(s => s.comfyIcuAccelerator) + const setComfyIcuAccelerator = useSettingsRendering(s => s.setComfyIcuAccelerator) + + const huggingFaceApiKey = useSettingsRendering(s => s.huggingFaceApiKey) + const setHuggingFaceApiKey = useSettingsRendering(s => s.setHuggingFaceApiKey) + + const falAiApiKey = useSettingsRendering(s => s.falAiApiKey) + const setFalAiApiKey = useSettingsRendering(s => s.setFalAiApiKey) + + const modelsLabApiKey = useSettingsRendering(s => s.modelsLabApiKey) + const setModelsLabApiKey = useSettingsRendering(s => s.setModelsLabApiKey) + + return ( +
+ + +

+ Note: preferences and credentials are stored inside your browser local storage.
{APP_NAME} uses them to perform API calls on your behalf, but forgets them immediately. +

+ + + + + + + + label="Comfy.icu hardware accelerator" + selectedItemId={comfyIcuAccelerator} + selectedItemLabel={ + (availableComfyIcuAccelerators as any)[comfyIcuAccelerator] + || ComfyIcuAccelerator.L4 + } + items={Object.entries(availableComfyIcuAccelerators).map(([accelerator, label]) => ({ + id: accelerator, + label, + disabled: false, + value: accelerator as ComfyIcuAccelerator, + }))} + onSelect={setComfyIcuAccelerator} + horizontal + /> + + + + + + +
+ ) +} \ No newline at end of file diff --git a/src/components/settings/sound.tsx b/src/components/settings/sound.tsx new file mode 100644 index 0000000000000000000000000000000000000000..b2c994b4ca191cd95261c4f4a54ef72a3d49b3d5 --- /dev/null +++ b/src/components/settings/sound.tsx @@ -0,0 +1,87 @@ +import { FormSection } from "@/components/form/form-section" +import { getDefaultSettingsRendering, useSettingsRendering } from "@/settings/rendering" +import { ComputeProvider } from "@/types" + +import { FormSelect } from "../form/form-select" +import { availableComputeProvidersForSound } from "./constants" +import { FormInput } from "../form/form-input" + +export function SettingsSectionSound() { + const defaultSettings = getDefaultSettingsRendering() + + const soundProvider = useSettingsRendering(s => s.soundProvider) + const setSoundProvider = useSettingsRendering(s => s.setSoundProvider) + + const huggingFaceModelForSound = useSettingsRendering(s => s.huggingFaceModelForSound) + const setHuggingFaceModelForSound = useSettingsRendering(s => s.setHuggingFaceModelForSound) + + const replicateModelForSound = useSettingsRendering(s => s.replicateModelForSound) + const setReplicateModelForSound = useSettingsRendering(s => s.setReplicateModelForSound) + + const falAiModelForSound = useSettingsRendering(s => s.falAiModelForSound) + const setFalAiModelForSound = useSettingsRendering(s => s.setFalAiModelForSound) + + const modelsLabModelForSound = useSettingsRendering(s => s.modelsLabModelForSound) + const setModelsLabModelForSound = useSettingsRendering(s => s.setModelsLabModelForSound) + + const comfyWorkflowForSound = useSettingsRendering(s => s.comfyWorkflowForSound) + const setComfyWorkflowForSound = useSettingsRendering(s => s.setComfyWorkflowForSound) + + return ( +
+ + + + label="Sound provider" + selectedItemId={soundProvider} + selectedItemLabel={ + (availableComputeProvidersForSound as any)[soundProvider] + || ComputeProvider.NONE + } + items={Object.entries(availableComputeProvidersForSound).map(([provider, label]) => ({ + id: provider, + label, + disabled: false, + value: provider as ComputeProvider, + }))} + onSelect={setSoundProvider} + horizontal + /> + {soundProvider.startsWith("COMFY_") + ? + : // "proprietary" parameters + <> + {soundProvider === ComputeProvider.HUGGINGFACE && } + {soundProvider === ComputeProvider.REPLICATE && } + {soundProvider === ComputeProvider.FALAI && } + {soundProvider === ComputeProvider.MODELSLAB && } + } + +
+ ) +} \ No newline at end of file diff --git a/src/components/settings/speech.tsx b/src/components/settings/speech.tsx new file mode 100644 index 0000000000000000000000000000000000000000..faa31f4a37b29fe7afd61cd3de1c8fb7514fa9c6 --- /dev/null +++ b/src/components/settings/speech.tsx @@ -0,0 +1,87 @@ +import { FormSection } from "@/components/form/form-section" +import { getDefaultSettingsRendering, useSettingsRendering } from "@/settings/rendering" +import { ComputeProvider } from "@/types" + +import { FormSelect } from "../form/form-select" +import { availableComputeProvidersForSpeech } from "./constants" +import { FormInput } from "../form/form-input" + +export function SettingsSectionSpeech() { + const defaultSettings = getDefaultSettingsRendering() + + const speechProvider = useSettingsRendering(s => s.speechProvider) + const setSpeechProvider = useSettingsRendering(s => s.setSpeechProvider) + + const huggingFaceModelForSpeech = useSettingsRendering(s => s.huggingFaceModelForSpeech) + const setHuggingFaceModelForSpeech = useSettingsRendering(s => s.setHuggingFaceModelForSpeech) + + const replicateModelForSpeech = useSettingsRendering(s => s.replicateModelForSpeech) + const setReplicateModelForSpeech = useSettingsRendering(s => s.setReplicateModelForSpeech) + + const falAiModelForSpeech = useSettingsRendering(s => s.falAiModelForSpeech) + const setFalAiModelForSpeech = useSettingsRendering(s => s.setFalAiModelForSpeech) + + const modelsLabModelForSpeech = useSettingsRendering(s => s.modelsLabModelForSpeech) + const setModelsLabModelForSpeech = useSettingsRendering(s => s.setModelsLabModelForSpeech) + + const comfyWorkflowForSpeech = useSettingsRendering(s => s.comfyWorkflowForSpeech) + const setComfyWorkflowForSpeech = useSettingsRendering(s => s.setComfyWorkflowForSpeech) + + return ( +
+ + + + label="Speech provider" + selectedItemId={speechProvider} + selectedItemLabel={ + (availableComputeProvidersForSpeech as any)[speechProvider] + || ComputeProvider.NONE + } + items={Object.entries(availableComputeProvidersForSpeech).map(([provider, label]) => ({ + id: provider, + label, + disabled: false, + value: provider as ComputeProvider, + }))} + onSelect={setSpeechProvider} + horizontal + /> + {speechProvider.startsWith("COMFY_") + ? + : // "proprietary" parameters + <> + {speechProvider === ComputeProvider.HUGGINGFACE && } + {speechProvider === ComputeProvider.REPLICATE && } + {speechProvider === ComputeProvider.FALAI && } + {speechProvider === ComputeProvider.MODELSLAB && } + } + +
+ ) +} \ No newline at end of file diff --git a/src/components/settings/storyboard.tsx b/src/components/settings/storyboard.tsx new file mode 100644 index 0000000000000000000000000000000000000000..ff493108e590b547795594994cad58252eb56f96 --- /dev/null +++ b/src/components/settings/storyboard.tsx @@ -0,0 +1,99 @@ +import { FormSection } from "@/components/form/form-section" +import { getDefaultSettingsRendering, useSettingsRendering } from "@/settings/rendering" + +import { FormInput } from "../form/form-input" +import { ComputeProvider } from "@/types" +import { availableComputeProvidersForStoryboards } from "./constants" +import { FormSelect } from "../form/form-select" + +export function SettingsSectionStoryboard() { + const defaultSettings = getDefaultSettingsRendering() + + const storyboardProvider = useSettingsRendering(s => s.storyboardProvider) + const setStoryboardProvider = useSettingsRendering(s => s.setStoryboardProvider) + + const huggingFaceModelForImage = useSettingsRendering(s => s.huggingFaceModelForImage) + const setHuggingFaceModelForImage = useSettingsRendering(s => s.setHuggingFaceModelForImage) + + const replicateModelForImage = useSettingsRendering(s => s.replicateModelForImage) + const setReplicateModelForImage = useSettingsRendering(s => s.setReplicateModelForImage) + + const falAiModelForImage = useSettingsRendering(s => s.falAiModelForImage) + const setFalAiModelForImage = useSettingsRendering(s => s.setFalAiModelForImage) + + const modelsLabModelForImage = useSettingsRendering(s => s.modelsLabModelForImage) + const setModelsLabModelForImage = useSettingsRendering(s => s.setModelsLabModelForImage) + + const maxStoryboardsToGenerateInParallel = useSettingsRendering(s => s.maxStoryboardsToGenerateInParallel) + const setMaxStoryboardsToGenerateInParallel = useSettingsRendering(s => s.setMaxStoryboardsToGenerateInParallel) + + const comfyWorkflowForStoryboard = useSettingsRendering(s => s.comfyWorkflowForStoryboard) + const setComfyWorkflowForStoryboard = useSettingsRendering(s => s.setComfyWorkflowForStoryboard) + + return ( +
+ + + + label="Storyboard provider" + selectedItemId={storyboardProvider} + selectedItemLabel={ + (availableComputeProvidersForStoryboards as any)[storyboardProvider] + || ComputeProvider.NONE + } + items={Object.entries(availableComputeProvidersForStoryboards).map(([provider, label]) => ({ + id: provider, + label, + disabled: false, + value: provider as ComputeProvider, + }))} + onSelect={setStoryboardProvider} + horizontal + /> + + + + {storyboardProvider.startsWith("COMFY_") + ? + : // "proprietary" parameters + <> + {storyboardProvider === ComputeProvider.HUGGINGFACE && } + {storyboardProvider === ComputeProvider.REPLICATE && } + {storyboardProvider === ComputeProvider.FALAI && } + {storyboardProvider === ComputeProvider.MODELSLAB && } + } + + +
+ ) +} \ No newline at end of file diff --git a/src/components/settings/video.tsx b/src/components/settings/video.tsx new file mode 100644 index 0000000000000000000000000000000000000000..a0766b8b11733df66b5de9c6cb831f375318366c --- /dev/null +++ b/src/components/settings/video.tsx @@ -0,0 +1,103 @@ +import { FormSection } from "@/components/form/form-section" +import { getDefaultSettingsRendering, useSettingsRendering } from "@/settings/rendering" +import { FormSelect } from "../form/form-select" +import { ComputeProvider } from "@/types" +import { FormInput } from "../form/form-input" +import { APP_NAME } from "@/lib/core/constants" +import { availableComputeProvidersForVideos } from "./constants" + +export function SettingsSectionVideo() { + const defaultSettings = getDefaultSettingsRendering() + + const videoProvider = useSettingsRendering(s => s.videoProvider) + const setVideoProvider = useSettingsRendering(s => s.setVideoProvider) + + + const huggingFaceModelForVideo = useSettingsRendering(s => s.huggingFaceModelForVideo) + const setHuggingFaceModelForVideo = useSettingsRendering(s => s.setHuggingFaceModelForVideo) + + const replicateModelForVideo = useSettingsRendering(s => s.replicateModelForVideo) + const setReplicateModelForVideo = useSettingsRendering(s => s.setReplicateModelForVideo) + + const falAiModelForVideo = useSettingsRendering(s => s.falAiModelForVideo) + const setFalAiModelForVideo = useSettingsRendering(s => s.setFalAiModelForVideo) + + const modelsLabModelForVideo = useSettingsRendering(s => s.modelsLabModelForVideo) + const setModelsLabModelForVideo = useSettingsRendering(s => s.setModelsLabModelForVideo) + + const videoRenderingStrategy = useSettingsRendering(s => s.videoRenderingStrategy) + const setVideoRenderingStrategy = useSettingsRendering(s => s.setVideoRenderingStrategy) + + const maxVideosToGenerateInParallel = useSettingsRendering(s => s.maxVideosToGenerateInParallel) + const setMaxVideosToGenerateInParallel = useSettingsRendering(s => s.setMaxVideosToGenerateInParallel) + + const comfyWorkflowForVideo = useSettingsRendering(s => s.comfyWorkflowForVideo) + const setComfyWorkflowForVideo = useSettingsRendering(s => s.setComfyWorkflowForVideo) + + return ( +
+ + + + label="Video provider" + selectedItemId={videoProvider} + selectedItemLabel={ + (availableComputeProvidersForVideos as any)[videoProvider] + || ComputeProvider.NONE + } + items={Object.entries(availableComputeProvidersForVideos).map(([provider, label]) => ({ + id: provider, + label, + disabled: false, + value: provider as ComputeProvider, + }))} + onSelect={setVideoProvider} + horizontal + /> + + + + {videoProvider.startsWith("COMFY_") + ? + : // "proprietary" parameters + <> + {videoProvider === ComputeProvider.HUGGINGFACE && } + {videoProvider === ComputeProvider.REPLICATE && } + {videoProvider === ComputeProvider.FALAI && } + {videoProvider === ComputeProvider.MODELSLAB && } + } + + +
+ ) +} \ No newline at end of file diff --git a/src/components/ui/input.tsx b/src/components/ui/input.tsx index 1c55d2164e846105000383645b51420c19a6cd64..219354baa90e1dd8c522798dff3fafae3b9a29c1 100644 --- a/src/components/ui/input.tsx +++ b/src/components/ui/input.tsx @@ -11,7 +11,7 @@ const Input = React.forwardRef( = { - comfyVendor: `${version}COMFY_VENDOR`, - comfyApiKey: `${version}COMFY_API_KEY`, - - storyboardGenerationStrategy: `${version}STORYBOARD_GENERATION_STRATEGY`, - videoGenerationStrategy: `${version}VIDEO_GENERATION_STRATEEGY`, - - maxNbAssetsToGenerateInParallel: `${version}MAX_NB_ASSETS_TO_GENERATE_IN_PARQLLEL`, -} diff --git a/src/lib/core/constants.ts b/src/lib/core/constants.ts index 7d49f84764a0635f6836352f2bd0bd226c00a238..13f682a0a2f90c16c2c67836434e3d43e684f21c 100644 --- a/src/lib/core/constants.ts +++ b/src/lib/core/constants.ts @@ -4,7 +4,7 @@ export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32 export const APP_NAME = "Clapper" -export const APP_REVISION = "nightly 20240604" +export const APP_REVISION = "nightly 20240605" export const APP_DOMAIN = "Clapper.app" export const APP_LINK = "https://clapper.app" \ No newline at end of file diff --git a/src/lib/core/getDefaultSettings.ts b/src/lib/core/getDefaultSettings.ts deleted file mode 100644 index 962d1d1d1fc5550ae5926770af9320d447da7e4a..0000000000000000000000000000000000000000 --- a/src/lib/core/getDefaultSettings.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { RenderingStrategy, ComfyVendor, Settings } from "@/types" - -export function getDefaultSettings(): Settings { - return { - comfyVendor: ComfyVendor.CUSTOM, - comfyApiKey: "", - - storyboardGenerationStrategy: RenderingStrategy.ON_DEMAND, - videoGenerationStrategy: RenderingStrategy.ON_DEMAND, - - maxNbAssetsToGenerateInParallel: 1, - } - } \ No newline at end of file diff --git a/src/lib/utils/blobToBase64DataUri.ts b/src/lib/utils/blobToBase64DataUri.ts new file mode 100644 index 0000000000000000000000000000000000000000..d5ed19cb68b0befab36519f363d108e82128a838 --- /dev/null +++ b/src/lib/utils/blobToBase64DataUri.ts @@ -0,0 +1,4 @@ +export const blobToBase64DataUri = async (blob: Blob): Promise => { + const buffer = Buffer.from(await blob.arrayBuffer()) + return "data:" + blob.type + ';base64,' + buffer.toString('base64') +} diff --git a/src/lib/utils/fetchContentToBase64.ts b/src/lib/utils/fetchContentToBase64.ts new file mode 100644 index 0000000000000000000000000000000000000000..516a38a39cc273b574fbde58664b7624fc426be8 --- /dev/null +++ b/src/lib/utils/fetchContentToBase64.ts @@ -0,0 +1,15 @@ +export async function fetchContentToBase64(url: string) { + const res = await fetch(url, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + cache: "no-store", + // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) + // next: { revalidate: 1 } + }) + const blob = await res.blob() + const buffer = Buffer.from(await blob.arrayBuffer()) + + return "data:" + blob.type + ';base64,' + buffer.toString('base64') +} \ No newline at end of file diff --git a/src/lib/utils/getValidComfyWorkflowTemplate.ts b/src/lib/utils/getValidComfyWorkflowTemplate.ts new file mode 100644 index 0000000000000000000000000000000000000000..4f6d02329365b00e1e236243b272920b48ff14f5 --- /dev/null +++ b/src/lib/utils/getValidComfyWorkflowTemplate.ts @@ -0,0 +1,13 @@ +export function getValidComfyWorkflowTemplate(something: any, defaultComfyWorkflowTemplate: string) { + const strValue = `${something || defaultComfyWorkflowTemplate}` + try { + const workflow = JSON.parse(strValue) + if (typeof workflow === "object") { + return strValue + } else { + throw new Error(`this doesn't look like a ComfyUI workflow template string`) + } + } catch (err) { + return "{}" + } +} \ No newline at end of file diff --git a/src/lib/utils/index.ts b/src/lib/utils/index.ts index c4a0c6ac9731ebeaa57bdffaddea9dd9aee53965..2d51c8e1578050a3e6495bc8666ed1726ac8be1d 100644 --- a/src/lib/utils/index.ts +++ b/src/lib/utils/index.ts @@ -5,5 +5,5 @@ export { getValidBoolean } from "./getValidBoolean" export { getValidNumber } from "./getValidNumber" export { getValidString } from "./getValidString" export { isValidNumber } from "./isValidNumber" -export { parseComfyVendor } from "./parseComfyVendor" -export { parseRenderingStrategy } from "./parseRenderingStrategy" \ No newline at end of file +export { parseComfyIcuAccelerator } from "./parseComfyIcuAccelerator" +export { parseComputeProvider } from "./parseComputeProvider" diff --git a/src/lib/utils/parseComfyIcuAccelerator.ts b/src/lib/utils/parseComfyIcuAccelerator.ts new file mode 100644 index 0000000000000000000000000000000000000000..c9c97688593275d6ad7f0cf2baa79447abc144f9 --- /dev/null +++ b/src/lib/utils/parseComfyIcuAccelerator.ts @@ -0,0 +1,43 @@ +import { ComfyIcuAccelerator } from "@/types" + +export function parseComfyIcuAccelerator(input: any, defaultAccelerator?: ComfyIcuAccelerator): ComfyIcuAccelerator { + + let unknownString = `${input || ""}`.trim() + + // the "normal" case + if (Object.values(ComfyIcuAccelerator).includes(unknownString as ComfyIcuAccelerator)) { + return unknownString as ComfyIcuAccelerator + } + + let accelerator: ComfyIcuAccelerator = defaultAccelerator || ComfyIcuAccelerator.T4 + + unknownString = unknownString.toLowerCase() + + if (unknownString === "none" || unknownString === "undefined" || unknownString === "") { + accelerator = ComfyIcuAccelerator.L4 + } + else if (unknownString === "l4") { + accelerator = ComfyIcuAccelerator.L4 + } + else if (unknownString === "t4") { + accelerator = ComfyIcuAccelerator.T4 + } + else if (unknownString === "a10") { + accelerator = ComfyIcuAccelerator.A10 + } + else if (unknownString === "a100_40gb") { + accelerator = ComfyIcuAccelerator.A100_40GB + } + else if (unknownString === "a100_80gb") { + accelerator = ComfyIcuAccelerator.A100_80GB + } + else if (unknownString === "a100") { + accelerator = ComfyIcuAccelerator.A100_40GB + } + else if (unknownString === "h100") { + accelerator = ComfyIcuAccelerator.H100 + } else { + accelerator = ComfyIcuAccelerator.L4 + } + return accelerator +} \ No newline at end of file diff --git a/src/lib/utils/parseComfyVendor.ts b/src/lib/utils/parseComfyVendor.ts deleted file mode 100644 index f130a3cd21e26fe803761980b560566b1df0cbb5..0000000000000000000000000000000000000000 --- a/src/lib/utils/parseComfyVendor.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { ComfyVendor } from "@/types" - -export function parseComfyVendor(input: any, defaultVendor?: ComfyVendor): ComfyVendor { - - let unknownString = `${input || ""}`.trim() - - // the "normal" case - if (Object.values(ComfyVendor).includes(unknownString as ComfyVendor)) { - return unknownString as ComfyVendor - } - - let vendor: ComfyVendor = defaultVendor || ComfyVendor.NONE - - unknownString = unknownString.toLowerCase() - - if (unknownString === "none" || unknownString === "undefined" || unknownString === "") { - vendor = ComfyVendor.NONE - } - else if (unknownString === "huggingface" || unknownString === "hugging_face") { - vendor = ComfyVendor.HUGGINGFACE - } - else if (unknownString === "replicate") { - vendor = ComfyVendor.REPLICATE - } else { - vendor = ComfyVendor.NONE - } - return vendor -} \ No newline at end of file diff --git a/src/lib/utils/parseComputeProvider.ts b/src/lib/utils/parseComputeProvider.ts new file mode 100644 index 0000000000000000000000000000000000000000..1980f8875de06d5030e9a8739bf2db3431f8af73 --- /dev/null +++ b/src/lib/utils/parseComputeProvider.ts @@ -0,0 +1,50 @@ +import { ComputeProvider } from "@/types" + +export function parseComputeProvider(input: any, defaultVendor?: ComputeProvider): ComputeProvider { + + let unknownString = `${input || ""}`.trim() + + // the "normal" case + if (Object.values(ComputeProvider).includes(unknownString as ComputeProvider)) { + return unknownString as ComputeProvider + } + + let vendor: ComputeProvider = defaultVendor || ComputeProvider.NONE + + unknownString = unknownString.toLowerCase() + + if (unknownString === "none" || unknownString === "undefined" || unknownString === "") { + vendor = ComputeProvider.NONE + } + else if (unknownString === "huggingface" || unknownString === "hugging_face") { + vendor = ComputeProvider.HUGGINGFACE + } + else if (unknownString === "replicate") { + vendor = ComputeProvider.REPLICATE + } + else if (unknownString === "comfyicu" || unknownString === "comfy.icu") { + vendor = ComputeProvider.COMFY_COMFYICU + } + else if (unknownString === "eleven labs" || unknownString === "eleven labs") { + vendor = ComputeProvider.ELEVENLABS + } + else if (unknownString === "openai") { + vendor = ComputeProvider.OPENAI + } + else if (unknownString === "stabilityai") { + vendor = ComputeProvider.STABILITYAI + } + else if (unknownString === "groq") { + vendor = ComputeProvider.GROQ + } + else if (unknownString === "falai") { + vendor = ComputeProvider.FALAI + } + else if (unknownString === "modelslab") { + vendor = ComputeProvider.MODELSLAB + } + else { + vendor = ComputeProvider.NONE + } + return vendor +} \ No newline at end of file diff --git a/src/lib/utils/parseRenderingStrategy.ts b/src/lib/utils/parseRenderingStrategy.ts deleted file mode 100644 index 204f952462625c242570febef29c3e6fe11f000d..0000000000000000000000000000000000000000 --- a/src/lib/utils/parseRenderingStrategy.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { RenderingStrategy } from "@/types" - -export function parseRenderingStrategy(input: any, defaultStrategy?: RenderingStrategy): RenderingStrategy { - - let unknownString = `${input || ""}`.trim() - - // the "normal" case - if (Object.values(RenderingStrategy).includes(unknownString as RenderingStrategy)) { - return unknownString as RenderingStrategy - } - - let strategy: RenderingStrategy = defaultStrategy || RenderingStrategy.ON_DEMAND - - unknownString = unknownString.toLowerCase() - - if (unknownString === "on_demand") { - strategy = RenderingStrategy.ON_DEMAND - } - else if (unknownString === "on_screen_only") { - strategy = RenderingStrategy.ON_SCREEN_ONLY - } - else if (unknownString === "on_screen_then_surrounding") { - strategy = RenderingStrategy.ON_SCREEN_THEN_SURROUNDING - } - else if (unknownString === "on_screen_then_all") { - strategy = RenderingStrategy.ON_SCREEN_THEN_ALL - } else { - strategy = RenderingStrategy.ON_DEMAND - } - return strategy -} \ No newline at end of file diff --git a/src/server/comfy/huggingface.ts b/src/server/comfy/huggingface.ts deleted file mode 100644 index 5b62923bf176547dd23fc408724bd2b6d13a05b3..0000000000000000000000000000000000000000 --- a/src/server/comfy/huggingface.ts +++ /dev/null @@ -1,15 +0,0 @@ -"use server" - - -export async function run({ - apiKey, - workflow -}: { - apiKey: string - workflow: string -}): Promise { - - // TODO: call Hugging Face - - throw new Error(`Hugging Face isn't supported yet`) -} \ No newline at end of file diff --git a/src/server/comfy/index.ts b/src/server/comfy/index.ts deleted file mode 100644 index 33f3e56821f3e074fd6e04975b27096b17de97fd..0000000000000000000000000000000000000000 --- a/src/server/comfy/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -"use server" - -import { RenderRequest } from "@/types" - -import { run as runWithReplicate } from "./replicate" -import { run as runWithHuggingFace } from "./huggingface" -import { getComfyWorkflow } from "./getComfyWorkflow" - -// TODO: at some point in the future we will -// move src/server/comfy to @aitube/engine -export async function render(request: RenderRequest): Promise { - - const workflow = getComfyWorkflow(request) - - // TODO support Hugging Face as well - // const await runWithHuggingFace({ - - const result = await runWithReplicate({ - apiKey: request.comfyUiApiKey, - workflow, - }) - - return result -} \ No newline at end of file diff --git a/src/settings/rendering/getDefaultSettingsRendering.ts b/src/settings/rendering/getDefaultSettingsRendering.ts index 95a01253156f4a3691d916cbdb960dac72ea3bfd..ee286dab9ed971acfb299f71bdf0c69fb6cf4948 100644 --- a/src/settings/rendering/getDefaultSettingsRendering.ts +++ b/src/settings/rendering/getDefaultSettingsRendering.ts @@ -1,15 +1,68 @@ -import { ComfyVendor, RenderingStrategy } from "@/types" +import { RenderingStrategy } from "@aitube/timeline" + +import { ComfyIcuAccelerator, ComputeProvider } from "@/types" import { SettingsRenderingState } from "./types" +import { defaultWorkflowForStoryboards } from "./workflows/storyboard" export function getDefaultSettingsRendering(): SettingsRenderingState { const state: SettingsRenderingState = { - comfyUiApiVendor: ComfyVendor.NONE, - comfyUiApiKey: "", + assistantProvider: ComputeProvider.NONE, + storyboardProvider: ComputeProvider.NONE, + videoProvider: ComputeProvider.NONE, + soundProvider: ComputeProvider.NONE, + speechProvider: ComputeProvider.NONE, + musicProvider: ComputeProvider.NONE, + + customComfyUiApiKey: "", + replicateApiKey: "", + comfyIcuApiKey: "", + comfyIcuAccelerator: ComfyIcuAccelerator.L4, + huggingFaceApiKey: "", + modelsLabApiKey: "", + falAiApiKey: "", storyboardRenderingStrategy: RenderingStrategy.ON_DEMAND, videoRenderingStrategy: RenderingStrategy.ON_DEMAND, - maxNbAssetsToGenerateInParallel: 1, - } + maxStoryboardsToGenerateInParallel: 1, + maxVideosToGenerateInParallel: 1, + comfyWorkflowForStoryboard: defaultWorkflowForStoryboards, + comfyWorkflowForVideo: "{}", + comfyWorkflowForSpeech: "{}", + comfyWorkflowForSound: "{}", + comfyWorkflowForMusic: "{}", + + huggingFaceModelForImage: "sd-community/sdxl-flash", + huggingFaceModelForVideo: "", + huggingFaceModelForSpeech: "", + huggingFaceModelForSound: "", + huggingFaceModelForMusic: "", + + replicateModelForImage: "chenxwh/sdxl-flash:001bb81139b01780380407b4106ac681df46108e002eafbeb9ccb2d8faca42e1", + + // note: this model doesn't support width and height parameters + replicateModelForVideo: "camenduru/animatediff-lightning-4-step:be39c6d599942831314b770f03cfd062bfd0faa8cc52e9289bcce830b721fcb6", + replicateModelForSpeech: "", + replicateModelForSound: "", + replicateModelForMusic: "", + + stabilityAiModelForImage: "", + stabilityAiModelForVideo: "", + stabilityAiModelForSpeech: "", + stabilityAiModelForSound: "", + stabilityAiModelForMusic: "", + + falAiModelForImage: "", + falAiModelForVideo: "", + falAiModelForSpeech: "", + falAiModelForSound: "", + falAiModelForMusic: "", + + modelsLabModelForImage: "", + modelsLabModelForVideo: "", + modelsLabModelForSpeech: "", + modelsLabModelForSound: "", + modelsLabModelForMusic: "", + } return state } diff --git a/src/settings/rendering/types.ts b/src/settings/rendering/types.ts index 469a676db46a3f4f5a374990d04a7fc30f3e0f03..a115c2c1875125715c8e43706110223c18b5ec9c 100644 --- a/src/settings/rendering/types.ts +++ b/src/settings/rendering/types.ts @@ -1,19 +1,121 @@ -import { ComfyVendor, RenderingStrategy } from "@/types" +import { RenderingStrategy } from "@aitube/timeline" + +import { ComfyIcuAccelerator, ComputeProvider } from "@/types" export type SettingsRenderingState = { - comfyUiApiVendor: ComfyVendor - comfyUiApiKey: string + customComfyUiApiKey: string + replicateApiKey: string + comfyIcuApiKey: string + comfyIcuAccelerator: ComfyIcuAccelerator + falAiApiKey: string + modelsLabApiKey: string + huggingFaceApiKey: string + assistantProvider: ComputeProvider + videoProvider: ComputeProvider + storyboardProvider: ComputeProvider + speechProvider: ComputeProvider + soundProvider: ComputeProvider + musicProvider: ComputeProvider storyboardRenderingStrategy: RenderingStrategy videoRenderingStrategy: RenderingStrategy - maxNbAssetsToGenerateInParallel: number + maxStoryboardsToGenerateInParallel: number + maxVideosToGenerateInParallel: number + + comfyWorkflowForStoryboard: string + comfyWorkflowForVideo: string + comfyWorkflowForSpeech: string + comfyWorkflowForSound: string + comfyWorkflowForMusic: string + + huggingFaceModelForImage: string + huggingFaceModelForVideo: string + huggingFaceModelForSpeech: string + huggingFaceModelForSound: string + huggingFaceModelForMusic: string + + replicateModelForImage: string + replicateModelForVideo: string + replicateModelForSpeech: string + replicateModelForSound: string + replicateModelForMusic: string + + stabilityAiModelForImage: string + stabilityAiModelForVideo: string + stabilityAiModelForSpeech: string + stabilityAiModelForSound: string + stabilityAiModelForMusic: string + + falAiModelForImage: string + falAiModelForVideo: string + falAiModelForSpeech: string + falAiModelForSound: string + falAiModelForMusic: string + + modelsLabModelForImage: string + modelsLabModelForVideo: string + modelsLabModelForSpeech: string + modelsLabModelForSound: string + modelsLabModelForMusic: string } export type SettingsRenderingControls = { - setComfyUiApiVendor: (comfyUiApiVendor?: ComfyVendor) => void - setComfyUiApiKey: (comfyUiApiKey?: string) => void + setCustomComfyUiApiKey: (customComfyUiApiKey?: string) => void + setReplicateApiKey: (replicateApiKey?: string) => void + setComfyIcuApiKey: (comfyIcuApiKey?: string) => void + setComfyIcuAccelerator: (comfyIcuAccelerator?: ComfyIcuAccelerator) => void + setHuggingFaceApiKey: (huggingFaceApiKey?: string) => void + setModelsLabApiKey: (modelsLabApiKey?: string) => void + setFalAiApiKey: (falAiApiKey?: string) => void + + setAssistantProvider: (assistantProvider?: ComputeProvider) => void + setVideoProvider: (videoProvider?: ComputeProvider) => void + setStoryboardProvider: (storyboardProvider?: ComputeProvider) => void + setSpeechProvider: (speechProvider?: ComputeProvider) => void + setSoundProvider: (soundProvider?: ComputeProvider) => void + setMusicProvider: (musicProvider?: ComputeProvider) => void + setStoryboardRenderingStrategy: (storyboardRenderingStrategy?: RenderingStrategy) => void setVideoRenderingStrategy: (videoRenderingStrategy?: RenderingStrategy) => void - setMaxNbAssetsToGenerateInParallel: (maxNbAssetsToGenerateInParallel?: number) => void + setMaxStoryboardsToGenerateInParallel: (maxStoryboardsToGenerateInParallel?: number) => void + setMaxVideosToGenerateInParallel: (maxVideosToGenerateInParallel?: number) => void + + setComfyWorkflowForStoryboard: (comfyWorkflowForStoryboard?: string) => void + setComfyWorkflowForVideo: (comfyWorkflowForVideo?: string) => void + setComfyWorkflowForSpeech: (comfyWorkflowForSpeech?: string) => void + setComfyWorkflowForSound: (comfyWorkflowForSound?: string) => void + setComfyWorkflowForMusic: (comfyWorkflowForMusic?: string) => void + + setHuggingFaceModelForImage: (huggingFaceModelForImage?: string) => void + setHuggingFaceModelForVideo: (huggingFaceModelForVideo?: string) => void + setHuggingFaceModelForSpeech: (huggingFaceModelForSpeech?: string) => void + setHuggingFaceModelForSound: (huggingFaceModelForSound?: string) => void + setHuggingFaceModelForMusic: (huggingFaceModelForMusic?: string) => void + + setReplicateModelForImage: (replicateModelForImage?: string) => void + setReplicateModelForVideo: (replicateModelForVideo?: string) => void + setReplicateModelForSpeech: (replicateModelForSpeech?: string) => void + setReplicateModelForSound: (replicateModelForSound?: string) => void + setReplicateModelForMusic: (replicateModelForMusic?: string) => void + + setStabilityAiModelForImage: (stabilityAiModelForImage?: string) => void + setStabilityAiModelForVideo: (stabilityAiModelForVideo?: string) => void + setStabilityAiModelForSpeech: (stabilityAiModelForSpeech?: string) => void + setStabilityAiModelForSound: (stabilityAiModelForSound?: string) => void + setStabilityAiModelForMusic: (stabilityAiModelForMusic?: string) => void + + setFalAiModelForImage: (falAiModelForImage?: string) => void + setFalAiModelForVideo: (falAiModelForVideo?: string) => void + setFalAiModelForSpeech: (falAiModelForSpeech?: string) => void + setFalAiModelForSound: (falAiModelForSound?: string) => void + setFalAiModelForMusic: (falAiModelForMusic?: string) => void + + setModelsLabModelForImage: (modelsLabModelForImage?: string) => void + setModelsLabModelForVideo: (modelsLabModelForVideo?: string) => void + setModelsLabModelForSpeech: (modelsLabModelForSpeech?: string) => void + setModelsLabModelForSound: (modelsLabModelForSound?: string) => void + setModelsLabModelForMusic: (modelsLabModelForMusic?: string) => void + + getSettings: () => SettingsRenderingState } export type SettingsRenderingStore = diff --git a/src/settings/rendering/useSettingsRendering.ts b/src/settings/rendering/useSettingsRendering.ts index e4192c74829c54a14be49dc9c1ec57e7dd6c009a..c696d9f02148115f7ecc1ccd98302f5f0bcfc9e2 100644 --- a/src/settings/rendering/useSettingsRendering.ts +++ b/src/settings/rendering/useSettingsRendering.ts @@ -3,26 +3,71 @@ import { create } from "zustand" import { persist } from 'zustand/middleware' import { getValidNumber } from "@aitube/clap" +import { parseRenderingStrategy, RenderingStrategy } from "@aitube/timeline" -import { ComfyVendor, RenderingStrategy } from "@/types" -import { getValidString, parseRenderingStrategy, parseComfyVendor } from "@/lib/utils" +import { ComfyIcuAccelerator, ComputeProvider } from "@/types" +import { getValidString, parseComputeProvider } from "@/lib/utils" import { HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL } from "@/lib/core/constants" -import { SettingsRenderingStore } from "./types" +import { SettingsRenderingState, SettingsRenderingStore } from "./types" import { getDefaultSettingsRendering } from "./getDefaultSettingsRendering" +import { getValidComfyWorkflowTemplate } from "@/lib/utils/getValidComfyWorkflowTemplate" +import { parseComfyIcuAccelerator } from "@/lib/utils/parseComfyIcuAccelerator" export const useSettingsRendering = create()( persist( (set, get) => ({ ...getDefaultSettingsRendering(), - setComfyUiApiVendor: (comfyUiApiVendor?: ComfyVendor) => { - const { comfyUiApiVendor: defaultComfyUiApiVendor } = getDefaultSettingsRendering() - set({ comfyUiApiVendor: parseComfyVendor(comfyUiApiVendor, defaultComfyUiApiVendor) }) + setAssistantProvider: (assistantProvider?: ComputeProvider) => { + const { videoProvider: defaultAssistantProvider } = getDefaultSettingsRendering() + set({ assistantProvider: parseComputeProvider(assistantProvider, defaultAssistantProvider) }) }, - setComfyUiApiKey: (comfyUiApiKey?: string) => { - const { comfyUiApiKey: defaultComfyUiApiKey } = getDefaultSettingsRendering() - set({ comfyUiApiKey: getValidString(comfyUiApiKey, defaultComfyUiApiKey) }) + setVideoProvider: (videoProvider?: ComputeProvider) => { + const { videoProvider: defaultVideoProvider } = getDefaultSettingsRendering() + set({ videoProvider: parseComputeProvider(videoProvider, defaultVideoProvider) }) + }, + setStoryboardProvider: (storyboardProvider?: ComputeProvider) => { + const { storyboardProvider: defaultStoryboardProvider } = getDefaultSettingsRendering() + set({ storyboardProvider: parseComputeProvider(storyboardProvider, defaultStoryboardProvider) }) + }, + setSpeechProvider: (speechProvider?: ComputeProvider) => { + const { speechProvider: defaultSpeechProvider } = getDefaultSettingsRendering() + set({ speechProvider: parseComputeProvider(speechProvider, defaultSpeechProvider) }) + }, + setSoundProvider: (soundProvider?: ComputeProvider) => { + const { soundProvider: defaultSoundProvider } = getDefaultSettingsRendering() + set({ soundProvider: parseComputeProvider(soundProvider, defaultSoundProvider) }) + }, + setMusicProvider: (musicProvider?: ComputeProvider) => { + const { musicProvider: defaultMusicProvider } = getDefaultSettingsRendering() + set({ musicProvider: parseComputeProvider(musicProvider, defaultMusicProvider) }) + }, + setCustomComfyUiApiKey: (customComfyUiApiKey?: string) => { + const { customComfyUiApiKey: defaulCustomComfyUiApiKey } = getDefaultSettingsRendering() + set({ customComfyUiApiKey: getValidString(customComfyUiApiKey, defaulCustomComfyUiApiKey) }) + }, + setReplicateApiKey: (replicateApiKey?: string) => { + const { replicateApiKey: defaultReplicateApiKey } = getDefaultSettingsRendering() + set({ replicateApiKey: getValidString(replicateApiKey, defaultReplicateApiKey) }) + }, + setComfyIcuApiKey: (comfyIcuApiKey?: string) => { + const { comfyIcuApiKey: defaultComfyIcuApiKey } = getDefaultSettingsRendering() + set({ comfyIcuApiKey: getValidString(comfyIcuApiKey, defaultComfyIcuApiKey) }) + }, + setComfyIcuAccelerator: (comfyIcuAccelerator?: ComfyIcuAccelerator) => { + const { comfyIcuAccelerator: defaulComfyIcuAccelerator } = getDefaultSettingsRendering() + set({ comfyIcuAccelerator: parseComfyIcuAccelerator(comfyIcuAccelerator, defaulComfyIcuAccelerator) }) + }, + setHuggingFaceApiKey: (huggingFaceApiKey?: string) => { + const { huggingFaceApiKey: defaultHuggingFaceApiKey } = getDefaultSettingsRendering() + set({ huggingFaceApiKey: getValidString(huggingFaceApiKey, defaultHuggingFaceApiKey) }) + }, + setFalAiApiKey: (falAiApiKey?: string) => { + set({ falAiApiKey: getValidString(falAiApiKey, getDefaultSettingsRendering().falAiApiKey) }) + }, + setModelsLabApiKey: (modelsLabApiKey?: string) => { + set({ modelsLabApiKey: getValidString(modelsLabApiKey, getDefaultSettingsRendering().modelsLabApiKey) }) }, setStoryboardRenderingStrategy: (storyboardRenderingStrategy?: RenderingStrategy) => { const { storyboardRenderingStrategy: defaulStoryboardRenderingStrategy } = getDefaultSettingsRendering() @@ -32,17 +77,219 @@ export const useSettingsRendering = create()( const { videoRenderingStrategy: defaultVideoRenderingStrategy } = getDefaultSettingsRendering() set({ videoRenderingStrategy: parseRenderingStrategy(videoRenderingStrategy, defaultVideoRenderingStrategy) }) }, - setMaxNbAssetsToGenerateInParallel: (maxNbAssetsToGenerateInParallel?: number) => { - const { maxNbAssetsToGenerateInParallel: defaultMaxNbAssetsToGenerateInParallel } = getDefaultSettingsRendering() + setMaxStoryboardsToGenerateInParallel: (maxStoryboardsToGenerateInParallel?: number) => { + const { maxStoryboardsToGenerateInParallel: defaultMaxStoryboardsToGenerateInParallel } = getDefaultSettingsRendering() set({ - maxNbAssetsToGenerateInParallel: getValidNumber( - maxNbAssetsToGenerateInParallel, + maxStoryboardsToGenerateInParallel: getValidNumber( + maxStoryboardsToGenerateInParallel, 1, HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL, - defaultMaxNbAssetsToGenerateInParallel + defaultMaxStoryboardsToGenerateInParallel ) }) }, + setMaxVideosToGenerateInParallel: (maxVideosToGenerateInParallel?: number) => { + const { maxVideosToGenerateInParallel: defaultMaxVideosToGenerateInParallel } = getDefaultSettingsRendering() + set({ + maxVideosToGenerateInParallel: getValidNumber( + maxVideosToGenerateInParallel, + 1, + HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL, + defaultMaxVideosToGenerateInParallel + ) + }) + }, + setComfyWorkflowForStoryboard: (comfyWorkflowForStoryboard?: string) => { + set({ comfyWorkflowForStoryboard: getValidComfyWorkflowTemplate(comfyWorkflowForStoryboard, getDefaultSettingsRendering().comfyWorkflowForStoryboard) }) + }, + setComfyWorkflowForVideo: (comfyWorkflowForVideo?: string) => { + set({ comfyWorkflowForVideo: getValidComfyWorkflowTemplate(comfyWorkflowForVideo, getDefaultSettingsRendering().comfyWorkflowForVideo) }) + }, + setComfyWorkflowForSpeech: (comfyWorkflowForSpeech?: string) => { + set({ comfyWorkflowForSpeech: getValidComfyWorkflowTemplate(comfyWorkflowForSpeech, getDefaultSettingsRendering().comfyWorkflowForSpeech) }) + }, + setComfyWorkflowForSound: (comfyWorkflowForSound?: string) => { + set({ comfyWorkflowForSound: getValidComfyWorkflowTemplate(comfyWorkflowForSound, getDefaultSettingsRendering().comfyWorkflowForSound) }) + }, + setComfyWorkflowForMusic: (comfyWorkflowForMusic?: string) => { + set({ comfyWorkflowForMusic: getValidComfyWorkflowTemplate(comfyWorkflowForMusic, getDefaultSettingsRendering().comfyWorkflowForMusic) }) + }, + setHuggingFaceModelForImage: (huggingFaceModelForImage?: string) => { + set({ huggingFaceModelForImage: getValidString(huggingFaceModelForImage, getDefaultSettingsRendering().huggingFaceModelForImage) }) + }, + setHuggingFaceModelForVideo: (huggingFaceModelForVideo?: string) => { + set({ huggingFaceModelForVideo: getValidString(huggingFaceModelForVideo, getDefaultSettingsRendering().huggingFaceModelForVideo) }) + }, + setHuggingFaceModelForSpeech: (huggingFaceModelForSpeech?: string) => { + set({ huggingFaceModelForSpeech: getValidString(huggingFaceModelForSpeech, getDefaultSettingsRendering().huggingFaceModelForSpeech) }) + }, + setHuggingFaceModelForSound: (huggingFaceModelForSound?: string) => { + set({ huggingFaceModelForSound: getValidString(huggingFaceModelForSound, getDefaultSettingsRendering().huggingFaceModelForSound) }) + }, + setHuggingFaceModelForMusic: (huggingFaceModelForMusic?: string) => { + set({ huggingFaceModelForMusic: getValidString(huggingFaceModelForMusic, getDefaultSettingsRendering().huggingFaceModelForMusic) }) + }, + setReplicateModelForImage: (replicateModelForImage?: string) => { + set({ replicateModelForImage: getValidString(replicateModelForImage, getDefaultSettingsRendering().replicateModelForImage) }) + }, + setReplicateModelForVideo: (replicateModelForVideo?: string) => { + set({ replicateModelForVideo: getValidString(replicateModelForVideo, getDefaultSettingsRendering().replicateModelForVideo) }) + }, + setReplicateModelForSpeech: (replicateModelForSpeech?: string) => { + set({ replicateModelForSpeech: getValidString(replicateModelForSpeech, getDefaultSettingsRendering().replicateModelForSpeech) }) + }, + setReplicateModelForSound: (replicateModelForSound?: string) => { + set({ replicateModelForSound: getValidString(replicateModelForSound, getDefaultSettingsRendering().replicateModelForSound) }) + }, + setReplicateModelForMusic: (replicateModelForMusic?: string) => { + set({ replicateModelForMusic: getValidString(replicateModelForMusic, getDefaultSettingsRendering().replicateModelForMusic) }) + }, + setStabilityAiModelForImage: (stabilityAiModelForImage?: string) => { + set({ stabilityAiModelForImage: getValidString(stabilityAiModelForImage, getDefaultSettingsRendering().stabilityAiModelForImage) }) + }, + setStabilityAiModelForVideo: (stabilityAiModelForVideo?: string) => { + set({ stabilityAiModelForVideo: getValidString(stabilityAiModelForVideo, getDefaultSettingsRendering().stabilityAiModelForVideo) }) + }, + setStabilityAiModelForSpeech: (stabilityAiModelForSpeech?: string) => { + set({ stabilityAiModelForSpeech: getValidString(stabilityAiModelForSpeech, getDefaultSettingsRendering().stabilityAiModelForSpeech) }) + }, + setStabilityAiModelForSound: (stabilityAiModelForSound?: string) => { + set({ stabilityAiModelForSound: getValidString(stabilityAiModelForSound, getDefaultSettingsRendering().stabilityAiModelForSound) }) + }, + setStabilityAiModelForMusic: (stabilityAiModelForMusic?: string) => { + set({ stabilityAiModelForMusic: getValidString(stabilityAiModelForMusic, getDefaultSettingsRendering().stabilityAiModelForMusic) }) + }, + setFalAiModelForImage: (falAiModelForImage?: string) => { + set({ falAiModelForImage: getValidString(falAiModelForImage, getDefaultSettingsRendering().falAiModelForImage) }) + }, + setFalAiModelForVideo: (falAiModelForVideo?: string) => { + set({ falAiModelForVideo: getValidString(falAiModelForVideo, getDefaultSettingsRendering().falAiModelForVideo) }) + }, + setFalAiModelForSpeech: (falAiModelForSpeech?: string) => { + set({ falAiModelForSpeech: getValidString(falAiModelForSpeech, getDefaultSettingsRendering().falAiModelForSpeech) }) + }, + setFalAiModelForSound: (falAiModelForSound?: string) => { + set({ falAiModelForSound: getValidString(falAiModelForSound, getDefaultSettingsRendering().falAiModelForSound) }) + }, + setFalAiModelForMusic: (falAiModelForMusic?: string) => { + set({ falAiModelForMusic: getValidString(falAiModelForMusic, getDefaultSettingsRendering().falAiModelForMusic) }) + }, + setModelsLabModelForImage: (modelsLabModelForImage?: string) => { + set({ modelsLabModelForImage: getValidString(modelsLabModelForImage, getDefaultSettingsRendering().modelsLabModelForImage) }) + }, + setModelsLabModelForVideo: (modelsLabModelForVideo?: string) => { + set({ modelsLabModelForVideo: getValidString(modelsLabModelForVideo, getDefaultSettingsRendering().modelsLabModelForVideo) }) + }, + setModelsLabModelForSpeech: (modelsLabModelForSpeech?: string) => { + set({ modelsLabModelForSpeech: getValidString(modelsLabModelForSpeech, getDefaultSettingsRendering().modelsLabModelForSpeech) }) + }, + setModelsLabModelForSound: (modelsLabModelForSound?: string) => { + set({ modelsLabModelForSound: getValidString(modelsLabModelForSound, getDefaultSettingsRendering().modelsLabModelForSound) }) + }, + setModelsLabModelForMusic: (modelsLabModelForMusic?: string) => { + set({ modelsLabModelForMusic: getValidString(modelsLabModelForMusic, getDefaultSettingsRendering().modelsLabModelForMusic) }) + }, + + getSettings: (): SettingsRenderingState => { + const { + assistantProvider, + storyboardProvider, + videoProvider, + soundProvider, + speechProvider, + musicProvider, + customComfyUiApiKey, + replicateApiKey, + comfyIcuApiKey, + comfyIcuAccelerator, + huggingFaceApiKey, + falAiApiKey, + modelsLabApiKey, + storyboardRenderingStrategy, + videoRenderingStrategy, + maxStoryboardsToGenerateInParallel, + maxVideosToGenerateInParallel, + comfyWorkflowForStoryboard, + comfyWorkflowForVideo, + comfyWorkflowForSpeech, + comfyWorkflowForSound, + comfyWorkflowForMusic, + huggingFaceModelForImage, + huggingFaceModelForVideo, + huggingFaceModelForSpeech, + huggingFaceModelForSound, + huggingFaceModelForMusic, + replicateModelForImage, + replicateModelForVideo, + replicateModelForSpeech, + replicateModelForSound, + replicateModelForMusic, + stabilityAiModelForImage, + stabilityAiModelForVideo, + stabilityAiModelForSpeech, + stabilityAiModelForSound, + stabilityAiModelForMusic, + falAiModelForImage, + falAiModelForVideo, + falAiModelForSpeech, + falAiModelForSound, + falAiModelForMusic, + modelsLabModelForImage, + modelsLabModelForVideo, + modelsLabModelForSpeech, + modelsLabModelForSound, + modelsLabModelForMusic, + } = get() + return { + assistantProvider, + storyboardProvider, + videoProvider, + soundProvider, + speechProvider, + musicProvider, + customComfyUiApiKey, + replicateApiKey, + comfyIcuApiKey, + comfyIcuAccelerator, + huggingFaceApiKey, + falAiApiKey, + modelsLabApiKey, + storyboardRenderingStrategy, + videoRenderingStrategy, + maxStoryboardsToGenerateInParallel, + maxVideosToGenerateInParallel, + comfyWorkflowForStoryboard, + comfyWorkflowForVideo, + comfyWorkflowForSpeech, + comfyWorkflowForSound, + comfyWorkflowForMusic, + huggingFaceModelForImage, + huggingFaceModelForVideo, + huggingFaceModelForSpeech, + huggingFaceModelForSound, + huggingFaceModelForMusic, + replicateModelForImage, + replicateModelForVideo, + replicateModelForSpeech, + replicateModelForSound, + replicateModelForMusic, + stabilityAiModelForImage, + stabilityAiModelForVideo, + stabilityAiModelForSpeech, + stabilityAiModelForSound, + stabilityAiModelForMusic, + falAiModelForImage, + falAiModelForVideo, + falAiModelForSpeech, + falAiModelForSound, + falAiModelForMusic, + modelsLabModelForImage, + modelsLabModelForVideo, + modelsLabModelForSpeech, + modelsLabModelForSound, + modelsLabModelForMusic, + } + }, }), { name: 'CLAPPER_REVISION_0_SETTINGS_RENDERING' diff --git a/src/settings/rendering/workflows/storyboard.ts b/src/settings/rendering/workflows/storyboard.ts new file mode 100644 index 0000000000000000000000000000000000000000..55988a8a552dbefde0b6f0780605bb67927f0886 --- /dev/null +++ b/src/settings/rendering/workflows/storyboard.ts @@ -0,0 +1,110 @@ + + +// https://replicate.com/fofr/any-comfyui-workflow/examples#ps4f5zcthdrgm0cfqh586zed7r +export const defaultWorkflowForStoryboards = JSON.stringify({ + "3": { + "inputs": { + "seed": 156680208700286, + "steps": 10, + "cfg": 2.5, + "sampler_name": "dpmpp_2m_sde", + "scheduler": "karras", + "denoise": 1, + "model": [ + "4", + 0 + ], + "positive": [ + "6", + 0 + ], + "negative": [ + "7", + 0 + ], + "latent_image": [ + "5", + 0 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "4": { + "inputs": { + "ckpt_name": "SDXL-Flash.safetensors" + }, + "class_type": "CheckpointLoaderSimple", + "_meta": { + "title": "Load Checkpoint" + } + }, + "5": { + "inputs": { + "width": 1024, + "height": 1024, + "batch_size": 1 + }, + "class_type": "EmptyLatentImage", + "_meta": { + "title": "Empty Latent Image" + } + }, + "6": { + "inputs": { + "text": "beautiful scenery nature glass bottle landscape, purple galaxy bottle,", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "7": { + "inputs": { + "text": "text, watermark", + "clip": [ + "4", + 1 + ] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "8": { + "inputs": { + "samples": [ + "3", + 0 + ], + "vae": [ + "4", + 2 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "9": { + "inputs": { + "filename_prefix": "ComfyUI", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + } +}) \ No newline at end of file diff --git a/src/types.ts b/src/types.ts index d827d31a52002d7145c8b853d40db24508331562..49945b2e12fbc064fb226253890c75e70f100417 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,61 +1,33 @@ import { ClapEntity, ClapSegment } from "@aitube/clap" +import { SettingsRenderingState } from "./settings/rendering" -export enum ComfyVendor { +export enum ComputeProvider { NONE = "NONE", CUSTOM = "CUSTOM", - HUGGINGFACE = "HUGGINGFACE", - REPLICATE = "REPLICATE", + HUGGINGFACE = "HUGGINGFACE", // https://huggingface.co + COMFY_HUGGINGFACE = "COMFY_HUGGINGFACE", // https://huggingface.co + REPLICATE = "REPLICATE", // https://replicate.com + COMFY_REPLICATE = "COMFY_REPLICATE", // https://replicate.com + COMFY_COMFYICU = "COMFY_COMFYICU", // https://comfy.icu + ELEVENLABS = "ELEVENLABS", // https://elevenlabs.io + OPENAI = "OPENAI", // https://openai.com + STABILITYAI = "STABILITYAI", // https://stability.ai + GROQ = "GROQ", // https://groq.com + FALAI = "FALAI", // https://fal.ai + MODELSLAB = "MODELSLAB", // https://modelslab.com } -export enum RenderingStrategy { - - // render assets when the user asks for it (could be a click or mouse hover) - ON_DEMAND = "ON_DEMAND", - - // render assets currently visible on screen, never render invisible ones - ON_SCREEN_ONLY = "ON_SCREEN_ONLY", - - // render assets visible on screen in priority, - // then pre-render a few of the surrounding assets (but not the whole set) - ON_SCREEN_THEN_SURROUNDING = "ON_SCREEN_THEN_SURROUNDING", - - - // render assets visible on screen in priority, - // then pre-render *ALL* the remaining project's assets - // so yeah if you have 3000 storyboards, it will render that many ($$$) - // (note: there is a setting to cap the number of parallel renderings) - // - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - // !! this is hardcore! only GPU-rich people shoud use this feature! !! - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ON_SCREEN_THEN_ALL = "ON_SCREEN_THEN_ALL", -} - -// TODO: rename the *Model fields to better indicate if this is a LLM or RENDER mdoel -export type Settings = { - comfyVendor: ComfyVendor - comfyApiKey: string - - storyboardGenerationStrategy: RenderingStrategy - videoGenerationStrategy: RenderingStrategy - - maxNbAssetsToGenerateInParallel: number +export enum ComfyIcuAccelerator { + T4 = "T4", + L4 = "L4", + A10 = "A10", + A100_40GB = "A100_40GB", + A100_80GB = "A100_80GB", + H100 = "H100" } export type RenderRequest = { - comfyUiApiVendor: ComfyVendor - - // secret vendor api key to use (provided by the user) - comfyUiApiKey: string - - // available ComfyUI workflows - // a render request might use multiple workflows (eg. entity + storyboard) - // so we need to have them all readily available - entityWorkflow: string - dialogueWorkflow: string - storyboardWorkflow: string - videoWorkflow: string - musicWorkflow: string + settings: SettingsRenderingState // the reference segment to render (eg. storyboard or video) segment: ClapSegment