let's use @aitube/clap inside AiTube

.env

@@ -3,6 +3,8 @@ API_SECRET_JWT_KEY=""
 API_SECRET_JWT_ISSUER=""
 API_SECRET_JWT_AUDIENCE=""
 
+AI_TUBE_CLAP_EXPORTER_URL="https://jbilcke-hf-ai-tube-clap-exporter.hf.space"
+
 NEXT_PUBLIC_DOMAIN="https://aitube.at"
 
 NEXT_PUBLIC_SHOW_BETA_FEATURES="false"

next.config.js

@@ -1,6 +1,9 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   output: 'standalone',
+  transpilePackages: [
+    // '@aitube/clap'
+  ],
   experimental: {
     serverActions: {
 

package-lock.json

@@ -8,6 +8,7 @@
       "name": "ai-tube",
       "version": "0.0.0",
       "dependencies": {
+        "@aitube/clap": "^0.0.6",
         "@huggingface/hub": "0.12.3-oauth",
         "@huggingface/inference": "^2.6.7",
         "@jcoreio/async-throttle": "^1.6.0",
@@ -95,7 +96,7 @@
         "temp-dir": "^3.0.0",
         "ts-node": "^10.9.2",
         "type-fest": "^4.8.2",
-        "typescript": "5.4.4",
+        "typescript": "^5.4.5",
         "usehooks-ts": "^2.9.1",
         "uuid": "^9.0.1",
         "yaml": "^2.3.4",
@@ -118,6 +119,18 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/@aitube/clap": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.6.tgz",
+      "integrity": "sha512-SPo90RBnOJCmp+DqzxllNOcp38AbHSzqkAbYEudRiubqWHDF1GGqYi25gCdG7bFIWH+8evjSiiwsjkzedpbhoA==",
+      "dependencies": {
+        "pure-uuid": "^1.8.1",
+        "yaml": "^2.4.1"
+      },
+      "peerDependencies": {
+        "typescript": "^5.4.5"
+      }
+    },
     "node_modules/@alloc/quick-lru": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",
@@ -1494,14 +1507,14 @@
       }
     },
     "node_modules/@mediapipe/tasks-vision": {
-      "version": "0.10.13-rc.20240422",
-      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.13-rc.20240422.tgz",
-      "integrity": "sha512-yKUS+Qidsw0pttv8Bx/EOdGkcWLH0b1tO4D0HfM6PaBjBAFUr7l5OmRfToFh4k8/XPto6d3X8Ujvm37Da0n2nw=="
+      "version": "0.10.13-rc.20240426",
+      "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.13-rc.20240426.tgz",
+      "integrity": "sha512-YyickIMLXr2/pEOZ00bHYWfWmCAwC8uRv0Ek6haQvnzahwfiw4Evlka3XRa8SxL4X7p432puS558xAm5h0SJTA=="
     },
     "node_modules/@next/env": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.2.tgz",
-      "integrity": "sha512-sk72qRfM1Q90XZWYRoJKu/UWlTgihrASiYw/scb15u+tyzcze3bOuJ/UV6TBOQEeUaxOkRqGeuGUdiiuxc5oqw=="
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.3.tgz",
+      "integrity": "sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA=="
     },
     "node_modules/@next/eslint-plugin-next": {
       "version": "13.4.10",
@@ -1512,9 +1525,9 @@
       }
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.2.tgz",
-      "integrity": "sha512-3iPgMhzbalizGwHNFUcGnDhFPSgVBHQ8aqSTAMxB5BvJG0oYrDf1WOJZlbXBgunOEj/8KMVbejEur/FpvFsgFQ==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.3.tgz",
+      "integrity": "sha512-3pEYo/RaGqPP0YzwnlmPN2puaF2WMLM3apt5jLW2fFdXD9+pqcoTzRk+iZsf8ta7+quAe4Q6Ms0nR0SFGFdS1A==",
       "cpu": [
         "arm64"
       ],
@@ -1527,9 +1540,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.2.tgz",
-      "integrity": "sha512-x7Afi/jt0ZBRUZHTi49yyej4o8znfIMHO4RvThuoc0P+uli8Jd99y5GKjxoYunPKsXL09xBXEM1+OQy2xEL0Ag==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz",
+      "integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==",
       "cpu": [
         "x64"
       ],
@@ -1542,9 +1555,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.2.tgz",
-      "integrity": "sha512-zbfPtkk7L41ODMJwSp5VbmPozPmMMQrzAc0HAUomVeVIIwlDGs/UCqLJvLNDt4jpWgc21SjjyIn762lNGrMaUA==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz",
+      "integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==",
       "cpu": [
         "arm64"
       ],
@@ -1557,9 +1570,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.2.tgz",
-      "integrity": "sha512-wPbS3pI/JU16rm3XdLvvTmlsmm1nd+sBa2ohXgBZcShX4TgOjD4R+RqHKlI1cjo/jDZKXt6OxmcU0Iys0OC/yg==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz",
+      "integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==",
       "cpu": [
         "arm64"
       ],
@@ -1572,9 +1585,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.2.tgz",
-      "integrity": "sha512-NqWOHqqq8iC9tuHvZxjQ2tX+jWy2X9y8NX2mcB4sj2bIccuCxbIZrU/ThFPZZPauygajZuVQ6zediejQHwZHwQ==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz",
+      "integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==",
       "cpu": [
         "x64"
       ],
@@ -1587,9 +1600,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.2.tgz",
-      "integrity": "sha512-lGepHhwb9sGhCcU7999+iK1ZZT+6rrIoVg40MP7DZski9GIZP80wORSbt5kJzh9v2x2ev2lxC6VgwMQT0PcgTA==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz",
+      "integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==",
       "cpu": [
         "x64"
       ],
@@ -1602,9 +1615,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.2.tgz",
-      "integrity": "sha512-TZSh/48SfcLEQ4rD25VVn2kdIgUWmMflRX3OiyPwGNXn3NiyPqhqei/BaqCYXViIQ+6QsG9R0C8LftMqy8JPMA==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz",
+      "integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==",
       "cpu": [
         "arm64"
       ],
@@ -1617,9 +1630,9 @@
       }
     },
     "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.2.tgz",
-      "integrity": "sha512-M0tBVNMEBJN2ZNQWlcekMn6pvLria7Sa2Fai5znm7CCJz4pP3lrvlSxhKdkCerk0D9E0bqx5yAo3o2Q7RrD4gA==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz",
+      "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==",
       "cpu": [
         "ia32"
       ],
@@ -1632,9 +1645,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.2.tgz",
-      "integrity": "sha512-a/20E/wtTJZ3Ykv3f/8F0l7TtgQa2LWHU2oNB9bsu0VjqGuGGHmm/q6waoUNQYTVPYrrlxxaHjJcDV6aiSTt/w==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz",
+      "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==",
       "cpu": [
         "x64"
       ],
@@ -3792,9 +3805,9 @@
       "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="
     },
     "node_modules/clsx": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.0.tgz",
-      "integrity": "sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==",
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
+      "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
       "engines": {
         "node": ">=6"
       }
@@ -4274,9 +4287,9 @@
       "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.4.746",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.746.tgz",
-      "integrity": "sha512-jeWaIta2rIG2FzHaYIhSuVWqC6KJYo7oSBX4Jv7g+aVujKztfvdpf+n6MGwZdC5hQXbax4nntykLH2juIQrfPg=="
+      "version": "1.4.750",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.750.tgz",
+      "integrity": "sha512-9ItEpeu15hW5m8jKdriL+BQrgwDTXEL9pn4SkillWFu73ZNNNQ2BKKLS+ZHv2vC9UkNhosAeyfxOf/5OSeTCPA=="
     },
     "node_modules/elliptic": {
       "version": "6.5.4",
@@ -4404,13 +4417,13 @@
       }
     },
     "node_modules/es-iterator-helpers": {
-      "version": "1.0.18",
-      "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.0.18.tgz",
-      "integrity": "sha512-scxAJaewsahbqTYrGKJihhViaM6DDZDDoucfvzNbK0pOren1g/daDQ3IAhzn+1G14rBG7w+i5N+qul60++zlKA==",
+      "version": "1.0.19",
+      "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.0.19.tgz",
+      "integrity": "sha512-zoMwbCcH5hwUkKJkT8kDIBZSz9I6mVG//+lDCinLCGov4+r7NIy0ld8o03M0cJxl2spVf6ESYVS6/gpIfq1FFw==",
       "dependencies": {
         "call-bind": "^1.0.7",
         "define-properties": "^1.2.1",
-        "es-abstract": "^1.23.0",
+        "es-abstract": "^1.23.3",
         "es-errors": "^1.3.0",
         "es-set-tostringtag": "^2.0.3",
         "function-bind": "^1.1.2",
@@ -6335,11 +6348,11 @@
       "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="
     },
     "node_modules/next": {
-      "version": "14.2.2",
-      "resolved": "https://registry.npmjs.org/next/-/next-14.2.2.tgz",
-      "integrity": "sha512-oGwUaa2bCs47FbuxWMpOoXtBMPYpvTPgdZr3UAo+pu7Ns00z9otmYpoeV1HEiYL06AlRQQIA/ypK526KjJfaxg==",
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/next/-/next-14.2.3.tgz",
+      "integrity": "sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==",
       "dependencies": {
-        "@next/env": "14.2.2",
+        "@next/env": "14.2.3",
         "@swc/helpers": "0.5.5",
         "busboy": "1.6.0",
         "caniuse-lite": "^1.0.30001579",
@@ -6354,15 +6367,15 @@
         "node": ">=18.17.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "14.2.2",
-        "@next/swc-darwin-x64": "14.2.2",
-        "@next/swc-linux-arm64-gnu": "14.2.2",
-        "@next/swc-linux-arm64-musl": "14.2.2",
-        "@next/swc-linux-x64-gnu": "14.2.2",
-        "@next/swc-linux-x64-musl": "14.2.2",
-        "@next/swc-win32-arm64-msvc": "14.2.2",
-        "@next/swc-win32-ia32-msvc": "14.2.2",
-        "@next/swc-win32-x64-msvc": "14.2.2"
+        "@next/swc-darwin-arm64": "14.2.3",
+        "@next/swc-darwin-x64": "14.2.3",
+        "@next/swc-linux-arm64-gnu": "14.2.3",
+        "@next/swc-linux-arm64-musl": "14.2.3",
+        "@next/swc-linux-x64-gnu": "14.2.3",
+        "@next/swc-linux-x64-musl": "14.2.3",
+        "@next/swc-win32-arm64-msvc": "14.2.3",
+        "@next/swc-win32-ia32-msvc": "14.2.3",
+        "@next/swc-win32-x64-msvc": "14.2.3"
       },
       "peerDependencies": {
         "@opentelemetry/api": "^1.1.0",
@@ -6616,9 +6629,9 @@
       }
     },
     "node_modules/openai": {
-      "version": "4.38.3",
-      "resolved": "https://registry.npmjs.org/openai/-/openai-4.38.3.tgz",
-      "integrity": "sha512-mIL9WtrFNOanpx98mJ+X/wkoepcxdqqu0noWFoNQHl/yODQ47YM7NEYda7qp8JfjqpLFVxY9mQhshoS/Fqac0A==",
+      "version": "4.38.5",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-4.38.5.tgz",
+      "integrity": "sha512-Ym5GJL98ZhLJJ7enBx53jjG3vwN/fsB+Ozh46nnRZZS9W1NiYqbwkJ+sXd3dkCIiWIgcyyOPL2Zr8SQAzbpj3g==",
       "dependencies": {
         "@types/node": "^18.11.18",
         "@types/node-fetch": "^2.6.4",
@@ -6746,9 +6759,9 @@
       }
     },
     "node_modules/path-scurry/node_modules/lru-cache": {
-      "version": "10.2.0",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz",
-      "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==",
+      "version": "10.2.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.1.tgz",
+      "integrity": "sha512-tS24spDe/zXhWbNPErCHs/AGOzbKGHT+ybSBqmdLm8WZ1xXLWvH8Qn71QPAlqVhd0qUTWjy+Kl9JmISgDdEjsA==",
       "engines": {
         "node": "14 || >=16.14"
       }
@@ -6990,6 +7003,14 @@
         "node": ">=6"
       }
     },
+    "node_modules/pure-uuid": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/pure-uuid/-/pure-uuid-1.8.1.tgz",
+      "integrity": "sha512-PIwHXU7NZb/wTBwUfzCSjI85tfwx6DQOm74sRLtNLH8KHsFZEvAQbBQdz7E5ij8SNSv9WGdQPWiiM6NpNIeNfA==",
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
     "node_modules/qs": {
       "version": "6.12.1",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.12.1.tgz",
@@ -7433,9 +7454,9 @@
       }
     },
     "node_modules/scheduler": {
-      "version": "0.23.0",
-      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz",
-      "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==",
+      "version": "0.23.2",
+      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz",
+      "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==",
       "dependencies": {
         "loose-envify": "^1.1.0"
       }
@@ -8097,9 +8118,9 @@
       }
     },
     "node_modules/three": {
-      "version": "0.163.0",
-      "resolved": "https://registry.npmjs.org/three/-/three-0.163.0.tgz",
-      "integrity": "sha512-HlMgCb2TF/dTLRtknBnjUTsR8FsDqBY43itYop2+Zg822I+Kd0Ua2vs8CvfBVefXkBdNDrLMoRTGCIIpfCuDew==",
+      "version": "0.164.1",
+      "resolved": "https://registry.npmjs.org/three/-/three-0.164.1.tgz",
+      "integrity": "sha512-iC/hUBbl1vzFny7f5GtqzVXYjMJKaTPxiCxXfrvVdBi1Sf+jhd1CAkitiFwC7mIBFCo3MrDLJG97yisoaWig0w==",
       "peer": true
     },
     "node_modules/to-regex-range": {
@@ -8227,9 +8248,9 @@
       }
     },
     "node_modules/type-fest": {
-      "version": "4.16.0",
-      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.16.0.tgz",
-      "integrity": "sha512-z7Rf5PXxIhbI6eJBTwdqe5bO02nUUmctq4WqviFSstBAWV0YNtEQRhEnZw73WJ8sZOqgFG6Jdl8gYZu7NBJZnA==",
+      "version": "4.17.0",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.17.0.tgz",
+      "integrity": "sha512-9flrz1zkfLRH3jO3bLflmTxryzKMxVa7841VeMgBaNQGY6vH4RCcpN/sQLB7mQQYh1GZ5utT2deypMuCy4yicw==",
       "engines": {
         "node": ">=16"
       },
@@ -8315,9 +8336,9 @@
       }
     },
     "node_modules/typescript": {
-      "version": "5.4.4",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.4.tgz",
-      "integrity": "sha512-dGE2Vv8cpVvw28v8HCPqyb08EzbBURxDpuhJvTrusShUfGnhHBafDsLdS1EhhxyL6BJQE+2cT3dDPAv+MQ6oLw==",
+      "version": "5.4.5",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz",
+      "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==",
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "ai-tube",
+  "name": "@aitube/website",
   "version": "0.0.0",
   "private": true,
   "scripts": {
@@ -9,6 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@aitube/clap": "^0.0.6",
     "@huggingface/hub": "0.12.3-oauth",
     "@huggingface/inference": "^2.6.7",
     "@jcoreio/async-throttle": "^1.6.0",
@@ -96,7 +97,7 @@
     "temp-dir": "^3.0.0",
     "ts-node": "^10.9.2",
     "type-fest": "^4.8.2",
-    "typescript": "5.4.4",
+    "typescript": "^5.4.5",
     "usehooks-ts": "^2.9.1",
     "uuid": "^9.0.1",
     "yaml": "^2.3.4",

src/app/api/actions/ai-tube-hf/downloadClapProject.ts

@@ -1,10 +1,10 @@
 import { v4 as uuidv4 } from "uuid"
 import { Credentials } from "@/lib/huggingface/hub/src"
 
-import { ClapProject } from "@/lib/clap/types"
+import { ClapProject, parseClap } from "@aitube/clap"
+
 import { ChannelInfo, MediaInfo, VideoRequest } from "@/types/general"
 import { defaultVideoModel } from "@/app/config"
-import { parseClap } from "@/lib/clap/parseClap"
 
 import { parseVideoModelName } from "../../utils/parseVideoModelName"
 import { computeOrientationProjectionWidthHeight } from "../../utils/computeOrientationProjectionWidthHeight"
@@ -35,7 +35,7 @@ export async function downloadClapProject({
     expectedMimeType: "application/gzip"
   })
 
-  const clapProject = await parseClap(clapString)
+  const clapProject: ClapProject = await parseClap(clapString)
 
   const id = clapProject.meta.id || uuidv4()
   

src/app/api/generate/models/generateAudioID.ts

@@ -0,0 +1,12 @@
+export async function generateAudioID({
+  prompt,
+  // negativePrompt,
+  seed,
+}: {
+  prompt: string
+  // negativePrompt?: string
+  seed?: number
+}) {
+  // TODO: call Parler-TTS
+  return "TODO JULIAN"
+}

src/app/api/generate/models/generateImageID.ts

@@ -0,0 +1,68 @@
+
+import { generateSeed } from "@aitube/clap"
+
+import { sleep } from "@/lib/utils/sleep"
+import { getValidNumber } from "@/lib/utils/getValidNumber"
+
+import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
+import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
+
+export async function generateImageID({
+  prompt,
+  // negativePrompt,
+  seed,
+}: {
+  prompt: string
+  // negativePrompt?: string
+  seed?: number
+}) {
+  
+  // those can be constants for a face ID
+  // also we want something a bit portrait-ish
+  // but this risk creating a lot of variability in poses
+  // so perhaps we should use a controlnet to condition the face scale and position,
+  // to make sure it is uniform in size across all models
+  const width = 1024
+  const height = 768
+
+  // console.log("calling await newRender")
+  prompt = getPositivePrompt(prompt)
+  const negativePrompt = getNegativePrompt()
+
+  let render = await newRender({
+    prompt,
+    negativePrompt,
+    nbFrames: 1,
+    nbFPS: 1,
+
+    // note: for the model ID we might want to maximize things here,
+    // and maybe not use the "turbo" - but I'm not sure
+    width,
+    height,
+    nbSteps: 8,
+    turbo: true,
+
+    shouldRenewCache: true,
+    seed: seed || generateSeed()
+  })
+
+  let attempts = 10
+
+  while (attempts-- > 0) {
+    if (render.status === "completed") {
+      return render.assetUrl
+    }
+
+    if (render.status === "error") {
+      console.error(render.error)
+      throw new Error(`failed to generate the image ${render.error}`)
+    }
+
+    await sleep(2000) // minimum wait time
+
+    // console.log("asking getRender")
+    render = await getRender(render.renderId)
+  }
+
+  throw new Error(`failed to generate the image`)
+}

src/app/api/generate/models/route.ts

@@ -0,0 +1,75 @@
+import { NextResponse, NextRequest } from "next/server"
+import queryString from "query-string"
+
+import { parseClap, serializeClap, ClapModel } from "@aitube/clap"
+import { getToken } from "@/app/api/auth/getToken"
+
+import { generateImageID } from "./generateImageID"
+import { generateAudioID } from "./generateAudioID"
+
+export async function POST(req: NextRequest) {
+
+  const qs = queryString.parseUrl(req.url || "")
+  const query = (qs || {}).query
+
+  // TODO: use query parameters to determine *what* to generate:
+  /*
+  let prompt = ""
+  try {
+    prompt = decodeURIComponent(query?.p?.toString() || "").trim()
+  } catch (err) {}
+  if (!prompt) {
+    return NextResponse.json({ error: 'no prompt provided' }, { status: 400 });
+  }
+
+  if (!prompt.length) { throw new Error(`please provide a prompt`) }
+  */
+
+  console.log("[api/generate/models] request:", prompt)
+
+  const jwtToken = await getToken({ user: "anonymous" })
+
+  const blob = await req.blob()
+
+  const clap = await parseClap(blob)
+
+  if (!clap.models.length) { throw new Error(`please provide at least one model`) }
+
+  for (const model of clap.models) {
+
+    // TASK 1: GENERATE THE IMAGE PROMPT IF MISSING
+    if (!model.imagePrompt) {
+      model.imagePrompt = "a man with a beard"
+    }
+
+    // TASK 2: GENERATE THE IMAGE ID IF MISSING
+    if (!model.imageId) {
+      model.imageId = await generateImageID({
+        prompt: model.imagePrompt,
+        seed: model.seed
+      })
+    }
+
+    // TASK 3: GENERATE THE AUDIO PROMPT IF MISSING
+    if (!model.audioPrompt) {
+      model.audioPrompt = "a man with a beard"
+    }
+
+    // TASK 4: GENERATE THE AUDIO ID IF MISSING
+
+    // TODO here: call Parler-TTS or a generic audio generator
+    if (!model.audioId) {
+      model.audioId = await generateAudioID({
+        prompt: model.audioPrompt,
+        seed: model.seed
+      })
+    }
+  }
+
+  console.log(`[api/generate/models] returning the clap extended with the model`)
+
+  return new NextResponse(await serializeClap(clap), {
+    status: 200,
+    headers: new Headers({ "content-type": "application/x-gzip" }),
+  })
+}

src/app/api/generate/models/systemPrompt.ts

@@ -0,0 +1,3 @@
+
+
+export const systemPrompt = ""

src/app/api/generate/story/route.ts

@@ -1,13 +1,10 @@
 import { NextResponse, NextRequest } from "next/server"
+import { ClapProject, getValidNumber, newClap, newSegment, serializeClap } from "@aitube/clap"
 
-import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory"
-import { serializeClap } from "@/lib/clap/serializeClap"
-import { getValidNumber } from "@/lib/utils/getValidNumber"
-import { newClap } from "@/lib/clap/newClap"
 import { predict } from "../../providers/huggingface/predictWithHuggingFace"
-import { systemPrompt } from "./systemPrompt"
 import { parseRawStringToYAML } from "../../utils/parseRawStringToYAML"
-import { newSegment } from "@/lib/clap/newSegment"
+
+import { systemPrompt } from "./systemPrompt"
 
 export type LatentStory = {
   title: string
@@ -61,7 +58,7 @@ export async function POST(req: NextRequest) {
   let currentElapsedTimeInMs = 0
   let currentSegmentDurationInMs = defaultSegmentDurationInMs
 
-  const clap = newClap({
+  const clap: ClapProject = newClap({
     meta: {
       title: "Not needed", // we don't need a title actually
       description: "This video has been generated using AI",

src/app/api/generate/storyboards/generateStoryboard.ts

@@ -1,9 +1,8 @@
+import { generateSeed, getValidNumber } from "@aitube/clap"
 
 import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
-import { generateSeed } from "@/lib/utils/generateSeed"
 import { sleep } from "@/lib/utils/sleep"
 import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
-import { getValidNumber } from "@/lib/utils/getValidNumber"
 
 export async function generateStoryboard({
   prompt,

src/app/api/generate/storyboards/route.ts

@@ -1,13 +1,12 @@
 import { NextResponse, NextRequest } from "next/server"
 
-import { serializeClap } from "@/lib/clap/serializeClap"
-import { parseClap } from "@/lib/clap/parseClap"
+import { ClapProject, ClapSegment, newSegment, parseClap, serializeClap } from "@aitube/clap"
+
 import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2"
 import { getVideoPrompt } from "@/components/interface/latent-engine/core/prompts/getVideoPrompt"
-import { newSegment } from "@/lib/clap/newSegment"
-import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
 import { getToken } from "@/app/api/auth/getToken"
-import { generateSeed } from "@/lib/utils/generateSeed"
+
+import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
 import { getPositivePrompt } from "../../utils/imagePrompts"
 import { generateStoryboard } from "./generateStoryboard"
 
@@ -24,13 +23,13 @@ export async function POST(req: NextRequest) {
 
   const blob = await req.blob()
 
-  const clap = await parseClap(blob)
+  const clap: ClapProject = await parseClap(blob)
 
   if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) }
   
   console.log(`[api/generate/storyboards] detected ${clap.segments.length} segments`)
   
-  const shotsSegments = clap.segments.filter(s => s.category === "camera")
+  const shotsSegments: ClapSegment[] = clap.segments.filter(s => s.category === "camera")
   console.log(`[api/generate/storyboards] detected ${shotsSegments.length} shots`)
   
   if (shotsSegments.length > 32) {
@@ -39,15 +38,15 @@ export async function POST(req: NextRequest) {
 
   for (const shotSegment of shotsSegments) {
 
-    const shotSegments = clap.segments.filter(s =>
+    const shotSegments: ClapSegment[] = clap.segments.filter(s =>
       startOfSegment1IsWithinSegment2(s, shotSegment)
     )
 
-    const shotStoryboardSegments = shotSegments.filter(s =>
+    const shotStoryboardSegments: ClapSegment[] = shotSegments.filter(s =>
       s.category === "storyboard"
     )
 
-    let shotStoryboardSegment = shotStoryboardSegments.at(0)
+    let shotStoryboardSegment: ClapSegment | undefined = shotStoryboardSegments.at(0)
     
     console.log(`[api/generate/storyboards] shot [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}] has ${shotSegments.length} segments (${shotStoryboardSegments.length} storyboards)`)
   
@@ -67,14 +66,14 @@ export async function POST(req: NextRequest) {
     }
 
     // TASK 2: GENERATE MISSING STORYBOARD PROMPT
-    if (!shotStoryboardSegment.prompt) {
+    if (shotStoryboardSegment && !shotStoryboardSegment?.prompt) {
       // storyboard is missing, let's generate it
       shotStoryboardSegment.prompt = getVideoPrompt(shotSegments, {}, [])
       console.log(`[api/generate/storyboards] generating storyboard prompt: ${shotStoryboardSegment.prompt}`)
     }
 
     // TASK 3: GENERATE MISSING STORYBOARD BITMAP
-    if (!shotStoryboardSegment.assetUrl) {
+    if (shotStoryboardSegment && !shotStoryboardSegment.assetUrl) {
       console.log(`[api/generate/storyboards] generating image..`)
 
       try {
@@ -88,9 +87,9 @@ export async function POST(req: NextRequest) {
         throw err
       }
    
-      console.log(`[api/generate/storyboards] generated storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
+      console.log(`[api/generate/storyboards] generated storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
     } else {
-      console.log(`[api/generate/storyboards] there is already a storyboard image: ${shotStoryboardSegment.assetUrl.slice(0, 50)}...`)
+      console.log(`[api/generate/storyboards] there is already a storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
     }
   }
 

src/app/api/generate/video/route.ts

@@ -2,7 +2,8 @@ import { NextResponse, NextRequest } from "next/server"
 
 // we hide/wrap the micro-service under a unified AiTube API
 export async function POST(req: NextRequest, res: NextResponse) {
-  NextResponse.redirect("https://jbilcke-hf-ai-tube-clap-exporter.hf.space")
+
+  NextResponse.redirect(`${process.env.AI_TUBE_CLAP_EXPORTER_URL || "http://localhost:7860"}`)
 }
 /*
 Alternative solution (in case the redirect doesn't work):

src/app/api/generators/clap/addLatentScenesToClap.ts

@@ -1,12 +1,8 @@
 "use server"
 
-import { newClap } from "@/lib/clap/newClap"
-import { newSegment } from "@/lib/clap/newSegment"
+import { ClapProject, newSegment } from "@aitube/clap"
 
 import { LatentScenes } from "./types"
-import { serializeClap } from "@/lib/clap/serializeClap"
-import { getEmptyClap } from "@/lib/clap/emptyClap"
-import { ClapProject } from "@/lib/clap/types"
 
 let defaultSegmentDurationInMs = 2000
 

src/app/api/generators/clap/continueClap.ts

@@ -1,13 +1,10 @@
 "use server"
 
+import { ClapProject, serializeClap } from "@aitube/clap"
 
 import { LatentScenes } from "./types"
 import { addLatentScenesToClap } from "./addLatentScenesToClap"
 import { getLatentScenes } from "./getLatentScenes"
-import { serializeClap } from "@/lib/clap/serializeClap"
-import { newClap } from "@/lib/clap/newClap"
-import { getEmptyClap } from "@/lib/clap/emptyClap"
-import { ClapProject } from "@/lib/clap/types"
 
 /**
  * Imagine the continuity of a Clap file
@@ -60,7 +57,7 @@ export async function continueClap({
   // technically, it could also be transported as text
   // (and gzipped automatically between the HTTP server and browser)
   // but I think it is better to keep the idea of a dedicated file format
-  const archive = await serializeClap(clap)
+  const archive: Blob = await serializeClap(clap)
 
   return archive
 }

src/app/api/generators/clap/generateClap.ts

@@ -1,12 +1,10 @@
 "use server"
 
-import { serializeClap } from "@/lib/clap/serializeClap"
-import { newClap } from "@/lib/clap/newClap"
-import { getEmptyClap } from "@/lib/clap/emptyClap"
 
 import { LatentScenes } from "./types"
 import { addLatentScenesToClap } from "./addLatentScenesToClap"
 import { getLatentScenes } from "./getLatentScenes"
+import { ClapProject, getEmptyClap, newClap, serializeClap } from "@aitube/clap"
 
 /**
  * Generate a Clap file from scratch using a prompt
@@ -22,13 +20,13 @@ export async function generateClap({
   debug: false,
 }): Promise<Blob> {
 
-  const empty = await getEmptyClap()
+  const empty: Blob = await getEmptyClap()
 
   if (!prompt?.length) { 
     return empty
   }
 
-  let clap = newClap({
+  let clap: ClapProject = newClap({
     meta: {
       title: "Latent content", // TODO "
       description: "",
@@ -56,7 +54,7 @@ export async function generateClap({
     debug,
   })
 
-  const archive = await serializeClap(clap)
+  const archive: Blob = await serializeClap(clap)
 
   return archive
 }

src/app/api/generators/search/unknownObjectToLatentSearchResults.ts

@@ -1,4 +1,5 @@
-import { generateSeed } from "@/lib/utils/generateSeed"
+import { generateSeed } from "@aitube/clap"
+
 import { parseString } from "../../utils/parseString"
 import { parseStringArray } from "../../utils/parseStringArray"
 import { LatentSearchResult, LatentSearchResults } from "./types"

src/app/api/resolvers/clap/route.ts

@@ -1,5 +1,6 @@
 import { NextResponse, NextRequest } from "next/server"
 import queryString from "query-string"
+
 import { generateClap } from "../../generators/clap/generateClap"
 
 export async function GET(req: NextRequest) {

src/app/api/resolvers/image/route.ts

@@ -1,10 +1,10 @@
 import { NextResponse, NextRequest } from "next/server"
 import { createSecretKey } from "node:crypto"
 
+import { generateSeed } from "@aitube/clap"
 import queryString from "query-string"
 
 import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
-import { generateSeed } from "@/lib/utils/generateSeed"
 import { sleep } from "@/lib/utils/sleep"
 import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
 import { getContentType } from "@/lib/data/getContentType"

src/app/api/resolvers/video/route.ts

@@ -3,8 +3,9 @@ import queryString from "query-string"
 import { createSecretKey } from "crypto"
 import { jwtVerify } from "jose"
 
+import { generateSeed } from "@aitube/clap"
+
 import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain"
-import { generateSeed } from "@/lib/utils/generateSeed"
 import { sleep } from "@/lib/utils/sleep"
 import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts"
 import { getContentType } from "@/lib/data/getContentType"

src/app/dream/spoiler.tsx

@@ -1,8 +1,7 @@
 import { LatentQueryProps } from "@/types/general"
+import { clapToDataUri, generateClapFromSimpleStory } from "@aitube/clap"
 
 import { Main } from "../main"
-import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory"
-import { clapToDataUri } from "@/lib/clap/clapToDataUri"
 import { getNewMediaInfo } from "../api/generators/search/getNewMediaInfo"
 import { getToken } from "../api/auth/getToken"
 

src/components/interface/latent-engine/core/engine.tsx

@@ -5,8 +5,6 @@ import { useLocalStorage } from "usehooks-ts"
 
 import { cn } from "@/lib/utils/cn"
 import { MediaInfo } from "@/types/general"
-import { serializeClap } from "@/lib/clap/serializeClap"
-import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory"
 
 import { useLatentEngine } from "./useLatentEngine"
 import { PlayPauseButton } from "../components/play-pause-button"
@@ -15,6 +13,7 @@ import { ContentLayer } from "../components/content-layer"
 import { localStorageKeys } from "@/app/state/localStorageKeys"
 import { defaultSettings } from "@/app/state/defaultSettings"
 import { useStore } from "@/app/state/useStore"
+import { ClapProject, generateClapFromSimpleStory, serializeClap } from "@aitube/clap"
 
 function LatentEngine({
   media,
@@ -87,8 +86,8 @@ function LatentEngine({
         // there is a bug, we can't unpack the .clap when it's from a data-uri :/
         
         // open(mediaUrl)
-        const mockClap = generateClapFromSimpleStory()
-        const mockArchive = await serializeClap(mockClap)
+        const mockClap: ClapProject = generateClapFromSimpleStory()
+        const mockArchive: Blob = await serializeClap(mockClap)
         // for some reason conversion to data uri doesn't work
         // const mockDataUri = await blobToDataUri(mockArchive, "application/x-gzip")
         // console.log("mockDataUri:", mockDataUri)

src/components/interface/latent-engine/core/generators/fetchLatentClap.ts

@@ -1,5 +1,4 @@
-import { parseClap } from "@/lib/clap/parseClap"
-import { ClapProject } from "@/lib/clap/types"
+import { ClapProject, parseClap } from "@aitube/clap"
 
 export async function fetchLatentClap(prompt: string): Promise<ClapProject> {
 

src/components/interface/latent-engine/core/prompts/getCharacterPrompt.ts

@@ -1,4 +1,4 @@
-import { ClapModel } from "@/lib/clap/types"
+import { ClapModel } from "@aitube/clap"
 
 export function getCharacterPrompt(model: ClapModel): string {
 

src/components/interface/latent-engine/core/prompts/getVideoPrompt.ts

@@ -1,4 +1,4 @@
-import { ClapModel, ClapSegment } from "@/lib/clap/types"
+import { ClapModel, ClapSegment } from "@aitube/clap"
 
 import { deduplicatePrompt } from "../../utils/prompting/deduplicatePrompt"
 

src/components/interface/latent-engine/core/types.ts

@@ -1,4 +1,4 @@
-import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 import { InteractiveSegmenterResult } from "@mediapipe/tasks-vision"
 import { MouseEventHandler, ReactNode } from "react"
 

src/components/interface/latent-engine/core/useLatentEngine.ts

@@ -1,16 +1,12 @@
 
 import { create } from "zustand"
 
-import { ClapModel, ClapProject } from "@/lib/clap/types"
-import { newClap } from "@/lib/clap/newClap"
-import { sleep } from "@/lib/utils/sleep"
-// import { getSegmentationCanvas } from "@/lib/on-device-ai/getSegmentationCanvas"
+import { ClapModel, ClapProject, ClapSegment, newClap, parseClap } from "@aitube/clap"
 
 import { LatentEngineStore } from "./types"
 import { resolveSegments } from "../resolvers/resolveSegments"
 import { fetchLatentClap } from "./generators/fetchLatentClap"
-import { dataUriToBlob } from "@/app/api/utils/dataUriToBlob"
-import { parseClap } from "@/lib/clap/parseClap"
+
 import { InteractiveSegmenterResult, MPMask } from "@mediapipe/tasks-vision"
 import { segmentFrame } from "@/lib/on-device-ai/segmentFrameOnClick"
 import { drawSegmentation } from "../utils/canvas/drawSegmentation"
@@ -396,7 +392,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
       segments: clap.segments
     })
     
-    const prefilterSegmentsForPerformanceReasons = clap.segments.filter(s =>
+    const prefilterSegmentsForPerformanceReasons: ClapSegment[] = clap.segments.filter(s =>
       s.startTimeInMs >= positionInMs &&
       s.startTimeInMs < maxBufferDurationInMs
     )
@@ -424,7 +420,7 @@ export const useLatentEngine = create<LatentEngineStore>((set, get) => ({
       
       // we select the segments in the current shot
 
-      const shotSegmentsToPreload = prefilterSegmentsForPerformanceReasons.filter(s =>
+      const shotSegmentsToPreload: ClapSegment[] = prefilterSegmentsForPerformanceReasons.filter(s =>
         s.startTimeInMs >= bufferAheadOfCurrentPositionInMs &&
         s.startTimeInMs < (bufferAheadOfCurrentPositionInMs + videoDurationInMs)
       )

src/components/interface/latent-engine/resolvers/deprecated.txt

@@ -2,7 +2,7 @@
 
 import { useEffect, useState } from "react";
 
-import { ClapProject, ClapSegment } from "@/lib/clap/types";
+import { ClapProject, ClapSegment } from "@aitube/clap";
 import { resolveSegment } from "./resolveSegment";
 
 export function LatentComponent({

src/components/interface/latent-engine/resolvers/generic/index.tsx

@@ -2,7 +2,7 @@
 
 import { v4 as uuidv4 } from "uuid"
 
-import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 
 import { LayerElement } from "../../core/types"
 

src/components/interface/latent-engine/resolvers/image/index.tsx

@@ -1,6 +1,6 @@
 "use client"
 
-import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 import { generateImage } from "./generateImage"
 import { LayerElement } from "../../core/types"
 import { useStore } from "@/app/state/useStore"

src/components/interface/latent-engine/resolvers/interface/index.tsx

@@ -2,7 +2,7 @@
 
 import RunCSS, { extendRunCSS } from "runcss"
 
-import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 import { generateHtml } from "./generateHtml"
 import { AIContentDisclaimer } from "../../components/intros/ai-content-disclaimer"
 import { LayerElement } from "../../core/types"

src/components/interface/latent-engine/resolvers/resolveSegment.ts

@@ -1,4 +1,4 @@
-import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 
 import { LatentComponentResolver, LayerElement } from "../core/types"
 

src/components/interface/latent-engine/resolvers/resolveSegments.ts

@@ -1,4 +1,4 @@
-import { ClapProject, ClapSegmentCategory } from "@/lib/clap/types"
+import { ClapProject, ClapSegmentCategory } from "@aitube/clap"
 
 import { resolveSegment } from "./resolveSegment"
 import { LayerElement } from "../core/types"

src/components/interface/latent-engine/resolvers/video/index.tsx

@@ -1,6 +1,6 @@
 "use client"
 
-import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 
 import { LayerElement } from "../../core/types"
 

src/components/interface/latent-engine/resolvers/video/index_legacy.tsx

@@ -1,6 +1,6 @@
 "use client"
 
-import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 import { PromiseResponseType, waitPromisesUntil } from "@/lib/utils/waitPromisesUntil"
 import { generateVideo } from "./generateVideo"
 import { BasicVideo } from "./basic-video"

src/components/interface/latent-engine/resolvers/video/index_notSoGood.tsx

@@ -1,6 +1,6 @@
 "use client"
 
-import { ClapProject, ClapSegment } from "@/lib/clap/types"
+import { ClapProject, ClapSegment } from "@aitube/clap"
 import { PromiseResponseType, waitPromisesUntil } from "@/lib/utils/waitPromisesUntil"
 import { VideoLoop } from "./video-loop"
 import { generateVideo } from "./generateVideo"

src/lib/business/getClapAssetSourceType.ts

@@ -1,4 +1,4 @@
-import { ClapAssetSource } from "@/lib/clap/types"
+import { ClapAssetSource } from "@aitube/clap"
 
 export function getClapAssetSourceSource(input: string = ""): ClapAssetSource {
   

src/lib/clap/clap-specification-draft.md

@@ -1,162 +0,0 @@
-# CLAP Format Specification
-
-- Status: DRAFT
-- Document revision: 0.0.1
-- Last updated: Feb 6th, 2024
-- Author(s): Julian BILCKE (@flngr)
-
-## BEFORE YOU READ
-
-The CLAP format spec is experimental and not finished yet!
-There might be inconsistencies, unnecessary redundancies or blatant omissions.
-
-## What are CLAP files?
-
-The CLAP format (.clap) is a file format designed for AI video projects.
-
-It preserves prompts and assets into the same container, making it easier to share an AI video project between different people or applications.
-
-## Structure
-
-A CLAP is an array of objects serialized into a YAML text string, then finally compressed using gzip to a binary file.
-
-The file extension is `.clap`
-The mime type is `application/x-yaml`
-
-There can be 5 different types of objects:
-
-- one HEADER
-- one METADATA
-- zero, one or more MODEL(s)
-- zero, one or more SCENE(s)
-- zero, one or more SEGMENT(s)
-
-This can be represented in javascript like this:
-
-```javascript
-[
-  clapHeader, // one metadata object
-  clapMeta, // one metadata object
-  ...clapModels, // optional array of models
-  ...clapScenes, // optional array of scenes
-  ...clapSegments // optional array of segments
-]
-```
-
-## Header
-
-The HEADER provides information about how to decode a CLAP.
-
-Knowing in advance the number of models, scenes and segments helps the decoder parsing the information,
-and in some implementation, help with debugging, logging, and provisioning memory usage.
-
-However in the future, it is possible that a different scheme is used, in order to support streaming.
-
-Either by recognizing the shape of each object (fields), or by using a specific field eg. a `_type`.
-
-```typescript
-{
-  // used to know which format version is used.
-  // CLAP is still in development and the format is not fully specified yet,
-  // during the period most .clap file will have the "clap-0" format
-  format: "clap-0"
-  
-  numberOfModels: number // integer
-  numberOfScenes: number // integer
-  numberOfSegments: number // integer
-}
-```
-
-## Metadata
-
-```typescript
-{
-  id: string // "<a valid UUID V4>"
-  title: string // "project title"
-  description: string // "project description"
-  licence: string // "information about licensing"
-
-  // this provides information about the image ratio
-  // this might be removed in the final spec, as this
-  // can be re-computed from width and height
-  orientation: "landscape" | "vertical" | "square"
-
-  // the expected duration of the project
-  durationInMs: number
-
-  // the suggested width and height of the video
-  // note that this is just an indicator,
-  // and might be superseeded by the application reading the .clap file
-  width: number // integer between 256 and 8192 (value in pixels)
-  height: number // integer between 256 and 8192 (value in pixels)
-
-  // name of the suggested video model to use
-  // note that this is just an indicator,
-  // and might be superseeded by the application reading the .clap file
-  defaultVideoModel: string
-
-  // additional prompt to use in the video generation
-  // this helps adding some magic touch and flair to the videos,
-  // but perhaps the field should be renamed
-  extraPositivePrompt: string
-
-  // the screenplay (script) of the video
-  screenplay: string
-
-  // whether to loop the content by default or not
-  isLoop: boolean
-  
-  // helper to indicate whether the .clap might contain interactive elements
-  isInteractive: boolean
-}
-```
-
-## Models
-
-Before talking about models, first we should describe the concept of entity:
-
-in a story, an entity is something (person, place, vehicle, animal, robot, alien, object) with a name, a description of the appearance, an age, mileage or quality, an origin, and so on.
-
-An example could be "a giant magical school bus, with appearance of a cat with wheels, and which talks"
-
-The CLAP model would be an instance (an interpretation) of this entity, where we would assign it an identity:
-- a name and age
-- a visual style (a photo of the magic school bus cat)
-- a voice style
-- and maybe other things eg. an origin or background story
-
-As you can see, it can be difficult to create clearly separated categories, like "vehicule", "character", or "location"
-(the magical cat bus could turn into a location in some scene, a speaking character in another etc)
-
-This is why there is a common schema for all models:
-
-```typescript
-{
-  id: string
-  category: ClapSegmentCategory
-  triggerName: string
-  label: string
-  description: string
-  author: string
-  thumbnailUrl: string
-  seed: number
-
-  assetSourceType: ClapAssetSource
-  assetUrl: string
-  
-  age: number
-  gender: ClapModelGender
-  region: ClapModelRegion
-  appearance: ClapModelAppearance
-  voiceVendor: ClapVoiceVendor
-  voiceId: string
-}
-```
-
-## Atomic types
-
-...
-
-## TO BE CONTINUED
-
-(you can read "./types.ts" for more information)

src/lib/clap/clapToDataUri.ts

@@ -1,10 +0,0 @@
-import { blobToDataUri } from "@/app/api/utils/blobToDataUri"
-
-import { serializeClap } from "./serializeClap"
-import { ClapProject } from "./types"
-
-export async function clapToDataUri(clap: ClapProject): Promise<string> {
-  const archive = await serializeClap(clap)
-  const dataUri = await blobToDataUri(archive, "application/x-gzip")
-  return dataUri
-}

src/lib/clap/emptyClap.ts

@@ -1,18 +0,0 @@
-import { newClap } from "./newClap"
-import { serializeClap } from "./serializeClap"
-
-let globalState: {
-  blob?: Blob
-} = {
-  blob: undefined
-}
-
-export async function getEmptyClap(): Promise<Blob> {
-  if (globalState.blob) { return globalState.blob }
-
-  const clap = newClap()
-
-  globalState.blob = await serializeClap(clap)
-
-  return globalState.blob
-}

src/lib/clap/generateClapFromSimpleStory.ts

@@ -1,149 +0,0 @@
-import { newClap } from "./newClap"
-import { newSegment } from "./newSegment"
-import { ClapProject } from "./types"
-
-let defaultSegmentDurationInMs = 2500 // 2584
-
-const fishDemoStory = [
-  "Siamese fighting fish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-  
-  // this one is magnificient!
-  "princess parrot fish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-  
-  "pacific ocean perch, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-  
-  "Queen angelfish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-  
-  "sea turtle, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-
-  "hippocampus, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-]
-
-let demoStory = [
-  ...fishDemoStory,
-
-  // "portrait of one man news anchor, 60yo, thin, fit, american, mustache, beard, wearing a suit, medium-shot, central park, outside, serious, bokeh, perfect",
-  
-  // "screenshot from Call of Duty, FPS game, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
-
-  // "screenshot from a flight simulator, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
-  // "screenshot from fallout3, fallout4, western, wasteland, 3rd person RPG, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
-  // "portrait of single influencer woman, 30yo, thin, fit, american, wearing a red tshirt, medium-shot, central park, outside, serious, bokeh, perfect",
-]
-
-
-export function generateClapFromSimpleStory({
-  story = demoStory,
-  showIntroPoweredByEngine = false,
-  showIntroDisclaimerAboutAI = false,
-}: {
-  story?: string[]
-  showIntroPoweredByEngine?: boolean
-  showIntroDisclaimerAboutAI?: boolean
-} = {
-  story: demoStory,
-  showIntroPoweredByEngine: false,
-  showIntroDisclaimerAboutAI: false,
-}): ClapProject {
-
-  const clap = newClap({
-    meta: {
-      title: "Interactive Demo",
-      isInteractive: true,
-      isLoop: true,
-    }
-  })
-
-  let currentElapsedTimeInMs = 0
-  let currentSegmentDurationInMs = defaultSegmentDurationInMs
-
-  if (showIntroPoweredByEngine) {
-    clap.segments.push(newSegment({
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "interface",
-      prompt: "<BUILTIN:POWERED_BY_ENGINE>",
-      label: "disclaimer",
-      outputType: "interface",
-    }))
-    currentElapsedTimeInMs += currentSegmentDurationInMs
-  }
-
-  if (showIntroDisclaimerAboutAI) {
-    clap.segments.push(newSegment({
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "interface",
-      prompt: "<BUILTIN:DISCLAIMER_ABOUT_AI>",
-      label: "disclaimer",
-      outputType: "interface",
-    }))
-    currentElapsedTimeInMs += currentSegmentDurationInMs
-  }
-
-  /*
-  clap.segments.push(
-    newSegment({
-      // id: string
-      // track: number
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "interface",
-      // modelId: string
-      // sceneId: string
-      prompt: "a hello world",
-      label: "hello world",
-      outputType: "interface"
-      // renderId: string
-      // status: ClapSegmentStatus
-      // assetUrl: string
-      // assetDurationInMs: number
-      // createdBy: ClapAuthor
-      // editedBy: ClapAuthor
-      // outputGain: number
-      // seed: number
-    })
-  )
-
-  currentElapsedTimeInMs += currentSegmentDurationInMs
-  */
-  
- 
-
-  for (let prompt of story) {
-
-    clap.segments.push(newSegment({
-      track: 0,
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "video",
-      prompt: "",
-      label: "video",
-      outputType: "video",
-    }))
-    clap.segments.push(newSegment({
-      track: 1,
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "generic",
-      prompt,
-      label: prompt,
-      outputType: "text",
-    }))
-    clap.segments.push(newSegment({
-      track: 2,
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "camera",
-      prompt: "medium-shot",
-      label: "medium-shot",
-      outputType: "text",
-    }))
-
-    currentElapsedTimeInMs += currentSegmentDurationInMs
-  }
-
-  clap.meta.durationInMs = currentElapsedTimeInMs
-
-  return clap
-}

src/lib/clap/getClapAssetSource.ts

@@ -1,25 +0,0 @@
-import { ClapAssetSource } from "./types"
-
-export function getClapAssetSourceType(input: string = ""): ClapAssetSource {
-  
-  const str = `${input || ""}`.trim()
-
-  if (!str || !str.length) {
-    return "EMPTY"
-  }
-
-  if (str.startsWith("https://") || str.startsWith("http://")) {
-    return "REMOTE"
-  }
-
-  // note that "path" assets are potentially a security risk, they need to be treated with care
-  if (str.startsWith("/") || str.startsWith("../") || str.startsWith("./")) {
-    return "PATH"
-  }
-
-  if (str.startsWith("data:")) {
-    return "DATA"
-  }
-
-  return "PROMPT"
-}

src/lib/clap/newClap.ts

@@ -1,37 +0,0 @@
-
-import { v4 as uuidv4 } from "uuid"
-
-import { ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types"
-import { getValidNumber } from "@/lib/utils/getValidNumber"
-
-// generate an empty clap file, or copy one from a source
-export function newClap(clap: {
-    meta?: Partial<ClapMeta>
-    models?: ClapModel[]
-    scenes?: ClapScene[]
-    segments?: ClapSegment[]
-  } = {}): ClapProject {
-
-  const meta: ClapMeta = {
-    id: clap?.meta?.id === "string" ? clap.meta.id : uuidv4(),
-    title: clap?.meta?.title === "string" ? clap.meta.title : "",
-    description: typeof clap?.meta?.description === "string" ? clap.meta.description : "",
-    synopsis: typeof clap?.meta?.synopsis === "string" ? clap.meta.synopsis : "",
-    licence: typeof clap?.meta?.licence === "string" ? clap.meta.licence : "",
-    orientation: clap?.meta?.orientation === "portrait" ? "portrait" : clap?.meta?.orientation === "square" ? "square" : "landscape",
-    durationInMs: getValidNumber(clap?.meta?.durationInMs, 1000, Number.MAX_SAFE_INTEGER, 4000),
-    width: getValidNumber(clap?.meta?.width, 256, 8192, 1024),
-    height: getValidNumber(clap?.meta?.height, 256, 8192, 576),
-    defaultVideoModel: typeof clap?.meta?.defaultVideoModel === "string" ? clap.meta.defaultVideoModel : "SVD",
-    extraPositivePrompt: Array.isArray(clap?.meta?.extraPositivePrompt) ? clap.meta.extraPositivePrompt : [],
-    screenplay: typeof clap?.meta?.screenplay === "string" ? clap.meta.screenplay : "",
-    isLoop: typeof clap?.meta?.isLoop === "boolean" ? clap.meta.isLoop : false,
-    isInteractive: typeof clap?.meta?.isInteractive === "boolean" ? clap.meta.isInteractive : false,
-  }
-
-  const models: ClapModel[] = clap?.models && Array.isArray(clap.models) ? clap.models : []
-  const scenes: ClapScene[] =  clap?.scenes && Array.isArray(clap.scenes) ? clap.scenes : []
-  const segments: ClapSegment[] =  clap?.segments && Array.isArray(clap.segments) ? clap.segments : []
-
-  return { meta, models, scenes, segments }
-}

src/lib/clap/newSegment.ts

@@ -1,46 +0,0 @@
-import { v4 as uuidv4 } from "uuid"
-
-import { generateSeed } from "../utils/generateSeed"
-import { ClapSegment } from "./types"
-import { isValidNumber } from "../utils/isValidNumber"
-
-export function newSegment(maybeSegment?: Partial<ClapSegment>) {
-
-  const startTimeInMs =
-    isValidNumber(maybeSegment?.startTimeInMs)
-    ? (maybeSegment?.startTimeInMs || 0)
-    : 0
-
-  const assetDurationInMs =
-    isValidNumber(maybeSegment?.assetDurationInMs)
-    ? (maybeSegment?.assetDurationInMs || 0)
-    : 1000
-
-  const endTimeInMs =
-    isValidNumber(maybeSegment?.endTimeInMs)
-    ? (maybeSegment?.endTimeInMs || 0)
-    : (startTimeInMs + assetDurationInMs)
-
-  const segment: ClapSegment = {
-    id: typeof maybeSegment?.id === "string" ? maybeSegment.id : uuidv4(),
-    track: isValidNumber(maybeSegment?.track) ? (maybeSegment?.track || 0) : 0,
-    startTimeInMs,
-    endTimeInMs,
-    category: typeof maybeSegment?.category === "string" ? maybeSegment.category : "generic",
-    modelId: typeof maybeSegment?.modelId === "string" ? maybeSegment.modelId : "",
-    sceneId: typeof maybeSegment?.sceneId === "string" ? maybeSegment.sceneId : "",
-    prompt: typeof maybeSegment?.prompt === "string" ? maybeSegment.prompt : "",
-    label: typeof maybeSegment?.label === "string" ? maybeSegment.label : "",
-    outputType: typeof maybeSegment?.outputType === "string" ? maybeSegment.outputType : "text",
-    renderId: typeof maybeSegment?.renderId === "string" ? maybeSegment.renderId : "",
-    status: typeof maybeSegment?.status === "string" ? maybeSegment.status : "to_generate",
-    assetUrl: typeof maybeSegment?.assetUrl === "string" ? maybeSegment.assetUrl : "",
-    assetDurationInMs,
-    createdBy: typeof maybeSegment?.createdBy === "string" ?  maybeSegment.createdBy : "ai",
-    editedBy: typeof maybeSegment?.editedBy === "string" ?  maybeSegment.editedBy : "ai",
-    outputGain: isValidNumber(maybeSegment?.outputGain) ? (maybeSegment?.outputGain || 0) : 0,
-    seed: isValidNumber(maybeSegment?.seed) ? (maybeSegment?.seed || 0) : generateSeed()
-  }
-
-  return segment
-}

src/lib/clap/parseClap.ts

@@ -1,319 +0,0 @@
-import YAML from "yaml"
-import { v4 as uuidv4 } from "uuid"
-
-import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types"
-import { getValidNumber } from "@/lib/utils/getValidNumber"
-import { dataUriToBlob } from "@/app/api/utils/dataUriToBlob"
-
-type StringOrBlob = string | Blob
-
-/**
- * Import a clap file from various data sources into an ClapProject
- * 
- * Inputs can be:
- * - a Clap project (which is an object)
- * - an URL to a remote .clap file
- * - a string containing a YAML array
- * - a data uri containing a gzipped YAML array
- * - a Blob containing a gzipped YAML array
- * 
- * note: it is not really async, because for some reason YAML.parse is a blocking call like for JSON,
- * there is no async version although we are now in the 20s not 90s
- */
-export async function parseClap(src?: ClapProject | string | Blob, debug = false): Promise<ClapProject> {
-
-  try {
-    if (
-      typeof src === "object" &&
-      Array.isArray( (src as any)?.scenes) &&
-      Array.isArray((src as any)?.models)
-    ) {
-      if (debug) {
-        console.log("parseClap: input is already a Clap file, nothing to do:", src)
-      }
-      // we can skip verification
-      return src as ClapProject
-    }
-  } catch (err) {
-    // well, this is not a clap project
-  }
-
-  let stringOrBlob = (src || "") as StringOrBlob
-
-  // both should work
-  const dataUriHeader1 = "data:application/x-gzip;base64,"
-  const dataUriHeader2 = "data:application/octet-stream;base64,"
-
-  const inputIsString = typeof stringOrBlob === "string"
-  const inputIsDataUri = typeof stringOrBlob === "string" ? stringOrBlob.startsWith(dataUriHeader1) || stringOrBlob.startsWith(dataUriHeader2) : false
-  const inputIsRemoteFile = typeof stringOrBlob === "string" ? (stringOrBlob.startsWith("http://") || stringOrBlob.startsWith("https://")) : false
-
-  let inputIsBlob = typeof stringOrBlob !== "string"
-
-  let inputYamlArrayString = ""
-
-  if (debug) {
-    console.log(`parseClap: pre-analysis: ${JSON.stringify({
-      inputIsString,
-      inputIsBlob,
-      inputIsDataUri,
-      inputIsRemoteFile
-    }, null, 2)}`)
-  }
-
-  if (typeof stringOrBlob === "string") {
-    if (debug) {
-      console.log("parseClap: input is a string ", stringOrBlob.slice(0, 120))
-    }
-    if (inputIsDataUri) {
-      if (debug) {
-        console.log(`parseClap: input is a data uri archive`)
-      }
-      stringOrBlob = dataUriToBlob(stringOrBlob, "application/x-gzip")
-      if (debug) {
-        console.log(`parseClap: inputBlob = `, stringOrBlob)
-      }
-      inputIsBlob = true
-    } else if (inputIsRemoteFile) {
-      try {
-        if (debug) {
-          console.log(`parseClap: input is a remote .clap file`)
-        }
-        const res = await fetch(stringOrBlob)
-        stringOrBlob = await res.blob()
-        if (!stringOrBlob) { throw new Error("blob is empty") }
-        inputIsBlob = true
-      } catch (err) {
-        // url seems invalid
-        throw new Error(`failed to download the .clap file (${err})`)
-      }
-    } else {
-      if (debug) {
-        console.log("parseClap: input is a text string containing a YAML array")
-      }
-      inputYamlArrayString = stringOrBlob
-      inputIsBlob = false
-    }
-  }
-
-  if (typeof stringOrBlob !== "string" && stringOrBlob) {
-    if (debug) {
-      console.log("parseClap: decompressing the blob..")
-    }
-    // Decompress the input blob using gzip
-    const decompressedStream = stringOrBlob.stream().pipeThrough(new DecompressionStream('gzip'))
-
-    try {
-      // Convert the stream to text using a Response object
-      const decompressedOutput = new Response(decompressedStream)
-      // decompressedOutput.headers.set("Content-Type", "application/x-gzip")
-      if (debug) {
-        console.log("parseClap: decompressedOutput: ", decompressedOutput)
-      }
-      // const blobAgain = await decompressedOutput.blob()
-      inputYamlArrayString = await decompressedOutput.text()
-
-      if (debug && inputYamlArrayString) {
-        console.log("parseClap: successfully decompressed the blob!")
-      }
-    } catch (err) {
-      const message = `parseClap: failed to decompress (${err})`
-      console.error(message)
-      throw new Error(message)
-    }
-  }
-
-  // we don't need this anymore I think
-  // new Blob([inputStringOrBlob], { type: "application/x-yaml" })
-
-  let maybeArray: any = {}
-  try {
-    if (debug) {
-      console.log("parseClap: parsing the YAML array..")
-    }
-    // Parse YAML string to raw data
-    maybeArray = YAML.parse(inputYamlArrayString)
-  } catch (err) {
-    throw new Error("invalid clap file (input string is not YAML)")
-  }
-
-  if (!Array.isArray(maybeArray) || maybeArray.length < 2) {
-    throw new Error("invalid clap file (need a clap format header block and project metadata block)")
-  }
-
-  if (debug) {
-    console.log("parseClap: the YAML seems okay, continuing decoding..")
-  }
-
-  const maybeClapHeader = maybeArray[0] as ClapHeader
-
-  if (maybeClapHeader.format !== "clap-0") {
-    throw new Error("invalid clap file (sorry, but you can't make up version numbers like that)")
-  }
-
-
-  const maybeClapMeta = maybeArray[1] as ClapMeta
-
-  const clapMeta: ClapMeta = {
-    id: typeof maybeClapMeta.title === "string" ? maybeClapMeta.id : uuidv4(),
-    title: typeof maybeClapMeta.title === "string" ? maybeClapMeta.title : "",
-    description: typeof maybeClapMeta.description === "string" ? maybeClapMeta.description : "",
-    synopsis: typeof maybeClapMeta.synopsis === "string" ? maybeClapMeta.synopsis : "",
-    licence: typeof maybeClapMeta.licence === "string" ? maybeClapMeta.licence : "",
-    orientation: maybeClapMeta.orientation === "portrait" ? "portrait" : maybeClapMeta.orientation === "square" ? "square" : "landscape",
-    durationInMs: getValidNumber(maybeClapMeta.durationInMs, 1000, Number.MAX_SAFE_INTEGER, 4000),
-    width: getValidNumber(maybeClapMeta.width, 128, 8192, 1024),
-    height: getValidNumber(maybeClapMeta.height, 128, 8192, 576),
-    defaultVideoModel: typeof maybeClapMeta.defaultVideoModel === "string" ? maybeClapMeta.defaultVideoModel : "SVD",
-    extraPositivePrompt: Array.isArray(maybeClapMeta.extraPositivePrompt) ? maybeClapMeta.extraPositivePrompt : [],
-    screenplay: typeof maybeClapMeta.screenplay === "string" ? maybeClapMeta.screenplay : "",
-    isLoop: typeof maybeClapMeta.isLoop === "boolean" ? maybeClapMeta.isLoop : false,
-    isInteractive: typeof maybeClapMeta.isInteractive === "boolean" ? maybeClapMeta.isInteractive : false,
-  }
-
-  /*
-  in case we want to support streaming (mix of models and segments etc), we could do it this way:
-
-  const maybeModelsOrSegments = rawData.slice(2)
-  maybeModelsOrSegments.forEach((unknownElement: any) => {
-    if (isValidNumber(unknownElement?.track)) {
-      maybeSegments.push(unknownElement as ClapSegment)
-    } else {
-      maybeModels.push(unknownElement as ClapModel)
-    }
-  })
-  */
-
-
-  const expectedNumberOfModels = maybeClapHeader.numberOfModels || 0
-  const expectedNumberOfScenes = maybeClapHeader.numberOfScenes || 0
-  const expectedNumberOfSegments = maybeClapHeader.numberOfSegments || 0
-
-  // note: we assume the order is strictly enforced!
-  // if you implement streaming (mix of models and segments) you will have to rewrite this!
-
-  const afterTheHeaders = 2
-  const afterTheModels = afterTheHeaders + expectedNumberOfModels
-
-  const afterTheScenes = afterTheModels + expectedNumberOfScenes
-
-  // note: if there are no expected models, maybeModels will be empty
-  const maybeModels = maybeArray.slice(afterTheHeaders, afterTheModels) as ClapModel[]
-
-  // note: if there are no expected scenes, maybeScenes will be empty
-  const maybeScenes = maybeArray.slice(afterTheModels, afterTheScenes) as ClapScene[]
-
-  const maybeSegments = maybeArray.slice(afterTheScenes) as ClapSegment[]
-
-  const clapModels: ClapModel[] = maybeModels.map(({
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }) => ({
-    // TODO: we should verify each of those, probably
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }))
-
-  const clapScenes: ClapScene[] = maybeScenes.map(({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events,
-  }) => ({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events: events.map(e => e)
-  }))
-  
-  const clapSegments: ClapSegment[] = maybeSegments.map(({
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }) => ({
-    // TODO: we should verify each of those, probably
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }))
-
-  if (debug) {
-    console.log(`parseClap: successfully parsed ${clapModels.length} models, ${clapScenes.length} scenes and ${clapSegments.length} segments`)
-  }
-  return {
-    meta: clapMeta,
-    models: clapModels,
-    scenes: clapScenes,
-    segments: clapSegments
-  }
-}

src/lib/clap/serializeClap.ts

@@ -1,169 +0,0 @@
-import YAML from "yaml"
-import { v4 as uuidv4 } from "uuid"
-
-import { getValidNumber } from "@/lib/utils/getValidNumber"
-
-import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types"
-
-export async function serializeClap({
-  meta, // ClapMeta
-  models, // ClapModel[]
-  scenes, // ClapScene[]
-  segments, // ClapSegment[]
-}: ClapProject): Promise<Blob> {
-  
-  // we play it safe, and we verify the structure of the parameters,
-  // to make sure we generate a valid clap file
-  const clapModels: ClapModel[] = models.map(({
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }) => ({
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }))
-
-  const clapScenes: ClapScene[] = scenes.map(({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events,
-  }) => ({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events: events.map(e => e)
-  }))
-
-  const clapSegments: ClapSegment[] = segments.map(({
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }) => ({
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }))
-
-  const clapHeader: ClapHeader = {
-    format: "clap-0",
-    numberOfModels: clapModels.length,
-    numberOfScenes: clapScenes.length,
-    numberOfSegments: clapSegments.length,
-  }
-
-  const clapMeta: ClapMeta = {
-    id: meta.id || uuidv4(),
-    title: typeof meta.title === "string" ? meta.title : "Untitled",
-    description: typeof meta.description === "string" ? meta.description : "",
-    synopsis: typeof meta.synopsis === "string" ? meta.synopsis : "",
-    licence: typeof meta.licence === "string" ? meta.licence : "",
-    orientation: meta.orientation === "portrait" ? "portrait" : meta.orientation === "square" ? "square" : "landscape",
-    durationInMs: getValidNumber(meta.durationInMs, 1000, Number.MAX_SAFE_INTEGER, 4000),
-    width: getValidNumber(meta.width, 256, 8192, 1024),
-    height: getValidNumber(meta.height, 256, 8192, 576),
-    defaultVideoModel:  typeof meta.defaultVideoModel === "string" ? meta.defaultVideoModel : "SVD",
-    extraPositivePrompt: Array.isArray(meta.extraPositivePrompt) ? meta.extraPositivePrompt : [],
-    screenplay: typeof meta.screenplay === "string" ? meta.screenplay : "",
-    isLoop: typeof meta.screenplay === "boolean" ? meta.screenplay : false,
-    isInteractive: typeof meta.isInteractive === "boolean" ? meta.isInteractive : false,
-  }
-
-  const entries = [
-    clapHeader,
-    clapMeta,
-    ...clapModels,
-    ...clapScenes,
-    ...clapSegments
-  ]
-
-  const strigifiedResult = YAML.stringify(entries)
-
-  // Convert the string to a Blob
-  const blobResult = new Blob([strigifiedResult], { type: "application/x-yaml" })
-
-   // Create a stream for the blob
-   const readableStream = blobResult.stream()
-
-   // Compress the stream using gzip
-   const compressionStream = new CompressionStream('gzip')
-   const compressedStream = readableStream.pipeThrough(compressionStream)
-
-   // Create a new blob from the compressed stream
-   const response = new Response(compressedStream)
-   
-   response.headers.set("Content-Type", "application/x-gzip")
-
-   const compressedBlob = await response.blob()
-
-  return compressedBlob
-}

src/lib/clap/types.ts

@@ -1,203 +0,0 @@
-
-export type ClapSegmentCategory =
-  | "splat"
-  | "mesh"
-  | "depth"
-  | "event"
-  | "interface"
-  | "phenomenon"
-  | "video"
-  | "storyboard"
-  | "transition"
-  | "characters"
-  | "location"
-  | "time"
-  | "era"
-  | "lighting"
-  | "weather"
-  | "action"
-  | "music"
-  | "sound"
-  | "dialogue"
-  | "style"
-  | "camera"
-  | "generic"
-
-export type ClapOutputType =
-  | "text"
-  | "animation"
-  | "interface"
-  | "event"
-  | "phenomenon"
-  | "transition"
-  | "image"
-  | "video"
-  | "audio"
-
-export type ClapSegmentStatus =
-  | "to_generate"
-  | "to_interpolate"
-  | "to_upscale"
-  | "completed"
-  | "error"
-
-export type ClapAuthor =
-  | "auto" // the element was edited automatically using basic if/else logical rules
-  | "ai" // the element was edited using a large language model
-  | "human" // the element was edited by a human
-
-export type ClapAssetSource =
-  | "REMOTE" // http:// or https://
-
-    // note that "path" assets are potentially a security risk, they need to be treated with care
-  | "PATH" // a file path eg. /path or ./path/to/ or ../path/to/
-
-  | "DATA" // a data URI, starting with data:
-
-  | "PROMPT" // by default, a plain text prompt
-
-  | "EMPTY"
-
-export type ClapModelGender =
-  | "male"
-  | "female"
-  | "person"
-  | "object"
-
-export type ClapModelAppearance = "serious" | "neutral" | "friendly" | "chill"
-
-// this is used for accent, style..
-export type ClapModelRegion =
-  | "american"
-  | "british"
-  | "australian"
-  | "canadian"
-  | "indian"
-  | "french"
-  | "italian"
-  | "german"
-  | "chinese"
-
-// note: this is all very subjective, so please use good judgment
-//
-// "deep" might indicate a deeper voice tone, thicker, rich in harmonics
-// in this context, it is used to indicate voices that could
-// be associated with African American (AADOS) characters
-//
-// "high" could be used for some other countries, eg. asia
-export type ClapModelTimbre = "high" | "neutral" | "deep"
-
-export type ClapVoiceVendor = "ElevenLabs" | "XTTS"
-
-export type ClapVoice = {
-  name: string
-  gender: ClapModelGender
-  age: number
-  region: ClapModelRegion
-  timbre: ClapModelTimbre
-  appearance: ClapModelAppearance
-  voiceVendor: ClapVoiceVendor
-  voiceId: string
-}
-
-export type ClapHeader = {
-  format: "clap-0"
-  numberOfModels: number
-  numberOfScenes: number
-  numberOfSegments: number
-}
-
-export type ClapMeta = {
-  id: string
-  title: string
-  description: string
-  synopsis: string
-  licence: string
-  orientation: string
-
-  // the default duration of the experience
-  // the real one might last longer if made interactive
-  durationInMs: number
-
-  width: number
-  height: number
-  defaultVideoModel: string
-  extraPositivePrompt: string[]
-  screenplay: string
-  isLoop: boolean
-  isInteractive: boolean
-}
-
-export type ClapSceneEvent = {
-  id: string
-  type: "description" | "dialogue" | "action"
-  character?: string
-  description: string
-  behavior: string
-  startAtLine: number
-  endAtLine: number
-}
-
-export type ClapScene = {
-  id: string
-  scene: string
-  line: string
-  rawLine: string
-  sequenceFullText: string
-  sequenceStartAtLine: number
-  sequenceEndAtLine: number
-  startAtLine: number
-  endAtLine: number
-  events: ClapSceneEvent[]
-}
-
-export type ClapSegment = {
-  id: string
-  track: number // usually track 0 is the video, track 1 is the storyboard, track 2 is the camera
-  startTimeInMs: number
-  endTimeInMs: number
-  category: ClapSegmentCategory
-  modelId: string
-  sceneId: string
-  prompt: string
-  label: string
-  outputType: ClapOutputType
-  renderId: string
-  status: ClapSegmentStatus
-  assetUrl: string
-  assetDurationInMs: number
-  createdBy: ClapAuthor
-  editedBy: ClapAuthor
-  outputGain: number
-  seed: number
-}
-
-export type ClapModel = {
-  id: string
-  category: ClapSegmentCategory
-  triggerName: string
-  label: string
-  description: string
-  author: string
-  thumbnailUrl: string
-  seed: number
-
-  assetSourceType: ClapAssetSource
-  assetUrl: string
-  
-  // those are only used by certain types of models
-  age: number
-  gender: ClapModelGender
-  region: ClapModelRegion
-  appearance: ClapModelAppearance
-  voiceVendor: ClapVoiceVendor
-  voiceId: string
-}
-
-export type ClapProject = {
-  meta: ClapMeta
-  models: ClapModel[]
-  scenes: ClapScene[]
-  segments: ClapSegment[]
-  // let's keep room for other stuff (screenplay etc)
-}

src/lib/prompts/generateSeed.ts

@@ -1,3 +0,0 @@
-export function generateSeed() {
-  return Math.floor(Math.random() * Math.pow(2, 31));
-}

src/lib/utils/generateSeed.ts

@@ -1,3 +0,0 @@
-export function generateSeed() {
-  return Math.floor(Math.random() * Math.pow(2, 31));
-}

src/lib/utils/startOfSegment1IsWithinSegment2.ts

@@ -1,4 +1,4 @@
-import { ClapSegment } from "../clap/types"
+import { ClapSegment } from "@aitube/clap"
 
 export function startOfSegment1IsWithinSegment2(s1: ClapSegment, s2: ClapSegment) {
   const startOfSegment1 = s1.startTimeInMs