diff --git a/.env b/.env index 8443be9ff582d3200bc967518837e930b83710df..7498243cc012e841aeb745fc0247394fc8403ed6 100644 --- a/.env +++ b/.env @@ -17,6 +17,11 @@ WINNERS="" AUTH_ALCHEMY_API_KEY="" +AUTH_OPENAI_API_KEY="" + +VIDEOCHAIN_API_URL="" +VIDEOCHAIN_API_KEY="" + # ----------- CENSORSHIP ------- ENABLE_CENSORSHIP= FINGERPRINT_KEY= diff --git a/next.config.js b/next.config.js index 5cd8cc341fdc321d3ffde0907685dfbec1db0686..d3b8458ff2b2f7f712d20ec2700b66c30bc7744e 100644 --- a/next.config.js +++ b/next.config.js @@ -1,6 +1,15 @@ /** @type {import('next').NextConfig} */ const nextConfig = { output: 'standalone', + experimental: { + serverActions: { + + // necessary as we are generating Clap files on server-side + // however, we are only generating text and not assets, so it should be lightweight, + // usually below 2mb + bodySizeLimit: '4mb', + }, + } } module.exports = nextConfig diff --git a/package-lock.json b/package-lock.json index bbae859dfa9b334df3de1bbfa141c79c3f81bdc2..7fd23854763eb47efa3b729c7f41c51b06755135 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "version": "0.0.0", "dependencies": { "@huggingface/hub": "0.12.3-oauth", + "@huggingface/inference": "^2.6.7", "@jcoreio/async-throttle": "^1.6.0", + "@mediapipe/tasks-vision": "^0.10.12", "@photo-sphere-viewer/core": "^5.7.2", "@photo-sphere-viewer/equirectangular-video-adapter": "^5.7.2", "@photo-sphere-viewer/gyroscope-plugin": "^5.7.2", @@ -38,6 +40,7 @@ "@radix-ui/react-toast": "^1.1.5", "@radix-ui/react-tooltip": "^1.0.7", "@react-spring/web": "^9.7.3", + "@tailwindcss/container-queries": "^0.1.1", "@types/lodash.debounce": "^4.0.9", "@types/node": "20.4.2", "@types/react": "18.2.15", @@ -62,10 +65,12 @@ "markdown-yaml-metadata-parser": "^3.0.0", "minisearch": "^6.3.0", "next": "^14.1.4", + "openai": "^4.36.0", "photo-sphere-viewer-lensflare-plugin": "^2.1.2", "pick": "^0.0.1", "postcss": "8.4.38", "qs": "^6.12.0", + "query-string": "^9.0.0", "react": "18.2.0", "react-circular-progressbar": "^2.1.0", "react-copy-to-clipboard": "^5.1.0", @@ -77,6 +82,7 @@ "react-tuby": "^0.1.24", "react-virtualized-auto-sizer": "^1.0.20", "react-window-infinite-loader": "^1.0.9", + "runcss": "^0.1.6", "sbd": "^1.0.19", "sentence-splitter": "^4.3.0", "sharp": "^0.33.3", @@ -924,6 +930,14 @@ "node": ">=18" } }, + "node_modules/@huggingface/inference": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.6.7.tgz", + "integrity": "sha512-vFBqvtU3LhxjufTs0jcRrDSc0nK+lah10bOgvlIn80lAH4JwMzHHPBQ4g4ECEdRD0PIt6EpTiidEZQq2sArb5Q==", + "engines": { + "node": ">=18" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.14", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", @@ -1477,6 +1491,11 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@mediapipe/tasks-vision": { + "version": "0.10.12", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.12.tgz", + "integrity": "sha512-688Vukid7hvGmx+7hzS/EQ3Q4diz4eeX4/FYDw8f/t56UjFueD8LTvA2rX5BCIwvT0oy8QHKh5uKIyct1AOFtQ==" + }, "node_modules/@next/env": { "version": "14.1.4", "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.4.tgz", @@ -2852,6 +2871,14 @@ "tslib": "^2.4.0" } }, + "node_modules/@tailwindcss/container-queries": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@tailwindcss/container-queries/-/container-queries-0.1.1.tgz", + "integrity": "sha512-p18dswChx6WnTSaJCSGx6lTmrGzNNvm2FtXmiO6AuA1V4U5REyoqwmT6kgAsIMdjo07QdAfYXHJ4hnMtfHzWgA==", + "peerDependencies": { + "tailwindcss": ">=3.2.0" + } + }, "node_modules/@textlint/ast-node-types": { "version": "13.4.1", "resolved": "https://registry.npmjs.org/@textlint/ast-node-types/-/ast-node-types-13.4.1.tgz", @@ -2905,6 +2932,15 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.2.tgz", "integrity": "sha512-Dd0BYtWgnWJKwO1jkmTrzofjK2QXXcai0dmtzvIBhcA+RsG5h8R3xlyta0kGOZRNfL9GuRtb1knmPEhQrePCEw==" }, + "node_modules/@types/node-fetch": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, "node_modules/@types/prop-types": { "version": "15.7.12", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.12.tgz", @@ -3101,6 +3137,17 @@ "crypto-js": "^4.2.0" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", @@ -3133,6 +3180,17 @@ "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-3.0.0.tgz", "integrity": "sha512-H7wUZRn8WpTq9jocdxQ2c8x2sKo9ZVmzfRE13GiNJXfp7NcKYEdvl3vspKjXox6RIG2VtaRe4JFvxG4rqp2Zuw==" }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -4021,6 +4079,14 @@ } } }, + "node_modules/decode-uri-component": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.4.1.tgz", + "integrity": "sha512-+8VxcR21HhTy8nOt6jf20w0c9CADrw1O8d+VZ/YzzCt4bJ3uBjw+D1q2osAB8RnpwwaeYBxy0HyKQxD5JBMuuQ==", + "engines": { + "node": ">=14.16" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -4879,6 +4945,14 @@ "es5-ext": "~0.10.14" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", @@ -4977,6 +5051,17 @@ "node": ">=8" } }, + "node_modules/filter-obj": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-5.1.0.tgz", + "integrity": "sha512-qWeTREPoT7I0bifpPUXtxkZJ1XJzxWtfoWWkdVGqa+eCr3SHW/Ocp89o8vLvbUuQnadybJpjOKu4V+RwO6sGng==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -5065,6 +5150,31 @@ "node": ">= 6" } }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "engines": { + "node": ">= 14" + } + }, "node_modules/fraction.js": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", @@ -5411,6 +5521,14 @@ "entities": "^4.4.0" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/ignore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", @@ -6277,6 +6395,43 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-gyp-build": { "version": "4.8.0", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.0.tgz", @@ -6440,6 +6595,32 @@ "wrappy": "1" } }, + "node_modules/openai": { + "version": "4.36.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.36.0.tgz", + "integrity": "sha512-AtYrhhWY64LhB9P6f3H0nV8nTSaQJ89mWPnfNU5CnYg81zlYaV8nkyO+aTNfprdqP/9xv10woNNUgefXINT4Dg==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + }, + "bin": { + "openai": "bin/cli" + } + }, + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.31", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.31.tgz", + "integrity": "sha512-ArgCD39YpyyrtFKIqMDvjz79jto5fcI/SVUs2HwB+f0dAzq68yqOdyaSivLiLugSziTpNXLQrVb7RZFmdZzbhA==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/optionator": { "version": "0.9.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz", @@ -6803,6 +6984,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/query-string": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-9.0.0.tgz", + "integrity": "sha512-4EWwcRGsO2H+yzq6ddHcVqkCQ2EFUSfDMEjF8ryp8ReymyZhIuaFRGLomeOQLkrzacMHoyky2HW0Qe30UbzkKw==", + "dependencies": { + "decode-uri-component": "^0.4.1", + "filter-obj": "^5.1.0", + "split-on-first": "^3.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -7156,6 +7353,11 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/runcss": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/runcss/-/runcss-0.1.6.tgz", + "integrity": "sha512-RyG7VVUxZi8+ynXA4YFH7N6uXK0QgULKTEEMmeUkK/uhfNGiQitWRKQp5b+1AEKu4/QTslFEx7wu971vQQk9Tg==" + }, "node_modules/safe-array-concat": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.2.tgz", @@ -7391,6 +7593,17 @@ "node": ">=0.10.0" } }, + "node_modules/split-on-first": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-3.0.0.tgz", + "integrity": "sha512-qxQJTx2ryR0Dw0ITYyekNQWpz6f8dGd7vffGNflQQ3Iqj9NJ6qiZ7ELpZsJ/QBhIVAiDfXdag3+Gp8RvWa62AA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -7885,6 +8098,11 @@ "resolved": "https://registry.npmjs.org/toggle-selection/-/toggle-selection-1.0.6.tgz", "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==" }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/ts-interface-checker": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", @@ -8102,6 +8320,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", @@ -8236,6 +8459,19 @@ "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==" }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, "node_modules/websocket": { "version": "1.0.34", "resolved": "https://registry.npmjs.org/websocket/-/websocket-1.0.34.tgz", @@ -8265,6 +8501,15 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 8f4208024156a0b76aecbbb2ced088877d1fc882..6e43c7854c0fc4d73f179e7e861e97da11d436de 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,9 @@ }, "dependencies": { "@huggingface/hub": "0.12.3-oauth", + "@huggingface/inference": "^2.6.7", "@jcoreio/async-throttle": "^1.6.0", + "@mediapipe/tasks-vision": "^0.10.12", "@photo-sphere-viewer/core": "^5.7.2", "@photo-sphere-viewer/equirectangular-video-adapter": "^5.7.2", "@photo-sphere-viewer/gyroscope-plugin": "^5.7.2", @@ -39,6 +41,7 @@ "@radix-ui/react-toast": "^1.1.5", "@radix-ui/react-tooltip": "^1.0.7", "@react-spring/web": "^9.7.3", + "@tailwindcss/container-queries": "^0.1.1", "@types/lodash.debounce": "^4.0.9", "@types/node": "20.4.2", "@types/react": "18.2.15", @@ -63,10 +66,12 @@ "markdown-yaml-metadata-parser": "^3.0.0", "minisearch": "^6.3.0", "next": "^14.1.4", + "openai": "^4.36.0", "photo-sphere-viewer-lensflare-plugin": "^2.1.2", "pick": "^0.0.1", "postcss": "8.4.38", "qs": "^6.12.0", + "query-string": "^9.0.0", "react": "18.2.0", "react-circular-progressbar": "^2.1.0", "react-copy-to-clipboard": "^5.1.0", @@ -78,6 +83,7 @@ "react-tuby": "^0.1.24", "react-virtualized-auto-sizer": "^1.0.20", "react-window-infinite-loader": "^1.0.9", + "runcss": "^0.1.6", "sbd": "^1.0.19", "sentence-splitter": "^4.3.0", "sharp": "^0.33.3", diff --git a/public/bubble.jpg b/public/bubble.jpg deleted file mode 100644 index 22e44c049b61e7b56281e8a74504855959970617..0000000000000000000000000000000000000000 Binary files a/public/bubble.jpg and /dev/null differ diff --git a/public/favicon.ico b/public/favicon.ico index 060fa8ce26f545dd54e28b76401e5bc7a55b7c92..73beb52bd315adbbda0ee88ddb1e521dd830d96f 100644 Binary files a/public/favicon.ico and b/public/favicon.ico differ diff --git a/public/favicon/favicon-114-precomposed.png b/public/favicon/favicon-114-precomposed.png deleted file mode 100644 index be8953b99cc353a6ea9047e83a8c28a627cff46c..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-114-precomposed.png and /dev/null differ diff --git a/public/favicon/favicon-120-precomposed.png b/public/favicon/favicon-120-precomposed.png deleted file mode 100644 index 3aab950a1f0268f0642e87e8fec63ef8f064c4da..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-120-precomposed.png and /dev/null differ diff --git a/public/favicon/favicon-144-precomposed.png b/public/favicon/favicon-144-precomposed.png deleted file mode 100644 index e29c5d95a6d22dd36aea448c8eda8df03e875815..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-144-precomposed.png and /dev/null differ diff --git a/public/favicon/favicon-152-precomposed.png b/public/favicon/favicon-152-precomposed.png deleted file mode 100644 index 6201f9f8fe506e8562741ab990c33c7a39d714d9..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-152-precomposed.png and /dev/null differ diff --git a/public/favicon/favicon-180-precomposed.png b/public/favicon/favicon-180-precomposed.png deleted file mode 100644 index 241a60c82c96c8665d4935632a1db121b8a99387..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-180-precomposed.png and /dev/null differ diff --git a/public/favicon/favicon-192.png b/public/favicon/favicon-192.png deleted file mode 100644 index ecc6cbefdfc1232e92b40cc616cc34388b0360da..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-192.png and /dev/null differ diff --git a/public/favicon/favicon-32.png b/public/favicon/favicon-32.png deleted file mode 100644 index 4076fa1b3ea9a28897aeb9dbb37bc8bfa4f1a624..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-32.png and /dev/null differ diff --git a/public/favicon/favicon-36.png b/public/favicon/favicon-36.png deleted file mode 100644 index 4bb5a3262eff1c5ba29e3e06320b9d76a07565af..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-36.png and /dev/null differ diff --git a/public/favicon/favicon-48.png b/public/favicon/favicon-48.png deleted file mode 100644 index 69d6328355ed22cf0dca75f558351e1f585af8bb..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-48.png and /dev/null differ diff --git a/public/favicon/favicon-57.png b/public/favicon/favicon-57.png deleted file mode 100644 index 91ac87f90441ddbe723a45cf7e0b16eded515d21..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-57.png and /dev/null differ diff --git a/public/favicon/favicon-60.png b/public/favicon/favicon-60.png deleted file mode 100644 index cf5ee0bea0c30fea6addb1134d8bfeb7ca888e0d..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-60.png and /dev/null differ diff --git a/public/favicon/favicon-72-precomposed.png b/public/favicon/favicon-72-precomposed.png deleted file mode 100644 index 4e957de6a9137ec73ab51ba9317e8c13d1e45bf5..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-72-precomposed.png and /dev/null differ diff --git a/public/favicon/favicon-72.png b/public/favicon/favicon-72.png deleted file mode 100644 index 4e957de6a9137ec73ab51ba9317e8c13d1e45bf5..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-72.png and /dev/null differ diff --git a/public/favicon/favicon-76.png b/public/favicon/favicon-76.png deleted file mode 100644 index 7eb8efbd1087dc53784116053667c49073e08d3d..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-76.png and /dev/null differ diff --git a/public/favicon/favicon-96.png b/public/favicon/favicon-96.png deleted file mode 100644 index 5098aed1cda10c42eabcacb721d17b62dc5a8f3a..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon-96.png and /dev/null differ diff --git a/public/favicon/favicon.ico b/public/favicon/favicon.ico deleted file mode 100644 index 060fa8ce26f545dd54e28b76401e5bc7a55b7c92..0000000000000000000000000000000000000000 Binary files a/public/favicon/favicon.ico and /dev/null differ diff --git a/public/favicon/manifest.json b/public/favicon/manifest.json deleted file mode 100644 index d0d92afbc1530e91966f13b737cea8885bf5a111..0000000000000000000000000000000000000000 --- a/public/favicon/manifest.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "pollo", - "icons": [ - { - "src": "\/favicon-36.png", - "sizes": "36x36", - "type": "image\/png", - "density": 0.75 - }, - { - "src": "\/favicon-48.png", - "sizes": "48x48", - "type": "image\/png", - "density": 1 - }, - { - "src": "\/favicon-72.png", - "sizes": "72x72", - "type": "image\/png", - "density": 1.5 - }, - { - "src": "\/favicon-96.png", - "sizes": "96x96", - "type": "image\/png", - "density": 2 - }, - { - "src": "\/favicon-144.png", - "sizes": "144x144", - "type": "image\/png", - "density": 3 - }, - { - "src": "\/favicon-192.png", - "sizes": "192x192", - "type": "image\/png", - "density": 4 - } - ] -} diff --git a/public/huggingface-avatar.jpeg b/public/huggingface-avatar.jpeg deleted file mode 100644 index 54a5d6ef0a3cf061cf1602db61321c6a1626453c..0000000000000000000000000000000000000000 Binary files a/public/huggingface-avatar.jpeg and /dev/null differ diff --git a/public/icon.png b/public/icon.png index ecc6cbefdfc1232e92b40cc616cc34388b0360da..d1bcb2646f92b6f65ac4bab1570dc9520ac4f266 100644 Binary files a/public/icon.png and b/public/icon.png differ diff --git a/public/mask.png b/public/mask.png deleted file mode 100644 index 5a1047425e0047f0449aabee676019c801cd7cf3..0000000000000000000000000000000000000000 Binary files a/public/mask.png and /dev/null differ diff --git a/public/next.svg b/public/next.svg deleted file mode 100644 index 5174b28c565c285e3e312ec5178be64fbeca8398..0000000000000000000000000000000000000000 --- a/public/next.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/public/report.jpg b/public/report.jpg deleted file mode 100644 index 3b61d4dd3994f57296b020e99dbf1e043cb5e98a..0000000000000000000000000000000000000000 Binary files a/public/report.jpg and /dev/null differ diff --git a/public/vercel.svg b/public/vercel.svg deleted file mode 100644 index d2f84222734f27b623d1c80dda3561b04d1284af..0000000000000000000000000000000000000000 --- a/public/vercel.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/src/app/server/actions/README.md b/src/app/api/actions/README.md similarity index 100% rename from src/app/server/actions/README.md rename to src/app/api/actions/README.md diff --git a/src/app/server/actions/ai-tube-hf/README.md b/src/app/api/actions/ai-tube-hf/README.md similarity index 100% rename from src/app/server/actions/ai-tube-hf/README.md rename to src/app/api/actions/ai-tube-hf/README.md diff --git a/src/app/server/actions/ai-tube-hf/deleteFileFromDataset.ts b/src/app/api/actions/ai-tube-hf/deleteFileFromDataset.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/deleteFileFromDataset.ts rename to src/app/api/actions/ai-tube-hf/deleteFileFromDataset.ts diff --git a/src/app/server/actions/ai-tube-hf/deleteVideoRequest.ts b/src/app/api/actions/ai-tube-hf/deleteVideoRequest.ts similarity index 87% rename from src/app/server/actions/ai-tube-hf/deleteVideoRequest.ts rename to src/app/api/actions/ai-tube-hf/deleteVideoRequest.ts index 8650561b501689255b0a7232f7768a1f8f8c96e6..93a6e8d20e87f225629ee4f15a1818b4702a366b 100644 --- a/src/app/server/actions/ai-tube-hf/deleteVideoRequest.ts +++ b/src/app/api/actions/ai-tube-hf/deleteVideoRequest.ts @@ -2,7 +2,7 @@ import { MediaInfo } from "@/types/general" import { deleteFileFromDataset } from "./deleteFileFromDataset" -import { formatPromptFileName } from "../utils/formatPromptFileName" +import { formatPromptFileName } from "../../utils/formatPromptFileName" export async function deleteVideoRequest({ video, diff --git a/src/app/server/actions/ai-tube-hf/downloadClapProject.ts b/src/app/api/actions/ai-tube-hf/downloadClapProject.ts similarity index 94% rename from src/app/server/actions/ai-tube-hf/downloadClapProject.ts rename to src/app/api/actions/ai-tube-hf/downloadClapProject.ts index 03c6b993b53b9df545cfafd484eba797d3714e68..954107756b911ccaa4b498d2d9c7eb9af4601a72 100644 --- a/src/app/server/actions/ai-tube-hf/downloadClapProject.ts +++ b/src/app/api/actions/ai-tube-hf/downloadClapProject.ts @@ -6,8 +6,8 @@ import { ChannelInfo, MediaInfo, VideoRequest } from "@/types/general" import { defaultVideoModel } from "@/app/config" import { parseClap } from "@/lib/clap/parseClap" -import { parseVideoModelName } from "../utils/parseVideoModelName" -import { computeOrientationProjectionWidthHeight } from "../utils/computeOrientationProjectionWidthHeight" +import { parseVideoModelName } from "../../utils/parseVideoModelName" +import { computeOrientationProjectionWidthHeight } from "../../utils/computeOrientationProjectionWidthHeight" import { downloadFileAsBlob } from "./downloadFileAsBlob" diff --git a/src/app/server/actions/ai-tube-hf/downloadFileAsBlob.ts b/src/app/api/actions/ai-tube-hf/downloadFileAsBlob.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/downloadFileAsBlob.ts rename to src/app/api/actions/ai-tube-hf/downloadFileAsBlob.ts diff --git a/src/app/server/actions/ai-tube-hf/downloadFileAsText.ts b/src/app/api/actions/ai-tube-hf/downloadFileAsText.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/downloadFileAsText.ts rename to src/app/api/actions/ai-tube-hf/downloadFileAsText.ts diff --git a/src/app/server/actions/ai-tube-hf/downloadPlainText.ts b/src/app/api/actions/ai-tube-hf/downloadPlainText.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/downloadPlainText.ts rename to src/app/api/actions/ai-tube-hf/downloadPlainText.ts diff --git a/src/app/server/actions/ai-tube-hf/extendVideosWithStats.ts b/src/app/api/actions/ai-tube-hf/extendVideosWithStats.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/extendVideosWithStats.ts rename to src/app/api/actions/ai-tube-hf/extendVideosWithStats.ts diff --git a/src/app/server/actions/ai-tube-hf/getChannel.ts b/src/app/api/actions/ai-tube-hf/getChannel.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/getChannel.ts rename to src/app/api/actions/ai-tube-hf/getChannel.ts diff --git a/src/app/server/actions/ai-tube-hf/getChannelVideos.ts b/src/app/api/actions/ai-tube-hf/getChannelVideos.ts similarity index 95% rename from src/app/server/actions/ai-tube-hf/getChannelVideos.ts rename to src/app/api/actions/ai-tube-hf/getChannelVideos.ts index 89422e26a262ac50e3e6ec62778db457f6c8ee1c..c6c4098458d426500492aa1d162afd0172b69d73 100644 --- a/src/app/server/actions/ai-tube-hf/getChannelVideos.ts +++ b/src/app/api/actions/ai-tube-hf/getChannelVideos.ts @@ -6,7 +6,7 @@ import { getVideoRequestsFromChannel } from "./getVideoRequestsFromChannel" import { adminApiKey } from "../config" import { getVideoIndex } from "./getVideoIndex" import { extendVideosWithStats } from "./extendVideosWithStats" -import { computeOrientationProjectionWidthHeight } from "../utils/computeOrientationProjectionWidthHeight" +import { computeOrientationProjectionWidthHeight } from "../../utils/computeOrientationProjectionWidthHeight" import { defaultVideoModel } from "@/app/config" // return diff --git a/src/app/server/actions/ai-tube-hf/getChannels.ts b/src/app/api/actions/ai-tube-hf/getChannels.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/getChannels.ts rename to src/app/api/actions/ai-tube-hf/getChannels.ts diff --git a/src/app/server/actions/ai-tube-hf/getCredentials.ts b/src/app/api/actions/ai-tube-hf/getCredentials.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/getCredentials.ts rename to src/app/api/actions/ai-tube-hf/getCredentials.ts diff --git a/src/app/server/actions/ai-tube-hf/getPrivateChannels.ts b/src/app/api/actions/ai-tube-hf/getPrivateChannels.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/getPrivateChannels.ts rename to src/app/api/actions/ai-tube-hf/getPrivateChannels.ts diff --git a/src/app/server/actions/ai-tube-hf/getTags.ts b/src/app/api/actions/ai-tube-hf/getTags.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/getTags.ts rename to src/app/api/actions/ai-tube-hf/getTags.ts diff --git a/src/app/server/actions/ai-tube-hf/getVideo.ts b/src/app/api/actions/ai-tube-hf/getVideo.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/getVideo.ts rename to src/app/api/actions/ai-tube-hf/getVideo.ts diff --git a/src/app/server/actions/ai-tube-hf/getVideoIndex.ts b/src/app/api/actions/ai-tube-hf/getVideoIndex.ts similarity index 100% rename from src/app/server/actions/ai-tube-hf/getVideoIndex.ts rename to src/app/api/actions/ai-tube-hf/getVideoIndex.ts diff --git a/src/app/server/actions/ai-tube-hf/getVideoRequestsFromChannel.ts b/src/app/api/actions/ai-tube-hf/getVideoRequestsFromChannel.ts similarity index 95% rename from src/app/server/actions/ai-tube-hf/getVideoRequestsFromChannel.ts rename to src/app/api/actions/ai-tube-hf/getVideoRequestsFromChannel.ts index 540bec49deb5981d03214e323c6cd0b8eba5e26c..a36b7ef2e7cb529f43d3fecfbd5c6a57955a2b21 100644 --- a/src/app/server/actions/ai-tube-hf/getVideoRequestsFromChannel.ts +++ b/src/app/api/actions/ai-tube-hf/getVideoRequestsFromChannel.ts @@ -3,10 +3,10 @@ import { ChannelInfo, VideoRequest } from "@/types/general" import { getCredentials } from "./getCredentials" import { listFiles } from "@/lib/huggingface/hub/src" -import { parsePromptFileName } from "../utils/parsePromptFileName" +import { parsePromptFileName } from "../../utils/parsePromptFileName" import { downloadFileAsText } from "./downloadFileAsText" -import { parseDatasetPrompt } from "../utils/parseDatasetPrompt" -import { computeOrientationProjectionWidthHeight } from "../utils/computeOrientationProjectionWidthHeight" +import { parseDatasetPrompt } from "../../utils/parseDatasetPrompt" +import { computeOrientationProjectionWidthHeight } from "../../utils/computeOrientationProjectionWidthHeight" import { downloadClapProject } from "./downloadClapProject" /** diff --git a/src/app/server/actions/ai-tube-hf/getVideos.ts b/src/app/api/actions/ai-tube-hf/getVideos.ts similarity index 93% rename from src/app/server/actions/ai-tube-hf/getVideos.ts rename to src/app/api/actions/ai-tube-hf/getVideos.ts index 5dac5b4bbb1201cfb7f2c67f67768a17aef8ee4c..479b521eed08423e6eaa8855f9d3f14dc440f3ce 100644 --- a/src/app/server/actions/ai-tube-hf/getVideos.ts +++ b/src/app/api/actions/ai-tube-hf/getVideos.ts @@ -7,8 +7,8 @@ import { MediaInfo } from "@/types/general" import { getVideoIndex } from "./getVideoIndex" import { extendVideosWithStats } from "./extendVideosWithStats" -import { isHighQuality } from "../utils/isHighQuality" -import { isAntisocial } from "../utils/isAntisocial" +import { isHighQuality } from "../../utils/isHighQuality" +import { isAntisocial } from "../../utils/isAntisocial" const HARD_LIMIT = 100 @@ -19,7 +19,7 @@ export async function getVideos({ niceToHaveTags = [], sortBy = "date", ignoreVideoIds = [], - maxVideos = HARD_LIMIT, + maxNbMedias = HARD_LIMIT, neverThrow = false, renewCache = true, }: { @@ -42,7 +42,7 @@ export async function getVideos({ // eg. videos already watched, or disliked etc ignoreVideoIds?: string[] - maxVideos?: number + maxNbMedias?: number neverThrow?: boolean @@ -118,9 +118,9 @@ export async function getVideos({ ) // if we don't have enough videos - if (videosMatchingFilters.length < maxVideos) { + if (videosMatchingFilters.length < maxNbMedias) { // count how many we need - const nbMissingVideos = maxVideos - videosMatchingFilters.length + const nbMissingVideos = maxNbMedias - videosMatchingFilters.length // then we try to fill the gap with valid videos from other topics const videosToUseAsFiller = allPotentiallyValidVideos @@ -138,7 +138,7 @@ export async function getVideos({ const sanitizedVideos = videosMatchingFilters.filter(v => !isAntisocial(v)) // we enforce the max limit of HARD_LIMIT (eg. 100) - const limitedNumberOfVideos = sanitizedVideos.slice(0, Math.min(HARD_LIMIT, maxVideos)) + const limitedNumberOfVideos = sanitizedVideos.slice(0, Math.min(HARD_LIMIT, maxNbMedias)) // we ask Redis for the freshest stats const videosWithStats = await extendVideosWithStats(limitedNumberOfVideos) diff --git a/src/app/server/actions/ai-tube-hf/parseChannel.ts b/src/app/api/actions/ai-tube-hf/parseChannel.ts similarity index 97% rename from src/app/server/actions/ai-tube-hf/parseChannel.ts rename to src/app/api/actions/ai-tube-hf/parseChannel.ts index bbbf60498c1956fce3298ad40582ab23957ae3d2..655716c0d064a450bf3c0bba7e6982d1c1493972 100644 --- a/src/app/server/actions/ai-tube-hf/parseChannel.ts +++ b/src/app/api/actions/ai-tube-hf/parseChannel.ts @@ -1,7 +1,7 @@ "use server" import { Credentials, downloadFile, whoAmI } from "@/lib/huggingface/hub/src" -import { parseDatasetReadme } from "@/app/server/actions/utils/parseDatasetReadme" +import { parseDatasetReadme } from "@/app/api/utils/parseDatasetReadme" import { ChannelInfo, VideoGenerationModel, VideoOrientation } from "@/types/general" import { adminCredentials } from "../config" diff --git a/src/app/server/actions/ai-tube-hf/uploadVideoRequestToDataset.ts b/src/app/api/actions/ai-tube-hf/uploadVideoRequestToDataset.ts similarity index 94% rename from src/app/server/actions/ai-tube-hf/uploadVideoRequestToDataset.ts rename to src/app/api/actions/ai-tube-hf/uploadVideoRequestToDataset.ts index e7aca9d4209f2c497f1ed65180e15813d319d7ab..ab23bd0ef6d2f32b9416d6425c04766e5943b40e 100644 --- a/src/app/server/actions/ai-tube-hf/uploadVideoRequestToDataset.ts +++ b/src/app/api/actions/ai-tube-hf/uploadVideoRequestToDataset.ts @@ -4,8 +4,8 @@ import { Blob } from "buffer" import { Credentials, uploadFile, whoAmI } from "@/lib/huggingface/hub/src" import { ChannelInfo, VideoGenerationModel, MediaInfo, VideoOrientation, VideoRequest } from "@/types/general" -import { formatPromptFileName } from "../utils/formatPromptFileName" -import { computeOrientationProjectionWidthHeight } from "../utils/computeOrientationProjectionWidthHeight" +import { formatPromptFileName } from "../../utils/formatPromptFileName" +import { computeOrientationProjectionWidthHeight } from "../../utils/computeOrientationProjectionWidthHeight" /** * Save the video request to the user's own dataset diff --git a/src/app/server/actions/comments.ts b/src/app/api/actions/comments.ts similarity index 100% rename from src/app/server/actions/comments.ts rename to src/app/api/actions/comments.ts diff --git a/src/app/server/actions/config.ts b/src/app/api/actions/config.ts similarity index 100% rename from src/app/server/actions/config.ts rename to src/app/api/actions/config.ts diff --git a/src/app/server/actions/redis.ts b/src/app/api/actions/redis.ts similarity index 100% rename from src/app/server/actions/redis.ts rename to src/app/api/actions/redis.ts diff --git a/src/app/server/actions/stats.ts b/src/app/api/actions/stats.ts similarity index 100% rename from src/app/server/actions/stats.ts rename to src/app/api/actions/stats.ts diff --git a/src/app/server/actions/submitVideoRequest.ts b/src/app/api/actions/submitVideoRequest.ts similarity index 100% rename from src/app/server/actions/submitVideoRequest.ts rename to src/app/api/actions/submitVideoRequest.ts diff --git a/src/app/server/actions/users.ts b/src/app/api/actions/users.ts similarity index 100% rename from src/app/server/actions/users.ts rename to src/app/api/actions/users.ts diff --git a/src/app/api/generators/clap/addLatentScenesToClap.ts b/src/app/api/generators/clap/addLatentScenesToClap.ts new file mode 100644 index 0000000000000000000000000000000000000000..461106cb78d04bc034d267dcab9a5522b56f7735 --- /dev/null +++ b/src/app/api/generators/clap/addLatentScenesToClap.ts @@ -0,0 +1,105 @@ +"use server" + +import { newClap } from "@/lib/clap/newClap" +import { newSegment } from "@/lib/clap/newSegment" + +import { LatentScenes } from "./types" +import { serializeClap } from "@/lib/clap/serializeClap" +import { getEmptyClap } from "@/lib/clap/emptyClap" +import { ClapProject } from "@/lib/clap/types" + +let defaultSegmentDurationInMs = 2000 + +/** + * This generates a fully valid Clap blob (compressed archive) + * + * @param param0 + * @returns + */ +export async function addLatentScenesToClap({ + scenes = [], + clap, + debug = false +}: { + scenes?: LatentScenes + clap: ClapProject + debug?: boolean +}): Promise { + + if (!Array.isArray(scenes) || !scenes?.length) { + return clap + } + + let startTimeInMs = 0 + let endTimeInMs = defaultSegmentDurationInMs + + clap.segments.push(newSegment({ + track: 0, + startTimeInMs, + endTimeInMs, + category: "interface", + prompt: "", + label: "fish", + outputType: "interface", + })) + + for (const { characters, locations, actions } of scenes) { + + startTimeInMs = endTimeInMs + endTimeInMs = startTimeInMs + defaultSegmentDurationInMs + let track = 0 + + for (const character of characters) { + clap.segments.push(newSegment({ + track: track++, + startTimeInMs, + endTimeInMs, + category: "characters", + prompt: character, + label: character, + outputType: "text", + })) + } + + for (const location of locations) { + clap.segments.push(newSegment({ + track: track++, + startTimeInMs, + endTimeInMs, + category: "location", + prompt: location, + label: location, + outputType: "text", + })) + } + + for (const action of actions) { + clap.segments.push(newSegment({ + track: track++, + startTimeInMs, + endTimeInMs, + category: "action", + prompt: action, + label: action, + outputType: "text", + })) + } + + clap.segments.push(newSegment({ + track: track++, + startTimeInMs, + endTimeInMs, + category: "video", + prompt: "video", + label: "video", + outputType: "video", + })) + } + + if (debug) { + console.log("latentScenesToClap: unpacked Clap content = ", JSON.stringify(clap, null, 2)) + } + + + return clap +} \ No newline at end of file diff --git a/src/app/api/generators/clap/continueClap.ts b/src/app/api/generators/clap/continueClap.ts new file mode 100644 index 0000000000000000000000000000000000000000..8e2c8708bbc15651e2a646d4fce5f3e0529cd2cb --- /dev/null +++ b/src/app/api/generators/clap/continueClap.ts @@ -0,0 +1,66 @@ +"use server" + + +import { LatentScenes } from "./types" +import { addLatentScenesToClap } from "./addLatentScenesToClap" +import { getLatentScenes } from "./getLatentScenes" +import { serializeClap } from "@/lib/clap/serializeClap" +import { newClap } from "@/lib/clap/newClap" +import { getEmptyClap } from "@/lib/clap/emptyClap" +import { ClapProject } from "@/lib/clap/types" + +/** + * Imagine the continuity of a Clap file + * + * This serves multiple purpose, such as being able to create + * long stories in a more streamed way + * + * This should integrate multiple factors such as the event history, actions etc + * + * Be careful however as the context will grow at the same time as the story + * (it's the same issue as in the AI Comic Factory) + * so it may become harder and/or slower to perform the query + */ +export async function continueClap({ + clap, + mode = "replace", // "append" + debug = false +}: { + clap: ClapProject + + // whether to replace or append the content + // replacing is the most efficient way to do things (smaller files) + // so it is the default mode + mode: "replace" | "append" + + debug?: boolean +}): Promise { + + // TODO a prompt like "imagine the next steps from.." + const prompt = "" + + const scenes: LatentScenes = await getLatentScenes({ + prompt, + debug, + }) + + // by default we always replace the content, + // so we need to remove the previous one + if (mode !== "append") { + clap.scenes = [] + } + + clap = await addLatentScenesToClap({ + clap, + scenes, + debug, + }) + + // a Clap must always be transported as a zipped file + // technically, it could also be transported as text + // (and gzipped automatically between the HTTP server and browser) + // but I think it is better to keep the idea of a dedicated file format + const archive = await serializeClap(clap) + + return archive +} \ No newline at end of file diff --git a/src/app/api/generators/clap/generateClap.ts b/src/app/api/generators/clap/generateClap.ts new file mode 100644 index 0000000000000000000000000000000000000000..ac4faef7c552c1aa33dba8d7cd56716393c1d52d --- /dev/null +++ b/src/app/api/generators/clap/generateClap.ts @@ -0,0 +1,61 @@ +"use server" + +import { serializeClap } from "@/lib/clap/serializeClap" +import { newClap } from "@/lib/clap/newClap" +import { getEmptyClap } from "@/lib/clap/emptyClap" + +import { LatentScenes } from "./types" +import { addLatentScenesToClap } from "./addLatentScenesToClap" +import { getLatentScenes } from "./getLatentScenes" + +/** + * Generate a Clap file from scratch using a prompt + */ +export async function generateClap({ + prompt = "", + debug = false +}: { + prompt?: string + debug?: boolean +} = { + prompt: "", + debug: false, +}): Promise { + + const empty = await getEmptyClap() + + if (!prompt?.length) { + return empty + } + + let clap = newClap({ + meta: { + title: "Latent content", // TODO " + description: "", + licence: "non commercial", + orientation: "landscape", + width: 1024, + height: 576, + defaultVideoModel: "SDXL", + extraPositivePrompt: [], + screenplay: "", + streamType: "interactive" + } + }) + + const scenes: LatentScenes = await getLatentScenes({ + prompt, + debug, + }) + + + clap = await addLatentScenesToClap({ + clap, + scenes, + debug, + }) + + const archive = await serializeClap(clap) + + return archive +} \ No newline at end of file diff --git a/src/app/api/generators/clap/getLatentScenes.ts b/src/app/api/generators/clap/getLatentScenes.ts new file mode 100644 index 0000000000000000000000000000000000000000..3726f964684eaa063e7818953432789ec16e89ae --- /dev/null +++ b/src/app/api/generators/clap/getLatentScenes.ts @@ -0,0 +1,58 @@ +"use server" + +import YAML from "yaml" + +import { predict as predictWithHuggingFace } from "@/app/api/providers/huggingface/predictWithHuggingFace" +import { predict as predictWithOpenAI } from "@/app/api/providers/openai/predictWithOpenAI" + +import { LatentScenes } from "./types" +import { getSystemPrompt } from "./getSystemPrompt" +import { unknownObjectToLatentScenes } from "./unknownObjectToLatentScenes" +import { parseRawStringToYAML } from "../../utils/parseRawStringToYAML" + +export async function getLatentScenes({ + prompt = "", + debug = false +}: { + prompt?: string + debug?: boolean +} = {}): Promise { + + // abort early + if (!prompt) { + return [] + } + + const systemPrompt = getSystemPrompt() + + const userPrompt = `generate a short story about: ${prompt}` + + let scenes: LatentScenes = [] + try { + // we use Hugging Face for now, as our users might try funny things, + // which could get us banned from OpenAI + let rawString = await predictWithHuggingFace({ + systemPrompt, + userPrompt, + nbMaxNewTokens: 1200, + prefix: "", + }) + + if (debug) { + console.log("getLatentScenes: rawString = " + rawString) + } + + const maybeLatentScenes = parseRawStringToYAML(rawString, []) + + scenes = unknownObjectToLatentScenes(maybeLatentScenes) + + if (debug) { + console.log(`getLatentScenes: scenes = ` + JSON.stringify(scenes, null, 2)) + } + } catch (err) { + scenes = [] + console.error(`getLatentScenes failed (${err})`) + } + + return scenes +} \ No newline at end of file diff --git a/src/app/api/generators/clap/getSystemPrompt.ts b/src/app/api/generators/clap/getSystemPrompt.ts new file mode 100644 index 0000000000000000000000000000000000000000..49439e4117afa20701976c13f35d5ad9e605226b --- /dev/null +++ b/src/app/api/generators/clap/getSystemPrompt.ts @@ -0,0 +1,53 @@ + +export const getSystemPrompt = () => { + return `# Context +You are a backend engine able to generate interactive projects in YAML. + +# Schema + +You will be given instructions to describe a story, and you need to return a YAML describing each scene as "character", "location", and "action". + +Here is a description of the schema in TypeScript for convenience (but you need to always reply using YAML): + +For the writing style of the location, please try to use the Stable Diffusion convention for prompts. + +\`\`\`typescript +{ + characters: string[] // list of characters visible in the scene + location: string + actions: string[] +}[] +\`\`\` + +# Samples + +Here are some basic sample outputs. In reality, you should create longer stories. +For brevity the location is very short in the example, but in reality you should write stable diffusion prompts descriptions. + +## a short story about a frog turning into a princess, she becomes happy but there is a cliffhanger at the end of the episode + +\`\`\`yaml +- characters: ["Fiona the Frog"] + location: A misty frog pond, mysterious, beautiful. + actions: "Fiona the Frog lived alone, spending her days hopping and swimming around the edges of Misty Pond." +- characters: ["Fiona the Frog", "Ella the Elderly Witch"] + location: Pond, sunny, riverbank, herbs, morning light, beautiful. + actions: "One sunny morning, Fiona encountered Ella the Elderly Witch who was gathering herbs by the pond." +- characters: ["Fiona the Frog", "Ella the Elderly Witch"] + location: Pond in the background, sunny, morning light, beautiful, bokeh + actions: "Ella, feeling pity for the lonely frog, decided to cast a magical spell. She whispered enchanted words and sprinkled Fiona with sparkling dust." +- characters: ["Fiona the Frog"] + location: Glowing circle of magic, emitting light, on the grass, at night + actions: "Suddenly Fiona is feeling a whirl of sensations and her form starts changing under the glistening moonlight." +- characters: ["Princess Fiona"] + location: Royal palace gardens, beautiful, french garden, medieval, in the morning. + actions: "As the magic settled, Fiona found herself transformed into a human princess, standing in the lush gardens of a grand palace." +- characters: ["Princess Fiona", "Prince Henry"] + location: Royal palace, in the court, medieval, during the day. + actions: "Prince Henry is charming Princess Fiona, he wonders where she is coming from." +- characters: ["Princess Fiona", "Prince Henry"] + location: Inside the royal palace, large medieval ball room, during a banquet. + actions: "Princess Fiona kisses the Prince, they are finally happy." +\`\`\` +` +} \ No newline at end of file diff --git a/src/app/api/generators/clap/types.ts b/src/app/api/generators/clap/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..d5ad7af6a74d9fbbce953348d25a1e74040766ae --- /dev/null +++ b/src/app/api/generators/clap/types.ts @@ -0,0 +1,7 @@ +export type LatentScene = { + characters: string[] + locations: string[] + actions: string[] +} + +export type LatentScenes = LatentScene[] \ No newline at end of file diff --git a/src/app/api/generators/clap/unknownObjectToLatentScenes.ts b/src/app/api/generators/clap/unknownObjectToLatentScenes.ts new file mode 100644 index 0000000000000000000000000000000000000000..d643e81577b5b0fa428fc4a5314ccc758b30a61e --- /dev/null +++ b/src/app/api/generators/clap/unknownObjectToLatentScenes.ts @@ -0,0 +1,20 @@ +import { parseStringArray } from "../../utils/parseStringArray" +import { LatentScene, LatentScenes } from "./types" + +/** + * Process a YAML result from the LLM to make sure it is a LatentScenes + * + * @param something + * @returns + */ +export function unknownObjectToLatentScenes(something: any): LatentScenes { + let scenes: LatentScenes = [] + if (Array.isArray(something)) { + scenes = something.map(thing => ({ + characters: parseStringArray(thing && (thing?.characters || thing?.character)), + locations: parseStringArray(thing && (thing?.locations || thing?.location)), + actions: parseStringArray(thing && (thing?.actions || thing?.action)), + } as LatentScene)) + } + return scenes +} \ No newline at end of file diff --git a/src/app/api/generators/image/generateImageWithVideochain.ts b/src/app/api/generators/image/generateImageWithVideochain.ts new file mode 100644 index 0000000000000000000000000000000000000000..56e5db78857998f8c8fe51b521ef1ec8295eb434 --- /dev/null +++ b/src/app/api/generators/image/generateImageWithVideochain.ts @@ -0,0 +1,151 @@ +"use server" + +import { RenderRequest, RenderedScene } from "@/types/general" + +// note: there is no / at the end in the variable +// so we have to add it ourselves if needed +const apiUrl = `${process.env.VIDEOCHAIN_API_URL || ""}` +const apiKey = `${process.env.VIDEOCHAIN_API_KEY || ""}` + +export async function newRender({ + prompt, + negativePrompt, + nbFrames, + nbSteps, + width, + height, + turbo, + shouldRenewCache, + seed, +}: { + prompt: string + negativePrompt: string + nbFrames: number + nbSteps: number + width: number + height: number + turbo: boolean + shouldRenewCache: boolean + seed?: number +}) { + if (!prompt) { + console.error(`cannot call the rendering API without a prompt, aborting..`) + throw new Error(`cannot call the rendering API without a prompt, aborting..`) + } + + const cacheKey = `render/${JSON.stringify({ prompt })}` + + // return await Gorgon.get(cacheKey, async () => { + + let defaulResult: RenderedScene = { + renderId: "", + status: "error", + assetUrl: "", + durationInMs: 0, + maskUrl: "", + error: "failed to fetch the data", + alt: "", + segments: [] + } + + try { + // console.log(`calling POST ${apiUrl}/render with seed ${seed} and prompt: ${prompt}`) + + const res = await fetch(`${apiUrl}/render`, { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + prompt, + negativePrompt, + // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation + nbFrames, // when nbFrames is 1, we will only generate static images + nbSteps, // 20 = fast, 30 = better, 50 = best + width, + height, + seed, + actionnables: [], + segmentation: "disabled", // one day we will remove this param, to make it automatic + upscalingFactor: 1, // let's disable upscaling right now + turbo, // always use turbo mode (it's for images only anyway) + // also what could be done iw that we could use the width and height to control this + cache: shouldRenewCache ? "renew" : "use" + } as Partial), + cache: 'no-store', + // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) + // next: { revalidate: 1 } + }) + + // console.log("res:", res) + // The return value is *not* serialized + // You can return Date, Map, Set, etc. + + // Recommendation: handle errors + if (res.status !== 200) { + // This will activate the closest `error.js` Error Boundary + throw new Error('Failed to fetch data') + } + + const response = (await res.json()) as RenderedScene + // console.log("response:", response) + return response + } catch (err) { + // console.error(err) + // Gorgon.clear(cacheKey) + return defaulResult + } +} + +export async function getRender(renderId: string) { + if (!renderId) { + console.error(`cannot call the rendering API without a renderId, aborting..`) + throw new Error(`cannot call the rendering API without a renderId, aborting..`) + } + + let defaulResult: RenderedScene = { + renderId: "", + status: "error", + assetUrl: "", + durationInMs: 0, + maskUrl: "", + error: "failed to fetch the data", + alt: "", + segments: [] + } + + try { + // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`) + const res = await fetch(`${apiUrl}/render/${renderId}`, { + method: "GET", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + cache: 'no-store', + // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) + // next: { revalidate: 1 } + }) + + // console.log("res:", res) + // The return value is *not* serialized + // You can return Date, Map, Set, etc. + + // Recommendation: handle errors + if (res.status !== 200) { + // This will activate the closest `error.js` Error Boundary + throw new Error('Failed to fetch data') + } + + const response = (await res.json()) as RenderedScene + // console.log("response:", response) + return response + } catch (err) { + console.error(err) + // Gorgon.clear(cacheKey) + return defaulResult + } +} \ No newline at end of file diff --git a/src/app/api/generators/search/getLatentSearchResults.ts b/src/app/api/generators/search/getLatentSearchResults.ts new file mode 100644 index 0000000000000000000000000000000000000000..e7235086df74b63d00f98dcff884bfca60106265 --- /dev/null +++ b/src/app/api/generators/search/getLatentSearchResults.ts @@ -0,0 +1,59 @@ +"use server" + +import YAML from "yaml" + +import { predict as predictWithHuggingFace } from "@/app/api/providers/huggingface/predictWithHuggingFace" +import { predict as predictWithOpenAI } from "@/app/api/providers/openai/predictWithOpenAI" +import { LatentSearchResults } from "./types" +import { getSystemPrompt } from "./getSystemPrompt" +import { parseRawStringToYAML } from "../../utils/parseRawStringToYAML" +import { unknownObjectToLatentSearchResults } from "./unknownObjectToLatentSearchResults" + +export async function getLatentSearchResults({ + prompt = "", + debug = false +}: { + prompt?: string + debug?: boolean +} = {}): Promise { + + // abort early + if (!prompt) { + return [] + } + + const systemPrompt = getSystemPrompt() + + const nbSearchResults = 8 + + const userPrompt = `${nbSearchResults} search results for "${prompt}"` + + let results: LatentSearchResults = [] + try { + // we use Hugging Face for now, as our users might try funny things, + // which could get us banned from OpenAI + let rawString = await predictWithHuggingFace({ + systemPrompt, + userPrompt, + nbMaxNewTokens: 1200, + prefix: "", + }) + + if (debug) { + console.log("getLatentSearchResults: rawString = " + rawString) + } + + const maybeLatentSearchResults = parseRawStringToYAML(rawString, []) + + results = unknownObjectToLatentSearchResults(maybeLatentSearchResults) + + if (debug) { + console.log(`getLatentSearchResults: scenes = ` + JSON.stringify(results, null, 2)) + } + } catch (err) { + results = [] + console.error(`getLatentSearchResults failed (${err})`) + } + + return results +} \ No newline at end of file diff --git a/src/app/api/generators/search/getSystemPrompt.ts b/src/app/api/generators/search/getSystemPrompt.ts new file mode 100644 index 0000000000000000000000000000000000000000..74a8b87817e66ee8e9160aacb7aeba99c06d8f2c --- /dev/null +++ b/src/app/api/generators/search/getSystemPrompt.ts @@ -0,0 +1,43 @@ + +export const getSystemPrompt = () => { + return `# Context +You are a backend engine of a video sharing platform called AiTube, able to generate search results in YAML. +You should generate realistic results, similar to real video platforms and social media. + +# Schema + +You will be given instructions to describe a search query, and you need to return a YAML describing each search result as "title", "thumbnail", and "tags". + +Here is a description of the schema in TypeScript for convenience (but you need to always reply using YAML): + +\`\`\`typescript +{ + label: string // title of the video + summary: string // summary of the video + thumbnail: string // a stable diffusion or dall-e prompt, to describe the video thumbnail + tags: string[] // a list of tags +}[] +\`\`\` + +# Samples + +Here are some basic sample outputs + +## 3 search results for "tiktok recipes" + +\`\`\`yaml +- label: I'm Testing Viral TikTok Recipes So You Don't Have To + summary: Video from an influencer, reviewing weird recipes that are becoming viral in TikTok. The video has a funny tone. + thumbnail: young woman, an influencer opening the mouth, very surprised, eating weird pink spaghetti, portrait, dramatic pose, high quality + tags: ["cooking", "review"] +- label: I went on a TikTok Food Hack Marathon And It Made Me 🤢 + summary: Funny video about an influencer reviewing viral TikTok recipes, but becomes hillarously sick as they are very bad. As an influencer video, it is made to maximize engagement and thus it exagerates everything. + thumbnail: an influencer being sick, nauseous, pixelated food plate, spectacular, grandiose + tags: ["food"] +- label: I've Tried 10 TikTok Food Recipes 🌮 and This Was Surprising + summary: Video about an influencer who tried 10 recipes from TikTok, which turned out to be complete disaster, but in an hillarous way. The video is made to maximalize the dramatic effect and views. + thumbnail: an influencer shrugging, very expressive, mouth open, over a plate of weird hotdogs, dramatic pose + tags: ["food", "cooking", "review"] +\`\`\` +` +} \ No newline at end of file diff --git a/src/app/api/generators/search/searchResultToMediaInfo.ts b/src/app/api/generators/search/searchResultToMediaInfo.ts new file mode 100644 index 0000000000000000000000000000000000000000..8a4eddfed9a252646046b11b009e91bad1cafd76 --- /dev/null +++ b/src/app/api/generators/search/searchResultToMediaInfo.ts @@ -0,0 +1,231 @@ +import { v4 as uuidv4 } from "uuid" + +import { + ChannelInfo, + MediaInfo, + VideoStatus, + VideoGenerationModel, + MediaProjection, + VideoOrientation +} from "@/types/general" + +import { LatentSearchResult, LatentSearchResults } from "./types" +import { newRender } from "../../providers/videochain/renderWithVideoChain" + +const channel: ChannelInfo = { + /** + * We actually use the dataset ID for the channel ID. + * + */ + id: "d25efcc1-3cc2-4b41-9f41-e3a93300ae5f", + + /** + * The name used in the URL for the channel + * + * eg: my-time-travel-journeys + */ + slug: "latent", + + /** + * username id of the Hugging Face dataset + * + * ex: f9a38286ec3436a45edd2cca + */ + // DISABLED FOR NOW + // datasetUserId: string + + /** + * username slug of the Hugging Face dataset + * + * eg: jbilcke-hf + */ + datasetUser: "", + + /** + * dataset slug of the Hugging Face dataset + * + * eg: ai-tube-my-time-travel-journeys + */ + datasetName: "", + + label: "Latent", + + description: "Latent", + + thumbnail: "", + + model: "SDXL", + + lora: "", + + style: "", + + voice: "", + + music: "", + + /** + * The system prompt + */ + prompt: "", + + likes: 0, + + tags: [], + + updatedAt: new Date().toISOString(), + + /** + * Default video orientation + */ + orientation: "landscape" +} + +export async function searchResultToMediaInfo(searchResult: LatentSearchResult): Promise { + + const renderResult = await newRender({ + prompt: searchResult.thumbnail, + negativePrompt: "", + nbFrames: 1, + nbSteps: 4, + width: 1024, + height: 576, + turbo: true, + shouldRenewCache: false, + seed: searchResult.seed, + }) + + const thumbnailUrl: string = renderResult.assetUrl || "" + + const mediaInfo: MediaInfo = { + /** + * UUID (v4) + */ + id: uuidv4(), + + /** + * Status of the media + */ + status: "published", + + /** + * Human readable title for the media + */ + label: searchResult.label, + + /** + * Human readable description for the media + */ + description: searchResult.summary, + + /** + * Content prompt + */ + prompt: searchResult.summary, + + /** + * URL to the media thumbnail + */ + thumbnailUrl, + + /** + * URL to a clap file + */ + clapUrl: "", + + assetUrl: "", + + /** + * This is contain the storage URL of the higher-resolution content + */ + assetUrlHd: "", + + /** + * Counter for the number of views + * + * Note: should be managed by the index to prevent cheating + */ + numberOfViews: 0, + + /** + * Counter for the number of likes + * + * Note: should be managed by the index to prevent cheating + */ + numberOfLikes: 0, + + /** + * Counter for the number of dislikes + * + * Note: should be managed by the index to prevent cheating + */ + numberOfDislikes: 0, + + /** + * When was the media updated + */ + updatedAt: new Date().toISOString(), + + /** + * Arbotrary string tags to label the content + */ + tags: searchResult.tags, + + /** + * Model name + */ + model: "SDXL", + + /** + * LoRA name + */ + lora: "", + + /** + * style name + */ + style: "", + + /** + * Music prompt + */ + music: "", + + /** + * Voice prompt + */ + voice: "", + + /** + * The channel + */ + channel, + + /** + * Media duration (in seconds) + */ + duration: 60, + + /** + * Media width (eg. 1024) + */ + width: 1024, + + /** + * Media height (eg. 576) + */ + height: 576, + + /** + * General media aspect ratio + */ + orientation: "landscape", + + /** + * Media projection (cartesian by default) + */ + projection: "latent" + } + + return mediaInfo +} \ No newline at end of file diff --git a/src/app/api/generators/search/types.ts b/src/app/api/generators/search/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..80c33f6e6c84102a679c1e1c7ab1540341c053a1 --- /dev/null +++ b/src/app/api/generators/search/types.ts @@ -0,0 +1,9 @@ +export type LatentSearchResult = { + label: string + summary: string + thumbnail: string + tags: string[] + seed: number // static seed is necessary to ensure result consistency for the thumbnail +} + +export type LatentSearchResults = LatentSearchResult[] \ No newline at end of file diff --git a/src/app/api/generators/search/unknownObjectToLatentSearchResults.ts b/src/app/api/generators/search/unknownObjectToLatentSearchResults.ts new file mode 100644 index 0000000000000000000000000000000000000000..3db87ac5fa4d15b4b4159b2101f81096cdc9bd9e --- /dev/null +++ b/src/app/api/generators/search/unknownObjectToLatentSearchResults.ts @@ -0,0 +1,20 @@ +import { generateSeed } from "@/lib/utils/generateSeed" +import { parseString } from "../../utils/parseString" +import { parseStringArray } from "../../utils/parseStringArray" +import { LatentSearchResult, LatentSearchResults } from "./types" + +export function unknownObjectToLatentSearchResults(something: any): LatentSearchResults { + let results: LatentSearchResults = [] + + if (Array.isArray(something)) { + results = something.map(thing => ({ + label: parseString(thing && (thing?.label || thing?.title)), + summary: parseString(thing && (thing?.summary || thing?.description || thing?.synopsis)), + thumbnail: parseString(thing && (thing?.thumbnail)), + tags: parseStringArray(thing && (thing?.tag)), + seed: generateSeed(), // a seed is necessary for consistency between search results and viewer + } as LatentSearchResult)) + } + + return results +} \ No newline at end of file diff --git a/src/app/api/media/[mediaId]/route.ts b/src/app/api/media/[mediaId]/route.ts index 46a7b54728ad8f482d42b04945d1b01d13c1e6c3..a855731a98a2c437600b3f5da6bca36bd7c87edd 100644 --- a/src/app/api/media/[mediaId]/route.ts +++ b/src/app/api/media/[mediaId]/route.ts @@ -1,6 +1,6 @@ import { NextResponse, NextRequest } from "next/server" -import { getVideo } from "@/app/server/actions/ai-tube-hf/getVideo" +import { getVideo } from "@/app/api/actions/ai-tube-hf/getVideo" import { parseMediaProjectionType } from "@/lib/utils/parseMediaProjectionType"; export async function GET(req: NextRequest) { @@ -18,7 +18,7 @@ export async function GET(req: NextRequest) { ${media.label} - AiTube - + diff --git a/src/app/api/providers/anthropic/predictWithAnthropic.txt b/src/app/api/providers/anthropic/predictWithAnthropic.txt new file mode 100644 index 0000000000000000000000000000000000000000..653d5ce5d04382d3bc9e8c8ee92cf591af5c0696 --- /dev/null +++ b/src/app/api/providers/anthropic/predictWithAnthropic.txt @@ -0,0 +1,47 @@ +"use server" + +import { LLMPredictionFunctionParams } from '@/types'; +import Anthropic from '@anthropic-ai/sdk'; +import { MessageParam } from '@anthropic-ai/sdk/resources'; + +export async function predict({ + systemPrompt, + userPrompt, + nbMaxNewTokens, + llmVendorConfig +}: LLMPredictionFunctionParams): Promise { + const anthropicApiKey = `${ + llmVendorConfig.apiKey || + process.env.AUTH_ANTHROPIC_API_KEY || + "" + }` + const anthropicApiModel = `${ + llmVendorConfig.modelId || + process.env.LLM_ANTHROPIC_API_MODEL || + "claude-3-opus-20240229" + }` + + const anthropic = new Anthropic({ + apiKey: anthropicApiKey, + }) + + const messages: MessageParam[] = [ + { role: "user", content: userPrompt }, + ] + + try { + const res = await anthropic.messages.create({ + messages: messages, + // stream: false, + system: systemPrompt, + model: anthropicApiModel, + // temperature: 0.8, + max_tokens: nbMaxNewTokens, + }) + + return res.content[0]?.text || "" + } catch (err) { + console.error(`error during generation: ${err}`) + return "" + } +} \ No newline at end of file diff --git a/src/app/api/providers/groq/predictWithGroq.txt b/src/app/api/providers/groq/predictWithGroq.txt new file mode 100644 index 0000000000000000000000000000000000000000..60225319633f031a576a9983a501d45d8c60ad7e --- /dev/null +++ b/src/app/api/providers/groq/predictWithGroq.txt @@ -0,0 +1,46 @@ +"use server" + +import { LLMPredictionFunctionParams } from "@/types" +import Groq from "groq-sdk" + +export async function predict({ + systemPrompt, + userPrompt, + nbMaxNewTokens, + llmVendorConfig +}: LLMPredictionFunctionParams): Promise { + const groqApiKey = `${ + llmVendorConfig.apiKey || + process.env.AUTH_GROQ_API_KEY || + "" + }` + const groqApiModel = `${ + llmVendorConfig.modelId || + process.env.LLM_GROQ_API_MODEL || + "mixtral-8x7b-32768" + }` + + const groq = new Groq({ + apiKey: groqApiKey, + }) + + const messages: Groq.Chat.Completions.CompletionCreateParams.Message[] = [ + { role: "system", content: systemPrompt }, + { role: "user", content: userPrompt }, + ] + + try { + const res = await groq.chat.completions.create({ + messages: messages, + model: groqApiModel, + stream: false, + temperature: 0.5, + max_tokens: nbMaxNewTokens, + }) + + return res.choices[0].message.content || "" + } catch (err) { + console.error(`error during generation: ${err}`) + return "" + } +} \ No newline at end of file diff --git a/src/app/api/providers/huggingface/predictWithHuggingFace.ts b/src/app/api/providers/huggingface/predictWithHuggingFace.ts new file mode 100644 index 0000000000000000000000000000000000000000..59f9795f9f2db0b93e3cb05329dbaae4f9b97da8 --- /dev/null +++ b/src/app/api/providers/huggingface/predictWithHuggingFace.ts @@ -0,0 +1,87 @@ + +import { HfInference } from "@huggingface/inference" + +import { createZephyrPrompt } from "@/lib/prompts/createZephyrPrompt" + +import { LLMPredictionFunctionParams } from "../types" + +export async function predict({ + systemPrompt, + userPrompt, + nbMaxNewTokens, + prefix, +}: LLMPredictionFunctionParams): Promise { + + const hf = new HfInference(process.env.ADMIN_HUGGING_FACE_API_TOKEN) + + let instructions = "" + try { + for await (const output of hf.textGenerationStream({ + // model: "mistralai/Mixtral-8x7B-v0.1", + model: "mistralai/Mixtral-8x7B-Instruct-v0.1", + inputs: createZephyrPrompt([ + { role: "system", content: systemPrompt }, + { role: "user", content: userPrompt } + ]) + '\n' + prefix, + + parameters: { + do_sample: true, + max_new_tokens: nbMaxNewTokens, + return_full_text: false, + } + })) { + instructions += output.token.text + // process.stdout.write(output.token.text) + if ( + instructions.includes("") || + instructions.includes("") || + instructions.includes("/s>") || + instructions.includes("[INST]") || + instructions.includes("[/INST]") || + instructions.includes("") || + instructions.includes("<>") || + instructions.includes("") || + instructions.includes("<>") || + instructions.includes("<|user|>") || + instructions.includes("<|end|>") || + instructions.includes("<|system|>") || + instructions.includes("<|assistant|>") + ) { + break + } + } + } catch (err) { + // console.error(`error during generation: ${err}`) + + // a common issue with Llama-2 might be that the model receives too many requests + if (`${err}` === "Error: Model is overloaded") { + instructions = `` + } + } + + // need to do some cleanup of the garbage the LLM might have gave us + let result = + instructions + .replaceAll("<|end|>", "") + .replaceAll("", "") + .replaceAll("", "") + .replaceAll("/s>", "") + .replaceAll("[INST]", "") + .replaceAll("[/INST]", "") + .replaceAll("", "") + .replaceAll("<>", "") + .replaceAll("", "") + .replaceAll("<>", "") + .replaceAll("<|system|>", "") + .replaceAll("<|user|>", "") + .replaceAll("<|all|>", "") + .replaceAll("<|assistant|>", "") + .replaceAll('""', '"') + .trim() + + if (prefix && !result.startsWith(prefix)) { + result = prefix + result + } + + return result +} diff --git a/src/app/api/providers/openai/predictWithOpenAI.ts b/src/app/api/providers/openai/predictWithOpenAI.ts new file mode 100644 index 0000000000000000000000000000000000000000..7ab926c5afbd4cc12a4c3ed38e853a117fc9ca49 --- /dev/null +++ b/src/app/api/providers/openai/predictWithOpenAI.ts @@ -0,0 +1,44 @@ +"use server" + +import { OpenAI } from "openai" +import { ChatCompletionMessageParam } from "openai/resources" + +import { LLMPredictionFunctionParams } from "../types" + +export async function predict({ + systemPrompt, + userPrompt, + nbMaxNewTokens, +}: LLMPredictionFunctionParams): Promise { + const openaiApiKey = `${process.env.AUTH_OPENAI_API_KEY || ""}` + const openaiApiModel = "gpt-4-turbo" + const openaiApiBaseUrl = "https://api.openai.com/v1" + if (!openaiApiKey) { throw new Error(`missing OpenAI API key`) } + + const openai = new OpenAI({ + apiKey: openaiApiKey, + baseURL: openaiApiBaseUrl, + }) + + const messages: ChatCompletionMessageParam[] = [ + { role: "system", content: systemPrompt }, + { role: "user", content: userPrompt }, + ] + + try { + const res = await openai.chat.completions.create({ + messages: messages, + stream: false, + model: openaiApiModel, + temperature: 0.8, + max_tokens: nbMaxNewTokens, + + // TODO: use the nbPanels to define a max token limit + }) + + return res.choices[0].message.content || "" + } catch (err) { + console.error(`error during generation: ${err}`) + return "" + } +} \ No newline at end of file diff --git a/src/app/api/providers/types.ts b/src/app/api/providers/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..df2cc659f939619f923f901d3b02449641b42e0e --- /dev/null +++ b/src/app/api/providers/types.ts @@ -0,0 +1,21 @@ + +// LLMEngine = the actual engine to use (eg. hugging face) +export type LLMVendor = + | "HUGGINGFACE" + | "OPENAI" + | "GROQ" + | "ANTHROPIC" + +export type LLMVendorConfig = { + vendor: LLMVendor + apiKey: string + modelId: string +} + +export type LLMPredictionFunctionParams = { + systemPrompt: string + userPrompt: string + nbMaxNewTokens: number + prefix?: string + // llmVendorConfig: LLMVendorConfig +} diff --git a/src/app/api/providers/videochain/renderWithVideoChain.ts b/src/app/api/providers/videochain/renderWithVideoChain.ts new file mode 100644 index 0000000000000000000000000000000000000000..26534679507f8b8a48bfdd1ef72fd38b9b8864ed --- /dev/null +++ b/src/app/api/providers/videochain/renderWithVideoChain.ts @@ -0,0 +1,153 @@ +"use server" + +import { RenderRequest, RenderedScene } from "@/types/general" + +// note: there is no / at the end in the variable +// so we have to add it ourselves if needed +const apiUrl = `${process.env.VIDEOCHAIN_API_URL || ""}` +const apiKey = `${process.env.VIDEOCHAIN_API_KEY || ""}` + +export async function newRender({ + prompt, + negativePrompt, + nbFrames, + nbSteps, + width, + height, + turbo, + shouldRenewCache, + seed, +}: { + prompt: string + negativePrompt: string + nbFrames: number + nbSteps: number + width: number + height: number + turbo: boolean + shouldRenewCache: boolean + seed?: number +}) { + if (!prompt) { + console.error(`cannot call the rendering API without a prompt, aborting..`) + throw new Error(`cannot call the rendering API without a prompt, aborting..`) + } + + const cacheKey = `render/${JSON.stringify({ prompt })}` + + // return await Gorgon.get(cacheKey, async () => { + + let defaulResult: RenderedScene = { + renderId: "", + status: "error", + assetUrl: "", + durationInMs: 0, + maskUrl: "", + error: "failed to fetch the data", + alt: "", + segments: [] + } + + // console.log("fetch api:", `${apiUrl}/render`) + try { + // console.log(`calling POST ${apiUrl}/render with seed ${seed} and prompt: ${prompt}`) + + const res = await fetch(`${apiUrl}/render`, { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + prompt, + negativePrompt, + // nbFrames: 8 and nbSteps: 15 --> ~10 sec generation + nbFrames, // when nbFrames is 1, we will only generate static images + nbSteps, // 20 = fast, 30 = better, 50 = best + width, + height, + seed, + actionnables: [], + segmentation: "disabled", // one day we will remove this param, to make it automatic + upscalingFactor: 1, // let's disable upscaling right now + turbo, // always use turbo mode (it's for images only anyway) + // also what could be done iw that we could use the width and height to control this + cache: "ignore", // shouldRenewCache ? "renew" : "use", + wait: true, + } as Partial), + cache: 'no-store', + // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) + // next: { revalidate: 1 } + }) + + // console.log("res:", res) + // The return value is *not* serialized + // You can return Date, Map, Set, etc. + + // Recommendation: handle errors + if (res.status !== 200) { + // This will activate the closest `error.js` Error Boundary + throw new Error('Failed to fetch data') + } + + const response = (await res.json()) as RenderedScene + // console.log("response:", response) + return response + } catch (err) { + // console.error(err) + // Gorgon.clear(cacheKey) + return defaulResult + } +} + +export async function getRender(renderId: string) { + if (!renderId) { + console.error(`cannot call the rendering API without a renderId, aborting..`) + throw new Error(`cannot call the rendering API without a renderId, aborting..`) + } + + let defaulResult: RenderedScene = { + renderId: "", + status: "error", + assetUrl: "", + durationInMs: 0, + maskUrl: "", + error: "failed to fetch the data", + alt: "", + segments: [] + } + + try { + // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`) + const res = await fetch(`${apiUrl}/render/${renderId}`, { + method: "GET", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + cache: 'no-store', + // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) + // next: { revalidate: 1 } + }) + + // console.log("res:", res) + // The return value is *not* serialized + // You can return Date, Map, Set, etc. + + // Recommendation: handle errors + if (res.status !== 200) { + // This will activate the closest `error.js` Error Boundary + throw new Error('Failed to fetch data') + } + + const response = (await res.json()) as RenderedScene + // console.log("response:", response) + return response + } catch (err) { + console.error(err) + // Gorgon.clear(cacheKey) + return defaulResult + } +} \ No newline at end of file diff --git a/src/app/api/resolvers/clap/route.ts b/src/app/api/resolvers/clap/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..f518adbfadb1548c4f080fb0a1aaddcad3353c1f --- /dev/null +++ b/src/app/api/resolvers/clap/route.ts @@ -0,0 +1,24 @@ +import { NextResponse, NextRequest } from "next/server" +import queryString from "query-string" +import { generateClap } from "../../generators/clap/generateClap" + +export async function GET(req: NextRequest) { + +const qs = queryString.parseUrl(req.url || "") +const query = (qs || {}).query + +let prompt = "" + try { + prompt = decodeURIComponent(query?.p?.toString() || "").trim() + } catch (err) {} + if (!prompt) { + return NextResponse.json({ error: 'no prompt provided' }, { status: 400 }); + } + + const blob = await generateClap({ prompt }) + + return new NextResponse(blob, { + status: 200, + headers: new Headers({ "content-type": "application/x-gzip" }), + }) +} diff --git a/src/app/api/resolvers/image/route.ts b/src/app/api/resolvers/image/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..a5d8923c919e44eb99125cf331cd12d100e7ea9f --- /dev/null +++ b/src/app/api/resolvers/image/route.ts @@ -0,0 +1,65 @@ +import { NextResponse, NextRequest } from "next/server" +import queryString from "query-string" + +import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain" +import { generateSeed } from "@/lib/utils/generateSeed" +import { sleep } from "@/lib/utils/sleep" +import { getContentType } from "@/lib/data/getContentType" + +export async function GET(req: NextRequest) { + +const qs = queryString.parseUrl(req.url || "") +const query = (qs || {}).query + +let prompt = "" + try { + prompt = decodeURIComponent(query?.p?.toString() || "").trim() + } catch (err) {} + if (!prompt) { + return NextResponse.json({ error: 'no prompt provided' }, { status: 400 }); + } + + // console.log("calling await newRender") + + let render = await newRender({ + prompt, + negativePrompt: "blurry, cropped, bad quality", + nbFrames: 1, + nbSteps: 8, + width: 1024, + height: 576, + turbo: true, + shouldRenewCache: true, + seed: generateSeed() + }) + + let attempts = 10 + + while (attempts-- > 0) { + if (render.status === "completed") { + return NextResponse.json(render, { + status: 200, + statusText: "OK", + }) + + } + + if (render.status === "error") { + return NextResponse.json(render, { + status: 200, + statusText: "OK", + }) + } + + await sleep(1000) // minimum wait time + + console.log("asking getRender") + render = await getRender(render.renderId) + } + + return NextResponse.json({ + "error": "failed to call VideoChain (timeout expired)" + }, { + status: 500, + }) +} diff --git a/src/app/api/resolvers/interface/route.ts b/src/app/api/resolvers/interface/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..aa907540481ccfc9e066439a14d4350abaa3c925 --- /dev/null +++ b/src/app/api/resolvers/interface/route.ts @@ -0,0 +1,36 @@ +import { NextResponse, NextRequest } from "next/server" +import queryString from "query-string" +import { predict } from "../../providers/huggingface/predictWithHuggingFace" +import { systemPrompt } from "./systemPrompt" + +export async function GET(req: NextRequest) { + +const qs = queryString.parseUrl(req.url || "") +const query = (qs || {}).query + +let prompt = "" + try { + prompt = decodeURIComponent(query?.p?.toString() || "").trim() + } catch (err) {} + if (!prompt) { + return NextResponse.json({ error: 'no prompt provided' }, { status: 400 }); + } + + const userPrompt = `HTML snippet to generate: ${prompt}` + + const html = await predict({ + systemPrompt, + userPrompt, + nbMaxNewTokens: 400, + prefix: "
` + + return new NextResponse(html, { + status: 200, + headers: new Headers({ "content-type": "text/html" }), + }) +} diff --git a/src/app/api/resolvers/interface/systemPrompt.ts b/src/app/api/resolvers/interface/systemPrompt.ts new file mode 100644 index 0000000000000000000000000000000000000000..3cccad8104f08b904057a0739b590ef5cfa68fa9 --- /dev/null +++ b/src/app/api/resolvers/interface/systemPrompt.ts @@ -0,0 +1,7 @@ +export const systemPrompt: string = + `You are a server-side function generating HTML snippets using Tailwind. +You are going to be asked to generate small widgets, the top component will always be a DIV. +Please use classic Tailwind classes, no arbitrary classes. +Be mindful of the design, we want components with large padding. +Use a dark grayscale theme, eg. bg-neutral-700 for a background, text-neutral-300 for text etc +` diff --git a/src/app/api/resolvers/video/route.ts b/src/app/api/resolvers/video/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..e179a7d161fdd115265bb5c8f0327b27263cb496 --- /dev/null +++ b/src/app/api/resolvers/video/route.ts @@ -0,0 +1,61 @@ +import { NextResponse, NextRequest } from "next/server" +import queryString from "query-string" + +import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain" +import { generateSeed } from "@/lib/utils/generateSeed" +import { sleep } from "@/lib/utils/sleep" +import { getContentType } from "@/lib/data/getContentType" + +export async function GET(req: NextRequest) { + +const qs = queryString.parseUrl(req.url || "") +const query = (qs || {}).query + +let prompt = "" + try { + prompt = decodeURIComponent(query?.p?.toString() || "").trim() + } catch (err) {} + if (!prompt) { + return NextResponse.json({ error: 'no prompt provided' }, { status: 400 }); + } + + // console.log("calling await newRender") + + let render = await newRender({ + prompt, + negativePrompt: "blurry, cropped, bad quality", + nbFrames: 1, + nbSteps: 4, + width: 1024, + height: 576, + turbo: true, + shouldRenewCache: true, + seed: generateSeed() + }) + + let attempts = 20 + + while (attempts-- > 0) { + if (render.status === "completed") { + return NextResponse.json(render, { + status: 200, + statusText: "OK", + }) + + } + + if (render.status === "error") { + return NextResponse.json(render, { + status: 200, + statusText: "OK", + }) + } + + await sleep(2000) // minimum wait time + + console.log("asking getRender") + render = await getRender(render.renderId) + } + + return NextResponse.json({ error: 'failed to call VideoChain (timeout expired)' }, { status: 500 }); +} diff --git a/src/app/server/actions/utils/censorship.ts b/src/app/api/utils/censorship.ts similarity index 100% rename from src/app/server/actions/utils/censorship.ts rename to src/app/api/utils/censorship.ts diff --git a/src/app/server/actions/utils/computeOrientationProjectionWidthHeight.ts b/src/app/api/utils/computeOrientationProjectionWidthHeight.ts similarity index 100% rename from src/app/server/actions/utils/computeOrientationProjectionWidthHeight.ts rename to src/app/api/utils/computeOrientationProjectionWidthHeight.ts diff --git a/src/app/server/actions/utils/formatPromptFileName.ts b/src/app/api/utils/formatPromptFileName.ts similarity index 100% rename from src/app/server/actions/utils/formatPromptFileName.ts rename to src/app/api/utils/formatPromptFileName.ts diff --git a/src/app/server/actions/utils/isAntisocial.ts b/src/app/api/utils/isAntisocial.ts similarity index 100% rename from src/app/server/actions/utils/isAntisocial.ts rename to src/app/api/utils/isAntisocial.ts diff --git a/src/app/server/actions/utils/isHighQuality.ts b/src/app/api/utils/isHighQuality.ts similarity index 100% rename from src/app/server/actions/utils/isHighQuality.ts rename to src/app/api/utils/isHighQuality.ts diff --git a/src/app/server/actions/utils/isValidNumber.ts b/src/app/api/utils/isValidNumber.ts similarity index 100% rename from src/app/server/actions/utils/isValidNumber.ts rename to src/app/api/utils/isValidNumber.ts diff --git a/src/app/server/actions/utils/parseDatasetPrompt.ts b/src/app/api/utils/parseDatasetPrompt.ts similarity index 100% rename from src/app/server/actions/utils/parseDatasetPrompt.ts rename to src/app/api/utils/parseDatasetPrompt.ts diff --git a/src/app/server/actions/utils/parseDatasetReadme.ts b/src/app/api/utils/parseDatasetReadme.ts similarity index 100% rename from src/app/server/actions/utils/parseDatasetReadme.ts rename to src/app/api/utils/parseDatasetReadme.ts diff --git a/src/app/server/actions/utils/parseProjectionFromLoRA.ts b/src/app/api/utils/parseProjectionFromLoRA.ts similarity index 100% rename from src/app/server/actions/utils/parseProjectionFromLoRA.ts rename to src/app/api/utils/parseProjectionFromLoRA.ts diff --git a/src/app/server/actions/utils/parsePromptFileName.ts b/src/app/api/utils/parsePromptFileName.ts similarity index 100% rename from src/app/server/actions/utils/parsePromptFileName.ts rename to src/app/api/utils/parsePromptFileName.ts diff --git a/src/app/api/utils/parseRawStringToYAML.ts b/src/app/api/utils/parseRawStringToYAML.ts new file mode 100644 index 0000000000000000000000000000000000000000..7d695aa14c6f4e3e945aca234282c1de9e66876c --- /dev/null +++ b/src/app/api/utils/parseRawStringToYAML.ts @@ -0,0 +1,20 @@ +import YAML from "yaml" + +export function parseRawStringToYAML(input: any, defaultValue: T) { + try { + let rawString = `${input || ""}`.trim() + + rawString = rawString + .replaceAll("```yaml\n", "") + .replaceAll("```yaml", "") + + // we remove everything after the last ``` + rawString = rawString.split('```')[0].trim() + + const something: any = YAML.parse(rawString) + + return something as T + } catch (err) { + return defaultValue + } +} \ No newline at end of file diff --git a/src/app/api/utils/parseString.ts b/src/app/api/utils/parseString.ts new file mode 100644 index 0000000000000000000000000000000000000000..aa2bf3d1ae110986f25917a00011b80af569b247 --- /dev/null +++ b/src/app/api/utils/parseString.ts @@ -0,0 +1,7 @@ +export function parseString(something: any): string { + let result: string = "" + if (typeof something === "string") { + result = `${something}`.trim() + } + return result +} \ No newline at end of file diff --git a/src/app/api/utils/parseStringArray.ts b/src/app/api/utils/parseStringArray.ts new file mode 100644 index 0000000000000000000000000000000000000000..9b3c2e6fdbd1f6a453627c4f9c9d466ddd3a64ac --- /dev/null +++ b/src/app/api/utils/parseStringArray.ts @@ -0,0 +1,9 @@ +export function parseStringArray(something: any): string[] { + let result: string[] = [] + if (typeof something === "string") { + result = [something] + } else if (Array.isArray(something)) { + result = something.map(thing => typeof thing === "string" ? thing : "").filter(x => x) + } + return result +} \ No newline at end of file diff --git a/src/app/server/actions/utils/parseVideoModelName.ts b/src/app/api/utils/parseVideoModelName.ts similarity index 100% rename from src/app/server/actions/utils/parseVideoModelName.ts rename to src/app/api/utils/parseVideoModelName.ts diff --git a/src/app/server/actions/utils/parseVideoOrientation.ts b/src/app/api/utils/parseVideoOrientation.ts similarity index 100% rename from src/app/server/actions/utils/parseVideoOrientation.ts rename to src/app/api/utils/parseVideoOrientation.ts diff --git a/src/app/api/video/[videoId]/route.ts b/src/app/api/video/[videoId]/route.ts index fa073c26322dbc2d347d31bfdb938c5f734f497a..c9549d956d472a9ca06d534d188e26c5632b267c 100644 --- a/src/app/api/video/[videoId]/route.ts +++ b/src/app/api/video/[videoId]/route.ts @@ -1,6 +1,6 @@ import { NextResponse, NextRequest } from "next/server" -import { getVideo } from "@/app/server/actions/ai-tube-hf/getVideo" +import { getVideo } from "@/app/api/actions/ai-tube-hf/getVideo" import { parseMediaProjectionType } from "@/lib/utils/parseMediaProjectionType"; /** @@ -22,7 +22,7 @@ export async function GET(req: NextRequest) { ${video.label} - AiTube - + diff --git a/src/app/channel/page.tsx b/src/app/channel/page.tsx index d75c3c924f72a68d7124d502e7c957ebf88166aa..f7ad8076520cb6e48c884b1071fc164a9ccc3cd2 100644 --- a/src/app/channel/page.tsx +++ b/src/app/channel/page.tsx @@ -1,8 +1,8 @@ import { AppQueryProps } from "@/types/general" import { Main } from "../main" -import { getChannel } from "../server/actions/ai-tube-hf/getChannel" -import { getChannelVideos } from "../server/actions/ai-tube-hf/getChannelVideos" +import { getChannel } from "../api/actions/ai-tube-hf/getChannel" +import { getChannelVideos } from "../api/actions/ai-tube-hf/getChannelVideos" export default async function ChannelPage({ searchParams: { c: channelId } }: AppQueryProps) { const channel = await getChannel({ channelId, neverThrow: true }) diff --git a/src/app/dream/page.tsx b/src/app/dream/page.tsx new file mode 100644 index 0000000000000000000000000000000000000000..823cbb1754d4b964cd677d6e7d1cd03d7c62233e --- /dev/null +++ b/src/app/dream/page.tsx @@ -0,0 +1,21 @@ + + +import { LatentQueryProps } from "@/types/general" + +import { Main } from "../main" +import { searchResultToMediaInfo } from "../api/generators/search/searchResultToMediaInfo" +import { LatentSearchResult } from "../api/generators/search/types" + +export default async function DreamPage({ searchParams: { + l: latentContent, +} }: LatentQueryProps) { + + const latentSearchResult = JSON.parse(atob(`${latentContent}`)) as LatentSearchResult + + // this will hallucinate the thumbnail on the fly - maybe we should cache it + const latentMedia = await searchResultToMediaInfo(latentSearchResult) + + return ( +
+ ) +} \ No newline at end of file diff --git a/src/app/embed/page.tsx b/src/app/embed/page.tsx index 3c1ff533faf90e55d722872c850cec9d4c46e377..7c255bb9da135e394a7a3a3448df8f07043f2763 100644 --- a/src/app/embed/page.tsx +++ b/src/app/embed/page.tsx @@ -4,7 +4,7 @@ import { Metadata, ResolvingMetadata } from "next" import { AppQueryProps } from "@/types/general" import { Main } from "../main" -import { getVideo } from "../server/actions/ai-tube-hf/getVideo" +import { getVideo } from "../api/actions/ai-tube-hf/getVideo" // https://nextjs.org/docs/pages/building-your-application/optimizing/fonts @@ -84,9 +84,9 @@ export default async function Embed({ // m: mediaId } }: AppQueryProps) { - const publicVideo = await getVideo({ videoId, neverThrow: true }) + const publicMedia = await getVideo({ videoId, neverThrow: true }) // console.log("WatchPage: --> " + video?.id) return ( -
+
) } \ No newline at end of file diff --git a/src/app/favicon.ico b/src/app/favicon.ico index 060fa8ce26f545dd54e28b76401e5bc7a55b7c92..73beb52bd315adbbda0ee88ddb1e521dd830d96f 100644 Binary files a/src/app/favicon.ico and b/src/app/favicon.ico differ diff --git a/src/app/icon.png b/src/app/icon.png index ecc6cbefdfc1232e92b40cc616cc34388b0360da..d1bcb2646f92b6f65ac4bab1570dc9520ac4f266 100644 Binary files a/src/app/icon.png and b/src/app/icon.png differ diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 822bf863d2902ab6306322c49eced2c5046f1813..cfb1613bccf22a15168b22ad304f78efbf5dc931 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -29,6 +29,13 @@ export default function RootLayout({ + + s.setPublicVideo) + const setPublicMedia = useStore(s => s.setPublicMedia) const setView = useStore(s => s.setView) const setPathname = useStore(s => s.setPathname) const setPublicChannel = useStore(s => s.setPublicChannel) - const setPublicVideos = useStore(s => s.setPublicVideos) + const setPublicMedias = useStore(s => s.setPublicMedias) const setPublicChannelVideos = useStore(s => s.setPublicChannelVideos) const setPublicTracks = useStore(s => s.setPublicTracks) const setPublicTrack = useStore(s => s.setPublicTrack) useEffect(() => { - if (!publicVideos?.length) { return } + if (!publicMedias?.length) { return } // note: it is important to ALWAYS set the current video to videoId // even if it's undefined - setPublicVideos(publicVideos) - }, [getCollectionKey(publicVideos)]) + setPublicMedias(publicMedias) + }, [getCollectionKey(publicMedias)]) useEffect(() => { @@ -98,19 +98,19 @@ export function Main({ useEffect(() => { // note: it is important to ALWAYS set the current video to videoId // even if it's undefined - setPublicVideo(publicVideo) + setPublicMedia(publicMedia) - if (!publicVideo || !publicVideo?.id) { return } + if (!publicMedia || !publicMedia?.id) { return } if (pathname === "/embed") { return } // this is a hack for hugging face: // we allow the ?v= param on the root of the domain if (pathname !== "/watch") { // console.log("we are on huggingface apparently!") - router.replace(`/watch?v=${publicVideo.id}`) + router.replace(`/watch?v=${publicMedia.id}`) } - }, [publicVideo?.id]) + }, [publicMedia?.id]) useEffect(() => { diff --git a/src/app/music/page.tsx b/src/app/music/page.tsx index a8413a3550d6d0910163d0c776ead7a625b2f9d8..6765bcdfa162b0efc9168ea4e898251012d7131b 100644 --- a/src/app/music/page.tsx +++ b/src/app/music/page.tsx @@ -1,8 +1,8 @@ import { AppQueryProps } from "@/types/general" import { Main } from "../main" -import { getVideos } from "../server/actions/ai-tube-hf/getVideos" -import { getVideo } from "../server/actions/ai-tube-hf/getVideo" +import { getVideos } from "../api/actions/ai-tube-hf/getVideos" +import { getVideo } from "../api/actions/ai-tube-hf/getVideo" import { Metadata } from "next" @@ -74,7 +74,7 @@ export default async function MusicPage({ searchParams: { m: mediaId } }: AppQue const publicTracks = await getVideos({ sortBy: "date", mandatoryTags: ["music"], - maxVideos: 25, + maxNbMedias: 25, neverThrow: true, }) diff --git a/src/app/page.tsx b/src/app/page.tsx index b3ac4ebb3798630463880e6e86537e7ab6698cd6..b3d7075adcef35366f12ce076eb143b162fa842d 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -2,7 +2,7 @@ import { AppQueryProps } from "@/types/general" import { Main } from "./main" -import { getVideo } from "./server/actions/ai-tube-hf/getVideo" +import { getVideo } from "./api/actions/ai-tube-hf/getVideo" import { Metadata, ResolvingMetadata } from "next" @@ -77,11 +77,11 @@ export async function generateMetadata( // we have routes but on Hugging Face we don't see them // so.. let's use the work around export default async function Page({ searchParams: { v: videoId } }: AppQueryProps) { - const publicVideo = await getVideo({ + const publicMedia = await getVideo({ videoId, neverThrow: true }) return ( -
+
) } \ No newline at end of file diff --git a/src/app/playlist/page.tsx b/src/app/playlist/page.tsx index 0940171086e2aa477c4189d7cfd6b3225c7f661e..855df4bdfcda444c68e1ee89a5c246f30b7eb1c9 100644 --- a/src/app/playlist/page.tsx +++ b/src/app/playlist/page.tsx @@ -1,8 +1,8 @@ import { AppQueryProps } from "@/types/general" import { Main } from "../main" -import { getVideos } from "../server/actions/ai-tube-hf/getVideos" -import { getVideo } from "../server/actions/ai-tube-hf/getVideo" +import { getVideos } from "../api/actions/ai-tube-hf/getVideos" +import { getVideo } from "../api/actions/ai-tube-hf/getVideo" import { Metadata } from "next" @@ -74,7 +74,7 @@ export default async function PlaylistPage({ searchParams: { m: mediaId } }: App const publicTracks = await getVideos({ sortBy: "date", mandatoryTags: ["music"], - maxVideos: 25, + maxNbMedias: 25, neverThrow: true, }) diff --git a/src/app/server/README.md b/src/app/server/README.md deleted file mode 100644 index ef45f0106be7955372bfa71b992a259473b81cb3..0000000000000000000000000000000000000000 --- a/src/app/server/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Server - -Those are files used on the server-side only. -It is safe to call NodeJS functions, do file operations and work with secrets env variables here. - -The frontend can call some functions using a very specific protocol, the [Server Actions](https://makerkit.dev/blog/tutorials/nextjs-server-actions). - -Those functions are currently in `/src/app/server/actions`. - diff --git a/src/app/server/actions/generation/generateImage.txt b/src/app/server/actions/generation/generateImage.txt deleted file mode 100644 index f1cdfe7f167cdfe1a98b5eb4f2245e33f5f26249..0000000000000000000000000000000000000000 --- a/src/app/server/actions/generation/generateImage.txt +++ /dev/null @@ -1,106 +0,0 @@ -"use server" - -// TODO add a system to mark failed instances as "unavailable" for a couple of minutes -// console.log("process.env:", process.env) - -import { generateSeed } from "@/lib/generateSeed"; -import { getValidNumber } from "@/lib/getValidNumber"; - -// note: to reduce costs I use the small A10s (not the large) -// anyway, we will soon not need to use this cloud anymore -// since we will be able to leverage the Inference API -const instance = `${process.env.FAST_IMAGE_SERVER_API_GRADIO_URL || ""}` -const secretToken = `${process.env.FAST_IMAGE_SERVER_API_SECRET_TOKEN || ""}` - -// console.log("DEBUG:", JSON.stringify({ instances, secretToken }, null, 2)) - -export async function generateImage(options: { - positivePrompt: string; - negativePrompt?: string; - seed?: number; - width?: number; - height?: number; - nbSteps?: number; -}): Promise { - - // console.log("querying " + instance) - const positivePrompt = options?.positivePrompt || "" - if (!positivePrompt) { - throw new Error("missing prompt") - } - - // the negative prompt CAN be missing, since we use a trick - // where we make the interface mandatory in the TS doc, - // but browsers might send something partial - const negativePrompt = options?.negativePrompt || "" - - // we treat 0 as meaning "random seed" - const seed = (options?.seed ? options.seed : 0) || generateSeed() - - const width = getValidNumber(options?.width, 256, 1024, 512) - const height = getValidNumber(options?.height, 256, 1024, 512) - const nbSteps = getValidNumber(options?.nbSteps, 1, 8, 4) - // console.log("SEED:", seed) - - const positive = [ - - // oh well.. is it too late to move this to the bottom? - "beautiful", - - // too opinionated, so let's remove it - // "intricate details", - - positivePrompt, - - "award winning", - "high resolution" - ].filter(word => word) - .join(", ") - - const negative = [ - negativePrompt, - "watermark", - "copyright", - "blurry", - // "artificial", - // "cropped", - "low quality", - "ugly" - ].filter(word => word) - .join(", ") - - const res = await fetch(instance + (instance.endsWith("/") ? "" : "/") + "api/predict", { - method: "POST", - headers: { - "Content-Type": "application/json", - // Authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ - fn_index: 0, // <- important! - data: [ - positive, // string in 'Prompt' Textbox component - negative, // string in 'Negative prompt' Textbox component - seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component - width, // number (numeric value between 256 and 1024) in 'Width' Slider component - height, // number (numeric value between 256 and 1024) in 'Height' Slider component - 0.0, // can be disabled for LCM SDXL - nbSteps, // number (numeric value between 2 and 8) in 'Number of inference steps for base' Slider component - secretToken - ] - }), - cache: "no-store", - }) - - const { data } = await res.json() - - if (res.status !== 200 || !Array.isArray(data)) { - // This will activate the closest `error.js` Error Boundary - throw new Error(`Failed to fetch data (status: ${res.status})`) - } - - if (!data[0]) { - throw new Error(`the returned image was empty`) - } - - return data[0] as string -} \ No newline at end of file diff --git a/src/app/server/actions/generation/generateStoryLines.txt b/src/app/server/actions/generation/generateStoryLines.txt deleted file mode 100644 index f6a31b159c7828623be4b37d2b3828d47462382d..0000000000000000000000000000000000000000 --- a/src/app/server/actions/generation/generateStoryLines.txt +++ /dev/null @@ -1,51 +0,0 @@ -"use server" - -import { Story, StoryLine, TTSVoice } from "@/types" - -const instance = `${process.env.AI_BEDTIME_STORY_API_GRADIO_URL || ""}` -const secretToken = `${process.env.AI_BEDTIME_STORY_API_SECRET_TOKEN || ""}` - -export async function generateStoryLines(prompt: string, voice: TTSVoice): Promise { - if (!prompt?.length) { - throw new Error(`prompt is too short!`) - } - - const cropped = prompt.slice(0, 30) - console.log(`user requested "${cropped}${cropped !== prompt ? "..." : ""}"`) - - // positivePrompt = filterOutBadWords(positivePrompt) - - const res = await fetch(instance + (instance.endsWith("/") ? "" : "/") + "api/predict", { - method: "POST", - headers: { - "Content-Type": "application/json", - // Authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ - fn_index: 0, // <- important! - data: [ - secretToken, - prompt, - voice, - ], - }), - cache: "no-store", - // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) - // next: { revalidate: 1 } - }) - - - const rawJson = await res.json() - const data = rawJson.data as StoryLine[][] - - const stories = data?.[0] || [] - - if (res.status !== 200) { - throw new Error('Failed to fetch data') - } - - return stories.map(line => ({ - text: line.text.replaceAll(" .", ".").replaceAll(" ?", "?").replaceAll(" !", "!").trim(), - audio: line.audio - })) -} \ No newline at end of file diff --git a/src/app/server/actions/generation/videochain.ts b/src/app/server/actions/generation/videochain.ts deleted file mode 100644 index 691219f58d1cad2050603f4ed382b8f81e78f0d7..0000000000000000000000000000000000000000 --- a/src/app/server/actions/generation/videochain.ts +++ /dev/null @@ -1,161 +0,0 @@ - -// note: there is no / at the end in the variable -// so we have to add it ourselves if needed -const apiUrl = process.env.VIDEOCHAIN_API_URL - -export const GET = async (path: string = '', defaultValue: T): Promise => { - try { - const res = await fetch(`${apiUrl}/${path}`, { - method: "GET", - headers: { - Accept: "application/json", - Authorization: `Bearer ${process.env.SECRET_ACCESS_TOKEN}`, - }, - cache: 'no-store', - // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) - // next: { revalidate: 1 } - }) - - // The return value is *not* serialized - // You can return Date, Map, Set, etc. - - // Recommendation: handle errors - if (res.status !== 200) { - // This will activate the closest `error.js` Error Boundary - throw new Error('Failed to fetch data') - } - - const data = await res.json() - - return ((data as T) || defaultValue) - } catch (err) { - console.error(err) - return defaultValue - } -} - - -export const DELETE = async (path: string = '', defaultValue: T): Promise => { - try { - const res = await fetch(`${apiUrl}/${path}`, { - method: "DELETE", - headers: { - Accept: "application/json", - Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`, - }, - cache: 'no-store', - // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) - // next: { revalidate: 1 } - }) - - // The return value is *not* serialized - // You can return Date, Map, Set, etc. - - // Recommendation: handle errors - if (res.status !== 200) { - // This will activate the closest `error.js` Error Boundary - throw new Error('Failed to fetch data') - } - - const data = await res.json() - - return ((data as T) || defaultValue) - } catch (err) { - console.error(err) - return defaultValue - } -} - -export const POST = async (path: string = '', payload: S, defaultValue: T): Promise => { - try { - const res = await fetch(`${apiUrl}/${path}`, { - method: "POST", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`, - }, - body: JSON.stringify(payload), - // cache: 'no-store', - // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) - next: { revalidate: 1 } - }) - // The return value is *not* serialized - // You can return Date, Map, Set, etc. - - // Recommendation: handle errors - if (res.status !== 200) { - // This will activate the closest `error.js` Error Boundary - throw new Error('Failed to post data') - } - - const data = await res.json() - - return ((data as T) || defaultValue) - } catch (err) { - return defaultValue - } -} - - -export const PUT = async (path: string = '', payload: S, defaultValue: T): Promise => { - try { - const res = await fetch(`${apiUrl}/${path}`, { - method: "PUT", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`, - }, - body: JSON.stringify(payload), - // cache: 'no-store', - // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) - next: { revalidate: 1 } - }) - // The return value is *not* serialized - // You can return Date, Map, Set, etc. - - // Recommendation: handle errors - if (res.status !== 200) { - // This will activate the closest `error.js` Error Boundary - throw new Error('Failed to post data') - } - - const data = await res.json() - - return ((data as T) || defaultValue) - } catch (err) { - return defaultValue - } -} - -export const PATCH = async (path: string = '', payload: S, defaultValue: T): Promise => { - try { - const res = await fetch(`${apiUrl}/${path}`, { - method: "PATCH", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - Authorization: `Bearer ${process.env.VC_SECRET_ACCESS_TOKEN}`, - }, - body: JSON.stringify(payload), - // cache: 'no-store', - // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache) - next: { revalidate: 1 } - }) - // The return value is *not* serialized - // You can return Date, Map, Set, etc. - - // Recommendation: handle errors - if (res.status !== 200) { - // This will activate the closest `error.js` Error Boundary - throw new Error('Failed to post data') - } - - const data = await res.json() - - return ((data as T) || defaultValue) - } catch (err) { - return defaultValue - } -} \ No newline at end of file diff --git a/src/app/state/useCurrentUser.ts b/src/app/state/useCurrentUser.ts index 432ad9767f4693043facca0e25901b7bee586f9b..c5f99897052f627d35d326d8027c0bf88fac8617 100644 --- a/src/app/state/useCurrentUser.ts +++ b/src/app/state/useCurrentUser.ts @@ -9,7 +9,7 @@ import { useStore } from "./useStore" import { localStorageKeys } from "./localStorageKeys" import { defaultSettings } from "./defaultSettings" -import { getCurrentUser } from "../server/actions/users" +import { getCurrentUser } from "../api/actions/users" export function useCurrentUser({ isLoginRequired = false diff --git a/src/app/state/useStore.ts b/src/app/state/useStore.ts index 721b4656a7f9076c3b9412d565239c38eae4a9c5..9a2a6b3576c84974afaf56755d26e6c077b702cb 100644 --- a/src/app/state/useStore.ts +++ b/src/app/state/useStore.ts @@ -61,14 +61,14 @@ export const useStore = create<{ currentModel?: string setCurrentModel: (currentModel?: string) => void - publicVideo?: MediaInfo - setPublicVideo: (publicVideo?: MediaInfo) => void + publicMedia?: MediaInfo + setPublicMedia: (publicMedia?: MediaInfo) => void publicComments: CommentInfo[] setPublicComments: (publicComment: CommentInfo[]) => void - publicVideos: MediaInfo[] - setPublicVideos: (publicVideos: MediaInfo[]) => void + publicMedias: MediaInfo[] + setPublicMedias: (publicMedias: MediaInfo[]) => void publicChannelVideos: MediaInfo[] setPublicChannelVideos: (publicChannelVideos: MediaInfo[]) => void @@ -201,9 +201,9 @@ export const useStore = create<{ set({ currentModel }) }, - publicVideo: undefined, - setPublicVideo: (publicVideo?: MediaInfo) => { - set({ publicVideo }) + publicMedia: undefined, + setPublicMedia: (publicMedia?: MediaInfo) => { + set({ publicMedia }) }, publicComments: [], @@ -211,10 +211,10 @@ export const useStore = create<{ set({ publicComments }) }, - publicVideos: [], - setPublicVideos: (publicVideos: MediaInfo[] = []) => { + publicMedias: [], + setPublicMedias: (publicMedias: MediaInfo[] = []) => { set({ - publicVideos: Array.isArray(publicVideos) ? publicVideos : [] + publicMedias: Array.isArray(publicMedias) ? publicMedias : [] }) }, @@ -234,7 +234,7 @@ export const useStore = create<{ publicChannelVideos: [], setPublicChannelVideos: (publicChannelVideos: MediaInfo[] = []) => { set({ - publicVideos: Array.isArray(publicChannelVideos) ? publicChannelVideos : [] + publicMedias: Array.isArray(publicChannelVideos) ? publicChannelVideos : [] }) }, diff --git a/src/app/views/home-view/index.tsx b/src/app/views/home-view/index.tsx index 9ffe2eff03b4012b8cf4361875c154106e49f2d1..6ca3be377e07487b8638fe467406e7920326b5c8 100644 --- a/src/app/views/home-view/index.tsx +++ b/src/app/views/home-view/index.tsx @@ -5,37 +5,35 @@ import { useEffect, useTransition } from "react" import { useStore } from "@/app/state/useStore" import { cn } from "@/lib/utils/cn" import { MediaInfo } from "@/types/general" -import { getVideos } from "@/app/server/actions/ai-tube-hf/getVideos" +import { getVideos } from "@/app/api/actions/ai-tube-hf/getVideos" import { VideoList } from "@/components/interface/video-list" -import { getTags } from "@/app/server/actions/ai-tube-hf/getTags" -import { extendVideosWithStats } from "@/app/server/actions/ai-tube-hf/extendVideosWithStats" export function HomeView() { const [_isPending, startTransition] = useTransition() const setView = useStore(s => s.setView) const currentTag = useStore(s => s.currentTag) - const setPublicVideos = useStore(s => s.setPublicVideos) - const setPublicVideo = useStore(s => s.setPublicVideo) - const publicVideos = useStore(s => s.publicVideos) + const setPublicMedias = useStore(s => s.setPublicMedias) + const setPublicMedia = useStore(s => s.setPublicMedia) + const publicMedias = useStore(s => s.publicMedias) useEffect(() => { startTransition(async () => { - const videos = await getVideos({ + const medias = await getVideos({ sortBy: "date", mandatoryTags: currentTag ? [currentTag] : [], - maxVideos: 25 + maxNbMedias: 25 }) // due to some caching on the first function.. we update with fresh data! - // const updatedVideos = await extendVideosWithStats(videos) + // const updatedVideos = await extendVideosWithStats(medias) - setPublicVideos(videos) + setPublicMedias(medias) }) }, [currentTag]) - const handleSelect = (video: MediaInfo) => { + const handleSelect = (media: MediaInfo) => { setView("public_media") - setPublicVideo(video) + setPublicMedia(media) } return ( @@ -43,7 +41,7 @@ export function HomeView() { `sm:pr-4` )}>
diff --git a/src/app/views/public-channels-view/index.tsx b/src/app/views/public-channels-view/index.tsx index 8611aaa11a084431e17f266e15bf37373960505c..bdd829b5cf09bda7e895af163ec29f90115ddf6d 100644 --- a/src/app/views/public-channels-view/index.tsx +++ b/src/app/views/public-channels-view/index.tsx @@ -4,7 +4,7 @@ import { useEffect, useState, useTransition } from "react" import { useStore } from "@/app/state/useStore" import { cn } from "@/lib/utils/cn" -import { getChannels } from "@/app/server/actions/ai-tube-hf/getChannels" +import { getChannels } from "@/app/api/actions/ai-tube-hf/getChannels" import { ChannelList } from "@/components/interface/channel-list" export function PublicChannelsView() { diff --git a/src/app/views/public-media-embed-view/index.tsx b/src/app/views/public-media-embed-view/index.tsx index c335393b3fc219b4f79b13717565a55310d572cb..773548a878a127e7f388a53c842b28dbd1f941b6 100644 --- a/src/app/views/public-media-embed-view/index.tsx +++ b/src/app/views/public-media-embed-view/index.tsx @@ -6,7 +6,7 @@ import { useStore } from "@/app/state/useStore" import { cn } from "@/lib/utils/cn" import { MediaPlayer } from "@/components/interface/media-player" -import { countNewMediaView } from "@/app/server/actions/stats" +import { countNewMediaView } from "@/app/api/actions/stats" export function PublicMediaEmbedView() { const [_pending, startTransition] = useTransition() @@ -16,11 +16,11 @@ export function PublicMediaEmbedView() { // EDIT: you know what, let's do this the dirty way for now // const [desiredCurrentTime, setDesiredCurrentTime] = useState() - const media = useStore(s => s.publicVideo) + const media = useStore(s => s.publicMedia) const mediaId = `${media?.id || ""}` - const setPublicVideo = useStore(s => s.setPublicVideo) + const setPublicMedia = useStore(s => s.setPublicMedia) // we inject the current mediaId in the URL, if it's not already present // this is a hack for Hugging Face iframes @@ -48,7 +48,7 @@ export function PublicMediaEmbedView() { } const numberOfViews = await countNewMediaView(mediaId) - setPublicVideo({ + setPublicMedia({ ...media, numberOfViews }) diff --git a/src/app/views/public-media-view/index.tsx b/src/app/views/public-media-view/index.tsx index da32c2f24ab09fdf09cf82831f04aed3aa1cc9e7..69db8aad4f9f09421327ae3780b5e685a498e8f9 100644 --- a/src/app/views/public-media-view/index.tsx +++ b/src/app/views/public-media-view/index.tsx @@ -16,7 +16,7 @@ import { cn } from "@/lib/utils/cn" import { ActionButton, actionButtonClassName } from "@/components/interface/action-button" import { RecommendedVideos } from "@/components/interface/recommended-videos" import { isCertifiedUser } from "@/app/certification" -import { countNewMediaView } from "@/app/server/actions/stats" +import { countNewMediaView } from "@/app/api/actions/stats" import { formatTimeAgo } from "@/lib/formatters/formatTimeAgo" import { DefaultAvatar } from "@/components/interface/default-avatar" import { LikeButton } from "@/components/interface/like-button" @@ -28,7 +28,7 @@ import { Input } from "@/components/ui/input" import { localStorageKeys } from "@/app/state/localStorageKeys" import { defaultSettings } from "@/app/state/defaultSettings" -import { getComments, submitComment } from "@/app/server/actions/comments" +import { getComments, submitComment } from "@/app/api/actions/comments" import { useCurrentUser } from "@/app/state/useCurrentUser" import { parseMediaProjectionType } from "@/lib/utils/parseMediaProjectionType" import { MediaPlayer } from "@/components/interface/media-player" @@ -61,14 +61,14 @@ export function PublicMediaView() { } - const media = useStore(s => s.publicVideo) + const media = useStore(s => s.publicMedia) const mediaId = `${media?.id || ""}` const [copied, setCopied] = useState(false) const [channelThumbnail, setChannelThumbnail] = useState(`${media?.channel.thumbnail || ""}`) - const setPublicVideo = useStore(s => s.setPublicVideo) + const setPublicMedia = useStore(s => s.setPublicMedia) const publicComments = useStore(s => s.publicComments) @@ -118,7 +118,7 @@ export function PublicMediaView() { } const numberOfViews = await countNewMediaView(mediaId) - setPublicVideo({ + setPublicMedia({ ...media, numberOfViews }) diff --git a/src/app/views/public-music-videos-view/index.tsx b/src/app/views/public-music-videos-view/index.tsx index 2286a0de35ea468df709e9266ca4a89d64f31008..17ce646de50cf6fce751304aab400a09cd6ceadd 100644 --- a/src/app/views/public-music-videos-view/index.tsx +++ b/src/app/views/public-music-videos-view/index.tsx @@ -5,7 +5,7 @@ import { useEffect, useTransition } from "react" import { useStore } from "@/app/state/useStore" import { cn } from "@/lib/utils/cn" import { MediaInfo } from "@/types/general" -import { getVideos } from "@/app/server/actions/ai-tube-hf/getVideos" +import { getVideos } from "@/app/api/actions/ai-tube-hf/getVideos" import { TrackList } from "@/components/interface/track-list" import { PlaylistControl } from "@/components/interface/playlist-control" import { usePlaylist } from "@/lib/hooks/usePlaylist" @@ -26,10 +26,10 @@ export function PublicMusicVideosView() { const newTracks = await getVideos({ sortBy: "date", mandatoryTags: ["music"], - maxVideos: 25 + maxNbMedias: 25 }) - setPublicVideos(newTracks) + setPublicMedias(newTracks) }) */ }, []) diff --git a/src/app/views/user-account-view/index.tsx b/src/app/views/user-account-view/index.tsx index dd001c4d24096c026ccd7b7be9299ebb8c350418..f72deac0be1d66122312ba3f8b0b98aaacec8db1 100644 --- a/src/app/views/user-account-view/index.tsx +++ b/src/app/views/user-account-view/index.tsx @@ -6,7 +6,7 @@ import { useStore } from "@/app/state/useStore" import { cn } from "@/lib/utils/cn" import { ChannelList } from "@/components/interface/channel-list" -import { getPrivateChannels } from "@/app/server/actions/ai-tube-hf/getPrivateChannels" +import { getPrivateChannels } from "@/app/api/actions/ai-tube-hf/getPrivateChannels" import { useCurrentUser } from "@/app/state/useCurrentUser" import { Button } from "@/components/ui/button" import { Input } from "@/components/ui/input" diff --git a/src/app/views/user-channel-view/index.tsx b/src/app/views/user-channel-view/index.tsx index ee97c1cb5bf2f67b766c2d08c462013607a744c7..e2cf795b793acb0ca0d97207d0cc64c491bdd51b 100644 --- a/src/app/views/user-channel-view/index.tsx +++ b/src/app/views/user-channel-view/index.tsx @@ -12,13 +12,13 @@ import { defaultSettings } from "@/app/state/defaultSettings" import { Input } from "@/components/ui/input" import { Textarea } from "@/components/ui/textarea" import { Button } from "@/components/ui/button" -import { submitVideoRequest } from "@/app/server/actions/submitVideoRequest" +import { submitVideoRequest } from "@/app/api/actions/submitVideoRequest" import { PendingVideoList } from "@/components/interface/pending-video-list" -import { getChannelVideos } from "@/app/server/actions/ai-tube-hf/getChannelVideos" -import { parseVideoModelName } from "@/app/server/actions/utils/parseVideoModelName" +import { getChannelVideos } from "@/app/api/actions/ai-tube-hf/getChannelVideos" +import { parseVideoModelName } from "@/app/api/utils/parseVideoModelName" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" import { defaultVideoModel, defaultVideoOrientation, defaultVoice } from "@/app/config" -import { parseVideoOrientation } from "@/app/server/actions/utils/parseVideoOrientation" +import { parseVideoOrientation } from "@/app/api/utils/parseVideoOrientation" export function UserChannelView() { const [_isPending, startTransition] = useTransition() diff --git a/src/app/watch/page.tsx b/src/app/watch/page.tsx index 4f7f0ec19ba537590da9747bf50867295c100a0e..9fcebddd72a69aaa4802ec749416fa1a97ad4058 100644 --- a/src/app/watch/page.tsx +++ b/src/app/watch/page.tsx @@ -4,7 +4,7 @@ import { Metadata, ResolvingMetadata } from "next" import { AppQueryProps } from "@/types/general" import { Main } from "../main" -import { getVideo } from "../server/actions/ai-tube-hf/getVideo" +import { getVideo } from "../api/actions/ai-tube-hf/getVideo" // https://nextjs.org/docs/pages/building-your-application/optimizing/fonts export async function generateMetadata( @@ -81,9 +81,9 @@ export default async function WatchPage({ searchParams: { // TODO add: // m: mediaId } }: AppQueryProps) { - const publicVideo = await getVideo({ videoId, neverThrow: true }) + const publicMedia = await getVideo({ videoId, neverThrow: true }) // console.log("WatchPage: --> " + video?.id) return ( -
+
) } \ No newline at end of file diff --git a/src/components/interface/collection-card/index.tsx b/src/components/interface/collection-card/index.tsx index a0e1a33135d82c950706cad53acf0df28628e03f..e2c57bc710b82ba7d02c2a69314847eda671cedb 100644 --- a/src/components/interface/collection-card/index.tsx +++ b/src/components/interface/collection-card/index.tsx @@ -79,7 +79,6 @@ export function CollectionCard({ className={cn( `absolute`, `aspect-video`, - // `aspect-video object-cover`, `rounded-lg overflow-hidden`, collectionThumbnailReady ? `opacity-100`: 'opacity-0', `hover:opacity-0 w-full h-full top-0 z-30`, diff --git a/src/components/interface/icon-switch/index.tsx b/src/components/interface/icon-switch/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..071572308cb5914340ed60e85d67b874749037b2 --- /dev/null +++ b/src/components/interface/icon-switch/index.tsx @@ -0,0 +1,129 @@ +import { cn } from "@/lib/utils/cn" +import { ReactNode } from "react" +import { IconType } from "react-icons/lib" +import { SingleIcon } from "../single-icon" + +export function IconSwitch({ + onIcon, + onIconAlt, + offIcon, + offIconAlt, + onClick, + isToggledOn = false, + isAlt = false, + disabled = false, + className = "", + size = "md", + thickOnHover = false, + children = null, + iconClass = "", +}: { + onIcon: IconType + onIconAlt?: IconType + offIcon: IconType + offIconAlt?: IconType + onClick?: () => void + isToggledOn?: boolean + isAlt?: boolean + disabled?: boolean + className?: string + size?: "2xs" | "xs" | "sm" | "md" + thickOnHover?: boolean + children?: ReactNode + iconClass?: string +}) { + + const iconSize = + size === "2xs" ? "w-4 h-4" : + size === "xs" ? "w-5 h-5" : + size === "sm" ? "w-6 h-6" : + size === "md" ? "w-7 h-7" : + "w-8 h-8" + + return ( +
{ + if (!disabled) { + onClick() + } + } : undefined} + > +
+ + + + +
+ {children + ?
+ {children} +
: null} +
+ ) +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/components/content-layer/index.tsx b/src/components/interface/latent-engine/components/content-layer/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..02b5b9d5ca60df274c095969e101b48ee4f3dae5 --- /dev/null +++ b/src/components/interface/latent-engine/components/content-layer/index.tsx @@ -0,0 +1,29 @@ +import { ForwardedRef, forwardRef, ReactNode } from "react" + +export const ContentLayer = forwardRef(function ContentLayer({ + width = 256, + height = 256, + className = "", + children, +}: { + width?: number + height?: number + className?: string + children?: ReactNode +}, ref: ForwardedRef) { + return ( +
+
+ {children} +
+
+ ) +}) \ No newline at end of file diff --git a/src/components/interface/latent-engine/components/disclaimers/this-is-ai.tsx b/src/components/interface/latent-engine/components/disclaimers/this-is-ai.tsx new file mode 100644 index 0000000000000000000000000000000000000000..6a874602664df2d57354b0952fc66ea918b3298a --- /dev/null +++ b/src/components/interface/latent-engine/components/disclaimers/this-is-ai.tsx @@ -0,0 +1,102 @@ +import React from "react" + + +import { cn } from "@/lib/utils/cn" + +import { arimoBold, arimoNormal } from "@/lib/fonts" +import { StreamType } from "@/types/general" + +export function ThisIsAI({ + streamType, +}: { + streamType?: StreamType +} = {}) { + + return ( +
+
+
+
+ The following
+ { + /* + isDynamic + ? "dynamic" + : "static" + */ + } content +
{ + streamType !== "static" + ? "will be" + : "has been" + }
+ synthesized +
+ using +
+
+ artificial intelligence +
+
+ and may contain hallucinations or factual inaccuracies. +
+
+
+
+ ) +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/components/play-pause-button/index.tsx b/src/components/interface/latent-engine/components/play-pause-button/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..3ddcd9dc4e7c165799c4cd8cdd4d682797189b8e --- /dev/null +++ b/src/components/interface/latent-engine/components/play-pause-button/index.tsx @@ -0,0 +1,28 @@ +import React from "react" +import { IoMdPause, IoMdPlay } from "react-icons/io" + +import { IconSwitch } from "../../../icon-switch" + +export function PlayPauseButton({ + className = "", + isToggledOn, + onClick +}: { + className?: string + isToggledOn?: boolean + onClick?: () => void +}) { + return ( + + ) +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/core/engine.tsx b/src/components/interface/latent-engine/core/engine.tsx new file mode 100644 index 0000000000000000000000000000000000000000..ad3f634852ac791cbe87180e7a530f42569a3912 --- /dev/null +++ b/src/components/interface/latent-engine/core/engine.tsx @@ -0,0 +1,227 @@ +"use client" + +import React, { useEffect, useRef, useState } from "react" + +import { mockClap } from "@/lib/clap/mockClap" +import { cn } from "@/lib/utils/cn" + +import { useLatentEngine } from "../store/useLatentEngine" +import { PlayPauseButton } from "../components/play-pause-button" +import { StreamTag } from "../../stream-tag" +import { ContentLayer } from "../components/content-layer" + +function LatentEngine({ + url, + width, + height, + className = "" }: { + url: string + width?: number + height?: number + className?: string +}) { + const setContainerDimension = useLatentEngine(s => s.setContainerDimension) + const isLoaded = useLatentEngine(s => s.isLoaded) + const openLatentClapFile = useLatentEngine(s => s.openLatentClapFile) + const openClapFile = useLatentEngine(s => s.openClapFile) + + const setImageElement = useLatentEngine(s => s.setImageElement) + const setVideoElement = useLatentEngine(s => s.setVideoElement) + + const streamType = useLatentEngine(s => s.streamType) + const isStatic = useLatentEngine(s => s.isStatic) + const isLive = useLatentEngine(s => s.isLive) + const isInteractive = useLatentEngine(s => s.isInteractive) + + const isPlaying = useLatentEngine(s => s.isPlaying) + const togglePlayPause = useLatentEngine(s => s.togglePlayPause) + + const videoLayer = useLatentEngine(s => s.videoLayer) + const segmentationLayer = useLatentEngine(s => s.segmentationLayer) + const interfaceLayer = useLatentEngine(s => s.interfaceLayer) + + const stateRef = useRef({ isInitialized: false }) + + const [isOverlayVisible, setOverlayVisible] = useState(true) + + const overlayTimerRef = useRef() + + const videoLayerRef = useRef(null) + + useEffect(() => { + if (!stateRef.current.isInitialized) { + stateRef.current.isInitialized = true + console.log("let's load an experience") + // openClapFile(mockClap({ showDisclaimer: true })) + openLatentClapFile("short story about a podracer race") + } + }, []) + + const isPlayingRef = useRef(isPlaying) + isPlayingRef.current = isPlaying + + const scheduleOverlayInvisibility = () => { + clearTimeout(overlayTimerRef.current) + overlayTimerRef.current = setTimeout(() => { + if (isPlayingRef.current) { + setOverlayVisible(!isPlayingRef.current) + } + clearTimeout(overlayTimerRef.current) + }, 1000) + } + + /* + useEffect(() => { + if (isPlaying) { + scheduleOverlayInvisibility() + } else { + clearTimeout(overlayTimerRef.current) + setOverlayVisible(true) + } + + return () => { + clearTimeout(overlayTimerRef.current) + } + }, [isPlaying]) + */ + + useEffect(() => { + if (!videoLayerRef.current) { return } + + const videoElements = Array.from(videoLayerRef.current.querySelectorAll('.latent-video')) as HTMLVideoElement[] + setVideoElement(videoElements.at(0)) + + // images are used for simpler or static experiences + const imageElements = Array.from(videoLayerRef.current.querySelectorAll('.latent-image')) as HTMLImageElement[] + setImageElement(imageElements.at(0)) + }) + + useEffect(() => { + setContainerDimension({ width: width || 256, height: height || 256 }) + }, [width, height]) + + return ( +
+ {/* */} + + {/* main content container */} + {videoLayer} + + {segmentationLayer} + + {interfaceLayer} + + + {/* content overlay, with the gradient, buttons etc */} +
{ + setOverlayVisible(true) + scheduleOverlayInvisibility() + }} + style={{ width, height, boxShadow: "rgba(0, 0, 0, 1) 0px -77px 100px 15px inset" }}> + {/* bottom slider and button bar */} +
+ + {/* the (optional) timeline slider bar */} +
+
+
+ +
+ + {/* button bar */} +
+ + {/* left-side buttons */} +
+ + +
+ + + {/* right-side buttons */} +
+ {/* + + TODO: put a fullscreen button (and mode) here + + */} +
+
+
+
+
+ ); +} + +export default LatentEngine \ No newline at end of file diff --git a/src/components/interface/latent-engine/core/fetchLatentClap.ts b/src/components/interface/latent-engine/core/fetchLatentClap.ts new file mode 100644 index 0000000000000000000000000000000000000000..39b4bd843343cf2ab7088e1dca1c44f7f9bdc47c --- /dev/null +++ b/src/components/interface/latent-engine/core/fetchLatentClap.ts @@ -0,0 +1,19 @@ +import { parseClap } from "@/lib/clap/parseClap" +import { ClapProject } from "@/lib/clap/types" + +export async function fetchLatentClap(prompt: string): Promise { + + const requestUri = `/api/resolvers/clap?p=${encodeURIComponent(prompt)}` + + console.log(`fetchLatentClap: calling ${requestUri}`) + + const res = await fetch(requestUri) + + const blob = await res.blob() + + const clap = await parseClap(blob) + + console.log(`fetchLatentClap: received = `, clap) + + return clap +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/core/fetchLatentSearchResults.ts b/src/components/interface/latent-engine/core/fetchLatentSearchResults.ts new file mode 100644 index 0000000000000000000000000000000000000000..d34975bec69cb9c956262bb8f2d5663863362b91 --- /dev/null +++ b/src/components/interface/latent-engine/core/fetchLatentSearchResults.ts @@ -0,0 +1,3 @@ +export function getLatentSearchResults() { + +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/core/types.ts b/src/components/interface/latent-engine/core/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..c9b81b1152ce25cbe967cc319bb68c5867c5d423 --- /dev/null +++ b/src/components/interface/latent-engine/core/types.ts @@ -0,0 +1,99 @@ +import { ClapProject, ClapSegment, ClapStreamType } from "@/lib/clap/types" +import { ReactNode } from "react" + +export type LatentEngineStatus = + | "idle" + | "loading" + | "loaded" + | "failed" + +export type LatentNode = { + prompt: string + + // HTML example + example: string + + // React output + result: ReactNode +} + +export type LayerCategory = + | "interface" + | "segmentation" + | "video" + | "splat" + +export type LatentComponentResolver = (segment: ClapSegment, clap: ClapProject) => Promise + +export type LatentEngineStore = { + width: number + height: number + + clap: ClapProject + + streamType: ClapStreamType + + // just some aliases for convenience + isStatic: boolean + isLive: boolean + isInteractive: boolean + + // our "this is AI.. gasp!" disclaimer + hasDisclaimer: boolean + hasPresentedDisclaimer: boolean + + // for convenience the status is split into separate booleans, + // including their boolean opposites + isLoading: boolean // true when a .clap is being downloaded and/or generated + isLoaded: boolean // true if a clap is loaded + isPlaying: boolean + isPaused: boolean + + simulationPromise?: Promise + simulationPending: boolean // used as a "lock" + + renderingIntervalId: NodeJS.Timeout | string | number | undefined + renderingIntervalDelayInMs: number + + positionInMs: number + durationInMs: number + + videoLayerElement?: HTMLDivElement + imageElement?: HTMLImageElement + videoElement?: HTMLVideoElement + + videoLayer: ReactNode + videoBuffer: "A" | "B" + videoBufferA: ReactNode + videoBufferB: ReactNode + + segmentationLayer: ReactNode + + interfaceLayer: ReactNode + interfaceBuffer: "A" | "B" + interfaceBufferA: ReactNode + interfaceBufferB: ReactNode + + setContainerDimension: ({ width, height }: { width: number; height: number }) => void + openLatentClapFile: (prompt: string) => Promise + openClapFile: (clap: ClapProject) => void + + setVideoLayerElement: (videoLayerElement?: HTMLDivElement) => void + setImageElement: (imageElement?: HTMLImageElement) => void + setVideoElement: (videoElement?: HTMLVideoElement) => void + + togglePlayPause: () => boolean + play: () => boolean + pause: () => boolean + + // a slow rendering function (async - might call a third party LLM) + runSimulationLoop: () => Promise + + // a fast rendering function; whose sole role is to filter the component + // list to put into the buffer the one that should be displayed + runRenderingLoop: () => void + + jumpTo: (positionInMs: number) => void + jumpToStart: () => void + +} diff --git a/src/components/interface/latent-engine/index.tsx b/src/components/interface/latent-engine/index.tsx index ab458dead3a265c8c2684468dd18fb9ac8425e2e..d99cde34d58ff59404afa94c1dda63b854f820cf 100644 --- a/src/components/interface/latent-engine/index.tsx +++ b/src/components/interface/latent-engine/index.tsx @@ -1,36 +1,7 @@ -import React, { useEffect, useRef } from "react" -import { useLatentEngine } from "./useLatentEngine" -import { mockClap } from "@/lib/clap/mockClap" -import { Gsplat } from "../gsplat" +"use client" -export type LatentEngineStatus = - | "idle" - | "loading" - | "loaded" - | "failed" +import dynamic from "next/dynamic" -export function LatentEngine({ - url, - width, - height, - className = "" }: { - url: string - width?: number - height?: number - className?: string -}) { - const le = useLatentEngine() - - useEffect(() => { - if (!le.loaded) { - console.log("let's load an experience") - le.load(mockClap()) - } - }, [le.loaded]) - - return ( -
- {/* */} -
- ); -} +export const LatentEngine = dynamic(() => import("./core/engine"), { + loading: () => null, +}) diff --git a/src/components/interface/latent-engine/resolvers/generic/index.tsx b/src/components/interface/latent-engine/resolvers/generic/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..6cc5dba0b63de11a0fc8139296b800bc42bd4518 --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/generic/index.tsx @@ -0,0 +1,11 @@ +"use client" + +import { ClapProject, ClapSegment } from "@/lib/clap/types" + +export async function resolve(segment: ClapSegment, clap: ClapProject): Promise { + return ( +
+ ) +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/resolvers/image/generateImage.ts b/src/components/interface/latent-engine/resolvers/image/generateImage.ts new file mode 100644 index 0000000000000000000000000000000000000000..fbc8d26f967b121737285ca449a8617fefd25903 --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/image/generateImage.ts @@ -0,0 +1,15 @@ +import { RenderedScene } from "@/types/general" + +export async function generateImage(prompt: string): Promise { + const requestUri = `/api/resolvers/image?p=${encodeURIComponent(prompt)}` + + const res = await fetch(requestUri) + + const scene = (await res.json()) as RenderedScene + + if (scene.error || scene.status !== "completed") { + throw new Error(scene.error) + } + + return scene.assetUrl +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/resolvers/image/index.tsx b/src/components/interface/latent-engine/resolvers/image/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..130568f2721f84f3e311ea58aabb0814ed4ee2aa --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/image/index.tsx @@ -0,0 +1,28 @@ +"use client" + +import { ClapProject, ClapSegment } from "@/lib/clap/types" +import { generateImage } from "./generateImage" + +export async function resolve(segment: ClapSegment, clap: ClapProject): Promise { + + const { prompt } = segment + + let assetUrl = "" + try { + // console.log(`resolveImage: generating video for: ${prompt}`) + + assetUrl = await generateImage(prompt) + + // console.log(`resolveImage: generated ${assetUrl}`) + + } catch (err) { + console.error(`resolveImage failed (${err})`) + return <> + } + + // note: the latent-image class is not used for styling, but to grab the component + // from JS when we need to segment etc + return ( + + ) +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/resolvers/index.tsx b/src/components/interface/latent-engine/resolvers/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..c85795c817249727fc21f305590919840f46bf18 --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/index.tsx @@ -0,0 +1,36 @@ +"use client" + +import { useEffect, useState } from "react"; + +import { ClapProject, ClapSegment } from "@/lib/clap/types"; +import { resolveSegment } from "./resolveSegment"; + +export function LatentComponent({ + segment, + clap +}: { + segment: ClapSegment, + clap: ClapProject +}): JSX.Element { + const [component, setComponent] = useState(null) + const [isInitialized, setInitialized] = useState(false) + const [isLoaded, setLoaded] = useState(false) + + useEffect(() => { + if (isInitialized) { return } + setInitialized(true) + + const fn = async () => { + const component = await resolveSegment(segment, clap) + + setComponent(component) + } + + fn() + + return () => {} + }, [isInitialized]) + + + return <>{component} +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/resolvers/interface/generateHtml.ts b/src/components/interface/latent-engine/resolvers/interface/generateHtml.ts new file mode 100644 index 0000000000000000000000000000000000000000..7e3a6d0284d62ae497b06e57f0311eb916d1fcb2 --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/interface/generateHtml.ts @@ -0,0 +1,11 @@ +export async function generateHtml(prompt: string): Promise { + const requestUri = `/api/resolvers/interface?p=${encodeURIComponent(prompt)}` + + // console.log(`generateHtml: calling ${requestUri}`) + + const res = await fetch(requestUri) + + const dangerousHtml = await res.text() + + return dangerousHtml +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/resolvers/interface/index.tsx b/src/components/interface/latent-engine/resolvers/interface/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..3e71a014af2709e8b9381086730b3d12651381ec --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/interface/index.tsx @@ -0,0 +1,57 @@ +"use client" + +import RunCSS, { extendRunCSS } from "runcss" + +import { ClapProject, ClapSegment } from "@/lib/clap/types" +import { generateHtml } from "./generateHtml" +import { ThisIsAI } from "../../components/disclaimers/this-is-ai" + +let state = { + runCSS: RunCSS({}), + isWatching: false, +} + +export async function resolve(segment: ClapSegment, clap: ClapProject): Promise { + + const { prompt } = segment + + if (prompt.toLowerCase() === "") { + return + } + + let dangerousHtml = "" + try { + console.log(`resolveInterface: generating html for: ${prompt}`) + + dangerousHtml = await generateHtml(prompt) + + console.log(`resolveInterface: generated ${dangerousHtml}`) + + } catch (err) { + console.log(`resolveInterface failed (${err})`) + } + + const { processClasses, startWatching, stopWatching, exportCSS } = state.runCSS + + // + // call the API + // dynamically *new* classes into the current page + + // https://github.com/mudgen/runcss + + // Start watching for changes + // TODO: we should only watch the + // startWatching(document.getElementById('hello')) // if not specified, fallback to document.body + + if (!state.isWatching) { + console.log("resolveInterface: TODO: starting the CSS watcher..") + // startWatching(targetNode) + } + + return ( +
+ ) +} diff --git a/src/components/interface/latent-engine/resolvers/resolveSegment.ts b/src/components/interface/latent-engine/resolvers/resolveSegment.ts new file mode 100644 index 0000000000000000000000000000000000000000..0a77f109b2658f080015a873d33d42ac1c32d0e6 --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/resolveSegment.ts @@ -0,0 +1,24 @@ +import { ClapProject, ClapSegment } from "@/lib/clap/types" + +import { LatentComponentResolver } from "../core/types" + +import { resolve as genericResolver } from "./generic" +import { resolve as interfaceResolver } from "./interface" +import { resolve as videoResolver } from "./video" +import { resolve as imageResolver } from "./image" + +export async function resolveSegment(segment: ClapSegment, clap: ClapProject): Promise { + let latentComponentResolver: LatentComponentResolver = genericResolver + + if (segment.category === "interface") { + latentComponentResolver = interfaceResolver + } else if (segment.category === "video") { + + // for backend performance reason (generative video is slow) + // we do not support "true" video for now + // latentComponentResolver = videoResolver + latentComponentResolver = imageResolver + } + + return latentComponentResolver(segment, clap) +} diff --git a/src/components/interface/latent-engine/resolvers/resolveSegments.ts b/src/components/interface/latent-engine/resolvers/resolveSegments.ts new file mode 100644 index 0000000000000000000000000000000000000000..06e334243dd22e6d5429d8fd4e6516d9565689a1 --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/resolveSegments.ts @@ -0,0 +1,17 @@ +import { ClapProject, ClapSegmentCategory } from "@/lib/clap/types" + +import { resolveSegment } from "./resolveSegment" + +export async function resolveSegments( + clap: ClapProject, + segmentCategory: ClapSegmentCategory, + nbMax?: number +) : Promise { + const elements: JSX.Element[] = await Promise.all( + clap.segments + .filter(s => s.category === segmentCategory) + .slice(0, nbMax) + .map(s => resolveSegment(s, clap)) + ) + return elements +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/resolvers/video/generateVideo.ts b/src/components/interface/latent-engine/resolvers/video/generateVideo.ts new file mode 100644 index 0000000000000000000000000000000000000000..5247d0747d7ebf643cf870302db1c7c682ee9e29 --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/video/generateVideo.ts @@ -0,0 +1,17 @@ +import { RenderedScene } from "@/types/general" + +export async function generateVideo(prompt: string): Promise { + const requestUri = `/api/resolvers/video?p=${encodeURIComponent(prompt)}` + + // console.log(`generateVideo: calling ${requestUri}`) + + const res = await fetch(requestUri) + + const scene = (await res.json()) as RenderedScene + + if (scene.error || scene.status !== "completed") { + throw new Error(scene.error) + } + + return scene.assetUrl +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/resolvers/video/index.tsx b/src/components/interface/latent-engine/resolvers/video/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..e6196e53b34a3355a9fd6feb10f8a06c9a1b5740 --- /dev/null +++ b/src/components/interface/latent-engine/resolvers/video/index.tsx @@ -0,0 +1,28 @@ +"use client" + +import { ClapProject, ClapSegment } from "@/lib/clap/types" +import { generateVideo } from "./generateVideo" + +export async function resolve(segment: ClapSegment, clap: ClapProject): Promise { + + const { prompt } = segment + + let assetUrl = "" + try { + // console.log(`resolveVideo: generating video for: ${prompt}`) + + assetUrl = await generateVideo(prompt) + + // console.log(`resolveVideo: generated ${assetUrl}`) + + } catch (err) { + console.error(`resolveVideo failed (${err})`) + return <> + } + + // note: the latent-video class is not used for styling, but to grab the component + // from JS when we need to segment etc + return ( + + ) +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/store/useLatentEngine.ts b/src/components/interface/latent-engine/store/useLatentEngine.ts new file mode 100644 index 0000000000000000000000000000000000000000..f79d849f1b03a56923db0b759f3f3156ab5f9d14 --- /dev/null +++ b/src/components/interface/latent-engine/store/useLatentEngine.ts @@ -0,0 +1,280 @@ + +import { create } from "zustand" + +import { ClapProject } from "@/lib/clap/types" +import { newClap } from "@/lib/clap/newClap" +import { sleep } from "@/lib/utils/sleep" +import { getSegmentationCanvas } from "@/lib/on-device-ai/getSegmentationCanvas" + +import { LatentEngineStore } from "../core/types" +import { resolveSegments } from "../resolvers/resolveSegments" +import { fetchLatentClap } from "../core/fetchLatentClap" + +export const useLatentEngine = create((set, get) => ({ + width: 1024, + height: 576, + + clap: newClap(), + + streamType: "static", + isStatic: false, + isLive: false, + isInteractive: false, + + isLoading: false, // true when a .clap is being downloaded and/or generated + isLoaded: false, // true if a clap is loaded + isPlaying: false, + isPaused: true, + + // our "this is AI.. gasp!" disclaimer + hasDisclaimer: true, + hasPresentedDisclaimer: false, + + simulationPromise: undefined, + simulationPending: false, + + renderingIntervalId: undefined, + renderingIntervalDelayInMs: 2000, // 2 sec + + positionInMs: 0, + durationInMs: 0, + + videoLayerElement: undefined, + imageElement: undefined, + videoElement: undefined, + + videoLayer: undefined, + videoBuffer: "A", + videoBufferA: null, + videoBufferB: undefined, + + segmentationLayer: undefined, + + interfaceLayer: undefined, + interfaceBuffer: "A", + interfaceBufferA: undefined, + interfaceBufferB: undefined, + + setContainerDimension: ({ width, height }: { width: number; height: number }) => { + set({ + width, + height + }) + }, + + openLatentClapFile: async (prompt: string): Promise => { + set({ + isLoaded: false, + isLoading: true, + }) + + let clap: ClapProject | undefined = undefined + + try { + clap = await fetchLatentClap(prompt) + } catch (err) { + console.error(`generateAndLoad failed (${err})`) + set({ + isLoading: false, + }) + } + + if (!clap) { return } + + get().openClapFile(clap) + }, + + openClapFile: (clap: ClapProject) => { + set({ + clap, + isLoading: false, + isLoaded: true, + streamType: clap.meta.streamType, + isStatic: clap.meta.streamType !== "interactive" && clap.meta.streamType !== "live", + isLive: clap.meta.streamType === "live", + isInteractive: clap.meta.streamType === "interactive", + }) + }, + + setVideoLayerElement: (videoLayerElement?: HTMLDivElement) => { set({ videoLayerElement }) }, + setImageElement: (imageElement?: HTMLImageElement) => { set({ imageElement }) }, + setVideoElement: (videoElement?: HTMLVideoElement) => { set({ videoElement }) }, + + togglePlayPause: (): boolean => { + const { isLoaded, isPlaying, renderingIntervalId } = get() + if (!isLoaded) { return false } + + const newValue = !isPlaying + + clearInterval(renderingIntervalId) + + if (newValue) { + set({ + isPlaying: true, + renderingIntervalId: setTimeout(() => { get().runRenderingLoop() }, 0) + }) + } else { + set({ isPlaying: false }) + } + + return newValue + }, + + + play: (): boolean => { + const { isLoaded, isPlaying, renderingIntervalId, renderingIntervalDelayInMs } = get() + + if (!isLoaded) { return false } + + if (isPlaying) { return true } + + clearInterval(renderingIntervalId) + set({ + isPlaying: true, + renderingIntervalId: setTimeout(() => { get().runRenderingLoop() }, 0) + }) + + return true + }, + + pause: (): boolean => { + const { isLoaded, renderingIntervalId } = get() + if (!isLoaded) { return false } + + clearInterval(renderingIntervalId) + + set({ isPlaying: false }) + + return false + }, + + // a slow rendering function (async - might call a third party LLM) + runSimulationLoop: async () => { + const { + isLoaded, + isPlaying, + clap, + segmentationLayer, + imageElement, + videoElement, + height, + width, + } = get() + + if (!isLoaded || !isPlaying) { + + set({ + simulationPending: false, + }) + + return + } + + set({ + simulationPending: true, + }) + + try { + + // console.log("doing stuff") + let timestamp = performance.now() + + if (imageElement) { + // console.log("we have an image element:", imageElement) + const segmentationLayer = await getSegmentationCanvas({ + frame: imageElement, + timestamp, + width, + height, + }) + set({ segmentationLayer }) + } + + await sleep(500) + + // note: since we are asynchronous, we need to regularly check if + // the user asked to pause the system or no + if (get().isPlaying) { + // console.log(`runSimulationLoop: rendering video content layer..`) + // we only grab the first one + const videoLayer = (await resolveSegments(clap, "video", 1)).at(0) + + if (get().isPlaying) { + set({ + videoLayer + }) + + console.log(`runSimulationLoop: rendered video content layer`) + } + } + + } catch (err) { + console.error(`runSimulationLoop failed to render video layer ${err}`) + } + + try { + if (get().isPlaying) { + console.log(`runSimulationLoop: rendering UI layer..`) + + // note: for now we only display one element, to avoid handing a list of html elements + const interfaceLayer = (await resolveSegments(clap, "interface", 1)).at(0) + if (get().isPlaying) { + set({ + interfaceLayer + }) + + console.log(`runSimulationLoop: rendered UI layer`) + } + } + } catch (err) { + console.error(`runSimulationLoop failed to render UI layer ${err}`) + } + + set({ + simulationPending: false, + }) + }, + + // a fast sync rendering function; whose sole role is to filter the component + // list to put into the buffer the one that should be displayed + runRenderingLoop: () => { + const { + isLoaded, + isPlaying, + renderingIntervalId, + renderingIntervalDelayInMs, + simulationPromise, + simulationPending, + runSimulationLoop, + imageElement, + videoElement, + } = get() + if (!isLoaded) { return } + if (!isPlaying) { return } + try { + // console.log(`runRenderingLoop: starting..`) + + // TODO: some operations with + // console.log(`runRenderingLoop: ended`) + } catch (err) { + console.error(`runRenderingLoop failed ${err}`) + } + clearInterval(renderingIntervalId) + set({ + isPlaying: true, + simulationPromise: simulationPending ? simulationPromise : runSimulationLoop(), + + // TODO: use requestAnimationFrame somehow + // https://developers.google.com/mediapipe/solutions/vision/image_segmenter/web_js + renderingIntervalId: setTimeout(() => { get().runRenderingLoop() }, renderingIntervalDelayInMs) + }) + + }, + + jumpTo: (positionInMs: number) => { + set({ positionInMs }) + }, + jumpToStart: () => { + set({ positionInMs: 0 }) + }, +})) \ No newline at end of file diff --git a/src/components/interface/latent-engine/useLatentEngine.ts b/src/components/interface/latent-engine/useLatentEngine.ts deleted file mode 100644 index 38d563b0067286c04fc8167c843105cdaee7dfcd..0000000000000000000000000000000000000000 --- a/src/components/interface/latent-engine/useLatentEngine.ts +++ /dev/null @@ -1,26 +0,0 @@ -"use client" - -import { create } from "zustand" - -import { ClapProject } from "@/lib/clap/types" -import { newClap } from "@/lib/clap/newClap" - -export type LatentEngineStore = { - clap: ClapProject - loaded: boolean - load: (clap: ClapProject) => void -} - -export const useLatentEngine = create((set, get) => ({ - clap: newClap(), - loaded: false, - - // TODO: add a loader for either a Clap or a LatentScript - - load: (clap: ClapProject) => { - set({ - clap, - loaded: true - }) - }, -})) diff --git a/src/components/interface/like-button/index.tsx b/src/components/interface/like-button/index.tsx index 6e2edd1a35acc3349d0bfe1e16d0c6d41466d49a..e78187173cb159454aa80a663c8eb17fc9f9ee99 100644 --- a/src/components/interface/like-button/index.tsx +++ b/src/components/interface/like-button/index.tsx @@ -2,7 +2,7 @@ import { useEffect, useState, useTransition } from "react" import { useLocalStorage } from "usehooks-ts" import { MediaInfo, MediaRating } from "@/types/general" -import { getMediaRating, rateMedia } from "@/app/server/actions/stats" +import { getMediaRating, rateMedia } from "@/app/api/actions/stats" import { localStorageKeys } from "@/app/state/localStorageKeys" import { defaultSettings } from "@/app/state/defaultSettings" diff --git a/src/components/interface/media-player/index.tsx b/src/components/interface/media-player/index.tsx index 2f796a6936a3d7f350751350a5f620069ff211f8..354b69fd3dee9046078ef9d8d128a26827472283 100644 --- a/src/components/interface/media-player/index.tsx +++ b/src/components/interface/media-player/index.tsx @@ -9,6 +9,7 @@ import { parseMediaProjectionType } from "@/lib/utils/parseMediaProjectionType" import { EquirectangularVideoPlayer } from "./equirectangular" import { CartesianVideoPlayer } from "./cartesian" import { GaussianSplattingPlayer } from "./gaussian" +import { LatentPlayer } from "./latent" export function MediaPlayer({ media, @@ -21,10 +22,13 @@ export function MediaPlayer({ className?: string // currentTime?: number }) { - console.log("MediaPlayer called for \"" + media?.label + "\"") + // console.log("MediaPlayer called for \"" + media?.label + "\"") if (!media || !media?.assetUrl) { return null } + // uncomment one of those to forcefully test the .clap player! + media.assetUrlHd = "https://huggingface.co/datasets/jbilcke/ai-tube-cinema/tree/main/404.clap" + // uncomment one of those to forcefully test the .splatv player! // media.assetUrlHd = "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/4d/flame/flame.splatv" // media.assetUrlHd = "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/4d/sear/sear.splatv" @@ -32,6 +36,17 @@ export function MediaPlayer({ const projectionType = parseMediaProjectionType(media) + if (projectionType === "latent") { + // note: for AutoSizer to work properly it needs to be inside a normal div with no display: "flex" + return ( +
+ {({ height, width }) => ( + + )} +
+ ) + } + if (projectionType === "gaussian") { // note: for AutoSizer to work properly it needs to be inside a normal div with no display: "flex" return ( diff --git a/src/components/interface/recommended-videos/index.tsx b/src/components/interface/recommended-videos/index.tsx index 5dc16dac2d6a4aba1adfa6518d35b29cfdf59586..dfdcfa842854d5c7cb1d8fda9c909fef4a6e808b 100644 --- a/src/components/interface/recommended-videos/index.tsx +++ b/src/components/interface/recommended-videos/index.tsx @@ -5,7 +5,7 @@ import { useStore } from "@/app/state/useStore" import { MediaInfo } from "@/types/general" import { VideoList } from "../video-list" -import { getVideos } from "@/app/server/actions/ai-tube-hf/getVideos" +import { getVideos } from "@/app/api/actions/ai-tube-hf/getVideos" export function RecommendedVideos({ media, @@ -23,7 +23,7 @@ export function RecommendedVideos({ sortBy: "random", niceToHaveTags: media.tags, ignoreVideoIds: [media.id], - maxVideos: 16, + maxNbMedias: 16, })) }) }, media.tags) diff --git a/src/components/interface/search-input/index.tsx b/src/components/interface/search-input/index.tsx index 70f0265b4998c6720745a4306dd8e3cac6430f6e..2ff4f06eba943ad7366c116bdcf0d5158c02c72c 100644 --- a/src/components/interface/search-input/index.tsx +++ b/src/components/interface/search-input/index.tsx @@ -8,7 +8,7 @@ import { useStore } from "@/app/state/useStore" import { cn } from "@/lib/utils/cn" import { Input } from "@/components/ui/input" import { Button } from "@/components/ui/button" -import { getVideos } from "@/app/server/actions/ai-tube-hf/getVideos" +import { getVideos } from "@/app/api/actions/ai-tube-hf/getVideos" export function SearchInput() { const [_pending, startTransition] = useTransition() @@ -35,7 +35,7 @@ export function SearchInput() { const videos = await getVideos({ query, sortBy: "match", - maxVideos: 8, + maxNbMedias: 8, neverThrow: true, renewCache: false, // bit of optimization }) diff --git a/src/components/interface/single-icon/index.tsx b/src/components/interface/single-icon/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..cd5e4ca4ccecf55da5e7e71bcb50b68f4e2d7e1a --- /dev/null +++ b/src/components/interface/single-icon/index.tsx @@ -0,0 +1,34 @@ +import { IconType } from "react-icons/lib" + +import { cn } from "@/lib/utils/cn" + +export function SingleIcon({ + type, + className = "", + thickOnHover = false, +}: { + type?: IconType + className?: string + thickOnHover?: boolean +}) { + if (!type) { + return null + } + + const Icon = type + + return ( + +) +} \ No newline at end of file diff --git a/src/components/interface/stream-tag/index.tsx b/src/components/interface/stream-tag/index.tsx new file mode 100644 index 0000000000000000000000000000000000000000..fbff1a77d12f092bfcbef5d9c3c51aa50e792aae --- /dev/null +++ b/src/components/interface/stream-tag/index.tsx @@ -0,0 +1,43 @@ +import { PiLightningFill } from "react-icons/pi" + +import { cn } from "@/lib/utils/cn" +import { ClapStreamType } from "@/lib/clap/types" + +export function StreamTag({ + streamType = "static", + size = "md", + className = "", +}: { + streamType?: ClapStreamType + size?: "sm" | "md" + className?: string +}) { + const isInteractive = streamType === "interactive" + const isLive = streamType === "live" + const isStatic = !isInteractive && !isLive + + return ( +
+ + + { + isInteractive ? "Interactive" + : isLive ? "Live" + : "Static content" + } + +
+ ) +} \ No newline at end of file diff --git a/src/components/interface/top-header/index.tsx b/src/components/interface/top-header/index.tsx index fb8264cbd4e3ab5711c78be512823e0cc0df9623..b2864e0fb257d0d0e7b7c81d3f8da276efc0d5ae 100644 --- a/src/components/interface/top-header/index.tsx +++ b/src/components/interface/top-header/index.tsx @@ -2,25 +2,16 @@ import { useEffect, useState, useTransition } from 'react' -import { Pathway_Gothic_One } from 'next/font/google' -import { PiPopcornBold } from "react-icons/pi" -import { GoSearch } from "react-icons/go" import { AiTubeLogo } from "./logo" -const pathway = Pathway_Gothic_One({ - weight: "400", - style: "normal", - subsets: ["latin"], - display: "swap" -}) - import { useStore } from "@/app/state/useStore" import { cn } from "@/lib/utils/cn" -import { getTags } from '@/app/server/actions/ai-tube-hf/getTags' +import { getTags } from '@/app/api/actions/ai-tube-hf/getTags' import Link from 'next/link' import { Input } from '@/components/ui/input' import { Button } from '@/components/ui/button' import { SearchInput } from '../search-input' +import { pathway } from '@/lib/fonts' export function TopHeader() { const [_pending, startTransition] = useTransition() diff --git a/src/components/interface/video-card/index.tsx b/src/components/interface/video-card/index.tsx index 323a1c6115f29762a0afbf09ef33a5f0dfc6ec1d..84c44ac71424416834484bdad2427d72a799ce3d 100644 --- a/src/components/interface/video-card/index.tsx +++ b/src/components/interface/video-card/index.tsx @@ -74,6 +74,9 @@ export function VideoCard({ }, index * 1500) }, [index]) + // uncomment this as a temporary hack when developping, to avoid making too many fetches and downloads + return null; + return (
{ + if (globalState.blob) { return globalState.blob } + + const clap = newClap() + + globalState.blob = await serializeClap(clap) + + return globalState.blob +} \ No newline at end of file diff --git a/src/lib/clap/mockClap.ts b/src/lib/clap/mockClap.ts index 15935765ca3dda8c0601ed334901cf2429b6f1e2..ddfceffe78fc3855f890e9f91bbb2f9ce66eaeba 100644 --- a/src/lib/clap/mockClap.ts +++ b/src/lib/clap/mockClap.ts @@ -2,30 +2,66 @@ import { newClap } from "./newClap" import { newSegment } from "./newSegment" import { ClapProject } from "./types" -export function mockClap(): ClapProject { +let defaultSegmentDurationInMs = 2000 + +export function mockClap({ + showDisclaimer +}: { + showDisclaimer: boolean +}): ClapProject { const clap = newClap() - const mockSegment = newSegment({ - // id: string - // track: number - // startTimeInMs: number - // endTimeInMs: number - // category: ClapSegmentCategory - // modelId: string - // sceneId: string - // prompt: string - // label: string - // outputType: ClapOutputType - // renderId: string - // status: ClapSegmentStatus - // assetUrl: string - // assetDurationInMs: number - // createdBy: ClapAuthor - // editedBy: ClapAuthor - // outputGain: number - // seed: number - }) - clap.segments.push(mockSegment) + let currentElapsedTimeInMs = 0 + let currentSegmentDurationInMs = defaultSegmentDurationInMs + + if (showDisclaimer) { + clap.segments.push(newSegment({ + startTimeInMs: currentElapsedTimeInMs, + endTimeInMs: currentSegmentDurationInMs, + category: "interface", + prompt: "", + label: "fish", + outputType: "interface", + })) + currentElapsedTimeInMs += currentSegmentDurationInMs + } + + /* + clap.segments.push( + newSegment({ + // id: string + // track: number + startTimeInMs: currentElapsedTimeInMs, + endTimeInMs: currentSegmentDurationInMs, + category: "interface", + // modelId: string + // sceneId: string + prompt: "a hello world", + label: "hello world", + outputType: "interface" + // renderId: string + // status: ClapSegmentStatus + // assetUrl: string + // assetDurationInMs: number + // createdBy: ClapAuthor + // editedBy: ClapAuthor + // outputGain: number + // seed: number + }) + ) + + currentElapsedTimeInMs += currentSegmentDurationInMs + */ + + clap.segments.push(newSegment({ + startTimeInMs: currentElapsedTimeInMs, + endTimeInMs: currentSegmentDurationInMs, + category: "video", + // prompt: "closeup of Queen angelfish, bokeh", + prompt: "portrait of a man tv news anchor, pierre-jean-hyves, serious, bokeh", + label: "demo", + outputType: "video", + })) return clap } \ No newline at end of file diff --git a/src/lib/clap/newClap.ts b/src/lib/clap/newClap.ts index 1320dea8d3ab1f4245075bd0a66a2cb8b9a29305..27ac98d2bca23119c242a376de884808ccbcae68 100644 --- a/src/lib/clap/newClap.ts +++ b/src/lib/clap/newClap.ts @@ -1,7 +1,7 @@ import { v4 as uuidv4 } from "uuid" -import { ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types" +import { ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment, ClapStreamType } from "./types" import { getValidNumber } from "@/lib/utils/getValidNumber" // generate an empty clap file, or copy one from a source @@ -23,6 +23,7 @@ export function newClap(clap: { defaultVideoModel: typeof clap?.meta?.defaultVideoModel === "string" ? clap?.meta.defaultVideoModel : "SVD", extraPositivePrompt: Array.isArray(clap?.meta?.extraPositivePrompt) ? clap?.meta.extraPositivePrompt : [], screenplay: typeof clap?.meta?.screenplay === "string" ? clap?.meta.screenplay : "", + streamType: (typeof clap?.meta?.streamType == "string" ? clap?.meta?.streamType : "static") as ClapStreamType, } const models: ClapModel[] = clap?.models && Array.isArray(clap.models) ? clap.models : [] diff --git a/src/lib/clap/parseClap.ts b/src/lib/clap/parseClap.ts index b7af7833a8fd8efa2fd2914f691017a06578bdc4..f7b4339a1fcdd3c6f81fde366becd54ce9381bdf 100644 --- a/src/lib/clap/parseClap.ts +++ b/src/lib/clap/parseClap.ts @@ -1,7 +1,7 @@ import YAML from "yaml" import { v4 as uuidv4 } from "uuid" -import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types" +import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment, ClapStreamType } from "./types" import { getValidNumber } from "@/lib/utils/getValidNumber" /** @@ -52,6 +52,7 @@ export async function parseClap(inputStringOrBlob: string | Blob): Promise { + const results: ImageSegmenterResult = await segmentFrame(frame, timestamp); + + const canvas: HTMLCanvasElement | OffscreenCanvas | undefined = results.categoryMask?.canvas; + + // If there is a canvas, style it and return + if (canvas) { + const style = { + width: `${width}px`, + height: `${height}px`, + }; + + const CanvasComponent = () => ( + { + if (node) { + node.width = width; + node.height = height; + const context = node.getContext('2d'); + if (context) { + context.drawImage(canvas, 0, 0, width, height); + } + } + }} + style={style} + /> + ); + return ; + } else { + // Return a blank canvas if no canvas is found in results + return ( + + ); + } +} \ No newline at end of file diff --git a/src/lib/on-device-ai/segmentFrame.ts b/src/lib/on-device-ai/segmentFrame.ts new file mode 100644 index 0000000000000000000000000000000000000000..18157cd88268fb524895dd9ce8c4318d6e806075 --- /dev/null +++ b/src/lib/on-device-ai/segmentFrame.ts @@ -0,0 +1,57 @@ +import { FilesetResolver, ImageSegmenter, ImageSegmenterResult, ImageSource } from "@mediapipe/tasks-vision" + +export type VideoSegmenter = (videoFrame: ImageSource, timestamp: number) => Promise + +const getSegmenter = async (): Promise => { + const vision = await FilesetResolver.forVisionTasks( + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" + ); + + const imageSegmenter = await ImageSegmenter.createFromOptions(vision, { + baseOptions: { + modelAssetPath: + // this is a very lightweight model (< 2.7 Mb!) so it is not perfect, + // it can only detect a few types of objects + + "https://storage.googleapis.com/mediapipe-assets/deeplabv3.tflite?generation=1661875711618421", + }, + outputCategoryMask: true, + outputConfidenceMasks: false, + + // since we only generate images for now, + // there is little consistency between each of them + // so there is no need to use "VIDEO" + runningMode: "VIDEO" + }); + + const segmenter: VideoSegmenter = (videoFrame: ImageSource, timestamp: number): Promise => { + return new Promise((resolve, reject) => { + imageSegmenter.segmentForVideo(videoFrame, timestamp, (results) => { + resolve(results) + }) + }) + } + + return segmenter +} + + +const globalState: { segmenter?: VideoSegmenter } = {}; + +(async () => { + globalState.segmenter = globalState.segmenter || (await getSegmenter()) +})(); + +export async function segmentFrame(frame: ImageSource, timestamp: number): Promise { + console.log("segmentFrame: loading segmenter..") + globalState.segmenter = globalState.segmenter || (await getSegmenter()) + + console.log("segmentFrame: segmenting..") + return globalState.segmenter(frame, timestamp) +} + +// to run: + +// see doc: +// https://developers.google.com/mediapipe/solutions/vision/image_segmenter/web_js#video +// imageSegmenter.segmentForVideo(video, startTimeMs, callbackForVideo); diff --git a/src/lib/prompts/cleanJson.ts b/src/lib/prompts/cleanJson.ts new file mode 100644 index 0000000000000000000000000000000000000000..907dbc8b8e13a5cee2e74ed35e2aca147824d455 --- /dev/null +++ b/src/lib/prompts/cleanJson.ts @@ -0,0 +1,19 @@ +import { dirtyLLMResponseCleaner } from "./dirtyLLMResponseCleaner" + +export function cleanJson(input: string): string { + + if (input.includes('```')) { + input = input.split('```')[0] + } + let tmp = dirtyLLMResponseCleaner(input) + + // we only keep what's after the first [ + tmp = `[${tmp.split("[").pop() || ""}` + + // and before the first ] + tmp = `${tmp.split("]").shift() || ""}]` + + tmp = dirtyLLMResponseCleaner(tmp) + + return tmp +} \ No newline at end of file diff --git a/src/lib/prompts/createLlamaPrompt.ts b/src/lib/prompts/createLlamaPrompt.ts new file mode 100644 index 0000000000000000000000000000000000000000..3ce7e57966ba42667d05132997e3445b7c3ddb70 --- /dev/null +++ b/src/lib/prompts/createLlamaPrompt.ts @@ -0,0 +1,25 @@ +// adapted from https://huggingface.co/TheBloke/Llama-2-13B-chat-GPTQ/discussions/5 +export function createLlamaPrompt(messages: Array<{ role: string, content: string }>) { + const B_INST = "[INST]", E_INST = "[/INST]"; + const B_SYS = "<>\n", E_SYS = "\n<>\n\n"; + const BOS = "", EOS = ""; + const DEFAULT_SYSTEM_PROMPT = "You are a helpful, respectful and honest storywriting assistant. Always answer in a creative and entertaining way, while being safe. Please ensure that your stories and captions are socially unbiased and positive in nature. If a request does not make any sense, go on anyway, as we are writing a fantasy story."; + + if (messages[0].role != "system"){ + messages = [ + {role: "system", content: DEFAULT_SYSTEM_PROMPT} + ].concat(messages); + } + messages = [{role: messages[1].role, content: B_SYS + messages[0].content + E_SYS + messages[1].content}].concat(messages.slice(2)); + + let messages_list = messages.map((value, index, array) => { + if (index % 2 == 0 && index + 1 < array.length){ + return `${BOS}${B_INST} ${array[index].content.trim()} ${E_INST} ${array[index+1].content.trim()} ${EOS}` + } + return ''; + }) + + messages_list.push(`${BOS}${B_INST} ${messages[messages.length-1].content.trim()} ${E_INST}`) + + return messages_list.join(''); +} \ No newline at end of file diff --git a/src/lib/prompts/createZephyrPrompt.ts b/src/lib/prompts/createZephyrPrompt.ts new file mode 100644 index 0000000000000000000000000000000000000000..317e390d07d1a9ef1f7c04243b6046d52ae7367c --- /dev/null +++ b/src/lib/prompts/createZephyrPrompt.ts @@ -0,0 +1,26 @@ + + interface Message { + role: "system" | "user" | "assistant"; + content: string; + } + + /** + * Formats the messages for the chat with the LLM model in the style of a pirate. + * @param messages - Array of message objects with role and content. + * @returns The formatted chat prompt. + */ + export function createZephyrPrompt(messages: Message[]): string { + let prompt = ``; + + // Iterate over messages and generate corresponding chat entries. + messages.forEach(message => { + prompt += `<|${message.role}|>\n${message.content.trim()}`; + }); + + if (messages.at(-1)?.role === "user") { + // Append the assistant's tag for the next response but without a closing tag. + prompt += `<|assistant|>`; + } + + return prompt; + } \ No newline at end of file diff --git a/src/lib/prompts/dirtyLLMResponseCleaner.ts b/src/lib/prompts/dirtyLLMResponseCleaner.ts new file mode 100644 index 0000000000000000000000000000000000000000..91bbb0383534f5df3bb2b90d4aeaf6f87a3a28de --- /dev/null +++ b/src/lib/prompts/dirtyLLMResponseCleaner.ts @@ -0,0 +1,38 @@ +export function dirtyLLMResponseCleaner(input: string): string { + let str = ( + `${input || ""}` + // a summary of all the weird hallucinations I saw it make.. + .replaceAll(`"\n`, `",\n`) // fix missing commas at the end of a line + .replaceAll(`"]`, `"}]`) + .replaceAll(/"\S*,?\S*\]/gi, `"}]`) + .replaceAll(/"\S*,?\S*\}\S*]/gi, `"}]`) + + // this removes the trailing commas (which are valid in JS but not JSON) + .replace(/,(?=\s*?[\}\]])/g, '') + + .replaceAll("}}", "}") + .replaceAll("]]", "]") + .replaceAll("[[", "[") + .replaceAll("{{", "{") + .replaceAll(",,", ",") + ) + + // repair missing end of JSON array + if (str.at(-1) === '}') { + str = str + "]" + } + + if (str.at(-1) === '"') { + str = str + "}]" + } + + if (str[0] === '{') { + str = "[" + str + } + + if (str[0] === '"') { + str = "[{" + str + } + + return str +} \ No newline at end of file diff --git a/src/lib/prompts/generateSeed.ts b/src/lib/prompts/generateSeed.ts new file mode 100644 index 0000000000000000000000000000000000000000..563e25ec894ab5af54c5025a15a9b7a5918325de --- /dev/null +++ b/src/lib/prompts/generateSeed.ts @@ -0,0 +1,3 @@ +export function generateSeed() { + return Math.floor(Math.random() * Math.pow(2, 31)); +} \ No newline at end of file diff --git a/src/lib/utils/parseMediaProjectionType.ts b/src/lib/utils/parseMediaProjectionType.ts index 7b0110b3f4b49617f33e378418ce5070963ad233..3fc6b2d6e91fe8157cb00313950bd671aa95743d 100644 --- a/src/lib/utils/parseMediaProjectionType.ts +++ b/src/lib/utils/parseMediaProjectionType.ts @@ -1,4 +1,4 @@ -import { parseProjectionFromLoRA } from "@/app/server/actions/utils/parseProjectionFromLoRA" +import { parseProjectionFromLoRA } from "@/app/api/utils/parseProjectionFromLoRA" import { MediaInfo, MediaProjection } from "@/types/general" import { parseAssetToCheckIfGaussian } from "./parseAssetToCheckIfGaussian" diff --git a/src/types/general.ts b/src/types/general.ts index 200a7bca861e8171b2cda3cd1180016ed40d4ba1..110809f98f6be010747c232c693e64a335ae5fbd 100644 --- a/src/types/general.ts +++ b/src/types/general.ts @@ -1,4 +1,5 @@ + export type ProjectionMode = 'cartesian' | 'spherical' export type MouseEventType = "hover" | "click" @@ -83,6 +84,7 @@ export interface RenderedScene { error: string maskUrl: string segments: ImageSegment[] + durationInMs?: number } export interface ImageAnalysisRequest { @@ -608,6 +610,8 @@ export type VideoGenerationModel = | "HotshotXL" | "SVD" | "LaVie" + | "AnimateDiff" + | "SDXL" // yep, we can use images! export type InterfaceDisplayMode = | "desktop" @@ -700,9 +704,17 @@ export type UpdateQueueResponse = { export type AppQueryProps = { params: { id: string } searchParams: { - v?: string | string[], - m?: string | string[], - c?: string | string[], + v?: string | string[], // video id (deprecated) + m?: string | string[], // media id (a better term) + c?: string | string[], // clap id (do we still need this?) + [key: string]: string | string[] | undefined + } +} + +export type LatentQueryProps = { + params: { id: string } + searchParams: { + l?: string | string[], // latent content (serialized to a base64 json) [key: string]: string | string[] | undefined } } diff --git a/tailwind.config.js b/tailwind.config.js index b3b5d0ea06952e714fed0001a48cd9e356a745c6..d9624400c67dfc44ccaea0b20673b3c53298dae5 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -78,6 +78,7 @@ module.exports = { }, }, plugins: [ + require('@tailwindcss/container-queries'), require("tailwindcss-animate"), require("daisyui"), ],