diff --git a/.env b/.env index 7498243cc012e841aeb745fc0247394fc8403ed6..011a4dee019e75adc718b6f2cbbf5f75bc441dc2 100644 --- a/.env +++ b/.env @@ -1,4 +1,8 @@ +API_SECRET_JWT_KEY="" +API_SECRET_JWT_ISSUER="" +API_SECRET_JWT_AUDIENCE="" + NEXT_PUBLIC_DOMAIN="https://aitube.at" NEXT_PUBLIC_SHOW_BETA_FEATURES="false" diff --git a/package-lock.json b/package-lock.json index b0b4fbcbaddb21d79f239016b851692b70b212bf..f02402e8875bfa93f22030d8e9b14da83bee866b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -58,13 +58,14 @@ "eslint": "8.45.0", "eslint-config-next": "13.4.10", "fastest-levenshtein": "^1.0.16", - "gsplat": "^1.2.3", + "gsplat": "^1.2.4", "hash-wasm": "^4.11.0", + "jose": "^5.2.4", "lodash.debounce": "^4.0.8", "lucide-react": "^0.260.0", "markdown-yaml-metadata-parser": "^3.0.0", "minisearch": "^6.3.0", - "next": "^14.1.4", + "next": "^14.2.2", "openai": "^4.36.0", "photo-sphere-viewer-lensflare-plugin": "^2.1.2", "pick": "^0.0.1", @@ -1492,9 +1493,9 @@ } }, "node_modules/@mediapipe/tasks-vision": { - "version": "0.10.13-rc.20240419", - "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.13-rc.20240419.tgz", - "integrity": "sha512-IfIFVSkektnlo9FjqhDYe5LzbOJTdx9kjs65lXmerqPQSOA3akYwp5zt2Rtm1G4JrG1Isv1A15yfiSlqm5pGdQ==" + "version": "0.10.13-rc.20240422", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.13-rc.20240422.tgz", + "integrity": "sha512-yKUS+Qidsw0pttv8Bx/EOdGkcWLH0b1tO4D0HfM6PaBjBAFUr7l5OmRfToFh4k8/XPto6d3X8Ujvm37Da0n2nw==" }, "node_modules/@next/env": { "version": "14.2.2", @@ -1677,9 +1678,9 @@ } }, "node_modules/@photo-sphere-viewer/core": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/core/-/core-5.7.2.tgz", - "integrity": "sha512-5RznXVRwuO+Izceae2SbwYM/H8GHtwxKlT26P4UcRFZYsYKllMAggAz9hhU729Vu+r1+il5PHvomIsmPHVTTaw==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/core/-/core-5.7.3.tgz", + "integrity": "sha512-F2YYQVHwRxrFFvBXdfx0o9rBVOiHgHyMCGgtnJvo4dKVtoUzJdTjXXKVYiOG1ZCVpx1jsyhZeY5DykHnU+7NSw==", "dependencies": { "three": "^0.161.0" } @@ -1690,77 +1691,77 @@ "integrity": "sha512-LC28VFtjbOyEu5b93K0bNRLw1rQlMJ85lilKsYj6dgTu+7i17W+JCCEbvrpmNHF1F3NAUqDSWq50UD7w9H2xQw==" }, "node_modules/@photo-sphere-viewer/equirectangular-video-adapter": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/equirectangular-video-adapter/-/equirectangular-video-adapter-5.7.2.tgz", - "integrity": "sha512-cAaot52nPqa2p77Xp1humRvuxRIa8cqbZ/XRhA8kBToFLT1Ugh9YBcDD7pM/358JtAjicUbLpT7Ioap9iEigxQ==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/equirectangular-video-adapter/-/equirectangular-video-adapter-5.7.3.tgz", + "integrity": "sha512-a0vUihauMhWuNVkp6dWvE1dkmHjMNIe1GEQYUwpduBJlGNabzx7mp3iJNll9NqmLpz85iHvGNpunn5J+zVhfsg==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3" } }, "node_modules/@photo-sphere-viewer/gyroscope-plugin": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/gyroscope-plugin/-/gyroscope-plugin-5.7.2.tgz", - "integrity": "sha512-GHbm96fBfcxzo1RXYsuuhCRxR0TIv3S+3HszyyNKF4x9Nc6dledl26s98ND5ivnviHtAzH/u4eIWHTO3t2+HMg==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/gyroscope-plugin/-/gyroscope-plugin-5.7.3.tgz", + "integrity": "sha512-hS5ePszcR80Lb6ItLRK5xcUMDHsHyf1ebWeEeIwgMfURMdpenRA3phrrlSz8nVmeG/IQB0NRahHubUFfrIv/8g==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3" } }, "node_modules/@photo-sphere-viewer/markers-plugin": { - "version": "5.7.2-fix.1", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/markers-plugin/-/markers-plugin-5.7.2-fix.1.tgz", - "integrity": "sha512-zNJet/ACLBfZgKEl1fCz1qNd6aYey5/2Zqr2++xts0hx179qzA8V3ka2w55SQb/X3WQEWMjomgjtroNd/4wSwg==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/markers-plugin/-/markers-plugin-5.7.3.tgz", + "integrity": "sha512-m4f/vqCAMnwEssHiN1akvnsmD6yWdREI2t7Hs/k+nsbWd/vJ2XKb0iio/JyZWbCJhgsKGZ5sasqzhdxSOQBI8A==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3" } }, "node_modules/@photo-sphere-viewer/overlays-plugin": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/overlays-plugin/-/overlays-plugin-5.7.2.tgz", - "integrity": "sha512-0Jp6yWqMzBWr+TRrp90Ua9+YqzmE1a/m+Trg+8FFnXWDc4u1SWPGvwCa4xk0N/x8b5Vx24UnMCRCTQo8B+Ldxw==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/overlays-plugin/-/overlays-plugin-5.7.3.tgz", + "integrity": "sha512-OaoDXjsG6r5RuC7sKft+tfFFTJE1dbQokZM3/rB34kKbVpLWt9L8NJNBr1oBYyZt+3Fv5EYhL0MsmpTqf/gZTw==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3" } }, "node_modules/@photo-sphere-viewer/resolution-plugin": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/resolution-plugin/-/resolution-plugin-5.7.2.tgz", - "integrity": "sha512-4gUd4yI8o4GEmHMmBeF3+rN+ElrMXxflDxXKRll+VBGCYAsx3vBw0cHm4kVjYh07ep+6uCa35tyg2VFi1P4mrA==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/resolution-plugin/-/resolution-plugin-5.7.3.tgz", + "integrity": "sha512-DbSnIWnwFNa6jeIMGnVsmmuQGn5yZfUN5J9Ew7Xg3CYn5y3HT8EMF/A+yIZQCe7J1AnA30BRhOK1sqgLivDHjA==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2", - "@photo-sphere-viewer/settings-plugin": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3", + "@photo-sphere-viewer/settings-plugin": "5.7.3" } }, "node_modules/@photo-sphere-viewer/settings-plugin": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/settings-plugin/-/settings-plugin-5.7.2.tgz", - "integrity": "sha512-RnDG0dPmLtZ6mWnUhL9dSDhztMYTVCd/PwwvhqL3rIHMxcteFxjVKRdVQyLvxUoWGj9O6bvrHYGLEQM1wMNX1g==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/settings-plugin/-/settings-plugin-5.7.3.tgz", + "integrity": "sha512-YYhDd2xKYpxwnwgTq2h04+Aq19UALJbySc7B7GNlLilWuGE9Uy/u47PfUHkA5rJiUdJ0cOaXi7DOtPQZvPKiBQ==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3" } }, "node_modules/@photo-sphere-viewer/stereo-plugin": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/stereo-plugin/-/stereo-plugin-5.7.2.tgz", - "integrity": "sha512-vL0a0An5rlcMpERkuWRWlAkt5KoS8LovbMImAB0zZ9YFuorp/xZCKuODQ8HADz3YknoH0ohTAnuxusV7c5FQmA==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/stereo-plugin/-/stereo-plugin-5.7.3.tgz", + "integrity": "sha512-UGl9M3YilHcb1HhlTSl+hK+wfVdHrqKj4xSseJ5WDfFV3EErWhpSbg796GX+KWRrvF11EamhkXzAlR7ei5Y4hw==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2", - "@photo-sphere-viewer/gyroscope-plugin": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3", + "@photo-sphere-viewer/gyroscope-plugin": "5.7.3" } }, "node_modules/@photo-sphere-viewer/video-plugin": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/video-plugin/-/video-plugin-5.7.2.tgz", - "integrity": "sha512-vrPV9RCr4HsYiORkto1unDPeUkbN2kbyogvNUoLiQ78M4xkPOqoKxtfxCxTYoM+7gECwNL9VTF81+okck498qA==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/video-plugin/-/video-plugin-5.7.3.tgz", + "integrity": "sha512-bUo6qLT2Tnbc8d/Q7iZEDor2jWqL91ZKgAozxtXB86FqkYtzVoqmjhP008fMRxtoNCJFe8biisjTryYJ7oYp9g==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3" } }, "node_modules/@photo-sphere-viewer/visible-range-plugin": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/visible-range-plugin/-/visible-range-plugin-5.7.2.tgz", - "integrity": "sha512-vvLoDOkQbuG0YQC8NgKPORmfbKXpAn4AFWi/bjB9ziYPsXU9LTVobiIjqAwbHxh6/yzCZoAscRFb601K85H1mA==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@photo-sphere-viewer/visible-range-plugin/-/visible-range-plugin-5.7.3.tgz", + "integrity": "sha512-dO+qHQsTxuiyrhd3ESRxHOZbE0ZCtAotzv8XSPcOrgnnbHp+2AMKKJhCwuQB58vUb9sxHf4NBo13MWU37vZm1g==", "peerDependencies": { - "@photo-sphere-viewer/core": "5.7.2" + "@photo-sphere-viewer/core": "5.7.3" } }, "node_modules/@pkgjs/parseargs": { @@ -3698,9 +3699,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001611", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001611.tgz", - "integrity": "sha512-19NuN1/3PjA3QI8Eki55N8my4LzfkMCRLgCVfrl/slbSAchQfV0+GwjPrK3rq37As4UCLlM/DHajbKkAqbv92Q==", + "version": "1.0.30001612", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001612.tgz", + "integrity": "sha512-lFgnZ07UhaCcsSZgWW0K5j4e69dK1u/ltrL9lTUiFOwNHs12S3UMIEYgBV0Z6C6hRDev7iRnMzzYmKabYdXF9g==", "funding": [ { "type": "opencollective", @@ -4272,9 +4273,9 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, "node_modules/electron-to-chromium": { - "version": "1.4.744", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.744.tgz", - "integrity": "sha512-nAGcF0yeKKfrP13LMFr5U1eghfFSvFLg302VUFzWlcjPOnUYd52yU5x6PBYrujhNbc4jYmZFrGZFK+xasaEzVA==" + "version": "1.4.746", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.746.tgz", + "integrity": "sha512-jeWaIta2rIG2FzHaYIhSuVWqC6KJYo7oSBX4Jv7g+aVujKztfvdpf+n6MGwZdC5hQXbax4nntykLH2juIQrfPg==" }, "node_modules/elliptic": { "version": "6.5.4", @@ -6005,6 +6006,14 @@ "jiti": "bin/jiti.js" } }, + "node_modules/jose": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.2.4.tgz", + "integrity": "sha512-6ScbIk2WWCeXkmzF6bRPmEuaqy1m8SbsRFMa/FLrSCkGIhj8OLVG/IH+XHVmNMx/KUo8cVWEE6oKR4dJ+S0Rkg==", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-sha3": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz", @@ -6606,9 +6615,9 @@ } }, "node_modules/openai": { - "version": "4.38.1", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.38.1.tgz", - "integrity": "sha512-nmSKE9O2piuoh9+AgDqwGHojIFSxToQ2jJqwaxjbzz2ebdD5LYY9s+bMe25b18t4QEgvtgW70JfK8BU3xf5dRw==", + "version": "4.38.3", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.38.3.tgz", + "integrity": "sha512-mIL9WtrFNOanpx98mJ+X/wkoepcxdqqu0noWFoNQHl/yODQ47YM7NEYda7qp8JfjqpLFVxY9mQhshoS/Fqac0A==", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", @@ -8217,9 +8226,9 @@ } }, "node_modules/type-fest": { - "version": "4.15.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.15.0.tgz", - "integrity": "sha512-tB9lu0pQpX5KJq54g+oHOLumOx+pMep4RaM6liXh2PKmVRFF+/vAtUP0ZaJ0kOySfVNjF6doBWPHhBhISKdlIA==", + "version": "4.16.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.16.0.tgz", + "integrity": "sha512-z7Rf5PXxIhbI6eJBTwdqe5bO02nUUmctq4WqviFSstBAWV0YNtEQRhEnZw73WJ8sZOqgFG6Jdl8gYZu7NBJZnA==", "engines": { "node": ">=16" }, diff --git a/package.json b/package.json index 1e63cc39dd0316bd960427d84248ddf9ad2c0d34..88a1c8f0462f2de189c2ff8334ffef7bc9e702be 100644 --- a/package.json +++ b/package.json @@ -59,13 +59,14 @@ "eslint": "8.45.0", "eslint-config-next": "13.4.10", "fastest-levenshtein": "^1.0.16", - "gsplat": "^1.2.3", + "gsplat": "^1.2.4", "hash-wasm": "^4.11.0", + "jose": "^5.2.4", "lodash.debounce": "^4.0.8", "lucide-react": "^0.260.0", "markdown-yaml-metadata-parser": "^3.0.0", "minisearch": "^6.3.0", - "next": "^14.1.4", + "next": "^14.2.2", "openai": "^4.36.0", "photo-sphere-viewer-lensflare-plugin": "^2.1.2", "pick": "^0.0.1", diff --git a/public/blanks/blank_1sec_1024x576.webm b/public/blanks/blank_1sec_1024x576.webm new file mode 100644 index 0000000000000000000000000000000000000000..d139bc08c1653ebe99b3af1dce1445a5491f2b97 Binary files /dev/null and b/public/blanks/blank_1sec_1024x576.webm differ diff --git a/public/blanks/blank_1sec_512x288.webm b/public/blanks/blank_1sec_512x288.webm new file mode 100644 index 0000000000000000000000000000000000000000..2be77d4395f12b067ac0101fb0eb5cf38c5c7f6e Binary files /dev/null and b/public/blanks/blank_1sec_512x288.webm differ diff --git a/public/logos/latent-engine/latent-engine.png b/public/logos/latent-engine/latent-engine.png new file mode 100644 index 0000000000000000000000000000000000000000..458a8013e16f96492e72ff66171babf3a1f16315 Binary files /dev/null and b/public/logos/latent-engine/latent-engine.png differ diff --git a/public/logos/latent-engine/latent-engine.xcf b/public/logos/latent-engine/latent-engine.xcf new file mode 100644 index 0000000000000000000000000000000000000000..ebf68bb237c82e2826da9222ddaa3ef8c92f3d75 Binary files /dev/null and b/public/logos/latent-engine/latent-engine.xcf differ diff --git a/src/app/api/auth/getToken.ts b/src/app/api/auth/getToken.ts new file mode 100644 index 0000000000000000000000000000000000000000..f2550f7006e8ff27deb88fdbfddfe75b48fc9b24 --- /dev/null +++ b/src/app/api/auth/getToken.ts @@ -0,0 +1,20 @@ +import { createSecretKey } from "crypto" +import { SignJWT } from "jose" + +// https://jmswrnr.com/blog/protecting-next-js-api-routes-query-parameters + +export async function getToken(data: Record = {}): Promise { + const secretKey = createSecretKey(`${process.env.API_SECRET_JWT_KEY || ""}`, 'utf-8'); + + const jwtToken = await new SignJWT(data) + .setProtectedHeader({ + alg: 'HS256' + }) // algorithm + .setIssuedAt() + .setIssuer(`${process.env.API_SECRET_JWT_ISSUER || ""}`) // issuer + .setAudience(`${process.env.API_SECRET_JWT_AUDIENCE || ""}`) // audience + .setExpirationTime("1 day") // token expiration time - to prevent hackers from re-using our URLs more than a day + .sign(secretKey); // secretKey generated from previous step + + return jwtToken +} diff --git a/src/app/api/generate/story/route.ts b/src/app/api/generate/story/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..0cec2273abc181fd2514a135e3d0276abeb508c0 --- /dev/null +++ b/src/app/api/generate/story/route.ts @@ -0,0 +1,28 @@ +import { NextResponse, NextRequest } from "next/server" + +import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory" +import { serializeClap } from "@/lib/clap/serializeClap" + +// a helper to generate Clap stories from a few sentences +// this is mostly used by external apps such as the Stories Factory +export async function POST(req: NextRequest) { + + const request = await req.json() as { + story: string[] + // can add more stuff for the V2 of Stories Factory + } + + const story = Array.isArray(request?.story) ? request.story : [] + + if (!story.length) { throw new Error(`please provide at least oen sentence for the story`) } + + const clap = generateClapFromSimpleStory({ + story, + // can add more stuff for the V2 of Stories Factory + }) + + return new NextResponse(await serializeClap(clap), { + status: 200, + headers: new Headers({ "content-type": "application/x-gzip" }), + }) +} diff --git a/src/app/api/generate/storyboards/route.ts b/src/app/api/generate/storyboards/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..bcec7074e11e8babe582ad62b216147c02367bf7 --- /dev/null +++ b/src/app/api/generate/storyboards/route.ts @@ -0,0 +1,84 @@ +import { NextResponse, NextRequest } from "next/server" + +import { serializeClap } from "@/lib/clap/serializeClap" +import { parseClap } from "@/lib/clap/parseClap" +import { startOfSegment1IsWithinSegment2 } from "@/lib/utils/startOfSegment1IsWithinSegment2" +import { getVideoPrompt } from "@/components/interface/latent-engine/core/prompts/getVideoPrompt" +import { newSegment } from "@/lib/clap/newSegment" +import { generateImage } from "@/components/interface/latent-engine/resolvers/image/generateImage" +import { getToken } from "@/app/api/auth/getToken" + +// a helper to generate storyboards for a Clap +// this is mostly used by external apps such as the Stories Factory +// this function will: +// +// - add missing storyboards to the shots +// - add missing storyboard prompts +// - add missing storyboard images +export async function POST(req: NextRequest) { + + const jwtToken = await getToken({ user: "anonymous" }) + + const blob = await req.blob() + const clap = await parseClap(blob) + + if (!clap.segments) { throw new Error(`no segment found in the provided clap!`) } + + const shotsSegments = clap.segments.filter(s => s.category === "camera") + + if (shotsSegments.length > 32) { + throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`) + } + + for (const shotSegment of shotsSegments) { + + const shotSegments = clap.segments.filter(s => + startOfSegment1IsWithinSegment2(s, shotSegment) + ) + + const shotStoryboardSegments = shotSegments.filter(s => + s.category === "storyboard" + ) + + let shotStoryboardSegment = shotStoryboardSegments.at(0) + + // TASK 1: GENERATE MISSING STORYBOARD SEGMENT + if (!shotStoryboardSegment) { + shotStoryboardSegment = newSegment({ + track: 1, + startTimeInMs: shotSegment.startTimeInMs, + endTimeInMs: shotSegment.endTimeInMs, + assetDurationInMs: shotSegment.assetDurationInMs, + category: "storyboard", + prompt: "", + assetUrl: "", + outputType: "image" + }) + } + + // TASK 2: GENERATE MISSING STORYBOARD PROMPT + if (!shotStoryboardSegment.prompt) { + // storyboard is missing, let's generate it + shotStoryboardSegment.prompt = getVideoPrompt(shotSegments, {}, []) + } + + // TASK 3: GENERATE MISSING STORYBOARD BITMAP + if (!shotStoryboardSegment.assetUrl) { + // note this will do a fetch to AiTube API + // which is a bit weird since we are already inside the API, but it works + //TODO Julian: maybe we could use an internal function call instead? + shotStoryboardSegment.assetUrl = await generateImage({ + prompt: shotStoryboardSegment.prompt, + width: clap.meta.width, + height: clap.meta.height, + token: jwtToken, + }) + } + } + // TODO: generate the storyboards for the clap + + return new NextResponse(await serializeClap(clap), { + status: 200, + headers: new Headers({ "content-type": "application/x-gzip" }), + }) +} diff --git a/src/app/api/generate/video/route.ts b/src/app/api/generate/video/route.ts new file mode 100644 index 0000000000000000000000000000000000000000..7e19009f162d8c7f10a121f6d7803f746a95766f --- /dev/null +++ b/src/app/api/generate/video/route.ts @@ -0,0 +1,19 @@ +import { NextResponse, NextRequest } from "next/server" + +// we hide/wrap the micro-service under a unified AiTube API +export async function POST(req: NextRequest, res: NextResponse) { + NextResponse.redirect("https://jbilcke-hf-ai-tube-clap-exporter.hf.space") +} +/* +Alternative solution (in case the redirect doesn't work): + +We could also grab the blob and forward it, like this: + + const data = fetch( + "https://jbilcke-hf-ai-tube-clap-exporter.hf.space", + { method: "POST", body: await req.blob() } + ) + const blob = data.blob() + +Then return the blob with the right Content-Type using NextResponse +*/ \ No newline at end of file diff --git a/src/app/api/generators/clap/generateClap.ts b/src/app/api/generators/clap/generateClap.ts index ac4faef7c552c1aa33dba8d7cd56716393c1d52d..fc29ba2ba1622053d4e29e4494846fb5838835f8 100644 --- a/src/app/api/generators/clap/generateClap.ts +++ b/src/app/api/generators/clap/generateClap.ts @@ -39,7 +39,8 @@ export async function generateClap({ defaultVideoModel: "SDXL", extraPositivePrompt: [], screenplay: "", - streamType: "interactive" + isLoop: true, + isInteractive: true, } }) diff --git a/src/app/api/resolvers/image/route.ts b/src/app/api/resolvers/image/route.ts index bd796605bb88f44eb055ed67a574f24c69e636f2..7f7f91d8aac1c46ea67880e7dbb5cbc4b2e6e7af 100644 --- a/src/app/api/resolvers/image/route.ts +++ b/src/app/api/resolvers/image/route.ts @@ -1,4 +1,6 @@ import { NextResponse, NextRequest } from "next/server" +import { createSecretKey } from "node:crypto" + import queryString from "query-string" import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain" @@ -6,13 +8,37 @@ import { generateSeed } from "@/lib/utils/generateSeed" import { sleep } from "@/lib/utils/sleep" import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts" import { getContentType } from "@/lib/data/getContentType" +import { getValidNumber } from "@/lib/utils/getValidNumber" + +const secretKey = createSecretKey(`${process.env.API_SECRET_JWT_KEY || ""}`, 'utf-8'); export async function GET(req: NextRequest) { -const qs = queryString.parseUrl(req.url || "") -const query = (qs || {}).query + const qs = queryString.parseUrl(req.url || "") + const query = (qs || {}).query -let prompt = "" + /* + TODO: check the validity of the JWT token + let token = "" + try { + token = decodeURIComponent(query?.t?.toString() || "").trim() + + // verify token + const { payload, protectedHeader } = await jwtVerify(token, secretKey, { + issuer: `${process.env.API_SECRET_JWT_ISSUER || ""}`, // issuer + audience: `${process.env.API_SECRET_JWT_AUDIENCE || ""}`, // audience + }); + // log values to console + console.log(payload); + console.log(protectedHeader); + } catch (err) { + // token verification failed + console.log("Token is invalid"); + return NextResponse.json({ error: `access denied ${err}` }, { status: 400 }); + } + */ + + let prompt = "" try { prompt = decodeURIComponent(query?.p?.toString() || "").trim() } catch (err) {} @@ -20,6 +46,19 @@ let prompt = "" return NextResponse.json({ error: 'no prompt provided' }, { status: 400 }); } + let width = 512 + try { + const rawString = decodeURIComponent(query?.w?.toString() || "").trim() + width = getValidNumber(rawString, 256, 8192, 512) + } catch (err) {} + + let height = 288 + try { + const rawString = decodeURIComponent(query?.h?.toString() || "").trim() + height = getValidNumber(rawString, 256, 8192, 288) + } catch (err) {} + + let format = "binary" try { const f = decodeURIComponent(query?.f?.toString() || "").trim() @@ -36,8 +75,8 @@ let prompt = "" nbFrames: 1, nbFPS: 1, nbSteps: 8, - width: 1024, - height: 576, + width, + height, turbo: true, shouldRenewCache: true, seed: generateSeed() diff --git a/src/app/api/resolvers/video/route.ts b/src/app/api/resolvers/video/route.ts index b57097a0ec9f27ec7ddb61801f2acd9221b8936b..1e94bdefdce471c2d78e12ba2a9deb03487d19dc 100644 --- a/src/app/api/resolvers/video/route.ts +++ b/src/app/api/resolvers/video/route.ts @@ -1,17 +1,43 @@ import { NextResponse, NextRequest } from "next/server" import queryString from "query-string" +import { createSecretKey } from "crypto" +import { jwtVerify } from "jose" import { newRender, getRender } from "../../providers/videochain/renderWithVideoChain" import { generateSeed } from "@/lib/utils/generateSeed" import { sleep } from "@/lib/utils/sleep" import { getNegativePrompt, getPositivePrompt } from "../../utils/imagePrompts" import { getContentType } from "@/lib/data/getContentType" +import { whoAmI, WhoAmIUser } from "@huggingface/hub" +import { getValidNumber } from "@/lib/utils/getValidNumber" + +const secretKey = createSecretKey(`${process.env.API_SECRET_JWT_KEY || ""}`, 'utf-8'); export async function GET(req: NextRequest) { const qs = queryString.parseUrl(req.url || "") const query = (qs || {}).query + /* + TODO Julian: check the validity of the JWT token + let token = "" + try { + token = decodeURIComponent(query?.t?.toString() || "").trim() + + // verify token + const { payload, protectedHeader } = await jwtVerify(token, secretKey, { + issuer: `${process.env.API_SECRET_JWT_ISSUER || ""}`, // issuer + audience: `${process.env.API_SECRET_JWT_AUDIENCE || ""}`, // audience + }); + // log values to console + console.log(payload); + console.log(protectedHeader); + } catch (err) { + // token verification failed + console.log("Token is invalid"); + return NextResponse.json({ error: `access denied ${err}` }, { status: 400 }); + } + */ let prompt = "" try { @@ -22,6 +48,19 @@ export async function GET(req: NextRequest) { return NextResponse.json({ error: 'no prompt provided' }, { status: 400 }); } + let width = 512 + try { + const rawString = decodeURIComponent(query?.w?.toString() || "").trim() + width = getValidNumber(rawString, 256, 8192, 512) + } catch (err) {} + + let height = 288 + try { + const rawString = decodeURIComponent(query?.h?.toString() || "").trim() + height = getValidNumber(rawString, 256, 8192, 288) + } catch (err) {} + + let format = "binary" try { const f = decodeURIComponent(query?.f?.toString() || "").trim() @@ -40,19 +79,52 @@ export async function GET(req: NextRequest) { // ATTENTION: changing those will slow things to 5-6s of loading time (compared to 3-4s) // and with no real visible change - nbFrames: 20, // apparently the model can only do 2 seconds at 10, so be it + // ATTENTION! if you change those values, + // please make sure that the backend API can support them, + // and also make sure to update the Zustand store values in the frontend: + // videoModelFPS: number + // videoModelNumOfFrames: number + // videoModelDurationInSec: number + // + // note: internally, the model can only do 16 frames at 10 FPS + // (1.6 second of video) + // but I have added a FFmpeg interpolation step, which adds some + // overhead (2-3 secs) but at least can help smooth things out, or make + // them artificially longer + + // those settings are pretty good, takes about 2.9,, 3.1 seconds to compute + // represents 3 secs of 16fps + + + // with those parameters, we can generate a 2.584s long video at 24 FPS + // note that there is a overhead due to smoothing, + // on the A100 it takes betwen 5.3 and 7 seconds to compute + // although it will appear a bit "slo-mo" + // since the original is a 1.6s long video at 10 FPS + nbFrames: 80, + nbFPS: 24, + + + // nbFrames: 48, + // nbFPS: 24, - nbFPS: 10, + // it generated about: + // 24 frames + // 2.56s run time // possibles values are 1, 2, 4, and 8 - // but I don't see much improvements with 8 to be honest - // the best one seems to be 4 + // but with 2 steps the video "flashes" and it creates monstruosity + // like fishes with 2 tails etc + // and with 8 steps I don't see much improvements with 8 to be honest nbSteps: 4, // this corresponds roughly to 16:9 // which is the aspect ratio video used by AiTube - // unfortunately, this is too compute intensive, so we have to take half of that + // unfortunately, this is too compute intensive, + // and it creates monsters like two-headed fishes + // (although this artifact could probably be fixed with more steps, + // but we cannot afford those) // width: 1024, // height: 576, @@ -68,10 +140,15 @@ export async function GET(req: NextRequest) { // // that's not the only constraint: you also need to respect this: // `height` and `width` have to be divisible by 8 (use 32 to be safe) - // width: 512, - // height: 288, - width: 456, // 512, - height: 256, // 288, + width, + height, + + // we save about 500ms if we go below, + // but there we will be some deformed artifacts as the model + // doesn't perform well below 512px + // it also makes things more "flashy" + // width: 456, // 512, + // height: 256, // 288, turbo: true, // without much effect for videos as of now, as we only supports turbo (AnimateDiff Lightning) shouldRenewCache: true, diff --git a/src/app/dream/page.tsx b/src/app/dream/spoiler.tsx similarity index 54% rename from src/app/dream/page.tsx rename to src/app/dream/spoiler.tsx index 932f1e346cabae775941b6c16da77163fa659778..e451af5c363652190eeda9959a4ed5556d2d54a0 100644 --- a/src/app/dream/page.tsx +++ b/src/app/dream/spoiler.tsx @@ -1,18 +1,20 @@ - - import { LatentQueryProps } from "@/types/general" import { Main } from "../main" -import { searchResultToMediaInfo } from "../api/generators/search/searchResultToMediaInfo" -import { LatentSearchResult } from "../api/generators/search/types" -import { serializeClap } from "@/lib/clap/serializeClap" -import { getMockClap } from "@/lib/clap/getMockClap" +import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory" import { clapToDataUri } from "@/lib/clap/clapToDataUri" import { getNewMediaInfo } from "../api/generators/search/getNewMediaInfo" +import { getToken } from "../api/auth/getToken" -export default async function DreamPage({ searchParams: { - l: latentContent, -} }: LatentQueryProps) { +// https://jmswrnr.com/blog/protecting-next-js-api-routes-query-parameters + +export default async function DreamPage({ + searchParams: { + l: latentContent, + }, + ...rest +}: LatentQueryProps) { + const jwtToken = await getToken({ user: "anonymous" }) // const latentSearchResult = JSON.parse(atob(`${latentContent}`)) as LatentSearchResult @@ -24,10 +26,13 @@ export default async function DreamPage({ searchParams: { const latentMedia = getNewMediaInfo() latentMedia.clapUrl = await clapToDataUri( - getMockClap({showDisclaimer: true }) + generateClapFromSimpleStory({ + showIntroPoweredByEngine: false, + showIntroDisclaimerAboutAI: false + }) ) return ( -
- ) +
+ ) } \ No newline at end of file diff --git a/src/app/main.tsx b/src/app/main.tsx index a95d4f79be09fa073f151ef61768ea05d7edc6c6..6c688daff5eacab0693824782bd7f75a8944cb26 100644 --- a/src/app/main.tsx +++ b/src/app/main.tsx @@ -29,6 +29,8 @@ import { PublicLatentMediaView } from "./views/public-latent-media-view" // one benefit of doing this is that we will able to add some animations/transitions // more easily export function Main({ + jwtToken, + // view, publicMedia, publicMedias, @@ -42,25 +44,31 @@ export function Main({ publicTrack, channel, }: { - // server side params - // view?: InterfaceView - publicMedia?: MediaInfo - publicMedias?: MediaInfo[] - - latentMedia?: MediaInfo - latentMedias?: MediaInfo[] - - publicChannelVideos?: MediaInfo[] - - publicTracks?: MediaInfo[] - publicTrack?: MediaInfo - - channel?: ChannelInfo + // token used to secure communications between the Next frontend and the Next API + // this doesn't necessarily mean the user has to be logged it: + // we can use this for anonymous visitors too. + jwtToken?: string + + // server side params + // view?: InterfaceView + publicMedia?: MediaInfo + publicMedias?: MediaInfo[] + + latentMedia?: MediaInfo + latentMedias?: MediaInfo[] + + publicChannelVideos?: MediaInfo[] + + publicTracks?: MediaInfo[] + publicTrack?: MediaInfo + + channel?: ChannelInfo }) { // this could be also a parameter of main, where we pass this manually const pathname = usePathname() const router = useRouter() + const setJwtToken = useStore(s => s.setJwtToken) const setPublicMedia = useStore(s => s.setPublicMedia) const setView = useStore(s => s.setView) const setPathname = useStore(s => s.setPathname) @@ -72,6 +80,12 @@ export function Main({ const setPublicTracks = useStore(s => s.setPublicTracks) const setPublicTrack = useStore(s => s.setPublicTrack) + useEffect(() => { + if (typeof jwtToken !== "string" && !jwtToken) { return } + setJwtToken(jwtToken) + }, [jwtToken]) + + useEffect(() => { if (!publicMedias?.length) { return } // note: it is important to ALWAYS set the current video to videoId diff --git a/src/app/state/useStore.ts b/src/app/state/useStore.ts index 8b9aa4c2690c89523973024584ee0c1765f020ec..47767afde15d1acd42b832c97ccf65425e67b9e9 100644 --- a/src/app/state/useStore.ts +++ b/src/app/state/useStore.ts @@ -19,6 +19,9 @@ export const useStore = create<{ setPathname: (pathname: string) => void + jwtToken: string + setJwtToken: (jwtToken: string) => void + searchQuery: string setSearchQuery: (searchQuery?: string) => void @@ -131,6 +134,11 @@ export const useStore = create<{ set({ view: routes[pathname] || "not_found" }) }, + jwtToken: "", + setJwtToken: (jwtToken: string) => { + set({ jwtToken }) + }, + searchAutocompleteQuery: "", setSearchAutocompleteQuery: (searchAutocompleteQuery?: string) => { set({ searchAutocompleteQuery }) diff --git a/src/components/interface/latent-engine/components/content-layer/index.tsx b/src/components/interface/latent-engine/components/content-layer/index.tsx index 731dc09d6f3eb985b4153dad8dc9a6265fc79219..1dd8956a902a66335bf317d191d684a505d16645 100644 --- a/src/components/interface/latent-engine/components/content-layer/index.tsx +++ b/src/components/interface/latent-engine/components/content-layer/index.tsx @@ -26,7 +26,7 @@ export const ContentLayer = forwardRef(function ContentLayer({ ref={ref} onClick={onClick} > -
+
{children}
diff --git a/src/components/interface/latent-engine/components/disclaimers/this-is-ai.tsx b/src/components/interface/latent-engine/components/intros/ai-content-disclaimer.tsx similarity index 95% rename from src/components/interface/latent-engine/components/disclaimers/this-is-ai.tsx rename to src/components/interface/latent-engine/components/intros/ai-content-disclaimer.tsx index f800e07aa20c7c550b3716cfa92f1680f6cb1c75..ebd47bdff5fd972c682081dcad71ce6ebde2b2f9 100644 --- a/src/components/interface/latent-engine/components/disclaimers/this-is-ai.tsx +++ b/src/components/interface/latent-engine/components/intros/ai-content-disclaimer.tsx @@ -4,12 +4,11 @@ import React from "react" import { cn } from "@/lib/utils/cn" import { arimoBold, arimoNormal } from "@/lib/fonts" -import { ClapStreamType } from "@/lib/clap/types" -export function ThisIsAI({ - streamType, +export function AIContentDisclaimer({ + isInteractive = false, }: { - streamType?: ClapStreamType + isInteractive?: boolean } = {}) { return ( @@ -59,7 +58,7 @@ export function ThisIsAI({ */ } content { - streamType !== "static" + isInteractive ? "will be" : "has been" }
+ +
+ Latent Engine logo +
+
+ ) +} \ No newline at end of file diff --git a/src/components/interface/latent-engine/core/engine.tsx b/src/components/interface/latent-engine/core/engine.tsx index a8a26acdc62ced50eed537b90a331668e79d3df2..3e218176bfa1f94efb8559d1ba8a61878fbdf972 100644 --- a/src/components/interface/latent-engine/core/engine.tsx +++ b/src/components/interface/latent-engine/core/engine.tsx @@ -1,16 +1,20 @@ "use client" import React, { MouseEventHandler, useEffect, useRef, useState } from "react" +import { useLocalStorage } from "usehooks-ts" import { cn } from "@/lib/utils/cn" +import { MediaInfo } from "@/types/general" +import { serializeClap } from "@/lib/clap/serializeClap" +import { generateClapFromSimpleStory } from "@/lib/clap/generateClapFromSimpleStory" -import { useLatentEngine } from "../store/useLatentEngine" +import { useLatentEngine } from "./useLatentEngine" import { PlayPauseButton } from "../components/play-pause-button" -import { StreamTag } from "../../stream-tag" +import { StaticOrInteractiveTag } from "../../static-or-interactive-tag" import { ContentLayer } from "../components/content-layer" -import { MediaInfo } from "@/types/general" -import { getMockClap } from "@/lib/clap/getMockClap" -import { serializeClap } from "@/lib/clap/serializeClap" +import { localStorageKeys } from "@/app/state/localStorageKeys" +import { defaultSettings } from "@/app/state/defaultSettings" +import { useStore } from "@/app/state/useStore" function LatentEngine({ media, @@ -22,19 +26,35 @@ function LatentEngine({ height?: number className?: string }) { + // used to prevent people from opening multiple sessions at the same time + // note: this should also be enforced with the Hugging Face ID + const [multiTabsLock, setMultiTabsLock] = useLocalStorage( + "AI_TUBE_ENGINE_MULTI_TABS_LOCK", + Date.now() + ) + + const [huggingfaceApiKey, setHuggingfaceApiKey] = useLocalStorage( + localStorageKeys.huggingfaceApiKey, + defaultSettings.huggingfaceApiKey + ) + + // note here how we transfer the info from one store to another + const jwtToken = useStore(s => s.jwtToken) + const setJwtToken = useLatentEngine(s => s.setJwtToken) + useEffect(() => { + setJwtToken(jwtToken) + }, [jwtToken]) + + const setContainerDimension = useLatentEngine(s => s.setContainerDimension) const isLoaded = useLatentEngine(s => s.isLoaded) const imagine = useLatentEngine(s => s.imagine) const open = useLatentEngine(s => s.open) - const setImageElement = useLatentEngine(s => s.setImageElement) - const setVideoElement = useLatentEngine(s => s.setVideoElement) - const setSegmentationElement = useLatentEngine(s => s.setSegmentationElement) - - const simulationVideoPlaybackFPS = useLatentEngine(s => s.simulationVideoPlaybackFPS) - const simulationRenderingTimeFPS = useLatentEngine(s => s.simulationRenderingTimeFPS) + const videoSimulationVideoPlaybackFPS = useLatentEngine(s => s.videoSimulationVideoPlaybackFPS) + const videoSimulationRenderingTimeFPS = useLatentEngine(s => s.videoSimulationRenderingTimeFPS) - const streamType = useLatentEngine(s => s.streamType) + const isLoop = useLatentEngine(s => s.isLoop) const isStatic = useLatentEngine(s => s.isStatic) const isLive = useLatentEngine(s => s.isLive) const isInteractive = useLatentEngine(s => s.isInteractive) @@ -42,11 +62,8 @@ function LatentEngine({ const isPlaying = useLatentEngine(s => s.isPlaying) const togglePlayPause = useLatentEngine(s => s.togglePlayPause) - const videoLayer = useLatentEngine(s => s.videoLayer) - const segmentationLayer = useLatentEngine(s => s.segmentationLayer) - const interfaceLayer = useLatentEngine(s => s.interfaceLayer) - const videoElement = useLatentEngine(s => s.videoElement) - const imageElement = useLatentEngine(s => s.imageElement) + const videoLayers = useLatentEngine(s => s.videoLayers) + const interfaceLayers = useLatentEngine(s => s.interfaceLayers) const onClickOnSegmentationLayer = useLatentEngine(s => s.onClickOnSegmentationLayer) @@ -70,7 +87,7 @@ function LatentEngine({ // there is a bug, we can't unpack the .clap when it's from a data-uri :/ // open(mediaUrl) - const mockClap = getMockClap() + const mockClap = generateClapFromSimpleStory() const mockArchive = await serializeClap(mockClap) // for some reason conversion to data uri doesn't work // const mockDataUri = await blobToDataUri(mockArchive, "application/x-gzip") @@ -91,7 +108,7 @@ function LatentEngine({ setOverlayVisible(!isPlayingRef.current) } clearTimeout(overlayTimerRef.current) - }, 1000) + }, 3000) } /* @@ -109,32 +126,6 @@ function LatentEngine({ }, [isPlaying]) */ - useEffect(() => { - if (!videoLayerRef.current) { return } - - // note how in both cases we are pulling from the videoLayerRef - // that's because one day everything will be a video, but for now we - // "fake it until we make it" - const videoElements = Array.from( - videoLayerRef.current.querySelectorAll('.latent-video') - ) as HTMLVideoElement[] - setVideoElement(videoElements.at(0)) - - // images are used for simpler or static experiences - const imageElements = Array.from( - videoLayerRef.current.querySelectorAll('.latent-image') - ) as HTMLImageElement[] - setImageElement(imageElements.at(0)) - - - if (!segmentationLayerRef.current) { return } - - const segmentationElements = Array.from( - segmentationLayerRef.current.querySelectorAll('.segmentation-canvas') - ) as HTMLCanvasElement[] - setSegmentationElement(segmentationElements.at(0)) - - }) useEffect(() => { setContainerDimension({ width: width || 256, height: height || 256 }) @@ -161,9 +152,26 @@ function LatentEngine({ height={height} ref={videoLayerRef} onClick={onClickOnSegmentationLayer} - >{videoLayer} - - + >{videoLayers.map(({ id }) => ( +