Commit
•
bf8d4d8
1
Parent(s):
637dd5c
working on the menu
Browse files- src/app/engines.ts +28 -0
- src/app/main.tsx +55 -16
- src/app/types.ts +0 -1
- src/components/business/{image-renderer.tsx → renderer.tsx} +42 -19
- src/components/business/video-renderer.tsx +0 -22
src/app/engines.ts
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
export type EngineType = "image" | "video"
|
3 |
+
|
4 |
+
export interface Engine {
|
5 |
+
type: EngineType
|
6 |
+
label: string
|
7 |
+
modelName: string
|
8 |
+
modelUrl: string
|
9 |
+
}
|
10 |
+
|
11 |
+
export const engines: Record<string, Engine> = {
|
12 |
+
image: {
|
13 |
+
type: "image",
|
14 |
+
label: "Image",
|
15 |
+
modelName: "SDXL 1.0",
|
16 |
+
modelUrl: "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
|
17 |
+
},
|
18 |
+
video: {
|
19 |
+
type: "video",
|
20 |
+
label: "Video",
|
21 |
+
modelName: "Zeroscope V2 576w",
|
22 |
+
modelUrl: "https://huggingface.co/cerspense/zeroscope_v2_576w",
|
23 |
+
}
|
24 |
+
}
|
25 |
+
|
26 |
+
export const defaultEngine: EngineType = "image"
|
27 |
+
|
28 |
+
export const getEngine = (type?: EngineType): Engine => engines[type || defaultEngine] || engines[defaultEngine]
|
src/app/main.tsx
CHANGED
@@ -4,7 +4,7 @@ import { useEffect, useRef, useState, useTransition } from "react"
|
|
4 |
import { usePathname, useRouter, useSearchParams } from "next/navigation"
|
5 |
|
6 |
|
7 |
-
import {
|
8 |
|
9 |
import {
|
10 |
Select,
|
@@ -22,6 +22,7 @@ import { defaultGame, games, getGame } from "./games"
|
|
22 |
import { getBackground } from "@/app/queries/getBackground"
|
23 |
import { getDialogue } from "@/app/queries/getDialogue"
|
24 |
import { getActionnables } from "@/app/queries/getActionnables"
|
|
|
25 |
|
26 |
export default function Main() {
|
27 |
const [isPending, startTransition] = useTransition()
|
@@ -36,10 +37,13 @@ export default function Main() {
|
|
36 |
const searchParams = useSearchParams()
|
37 |
|
38 |
const requestedGame = (searchParams.get('game') as GameType) || defaultGame
|
39 |
-
console.log("requestedGame: " + requestedGame)
|
40 |
const gameRef = useRef<GameType>(requestedGame)
|
41 |
-
console.log("gameRef.current: " + gameRef.current)
|
42 |
const [game, setGame] = useState<Game>(getGame(gameRef.current))
|
|
|
|
|
|
|
|
|
|
|
43 |
const [situation, setSituation] = useState("")
|
44 |
const [scene, setScene] = useState("")
|
45 |
const [dialogue, setDialogue] = useState("")
|
@@ -66,7 +70,7 @@ export default function Main() {
|
|
66 |
)
|
67 |
|
68 |
// detect if type game type changed while we were busy
|
69 |
-
if (game
|
70 |
console.log("game type changed! aborting..")
|
71 |
return
|
72 |
}
|
@@ -153,21 +157,35 @@ export default function Main() {
|
|
153 |
window.history.pushState({path: newurl}, '', newurl)
|
154 |
}
|
155 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
156 |
return (
|
157 |
<div
|
158 |
-
className=
|
159 |
-
"flex flex-col w-full pt-4",
|
160 |
-
getGame(gameRef.current).className // apply the game theme
|
161 |
-
].join(" ")}
|
162 |
>
|
163 |
-
<div className="flex flex-
|
164 |
-
|
165 |
-
<label className="flex">Select a story:</label>
|
166 |
<Select
|
167 |
defaultValue={gameRef.current}
|
168 |
onValueChange={(value) => { handleSelectGame(value as GameType) }}>
|
169 |
-
<SelectTrigger className="
|
170 |
-
<SelectValue placeholder="Type" />
|
171 |
</SelectTrigger>
|
172 |
<SelectContent>
|
173 |
{Object.entries(games).map(([key, game]) =>
|
@@ -176,23 +194,44 @@ export default function Main() {
|
|
176 |
</SelectContent>
|
177 |
</Select>
|
178 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
179 |
<p className="text-xl">A stable diffusion exploration game. Click on an item to explore a new scene!</p>
|
180 |
<div className="flex flex-row">
|
181 |
<div className="text-xl mr-2">🔎 Clickable items:</div>
|
182 |
{clickables.map((clickable, i) =>
|
183 |
<div key={i} className="flex flex-row text-xl mr-2">
|
184 |
<div className="">{clickable}</div>
|
185 |
-
{i < (
|
186 |
</div>)}
|
187 |
</div>
|
188 |
<p className="text-xl font-normal">You are looking at: <span className="font-bold">{hoveredActionnable || "nothing"}</span></p>
|
189 |
</div>
|
190 |
-
<
|
191 |
rendered={rendered}
|
192 |
onUserAction={handleUserAction}
|
193 |
onUserHover={setHoveredActionnable}
|
194 |
isLoading={isLoading}
|
195 |
-
|
|
|
196 |
/>
|
197 |
<p className="text-xl">{dialogue}</p>
|
198 |
</div>
|
|
|
4 |
import { usePathname, useRouter, useSearchParams } from "next/navigation"
|
5 |
|
6 |
|
7 |
+
import { Renderer } from "@/components/business/renderer"
|
8 |
|
9 |
import {
|
10 |
Select,
|
|
|
22 |
import { getBackground } from "@/app/queries/getBackground"
|
23 |
import { getDialogue } from "@/app/queries/getDialogue"
|
24 |
import { getActionnables } from "@/app/queries/getActionnables"
|
25 |
+
import { Engine, EngineType, defaultEngine, engines, getEngine } from "./engines"
|
26 |
|
27 |
export default function Main() {
|
28 |
const [isPending, startTransition] = useTransition()
|
|
|
37 |
const searchParams = useSearchParams()
|
38 |
|
39 |
const requestedGame = (searchParams.get('game') as GameType) || defaultGame
|
|
|
40 |
const gameRef = useRef<GameType>(requestedGame)
|
|
|
41 |
const [game, setGame] = useState<Game>(getGame(gameRef.current))
|
42 |
+
|
43 |
+
const requestedEngine = (searchParams.get('engine') as EngineType) || defaultEngine
|
44 |
+
// const engineRef = useRef<EngineType>(requestedEngine)
|
45 |
+
const [engine, setEngine] = useState<Engine>(getEngine(requestedEngine))
|
46 |
+
|
47 |
const [situation, setSituation] = useState("")
|
48 |
const [scene, setScene] = useState("")
|
49 |
const [dialogue, setDialogue] = useState("")
|
|
|
70 |
)
|
71 |
|
72 |
// detect if type game type changed while we were busy
|
73 |
+
if (game?.type !== gameRef?.current) {
|
74 |
console.log("game type changed! aborting..")
|
75 |
return
|
76 |
}
|
|
|
157 |
window.history.pushState({path: newurl}, '', newurl)
|
158 |
}
|
159 |
|
160 |
+
|
161 |
+
const handleSelectEngine = (newEngine: EngineType) => {
|
162 |
+
setEngine(getEngine(newEngine))
|
163 |
+
|
164 |
+
const current = new URLSearchParams(Array.from(searchParams.entries()))
|
165 |
+
current.set("engine", newEngine)
|
166 |
+
const search = current.toString()
|
167 |
+
const query = search ? `?${search}` : ""
|
168 |
+
|
169 |
+
// for some reason, this doesn't work?!
|
170 |
+
router.replace(`${pathname}${query}`, { })
|
171 |
+
|
172 |
+
// workaround.. but it is strange that router.replace doesn't work..
|
173 |
+
let newurl = window.location.protocol + "//" + window.location.host + window.location.pathname + '?' + search.toString()
|
174 |
+
window.history.pushState({path: newurl}, '', newurl)
|
175 |
+
}
|
176 |
+
|
177 |
return (
|
178 |
<div
|
179 |
+
className="flex flex-col w-full"
|
|
|
|
|
|
|
180 |
>
|
181 |
+
<div className="flex flex-row w-full justify-between items-center px-2 py-2 border-b-1 border-gray-50 bg-gray-800">
|
182 |
+
<div className="flex flex-row items-center space-x-3 font-mono">
|
183 |
+
<label className="flex text-sm">Select a story:</label>
|
184 |
<Select
|
185 |
defaultValue={gameRef.current}
|
186 |
onValueChange={(value) => { handleSelectGame(value as GameType) }}>
|
187 |
+
<SelectTrigger className="w-[180px]">
|
188 |
+
<SelectValue className="text-sm" placeholder="Type" />
|
189 |
</SelectTrigger>
|
190 |
<SelectContent>
|
191 |
{Object.entries(games).map(([key, game]) =>
|
|
|
194 |
</SelectContent>
|
195 |
</Select>
|
196 |
</div>
|
197 |
+
<div className="flex flex-row items-center space-x-3 font-mono">
|
198 |
+
<label className="flex text-sm">Rendering engine:</label>
|
199 |
+
<Select
|
200 |
+
defaultValue={engine.type}
|
201 |
+
onValueChange={(value) => { handleSelectEngine(value as EngineType) }}>
|
202 |
+
<SelectTrigger className="w-[300px]">
|
203 |
+
<SelectValue className="text-sm" placeholder="Type" />
|
204 |
+
</SelectTrigger>
|
205 |
+
<SelectContent>
|
206 |
+
{Object.entries(engines).map(([key, engine]) =>
|
207 |
+
<SelectItem key={key} value={key}>{engine.label} ({engine.modelName})</SelectItem>
|
208 |
+
)}
|
209 |
+
</SelectContent>
|
210 |
+
</Select>
|
211 |
+
</div>
|
212 |
+
</div>
|
213 |
+
<div className={[
|
214 |
+
"flex flex-col w-full pt-4 space-y-3 px-2",
|
215 |
+
getGame(gameRef.current).className // apply the game theme
|
216 |
+
].join(" ")}>
|
217 |
<p className="text-xl">A stable diffusion exploration game. Click on an item to explore a new scene!</p>
|
218 |
<div className="flex flex-row">
|
219 |
<div className="text-xl mr-2">🔎 Clickable items:</div>
|
220 |
{clickables.map((clickable, i) =>
|
221 |
<div key={i} className="flex flex-row text-xl mr-2">
|
222 |
<div className="">{clickable}</div>
|
223 |
+
{i < (clickables.length - 1) ? <div>,</div> : null}
|
224 |
</div>)}
|
225 |
</div>
|
226 |
<p className="text-xl font-normal">You are looking at: <span className="font-bold">{hoveredActionnable || "nothing"}</span></p>
|
227 |
</div>
|
228 |
+
<Renderer
|
229 |
rendered={rendered}
|
230 |
onUserAction={handleUserAction}
|
231 |
onUserHover={setHoveredActionnable}
|
232 |
isLoading={isLoading}
|
233 |
+
game={game}
|
234 |
+
engine={engine}
|
235 |
/>
|
236 |
<p className="text-xl">{dialogue}</p>
|
237 |
</div>
|
src/app/types.ts
CHANGED
@@ -1,4 +1,3 @@
|
|
1 |
-
|
2 |
export interface RenderRequest {
|
3 |
prompt: string
|
4 |
|
|
|
|
|
1 |
export interface RenderRequest {
|
2 |
prompt: string
|
3 |
|
src/components/business/{image-renderer.tsx → renderer.tsx}
RENAMED
@@ -2,9 +2,10 @@ import { useEffect, useRef, useState } from "react"
|
|
2 |
|
3 |
import { ImageSegment, RenderedScene } from "@/app/types"
|
4 |
import { ProgressBar } from "../misc/progress"
|
5 |
-
import {
|
|
|
6 |
|
7 |
-
export const
|
8 |
rendered: {
|
9 |
assetUrl = "",
|
10 |
maskBase64 = "",
|
@@ -13,16 +14,18 @@ export const ImageRenderer = ({
|
|
13 |
onUserAction,
|
14 |
onUserHover,
|
15 |
isLoading,
|
16 |
-
|
|
|
17 |
}: {
|
18 |
rendered: RenderedScene
|
19 |
onUserAction: (actionnable: string) => void
|
20 |
onUserHover: (actionnable: string) => void
|
21 |
isLoading: boolean
|
22 |
-
|
|
|
23 |
}) => {
|
24 |
const timeoutRef = useRef<any>()
|
25 |
-
const imgRef = useRef<HTMLImageElement | null>(null)
|
26 |
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
27 |
const contextRef = useRef<CanvasRenderingContext2D | null>(null)
|
28 |
const [actionnable, setActionnable] = useState<string>("")
|
@@ -158,7 +161,7 @@ export const ImageRenderer = ({
|
|
158 |
|
159 |
// normally it takes 45, and we will try to go below,
|
160 |
// but to be safe let's set the counter a 1 min
|
161 |
-
const nbSeconds =
|
162 |
const amountInPercent = 100 / (nbUpdatesPerSec * nbSeconds) // 0.333
|
163 |
|
164 |
progressRef.current = Math.min(100, progressRef.current + amountInPercent)
|
@@ -173,7 +176,7 @@ export const ImageRenderer = ({
|
|
173 |
if (isLoading) {
|
174 |
timeoutRef.current = setInterval(updateProgressBar, 200)
|
175 |
}
|
176 |
-
}, [isLoading, assetUrl, type])
|
177 |
|
178 |
|
179 |
/*
|
@@ -214,16 +217,22 @@ export const ImageRenderer = ({
|
|
214 |
// isLoading ? "animate-pulse" : ""
|
215 |
].join(" ")
|
216 |
}>
|
217 |
-
<div className="relative w-
|
218 |
-
{assetUrl
|
219 |
-
|
|
|
|
|
220 |
src={assetUrl}
|
221 |
-
|
222 |
-
|
|
|
|
|
223 |
width="1024px"
|
224 |
height="512px"
|
|
|
225 |
className={
|
226 |
[
|
|
|
227 |
// "absolute top-0 left-0",
|
228 |
actionnable && !isLoading ? "cursor-pointer" : ""
|
229 |
].join(" ")
|
@@ -231,16 +240,30 @@ export const ImageRenderer = ({
|
|
231 |
onMouseDown={(event) => handleMouseEvent(event, true)}
|
232 |
onMouseMove={handleMouseEvent}
|
233 |
/>
|
234 |
-
:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
235 |
</div>
|
236 |
|
237 |
-
|
238 |
? <div className="fixed flex w-20 h-20 bottom-8 right-0 mr-8">
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
244 |
: null}
|
245 |
</div>
|
246 |
)
|
|
|
2 |
|
3 |
import { ImageSegment, RenderedScene } from "@/app/types"
|
4 |
import { ProgressBar } from "../misc/progress"
|
5 |
+
import { Game } from "@/app/games/types"
|
6 |
+
import { Engine, EngineType } from "@/app/engines"
|
7 |
|
8 |
+
export const Renderer = ({
|
9 |
rendered: {
|
10 |
assetUrl = "",
|
11 |
maskBase64 = "",
|
|
|
14 |
onUserAction,
|
15 |
onUserHover,
|
16 |
isLoading,
|
17 |
+
game,
|
18 |
+
engine,
|
19 |
}: {
|
20 |
rendered: RenderedScene
|
21 |
onUserAction: (actionnable: string) => void
|
22 |
onUserHover: (actionnable: string) => void
|
23 |
isLoading: boolean
|
24 |
+
game: Game
|
25 |
+
engine: Engine
|
26 |
}) => {
|
27 |
const timeoutRef = useRef<any>()
|
28 |
+
const imgRef = useRef<HTMLImageElement | HTMLVideoElement | null>(null)
|
29 |
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
30 |
const contextRef = useRef<CanvasRenderingContext2D | null>(null)
|
31 |
const [actionnable, setActionnable] = useState<string>("")
|
|
|
161 |
|
162 |
// normally it takes 45, and we will try to go below,
|
163 |
// but to be safe let's set the counter a 1 min
|
164 |
+
const nbSeconds = 32 // 1 min
|
165 |
const amountInPercent = 100 / (nbUpdatesPerSec * nbSeconds) // 0.333
|
166 |
|
167 |
progressRef.current = Math.min(100, progressRef.current + amountInPercent)
|
|
|
176 |
if (isLoading) {
|
177 |
timeoutRef.current = setInterval(updateProgressBar, 200)
|
178 |
}
|
179 |
+
}, [isLoading, assetUrl, engine?.type])
|
180 |
|
181 |
|
182 |
/*
|
|
|
217 |
// isLoading ? "animate-pulse" : ""
|
218 |
].join(" ")
|
219 |
}>
|
220 |
+
<div className="relative w-[1024px] h-[512px] border-2 border-gray-50 rounded-xl overflow-hidden">
|
221 |
+
{assetUrl === null ?
|
222 |
+
null
|
223 |
+
: engine.type === "video"
|
224 |
+
? <video
|
225 |
src={assetUrl}
|
226 |
+
ref={imgRef as any}
|
227 |
+
muted
|
228 |
+
autoPlay
|
229 |
+
loop
|
230 |
width="1024px"
|
231 |
height="512px"
|
232 |
+
|
233 |
className={
|
234 |
[
|
235 |
+
// "border-1 border-gray-50",
|
236 |
// "absolute top-0 left-0",
|
237 |
actionnable && !isLoading ? "cursor-pointer" : ""
|
238 |
].join(" ")
|
|
|
240 |
onMouseDown={(event) => handleMouseEvent(event, true)}
|
241 |
onMouseMove={handleMouseEvent}
|
242 |
/>
|
243 |
+
: <img
|
244 |
+
src={assetUrl}
|
245 |
+
// src={"data:image/png;base64," + maskBase64}
|
246 |
+
ref={imgRef as any}
|
247 |
+
width="1024px"
|
248 |
+
height="512px"
|
249 |
+
className={
|
250 |
+
[
|
251 |
+
// "absolute top-0 left-0",
|
252 |
+
actionnable && !isLoading ? "cursor-pointer" : ""
|
253 |
+
].join(" ")
|
254 |
+
}
|
255 |
+
onMouseDown={(event) => handleMouseEvent(event, true)}
|
256 |
+
onMouseMove={handleMouseEvent}
|
257 |
+
/>}
|
258 |
</div>
|
259 |
|
260 |
+
{isLoading
|
261 |
? <div className="fixed flex w-20 h-20 bottom-8 right-0 mr-8">
|
262 |
+
<ProgressBar
|
263 |
+
text="⌛"
|
264 |
+
progressPercentage={progressPercent}
|
265 |
+
/>
|
266 |
+
</div>
|
267 |
: null}
|
268 |
</div>
|
269 |
)
|
src/components/business/video-renderer.tsx
DELETED
@@ -1,22 +0,0 @@
|
|
1 |
-
"use client"
|
2 |
-
|
3 |
-
export const VideoRenderer = ({ url }: { url?: string }) => {
|
4 |
-
|
5 |
-
if (!url) {
|
6 |
-
return <div className="flex w-full pt-8 items-center justify-center text-center">
|
7 |
-
<div>Rendering first frames.. (might take around 30s)</div>
|
8 |
-
</div>
|
9 |
-
}
|
10 |
-
|
11 |
-
return (
|
12 |
-
<div className="w-full py-8 px-2">
|
13 |
-
<video
|
14 |
-
src={url}
|
15 |
-
muted
|
16 |
-
autoPlay
|
17 |
-
loop
|
18 |
-
className="w-full rounded-md overflow-hidden"
|
19 |
-
/>
|
20 |
-
</div>
|
21 |
-
)
|
22 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|