jbilcke-hf HF staff commited on
Commit
048071f
β€’
1 Parent(s): 6463491

experimenting with things

Browse files
README.md CHANGED
@@ -36,7 +36,8 @@ Rendering config:
36
  - `RENDERING_HF_INFERENCE_ENDPOINT_URL`: necessary if you decide to use a custom inference endpoint
37
  - `RENDERING_REPLICATE_API_MODEL_VERSION`: url to the VideoChain API server
38
  - `RENDERING_HF_INFERENCE_ENDPOINT_URL`: optional, default to nothing
39
- - `RENDERING_HF_INFERENCE_API_MODEL`: optional, defaults to "stabilityai/stable-diffusion-xl-base-1.0"
 
40
  - `RENDERING_REPLICATE_API_MODEL`: optional, defaults to "stabilityai/sdxl"
41
  - `RENDERING_REPLICATE_API_MODEL_VERSION`: optional, in case you want to change the version
42
 
 
36
  - `RENDERING_HF_INFERENCE_ENDPOINT_URL`: necessary if you decide to use a custom inference endpoint
37
  - `RENDERING_REPLICATE_API_MODEL_VERSION`: url to the VideoChain API server
38
  - `RENDERING_HF_INFERENCE_ENDPOINT_URL`: optional, default to nothing
39
+ - `RENDERING_HF_INFERENCE_API_BASE_MODEL`: optional, defaults to "stabilityai/stable-diffusion-xl-base-1.0"
40
+ - `RENDERING_HF_INFERENCE_API_REFINER_MODEL`: optional, defaults to "stabilityai/stable-diffusion-xl-refiner-1.0"
41
  - `RENDERING_REPLICATE_API_MODEL`: optional, defaults to "stabilityai/sdxl"
42
  - `RENDERING_REPLICATE_API_MODEL_VERSION`: optional, in case you want to change the version
43
 
package-lock.json CHANGED
The diff for this file is too large to render. See raw diff
 
package.json CHANGED
@@ -42,6 +42,7 @@
42
  "eslint": "8.45.0",
43
  "eslint-config-next": "13.4.10",
44
  "html2canvas": "^1.4.1",
 
45
  "lucide-react": "^0.260.0",
46
  "next": "13.4.10",
47
  "openai": "^4.10.0",
@@ -51,6 +52,7 @@
51
  "react-circular-progressbar": "^2.1.0",
52
  "react-dom": "18.2.0",
53
  "react-icons": "^4.11.0",
 
54
  "react-virtualized-auto-sizer": "^1.0.20",
55
  "replicate": "^0.17.0",
56
  "sbd": "^1.0.19",
 
42
  "eslint": "8.45.0",
43
  "eslint-config-next": "13.4.10",
44
  "html2canvas": "^1.4.1",
45
+ "konva": "^9.2.2",
46
  "lucide-react": "^0.260.0",
47
  "next": "13.4.10",
48
  "openai": "^4.10.0",
 
52
  "react-circular-progressbar": "^2.1.0",
53
  "react-dom": "18.2.0",
54
  "react-icons": "^4.11.0",
55
+ "react-konva": "^18.2.10",
56
  "react-virtualized-auto-sizer": "^1.0.20",
57
  "replicate": "^0.17.0",
58
  "sbd": "^1.0.19",
public/layouts/layout4 hd.jpg ADDED
public/layouts/layout4.jpg ADDED
src/app/engine/render.ts CHANGED
@@ -169,6 +169,10 @@ export async function newRender({
169
 
170
  // note: there is no "refiner" step yet for custom inference endpoint
171
  // you probably don't need it anyway, as you probably want to deploy an all-in-one model instead for perf reasons
 
 
 
 
172
  if (renderingEngine === "INFERENCE_API") {
173
  try {
174
  const refinerModelUrl = `https://api-inference.huggingface.co/models/${huggingFaceInferenceApiRefinerModel}`
@@ -213,7 +217,8 @@ export async function newRender({
213
  } catch (err) {
214
  console.log(`Refiner step failed, but this is not a blocker. Error details: ${err}`)
215
  }
216
- }
 
217
 
218
  return {
219
  renderId: uuidv4(),
 
169
 
170
  // note: there is no "refiner" step yet for custom inference endpoint
171
  // you probably don't need it anyway, as you probably want to deploy an all-in-one model instead for perf reasons
172
+
173
+ // update: right now it is not possible to use it from the Inference API either:
174
+ // "Model type not found or pipeline not implemented"
175
+ /*
176
  if (renderingEngine === "INFERENCE_API") {
177
  try {
178
  const refinerModelUrl = `https://api-inference.huggingface.co/models/${huggingFaceInferenceApiRefinerModel}`
 
217
  } catch (err) {
218
  console.log(`Refiner step failed, but this is not a blocker. Error details: ${err}`)
219
  }
220
+ } */
221
+
222
 
223
  return {
224
  renderId: uuidv4(),
src/app/interface/page/index.tsx CHANGED
@@ -1,4 +1,4 @@
1
- import { allLayouts } from "@/app/layouts"
2
  import { useStore } from "@/app/store"
3
  import { cn } from "@/lib/utils"
4
  import { useEffect, useRef } from "react"
@@ -9,7 +9,7 @@ export function Page({ page }: { page: number }) {
9
  // const prompt = useStore(state => state.prompt)
10
 
11
  const LayoutElement = (allLayouts as any)[layouts[page]]
12
-
13
  /*
14
  const [canLoad, setCanLoad] = useState(false)
15
  useEffect(() => {
@@ -36,11 +36,7 @@ export function Page({ page }: { page: number }) {
36
  ref={pageRef}
37
  className={cn(
38
  `w-full`,
39
- // we are trying to reach a "book" look
40
- // we are using aspect-[297/210] because it matches A4 (297mm x 210mm)
41
- // `aspect-[210/297]`,
42
- `aspect-[250/297]`,
43
-
44
  `transition-all duration-100 ease-in-out`,
45
  `border border-stone-200`,
46
  `shadow-2xl`,
 
1
+ import { allLayoutAspectRatios, allLayouts } from "@/app/layouts"
2
  import { useStore } from "@/app/store"
3
  import { cn } from "@/lib/utils"
4
  import { useEffect, useRef } from "react"
 
9
  // const prompt = useStore(state => state.prompt)
10
 
11
  const LayoutElement = (allLayouts as any)[layouts[page]]
12
+ const aspectRatio = ((allLayoutAspectRatios as any)[layouts[page]] as string) || "aspect-[250/297]"
13
  /*
14
  const [canLoad, setCanLoad] = useState(false)
15
  useEffect(() => {
 
36
  ref={pageRef}
37
  className={cn(
38
  `w-full`,
39
+ aspectRatio,
 
 
 
 
40
  `transition-all duration-100 ease-in-out`,
41
  `border border-stone-200`,
42
  `shadow-2xl`,
src/app/interface/top-menu/index.tsx CHANGED
@@ -31,7 +31,8 @@ const layoutIcons: Partial<Record<LayoutName, StaticImageData>> = {
31
  Layout0: layoutPreview0,
32
  Layout1: layoutPreview1,
33
  Layout2: layoutPreview2,
34
- Layout3: layoutPreview3
 
35
  }
36
 
37
  export function TopMenu() {
 
31
  Layout0: layoutPreview0,
32
  Layout1: layoutPreview1,
33
  Layout2: layoutPreview2,
34
+ Layout3: layoutPreview3,
35
+ Layout4: undefined,
36
  }
37
 
38
  export function TopMenu() {
src/app/layouts/index.tsx CHANGED
@@ -252,21 +252,104 @@ export function Layout3() {
252
  )
253
  }
254
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
255
  // export const layouts = { Layout1, Layout2_todo, Layout3_todo, Layout4_todo, Layout2, Layout3 }
256
  export const allLayouts = {
257
  random: <></>,
258
  Layout0,
259
  Layout1,
260
  Layout2,
261
- Layout3
 
262
  }
263
 
264
  export const allLayoutLabels = {
265
  random: "Random layout",
266
- Layout0: "Layout 0",
267
- Layout1: "Layout 1",
268
- Layout2: "Layout 2",
269
- Layout3: "Layout 3",
 
 
 
 
 
 
 
 
 
 
270
  }
271
 
272
  export type LayoutName = keyof typeof allLayouts
 
252
  )
253
  }
254
 
255
+ // squares + vertical
256
+ export function Layout4() {
257
+ return (
258
+ <Grid className="grid-cols-8 grid-rows-8">
259
+ <div className="bg-zinc-100 col-start-1 col-end-7 row-start-1 row-end-3">
260
+ <Panel
261
+ panel={0}
262
+ width={512}
263
+ height={1024}
264
+ />
265
+ </div>
266
+ <div className="bg-zinc-100 col-start-3 col-end-9 row-start-3 row-end-4">
267
+ <Panel
268
+ panel={1}
269
+ width={1024}
270
+ height={768}
271
+ />
272
+ </div>
273
+ <div className="bg-stone-100 col-start-2 col-end-8 row-start-4 row-end-6">
274
+ <Panel
275
+ panel={2}
276
+ width={768}
277
+ height={1024}
278
+ />
279
+ </div>
280
+ <div className="bg-slate-100 col-start-1 col-end-9 row-start-6 row-end-8">
281
+ <Panel
282
+ panel={3}
283
+ width={1024}
284
+ height={512}
285
+ />
286
+ </div>
287
+ </Grid>
288
+ )
289
+ }
290
+
291
+ // squares + horizontal
292
+ export function Layout5() {
293
+ return (
294
+ <Grid className="grid-cols-4 grid-rows-4">
295
+ <div className="bg-zinc-100">
296
+ <Panel
297
+ panel={0}
298
+ width={1024}
299
+ height={1024}
300
+ />
301
+ </div>
302
+ <div className="bg-zinc-100">
303
+ <Panel
304
+ panel={1}
305
+ width={1024}
306
+ height={1024}
307
+ />
308
+ </div>
309
+ <div className="bg-stone-100">
310
+ <Panel
311
+ panel={2}
312
+ width={1024}
313
+ height={1024}
314
+ />
315
+ </div>
316
+ <div className="bg-slate-100">
317
+ <Panel
318
+ panel={3}
319
+ width={1024}
320
+ height={1024}
321
+ />
322
+ </div>
323
+ </Grid>
324
+ )
325
+ }
326
+
327
  // export const layouts = { Layout1, Layout2_todo, Layout3_todo, Layout4_todo, Layout2, Layout3 }
328
  export const allLayouts = {
329
  random: <></>,
330
  Layout0,
331
  Layout1,
332
  Layout2,
333
+ Layout3,
334
+ Layout4
335
  }
336
 
337
  export const allLayoutLabels = {
338
  random: "Random layout",
339
+ Layout0: "Grid 0",
340
+ Layout1: "Grid 1",
341
+ Layout2: "Grid 2",
342
+ Layout3: "Grid 3",
343
+ // Layout4: "Blocks 1",
344
+ }
345
+
346
+ // note for reference: A4 (297mm x 210mm)
347
+ export const allLayoutAspectRatios = {
348
+ Layout0: "aspect-[250/297]",
349
+ Layout1: "aspect-[250/297]",
350
+ Layout2: "aspect-[250/297]",
351
+ Layout3: "aspect-[250/297]",
352
+ // Layout4: "aspect-[1/3]",
353
  }
354
 
355
  export type LayoutName = keyof typeof allLayouts
src/app/queries/getStory.ts CHANGED
@@ -16,6 +16,9 @@ export const getStory = async ({
16
  prompt: string;
17
  }): Promise<LLMResponse> => {
18
 
 
 
 
19
  const query = createLlamaPrompt([
20
  {
21
  role: "system",
 
16
  prompt: string;
17
  }): Promise<LLMResponse> => {
18
 
19
+ // In case you need to quickly debug the RENDERING engine you can uncomment this:
20
+ // return mockLLMResponse
21
+
22
  const query = createLlamaPrompt([
23
  {
24
  role: "system",
src/app/queries/mockLLMResponse.ts ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { LLMResponse } from "@/types"
2
+
3
+ export const mockLLMResponse: LLMResponse = [
4
+ {
5
+ "panel": 1,
6
+ "instructions": "Close-up of cat's face, looking straight at reader with a smirk on its face",
7
+ "caption": "Feline mischief"
8
+ },
9
+ {
10
+ "panel": 2,
11
+ "instructions": "Medium shot of cat sniffing a glass of milk, with a surprised expression",
12
+ "caption": "Uh oh, what's this?"
13
+ },
14
+ {
15
+ "panel": 3,
16
+ "instructions": "Wide shot of cat knocking over the glass of milk, with a crazed look in its eyes",
17
+ "caption": "Cat-astrophe!"
18
+ },
19
+ {
20
+ "panel": 4,
21
+ "instructions": "Close-up of cat's face, looking satisfied with a milk moustache",
22
+ "caption": "Mission accomplished"
23
+ }
24
+ ]
src/components/ui/dialog.tsx CHANGED
@@ -11,10 +11,9 @@ const Dialog = DialogPrimitive.Root
11
  const DialogTrigger = DialogPrimitive.Trigger
12
 
13
  const DialogPortal = ({
14
- className,
15
  ...props
16
  }: DialogPrimitive.DialogPortalProps) => (
17
- <DialogPrimitive.Portal className={cn(className)} {...props} />
18
  )
19
  DialogPortal.displayName = DialogPrimitive.Portal.displayName
20
 
 
11
  const DialogTrigger = DialogPrimitive.Trigger
12
 
13
  const DialogPortal = ({
 
14
  ...props
15
  }: DialogPrimitive.DialogPortalProps) => (
16
+ <DialogPrimitive.Portal {...props} />
17
  )
18
  DialogPortal.displayName = DialogPrimitive.Portal.displayName
19
 
tailwind.config.js CHANGED
@@ -57,6 +57,14 @@ module.exports = {
57
  },
58
  screens: {
59
  'print': { 'raw': 'print' },
 
 
 
 
 
 
 
 
60
  }
61
  },
62
  },
 
57
  },
58
  screens: {
59
  'print': { 'raw': 'print' },
60
+ },
61
+ gridTemplateColumns: {
62
+ '12': 'repeat(12, minmax(0, 1fr))',
63
+ '16': 'repeat(16, minmax(0, 1fr))',
64
+ },
65
+ gridTemplateRows: {
66
+ '12': 'repeat(12, minmax(0, 1fr))',
67
+ '16': 'repeat(16, minmax(0, 1fr))',
68
  }
69
  },
70
  },