pagecontent from pipelines

app_init.py

@@ -3,6 +3,7 @@ from fastapi.responses import StreamingResponse, JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
 from fastapi import Request
+import markdown2
 
 import logging
 import traceback
@@ -144,13 +145,18 @@ def init_app(app: FastAPI, user_data: UserData, args: Args, pipeline):
     # route to setup frontend
     @app.get("/settings")
     async def settings():
-        info = pipeline.Info.schema()
+        info_schema = pipeline.Info.schema()
+        info = pipeline.Info()
+        if info.page_content:
+            page_content = markdown2.markdown(info.page_content)
+
         input_params = pipeline.InputParams.schema()
         return JSONResponse(
             {
-                "info": info,
+                "info": info_schema,
                 "input_params": input_params,
                 "max_queue_size": args.max_queue_size,
+                "page_content": page_content if info.page_content else "",
             }
         )
 

frontend/src/lib/components/PipelineOptions.svelte

@@ -1,6 +1,5 @@
 <script lang="ts">
-  import { createEventDispatcher } from 'svelte';
-  import type { FieldProps } from '$lib/types';
+  import type { Fields } from '$lib/types';
   import { FieldType } from '$lib/types';
   import InputRange from './InputRange.svelte';
   import SeedInput from './SeedInput.svelte';
@@ -9,10 +8,10 @@
   import Selectlist from './Selectlist.svelte';
   import { pipelineValues } from '$lib/store';
 
-  export let pipelineParams: FieldProps[];
+  export let pipelineParams: Fields;
 
-  $: advanceOptions = pipelineParams?.filter((e) => e?.hide == true);
-  $: featuredOptions = pipelineParams?.filter((e) => e?.hide !== true);
+  $: advanceOptions = Object.values(pipelineParams)?.filter((e) => e?.hide == true);
+  $: featuredOptions = Object.values(pipelineParams)?.filter((e) => e?.hide !== true);
 </script>
 
 <div class="flex flex-col gap-3">
@@ -37,7 +36,7 @@
   <details>
     <summary class="cursor-pointer font-medium">Advanced Options</summary>
     <div
-      class="grid grid-cols-1 items-center gap-3 {pipelineParams.length > 5
+      class="grid grid-cols-1 items-center gap-3 {Object.values(pipelineParams).length > 5
         ? 'sm:grid-cols-2'
         : ''}"
     >

frontend/src/lib/components/VideoInput.svelte

@@ -10,21 +10,23 @@
     mediaDevices
   } from '$lib/mediaStream';
   import MediaListSwitcher from './MediaListSwitcher.svelte';
+  export let width = 512;
+  export let height = 512;
+  const size = { width, height };
 
   let videoEl: HTMLVideoElement;
   let canvasEl: HTMLCanvasElement;
   let ctx: CanvasRenderingContext2D;
   let videoFrameCallbackId: number;
-  const WIDTH = 768;
-  const HEIGHT = 768;
+
   // ajust the throttle time to your needs
   const THROTTLE_TIME = 1000 / 15;
   let selectedDevice: string = '';
 
   onMount(() => {
     ctx = canvasEl.getContext('2d') as CanvasRenderingContext2D;
-    canvasEl.width = WIDTH;
-    canvasEl.height = HEIGHT;
+    canvasEl.width = size.width;
+    canvasEl.height = size.height;
   });
   $: {
     console.log(selectedDevice);
@@ -46,10 +48,10 @@
     const videoHeight = videoEl.videoHeight;
     const blob = await grapCropBlobImg(
       videoEl,
-      videoWidth / 2 - WIDTH / 2,
-      videoHeight / 2 - HEIGHT / 2,
-      WIDTH,
-      HEIGHT
+      videoWidth / 2 - size.width / 2,
+      videoHeight / 2 - size.height / 2,
+      size.width,
+      size.height
     );
 
     onFrameChangeStore.set({ blob });

frontend/src/lib/types.ts

@@ -11,6 +11,11 @@ export const enum PipelineMode {
     TEXT = "text",
 }
 
+
+export interface Fields {
+    [key: string]: FieldProps;
+}
+
 export interface FieldProps {
     default: number | string;
     max?: number;

frontend/src/routes/+page.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
   import { onMount } from 'svelte';
-  import type { FieldProps, PipelineInfo } from '$lib/types';
+  import type { Fields, PipelineInfo } from '$lib/types';
   import { PipelineMode } from '$lib/types';
   import ImagePlayer from '$lib/components/ImagePlayer.svelte';
   import VideoInput from '$lib/components/VideoInput.svelte';
@@ -11,8 +11,9 @@
   import { mediaStreamActions, onFrameChangeStore } from '$lib/mediaStream';
   import { getPipelineValues, deboucedPipelineValues } from '$lib/store';
 
-  let pipelineParams: FieldProps[];
+  let pipelineParams: Fields;
   let pipelineInfo: PipelineInfo;
+  let pageContent: string;
   let isImageMode: boolean = false;
   let maxQueueSize: number = 0;
   let currentQueueSize: number = 0;
@@ -22,11 +23,12 @@
 
   async function getSettings() {
     const settings = await fetch('/settings').then((r) => r.json());
-    pipelineParams = Object.values(settings.input_params.properties);
+    pipelineParams = settings.input_params.properties;
     pipelineInfo = settings.info.properties;
     isImageMode = pipelineInfo.input_mode.default === PipelineMode.IMAGE;
     maxQueueSize = settings.max_queue_size;
-    pipelineParams = pipelineParams.filter((e) => e?.disabled !== true);
+    pageContent = settings.page_content;
+    console.log(pipelineParams);
     if (maxQueueSize > 0) {
       getQueueSize();
       setInterval(() => {
@@ -70,31 +72,9 @@
 
 <main class="container mx-auto flex max-w-5xl flex-col gap-3 px-4 py-4">
   <article class="text-center">
-    <h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1>
-    {#if pipelineInfo?.title?.default}
-      <h3 class="text-xl font-bold">{pipelineInfo?.title?.default}</h3>
+    {#if pageContent}
+      {@html pageContent}
     {/if}
-    <p class="text-sm">
-      This demo showcases
-      <a
-        href="https://huggingface.co/blog/lcm_lora"
-        target="_blank"
-        class="text-blue-500 underline hover:no-underline">LCM LoRA</a
-      >
-      Image to Image pipeline using
-      <a
-        href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
-        target="_blank"
-        class="text-blue-500 underline hover:no-underline">Diffusers</a
-      > with a MJPEG stream server.
-    </p>
-    <p class="text-sm text-gray-500">
-      Change the prompt to generate different images, accepts <a
-        href="https://github.com/damian0815/compel/blob/main/doc/syntax.md"
-        target="_blank"
-        class="text-blue-500 underline hover:no-underline">Compel</a
-      > syntax.
-    </p>
     {#if maxQueueSize > 0}
       <p class="text-sm">
         There are <span id="queue_size" class="font-bold">{currentQueueSize}</span>
@@ -111,7 +91,10 @@
     <article class="my-3 grid grid-cols-1 gap-3 sm:grid-cols-2">
       {#if isImageMode}
         <div class="sm:col-start-1">
-          <VideoInput></VideoInput>
+          <VideoInput
+            width={Number(pipelineParams.width.default)}
+            height={Number(pipelineParams.height.default)}
+          ></VideoInput>
         </div>
       {/if}
       <div class={isImageMode ? 'sm:col-start-2' : 'col-span-2'}>

frontend/tailwind.config.js

@@ -1,6 +1,6 @@
 /** @type {import('tailwindcss').Config} */
 export default {
-  content: ['./src/**/*.{html,js,svelte,ts}'],
+  content: ['./src/**/*.{html,js,svelte,ts}', '../pipelines/**/*.py'],
   theme: {
     extend: {}
   },

pipelines/controlnet.py

@@ -22,6 +22,31 @@ taesd_model = "madebyollin/taesd"
 controlnet_model = "lllyasviel/control_v11p_sd15_canny"
 
 default_prompt = "Portrait of The Terminator with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece"
+page_content = """
+<h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1>
+<h3 class="text-xl font-bold">LCM + Controlnet Canny</h3>
+<p class="text-sm">
+    This demo showcases
+    <a
+    href="https://huggingface.co/blog/lcm_lora"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">LCM LoRA</a
+    >
+    ControlNet + Image to Image pipeline using
+    <a
+    href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Diffusers</a
+    > with a MJPEG stream server.
+</p>
+<p class="text-sm text-gray-500">
+    Change the prompt to generate different images, accepts <a
+    href="https://github.com/damian0815/compel/blob/main/doc/syntax.md"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Compel</a
+    > syntax.
+</p>
+"""
 
 
 class Pipeline:
@@ -30,6 +55,7 @@ class Pipeline:
         title: str = "LCM + Controlnet"
         description: str = "Generates an image from a text prompt"
         input_mode: str = "image"
+        page_content: str = page_content
 
     class InputParams(BaseModel):
         prompt: str = Field(

pipelines/controlnetLoraSD15.py

@@ -26,17 +26,40 @@ base_models = {
     "nitrosocke/mo-di-diffusion": "modern disney style",
 }
 lcm_lora_id = "latent-consistency/lcm-lora-sdv1-5"
-
-
 default_prompt = "Portrait of The Terminator with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece"
+page_content = """
+<h1 class="text-3xl font-bold">Real-Time Latent Consistency Model SDv1.5</h1>
+<h3 class="text-xl font-bold">LCM + LoRA + Controlnet + Canny</h3>
+<p class="text-sm">
+    This demo showcases
+    <a
+    href="https://huggingface.co/blog/lcm_lora"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">LCM LoRA</a>
++ ControlNet + Image to Imasge pipeline using
+    <a
+    href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Diffusers</a
+    > with a MJPEG stream server.
+</p>
+<p class="text-sm text-gray-500">
+    Change the prompt to generate different images, accepts <a
+    href="https://github.com/damian0815/compel/blob/main/doc/syntax.md"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Compel</a
+    > syntax.
+</p>
+"""
 
 
 class Pipeline:
     class Info(BaseModel):
         name: str = "controlnet+loras+sd15"
-        title: str = "LCM + LoRA + Controlnet "
+        title: str = "LCM + LoRA + Controlnet"
         description: str = "Generates an image from a text prompt"
         input_mode: str = "image"
+        page_content: str = page_content
 
     class InputParams(BaseModel):
         prompt: str = Field(

pipelines/controlnetLoraSDXL.py

@@ -22,25 +22,42 @@ controlnet_model = "diffusers/controlnet-canny-sdxl-1.0"
 model_id = "stabilityai/stable-diffusion-xl-base-1.0"
 lcm_lora_id = "latent-consistency/lcm-lora-sdxl"
 
-# # base model with activation token, it will prepend the prompt with the activation token
-base_models = {
-    "plasmo/woolitize": "woolitize",
-    "nitrosocke/Ghibli-Diffusion": "ghibli style",
-    "nitrosocke/mo-di-diffusion": "modern disney style",
-}
-# lcm_lora_id = "latent-consistency/lcm-lora-sdv1-5"
-
 
 default_prompt = "Portrait of The Terminator with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece"
 default_negative_prompt = "blurry, low quality, render, 3D, oversaturated"
+page_content = """
+<h1 class="text-3xl font-bold">Real-Time Latent Consistency Model SDXL</h1>
+<h3 class="text-xl font-bold">SDXL + LCM + LoRA + Controlnet</h3>
+<p class="text-sm">
+    This demo showcases
+    <a
+    href="https://huggingface.co/blog/lcm_lora"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">LCM LoRA</a>
++ SDXL + Controlnet + Image to Image pipeline using
+    <a
+    href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Diffusers</a
+    > with a MJPEG stream server.
+</p>
+<p class="text-sm text-gray-500">
+    Change the prompt to generate different images, accepts <a
+    href="https://github.com/damian0815/compel/blob/main/doc/syntax.md"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Compel</a
+    > syntax.
+</p>
+"""
 
 
 class Pipeline:
     class Info(BaseModel):
         name: str = "controlnet+loras+sdxl"
-        title: str = "SDXL + LCM + LoRA + Controlnet "
+        title: str = "SDXL + LCM + LoRA + Controlnet"
         description: str = "Generates an image from a text prompt"
         input_mode: str = "image"
+        page_content: str = page_content
 
     class InputParams(BaseModel):
         prompt: str = Field(
@@ -63,10 +80,10 @@ class Pipeline:
             4, min=2, max=15, title="Steps", field="range", hide=True, id="steps"
         )
         width: int = Field(
-            768, min=2, max=15, title="Width", disabled=True, hide=True, id="width"
+            1024, min=2, max=15, title="Width", disabled=True, hide=True, id="width"
         )
         height: int = Field(
-            768, min=2, max=15, title="Height", disabled=True, hide=True, id="height"
+            1024, min=2, max=15, title="Height", disabled=True, hide=True, id="height"
         )
         guidance_scale: float = Field(
             1.0,

pipelines/img2img.py

@@ -19,6 +19,30 @@ base_model = "SimianLuo/LCM_Dreamshaper_v7"
 taesd_model = "madebyollin/taesd"
 
 default_prompt = "Portrait of The Terminator with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece"
+page_content = """
+<h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1>
+<h3 class="text-xl font-bold">Image-to-Image LCM</h3>
+<p class="text-sm">
+    This demo showcases
+    <a
+    href="https://huggingface.co/blog/lcm_lora"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">LCM</a>
+Image to Image pipeline using
+    <a
+    href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Diffusers</a
+    > with a MJPEG stream server.
+</p>
+<p class="text-sm text-gray-500">
+    Change the prompt to generate different images, accepts <a
+    href="https://github.com/damian0815/compel/blob/main/doc/syntax.md"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Compel</a
+    > syntax.
+</p>
+"""
 
 
 class Pipeline:
@@ -27,6 +51,7 @@ class Pipeline:
         title: str = "Image-to-Image LCM"
         description: str = "Generates an image from a text prompt"
         input_mode: str = "image"
+        page_content: str = page_content
 
     class InputParams(BaseModel):
         prompt: str = Field(

pipelines/txt2img.py

@@ -17,6 +17,28 @@ taesd_model = "madebyollin/taesd"
 
 default_prompt = "Portrait of The Terminator with , glare pose, detailed, intricate, full of colour, cinematic lighting, trending on artstation, 8k, hyperrealistic, focused, extreme details, unreal engine 5 cinematic, masterpiece"
 
+page_content = """<h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1>
+<h3 class="text-xl font-bold">Text-to-Image</h3>
+<p class="text-sm">
+    This demo showcases
+    <a
+    href="https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">LCM</a>
+Image to Image pipeline using
+    <a
+    href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Diffusers</a> with a MJPEG stream server
+</p>
+<p class="text-sm text-gray-500">
+    Change the prompt to generate different images, accepts <a
+    href="https://github.com/damian0815/compel/blob/main/doc/syntax.md"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Compel</a
+    > syntax.
+</p>"""
+
 
 class Pipeline:
     class Info(BaseModel):
@@ -24,6 +46,7 @@ class Pipeline:
         title: str = "Text-to-Image LCM"
         description: str = "Generates an image from a text prompt"
         input_mode: str = "text"
+        page_content: str = page_content
 
     class InputParams(BaseModel):
         prompt: str = Field(

pipelines/txt2imgLora.py

@@ -18,6 +18,34 @@ taesd_model = "madebyollin/taesd"
 
 default_prompt = "Analog style photograph of young Harrison Ford as Han Solo, star wars behind the scenes"
 
+page_content = """
+<h1 class="text-3xl font-bold">Real-Time Latent Consistency Model SDv1.5</h1>
+<h3 class="text-xl font-bold">Text-to-Image LCM + LoRa</h3>
+<p class="text-sm">
+    This demo showcases
+    <a
+    href="https://huggingface.co/blog/lcm_lora"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">LCM</a>
+Image to Image pipeline using
+    <a
+    href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Diffusers</a
+    > with a MJPEG stream server. Featuring <a
+    href="https://huggingface.co/wavymulder/Analog-Diffusion"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Analog-Diffusion</a>
+</p>
+<p class="text-sm text-gray-500">
+    Change the prompt to generate different images, accepts <a
+    href="https://github.com/damian0815/compel/blob/main/doc/syntax.md"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Compel</a
+    > syntax.
+</p>
+"""
+
 
 class Pipeline:
     class Info(BaseModel):
@@ -25,6 +53,7 @@ class Pipeline:
         title: str = "Text-to-Image LCM + LoRa"
         description: str = "Generates an image from a text prompt"
         input_mode: str = "text"
+        page_content: str = page_content
 
     class InputParams(BaseModel):
         prompt: str = Field(

pipelines/txt2imgLoraSDXL.py

@@ -23,6 +23,31 @@ lcm_lora_id = "latent-consistency/lcm-lora-sdxl"
 
 default_prompt = "close-up photography of old man standing in the rain at night, in a street lit by lamps, leica 35mm summilux"
 default_negative_prompt = "blurry, low quality, render, 3D, oversaturated"
+page_content = """
+<h1 class="text-3xl font-bold">Real-Time Latent Consistency Model</h1>
+<h3 class="text-xl font-bold">Text-to-Image SDXL + LCM + LoRA</h3>
+<p class="text-sm">
+    This demo showcases
+    <a
+    href="https://huggingface.co/blog/lcm_lora"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">LCM LoRA</a
+    >
+    Text to Image pipeline using
+    <a
+    href="https://huggingface.co/docs/diffusers/main/en/using-diffusers/lcm#performing-inference-with-lcm"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Diffusers</a
+    > with a MJPEG stream server.
+</p>
+<p class="text-sm text-gray-500">
+    Change the prompt to generate different images, accepts <a
+    href="https://github.com/damian0815/compel/blob/main/doc/syntax.md"
+    target="_blank"
+    class="text-blue-500 underline hover:no-underline">Compel</a
+    > syntax.
+</p>
+"""
 
 
 class Pipeline:
@@ -30,6 +55,7 @@ class Pipeline:
         name: str = "LCM+Lora+SDXL"
         title: str = "Text-to-Image SDXL + LCM + LoRA"
         description: str = "Generates an image from a text prompt"
+        page_content: str = page_content
         input_mode: str = "text"
 
     class InputParams(BaseModel):

requirements.txt

@@ -1,4 +1,4 @@
-git+https://github.com/huggingface/diffusers@6f1435332bc74e286af5e88014236e4cc712b747
+git+https://github.com/huggingface/diffusers@141cd52d56f31a8653795e14d29a87071788dc15
 transformers==4.35.2
 --extra-index-url https://download.pytorch.org/whl/cu121;
 torch==2.1.0
@@ -9,4 +9,5 @@ accelerate==0.24.0
 compel==2.0.2
 controlnet-aux==0.0.7
 peft==0.6.0
-xformers; sys_platform != 'darwin' or platform_machine != 'arm64'
+xformers; sys_platform != 'darwin' or platform_machine != 'arm64'
+markdown2