use stable cascade

.env

@@ -4,11 +4,4 @@ HF_INFERENCE_API_BASE_MODEL="stabilityai/stable-diffusion-xl-base-1.0"
 HF_INFERENCE_API_REFINER_MODEL="stabilityai/stable-diffusion-xl-refiner-1.0"
 HF_INFERENCE_API_MODEL_TRIGGER=""
 HF_INFERENCE_API_FILE_TYPE="image/png"
-
-STABLE_CASCADE_API_GRADIO_TOKEN="<USE YOUR OWN>"
-
-IMAGE_UPSCALING_API_GRADIO_URL="https://jbilcke-hf-image-upscaling-api.hf.space"
-IMAGE_UPSCALING_API_GRADIO_TOKEN="<USE YOUR OWN>"
-
-BACKGROUND_REMOVAL_API_GRADIO_URL="https://jbilcke-hf-background-removal-api.hf.space"
-BACKGROUND_REMOVAL_API_GRADIO_TOKEN="<USE YOUR OWN>"
+MICROSERVICE_API_SECRET_TOKEN="<USE YOUR OWN>"

README.md

@@ -1,52 +1,10 @@
 ---
-title: AI Clip Factory
-emoji: 🧞
-colorFrom: yellow
-colorTo: purple
-sdk: docker
+title: Illustrateur (cloud)
+emoji: ⚡
+colorFrom: purple
+colorTo: green
 pinned: true
-app_port: 3000
-disable_embedding: true
-hf_oauth_redirect_path: /api/oauth/callback
+header: mini
 ---
 
-# The AI Clip Factory
-
-The AI Clip Factory is a space to create animated videos in an ultra simple and fun way. It is meant to be a child's play.
-
-## Text-to-video model
-
-The AI Clip Factory is a space about clip generation and providing a fun UI, and is not meant to promote a specific AI model.
-
-As a consequence, a model currently defined as default may be replaced at anytime by a newer SOTA model.
-
-Right now (2023-10-19) the default model is the base Hotshot-XL (use the official website for faster inference at [https://hotshot.co](https://hotshot.co)).
-
-# Interpolation model
-
-The default model used for interpolation is [ST-MFNet](https://github.com/zsxkib/ST-MFNet)
-
-## Setup
-
-If you run the app locally you need to create a `.env.local` file 
-(If you deploy to Hugging Face, just set the environment variable from the settings)
-
-### Video rendering engine
-
-Note: the app is in heavy development, not all backends are supported
-
-Set `VIDEO_ENGINE` to one of:
-
-- `VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_GRADIO"`
-- `VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_REPLICATE"`
-- `VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_NODE"` <- not working yet
-- `VIDEO_ENGINE="VIDEO_HOTSHOT_XL_API_OFFICIAL"` <- not working yet
-
-
-### Authentication
-
-If you intent to use a special provider (eg. Replicate) you need to setup your token
-
-- `AUTH_REPLICATE_API_TOKEN="<YOUR SECRET>"`
-
-
+This is the "cloud" version of Illustrateur, which means you do not need a GPU in your laptop!

next.config.js

@@ -1,6 +1,12 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   output: 'standalone',
+
+  experimental: {
+    serverActions: {
+      bodySizeLimit: '10mb'
+    }
+  }
 }
 
 module.exports = nextConfig

src/app/main.tsx

@@ -20,11 +20,14 @@ import { SliderField } from '@/components/form/slider-field'
 import { Toaster } from '@/components/ui/sonner'
 import { cn } from '@/lib/utils'
 import { CharacterExpression, CharacterGender, CharacterPose, MIN_AGE, DEFAULT_AGE, MAX_AGE, DEFAULT_WEIGHT, GenerationMode, HairColor, HairStyle, PhotoshootMode, characterExpressions, characterGenders, characterPoses, hairColors, hairStyles, photoshootModes } from '../lib/config'
-import { getGenerationParams } from '@/lib/getGenerationParams'
-import { generate } from './server/generate'
+import { getStableDiffusionParams } from '@/lib/getStableDiffusionParams'
+import { getStableCascadeParams } from '@/lib/getStableCascadeParams'
+import { stableDiffusion } from './server/stableDiffusion'
+import { stableCascade } from './server/stableCascade'
 import { generateSeed } from '@/lib/generateSeed'
 import { BackgroundRemovalParams, GenerationStatus, UpscalingParams } from '@/types'
 import { removeBackground } from './server/background'
+import { upscale } from './server/upscale'
 
 /*
 Maybe we should use this classification instead:
@@ -47,19 +50,21 @@ export function Main() {
   )
   const [characterWeight, setCharacterWeight] = useState(DEFAULT_WEIGHT)
   const [characterHairStyle, setCharacterHairStyle] = useState<HairStyle>(
-    "Cornrows" // pick<HairStyle>(hairStyles)
+    "Braids" // pick<HairStyle>(hairStyles)
   )
   const [characterHairColor, setCharacterHairColor] = useState<HairColor>(
-    "braids" // pick<HairColor>(hairColors)
+    "chesnut" // pick<HairColor>(hairColors)
   )
   const [characterExpression, setCharacterExpression] = useState<CharacterExpression>("neutral")
   const [characterPose, setCharacterPose] = useState<CharacterPose>("side-on pose")
-  const [photoshootMode, setPhotoshootMode] = useState<PhotoshootMode>(photoshootModes[1])
+  const [photoshootMode, setPhotoshootMode] = useState<PhotoshootMode>(photoshootModes[0])
   
   const [prompt, setPrompt] = useState('llamacookie')
   const [negativePrompt, setNegativePrompt] = useState('')
   const [inferenceSteps, setInferenceSteps] = useState(20)
-  const [guidanceScale, setGuidanceScale] = useState(5)
+  const [nbPriorInferenceSteps, setNbPriorInferenceSteps] = useState(20)
+  const [nbDecoderInferenceSteps, setNbDecoderInferenceSteps] = useState(10)
+  const [guidanceScale, setGuidanceScale] = useState(4)
   const [seed, setSeed] = useState('')
   const [strength, setStrength] = useState(0.8)
   const [runVaeOnEachStep, setRunVaeOnEachStep] = useState(false)
@@ -73,7 +78,8 @@ export function Main() {
 
   const showAdvancedSettings = true
 
-  const generationParams = getGenerationParams({
+  const stableCascadeParams = getStableCascadeParams({
+    prompt,
     generationMode,
     negativePrompt,
     characterAge,
@@ -85,21 +91,6 @@ export function Main() {
     characterPose
   })
 
-  const upscalingParams: UpscalingParams = {
-    imageAsBase64: "",
-    prompt: generationParams.prompt,
-    negativePrompt: generationParams.negativePrompt,
-    scaleFactor: 2,
-    seed: generateSeed(),
-  
-    // // for a single image we can afford a higher rate, such as 25
-    nbSteps: 25,
-  }
-
-  const backgroundRemovalParams: BackgroundRemovalParams = {
-    imageAsBase64: ""
-  }
-
   const onDraw = async () => {
     console.log("onRender")
 
@@ -107,53 +98,80 @@ export function Main() {
     let upscaledImageUrl = "" 
     let croppedImageUrl = "" 
 
-    // startTransition(() => {
-      setStatus("generating")
-      setBaseImageUrl(baseImageUrl)
-      setUpscaledImageUrl(upscaledImageUrl)
-      setCroppedImageUrl(croppedImageUrl)
-    // })
+    setStatus("generating")
+    setBaseImageUrl(baseImageUrl)
+    setUpscaledImageUrl(upscaledImageUrl)
+    setCroppedImageUrl(croppedImageUrl)
 
     try {
-      baseImageUrl = await generate(generationParams)
+      // baseImageUrl = await stableDiffusion(stableDiffusionParams)
+      baseImageUrl = await stableCascade(stableCascadeParams)
     } catch (err) {
       console.error(`failed to generate:`, err)
     }
   
     if (!baseImageUrl) {
-      // startTransition(() => {
-        setStatus("error")
-      // })
-
+      setStatus("error")
       return
     }
 
-    // startTransition(() => {
-      setBaseImageUrl(baseImageUrl)
-    // })
+    setBaseImageUrl(baseImageUrl)
 
+    /*
     try {
       croppedImageUrl = await removeBackground({
-        ...backgroundRemovalParams,
         imageAsBase64: baseImageUrl,
       })
     } catch (err) {
-      console.error(`failed to crop:`, err)
+      console.error(`failed to crop the low-resolution image:`, err)
     }
     
     if (!croppedImageUrl) {
-      // startTransition(() => {
-        setStatus( "error")
-      // })
+      setStatus( "error")
+      return
+    }
 
+    setCroppedImageUrl(croppedImageUrl)
+    */
+    setStatus("upscaling")
+
+    try {
+      upscaledImageUrl = await upscale({
+        imageAsBase64: baseImageUrl,
+        prompt: stableCascadeParams.prompt,
+        negativePrompt: stableCascadeParams.negativePrompt,
+        scaleFactor: 2,
+        seed: generateSeed(),
+  
+        // // for a single image we can afford a higher rate, such as 25
+        nbSteps: 20,
+      })
+    } catch (err) {
+      console.error(`failed to upscale:`, err)
+    }
+    
+    if (!upscaledImageUrl) {
+      setStatus( "error")
       return
     }
 
-    // startTransition(() => {
-      setCroppedImageUrl(croppedImageUrl)
-      setStatus("finished")
-    // })
+    setUpscaledImageUrl(upscaledImageUrl)
 
+    try {
+      croppedImageUrl = await removeBackground({
+        imageAsBase64: upscaledImageUrl,
+      })
+    } catch (err) {
+      console.error(`failed to crop the upscaled image:`, err)
+    }
+    
+    if (!croppedImageUrl) {
+      setStatus( "error")
+      return
+    }
+
+    setCroppedImageUrl(croppedImageUrl)
+    setStatus("finished")
   }
 
   const isBusy = status === "generating" || status === "cropping" || status === "upscaling"
@@ -343,13 +361,13 @@ export function Main() {
                         label="Guidance Scale"
                         type='number'
                         min={1}
-                        max={20}
-                        step={0.5}
+                        max={15}
+                        step={0.1}
                         // disabled={modelState != 'ready'}
                         onChange={(e) => setGuidanceScale(parseFloat(e.target.value))}
                         value={guidanceScale}
                         className={cn({
-                          hidden: !showAdvancedSettings
+                          hidden: true, // !showAdvancedSettings
                         })}
                       />
                       <SliderField
@@ -418,7 +436,7 @@ export function Main() {
                         defaultValue={[inferenceSteps]}
                         value={[inferenceSteps]}
                         className={cn({
-                         hidden: !showAdvancedSettings
+                         hidden: true, // !showAdvancedSettings
                         })}
                       />
 
@@ -432,7 +450,7 @@ export function Main() {
                         onChange={(e) => setGuidanceScale(parseFloat(e.target.value))}
                         value={guidanceScale}
                         className={cn({
-                          hidden: !showAdvancedSettings
+                          hidden: true, // !showAdvancedSettings
                         })}
                       />
                       {/*

src/app/server/background.ts

@@ -1,14 +1,15 @@
 "use server"
 
 import { BackgroundRemovalParams } from "@/types"
+
 import { addBase64HeaderToPng } from "./addBase64HeaderToPng"
 
-const gradioApi = `${process.env.BACKGROUND_REMOVAL_API_GRADIO_URL || ""}`
-const accessToken = `${process.env.BACKGROUND_REMOVAL_API_GRADIO_TOKEN || ""}`
+const gradioApi = `https://jbilcke-hf-background-removal-api.hf.space`
+const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
 
-export const removeBackground = async ({
+export async function removeBackground({
   imageAsBase64,
-}: BackgroundRemovalParams): Promise<string> => {
+}: BackgroundRemovalParams): Promise<string> {
 
   // remember: a space needs to be public for the classic fetch() to work
   const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
@@ -20,7 +21,7 @@ export const removeBackground = async ({
     body: JSON.stringify({
       fn_index: 0, // <- is it 0 or 1?
       data: [
-        accessToken,
+        microserviceApiKey,
         imageAsBase64,
       ],
     }),

src/app/server/stableCascade.ts

@@ -0,0 +1,66 @@
+"use server"
+
+import { generateSeed } from "@/lib/generateSeed"
+import { StableCascadeParams } from "@/types"
+import { addBase64HeaderToPng } from "./addBase64HeaderToPng"
+
+const gradioApi = `https://jbilcke-hf-stable-cascade-api.hf.space`
+const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
+
+export async function stableCascade({
+  prompt,
+  negativePrompt,
+  guidanceScale,
+  nbPriorInferenceSteps,
+  nbDecoderInferenceSteps,
+  seed,
+  width,
+  height,
+}: StableCascadeParams): Promise<string> {
+
+  // console.log(`calling `+ gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict")
+
+  // remember: a space needs to be public for the classic fetch() to work
+  const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      // Authorization: `Bearer ${hfApiToken}`,
+    },
+    body: JSON.stringify({
+      fn_index: 0, // <- is it 0 or 1?
+      data: [
+        microserviceApiKey,
+        prompt,
+        negativePrompt,
+        height,
+        width,
+        guidanceScale,
+        seed || generateSeed(),
+        nbPriorInferenceSteps,
+        nbDecoderInferenceSteps
+      ],
+    }),
+    cache: "no-store",
+    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+    // next: { revalidate: 1 }
+  })
+
+  const { data } = await res.json()
+  
+  // console.log("data:", data)
+  // Recommendation: handle errors
+  if (res.status !== 200 || !Array.isArray(data)) {
+    // This will activate the closest `error.js` Error Boundary
+    throw new Error(`Failed to fetch data (status: ${res.status})`)
+  }
+  // console.log("data:", data.slice(0, 50))
+
+  const base64Content = (data?.[0] || "") as string
+
+  if (!base64Content) {
+    throw new Error(`invalid response (no content)`)
+  }
+
+  return addBase64HeaderToPng(base64Content)
+}

src/app/server/{generate.ts → stableDiffusion.ts}

@@ -1,9 +1,9 @@
 "use server"
 
-import { Settings, GenerationParams } from "@/types"
+import { StableDiffusionParams } from "@/types"
 import { serverHuggingfaceApiKey, serverHuggingfaceInferenceApiFileType, serverHuggingfaceInferenceApiModel, serverHuggingfaceInferenceApiModelRefinerModel, serverHuggingfaceInferenceApiModelTrigger } from "./config"
 
-export async function generate({
+export async function stableDiffusion({
   prompt,
   negativePrompt,
   guidanceScale,
@@ -12,7 +12,7 @@ export async function generate({
   height,
   numInferenceSteps,
   hfApiKey,
-}: GenerationParams) {
+}: StableDiffusionParams) {
   // throw new Error("Planned maintenance")
   if (!prompt) {
     const error = `cannot call the rendering API without a prompt, aborting..`

src/app/server/upscale.ts

@@ -4,17 +4,17 @@ import { generateSeed } from "@/lib/generateSeed"
 import { UpscalingParams } from "@/types"
 import { addBase64HeaderToPng } from "./addBase64HeaderToPng"
 
-const gradioApi = `${process.env.IMAGE_UPSCALING_API_GRADIO_URL || ""}`
-const accessToken = `${process.env.IMAGE_UPSCALING_API_GRADIO_TOKEN || ""}`
+const gradioApi = `https://jbilcke-hf-image-upscaling-api.hf.space`
+const microserviceApiKey = `${process.env.MICROSERVICE_API_SECRET_TOKEN || ""}`
 
-export const upscale = async ({
+export async function upscale({
   imageAsBase64,
   prompt,
   negativePrompt,
   scaleFactor,
   nbSteps,
   seed,
-}: UpscalingParams): Promise<string> => {
+}: UpscalingParams): Promise<string> {
 
   const addedPrompt = [
     "clean",
@@ -66,7 +66,7 @@ export const upscale = async ({
     body: JSON.stringify({
       fn_index: 0, // <- is it 0 or 1?
       data: [
-        accessToken, // <- this is new, need a redeployment of the upscaler
+        microserviceApiKey,
         imageAsBase64, 	// blob in 'parameter_5' Image component		
 				prompt, // string  in 'Prompt' Textbox component		
 				addedPrompt, // string  in 'Added Prompt' Textbox component		

src/lib/config.ts

@@ -100,8 +100,10 @@ export const characterExpressions = [
 export type CharacterExpression = typeof characterExpressions[number]
 
 export const photoshootModes = [
-  "portrait photo",
-  "gopro webcam portrait photo",
+  "realistic photo",
+  "gopro webcam photo",
+  "instagram influencer"
 ]
+
 export type PhotoshootMode = typeof photoshootModes[number]
 

src/lib/getStableCascadeParams.ts

@@ -0,0 +1,119 @@
+import { StableCascadeParams } from "@/types"
+
+import { CharacterExpression, CharacterGender, CharacterPose, GenerationMode, HairColor, HairStyle } from './config'
+import { generateSeed } from "./generateSeed"
+
+export function getStableCascadeParams({
+  prompt,
+  generationMode,
+  negativePrompt,
+  characterAge,
+  characterExpression,
+  characterGender,
+  characterHairColor,
+  characterHairStyle,
+  characterName,
+  characterPose
+}: {
+  prompt: string
+  generationMode: GenerationMode
+  negativePrompt: string
+  characterAge: number
+  characterExpression: CharacterExpression
+  characterGender: CharacterGender
+  characterHairColor: HairColor
+  characterHairStyle: HairStyle
+  characterName: string
+  characterPose: CharacterPose
+}): StableCascadeParams {
+
+  const baseParams: StableCascadeParams = {
+    prompt: "",
+    negativePrompt: "",
+
+    // between 0.1 and 15 (default 4)
+    guidanceScale: 4,
+
+    seed: generateSeed(),
+    width: 1024,
+    height: 1024,
+
+    // between 1 and 50 (default 20)
+    nbPriorInferenceSteps: 20,
+
+    // between 1 and 50 (default 10)
+    nbDecoderInferenceSteps: 10,
+  }
+
+  const params: StableCascadeParams = {
+    ...baseParams,
+    ...generationMode === "characters" ? {
+      prompt: [
+        `${characterAge} years old $${characterGender}`,
+        `${characterHairColor} ${characterHairStyle}`,
+        `looking ${characterExpression}`,
+        `${characterPose}`,
+        `named ${characterName}`,
+        'beautiful',
+        'award winning',
+        'sharp',
+        'crisp',
+        'centered',
+        'aligned'
+      ].filter(x => x).join(", "),
+      negativePrompt: [
+        negativePrompt,
+        'drawing',
+        'painting',
+        'unrealistic',
+        'sitting',
+        'chair',
+        '3D render',
+        'unaligned',
+        'cropped',
+        'bad hands',
+        'wrong hands',
+        'deformed',
+        'glitch',
+        'blurry',
+        'overexposed'
+      ].join(", "),
+    } : {
+      prompt: [
+        'picture of a single',
+        prompt,
+        '3D render',
+        'logo',
+        'ios icon',
+        'illustration',
+        'vector graphics',
+        'svg',
+        'beautiful',
+        'award winning',
+        'sharp',
+        'crisp',
+        'centered',
+        'aligned',
+      ].join(", "),
+      negativePrompt: [
+        negativePrompt,
+        'photo',
+        'gradient',
+        'many',
+        'realistic',
+        'shadow',
+        'multiple',
+        'various',
+        'unaligned',
+        'cropped',
+        'bad hands',
+        'wrong hands',
+        'deformed',
+        'glitch',
+        'blurry',
+        'overexposed'
+      ].join(", "),
+    }
+  }
+  return params
+}

src/lib/{getGenerationParams.ts → getStableDiffusionParams.ts}

@@ -1,9 +1,9 @@
-import { GenerationParams } from "@/types"
+import { StableDiffusionParams } from "@/types"
 
 import { CharacterExpression, CharacterGender, CharacterPose, GenerationMode, HairColor, HairStyle } from './config'
 import { generateSeed } from "./generateSeed"
 
-export function getGenerationParams({
+export function getStableDiffusionParams({
   generationMode,
   negativePrompt,
   characterAge,
@@ -23,9 +23,9 @@ export function getGenerationParams({
   characterHairStyle: HairStyle
   characterName: string
   characterPose: CharacterPose
-}): GenerationParams {
+}): StableDiffusionParams {
 
-  const baseParams: GenerationParams = {
+  const baseParams: StableDiffusionParams = {
     prompt: "",
     negativePrompt: "",
     guidanceScale: 9,
@@ -36,7 +36,7 @@ export function getGenerationParams({
     hfApiKey: "",
   }
 
-  const params: GenerationParams = {
+  const params: StableDiffusionParams = {
     ...baseParams,
     ...generationMode === "characters" ? {
       prompt: [

src/types.ts

@@ -13,7 +13,8 @@ export type Settings = {
   huggingfaceInferenceApiFileType: string
 }
 
-export interface GenerationParams {
+
+export interface StableDiffusionParams {
   prompt: string
   negativePrompt: string
   guidanceScale: number
@@ -24,6 +25,24 @@ export interface GenerationParams {
   hfApiKey: string
 }
 
+export interface StableCascadeParams {
+  prompt: string
+  negativePrompt: string
+
+
+  // between 0.1 and 15 (default 4)
+  guidanceScale: number
+  seed: number
+  width: number
+  height: number
+
+  // between 1 and 50 (default 20)
+  nbPriorInferenceSteps: number
+
+  // between 1 and 50 (default 10)
+  nbDecoderInferenceSteps: number
+}
+
 export type UpscalingParams = {
   imageAsBase64: string
   prompt: string