add LCM model

src/index.mts

@@ -162,7 +162,7 @@ app.post("/render", async (req, res) => {
     return
   }
   
-  console.log(req.body)
+  // console.log(req.body)
 
   const request = parseRenderRequest(req.body as RenderRequest)
 

src/production/renderImage.mts

@@ -1,5 +1,5 @@
-import { generateImageLCMAsBase64 } from "../providers/image-generation/generateImageLCM.mts"
-import { generateImageSDXLAsBase64 } from "../providers/image-generation/generateImageSDXL.mts"
+import { generateImageLCMAsBase64 } from "../providers/image-generation/generateImageLCMGradio.mts"
+import { generateImageSDXLAsBase64 } from "../providers/image-generation/generateImageSDXLGradio.mts"
 import { generateImageSDXL360AsBase64 } from "../providers/image-generation/generateImageSDXL360.mts"
 import { RenderedScene, RenderRequest } from "../types.mts"
 
@@ -52,7 +52,7 @@ export async function renderImage(
           throw new Error(`the generated image is empty`)
         }
       } catch (err) {
-        console.error(`failed to generate the image, although ${err}`)
+        // console.error(`failed to generate the image, due to`, err)
         response.error = `failed to render scene: ${err}`
         response.status = "error"
         response.assetUrl = ""

src/providers/image-generation/generateImage.mts

@@ -30,7 +30,7 @@ export async function generateImage(options: {
   const blob = await hf.textToImage({
     inputs: [
       positivePrompt,
-      "bautiful",
+      "beautiful",
       "award winning",
       // "intricate details",
       "high resolution"

src/providers/image-generation/generateImageLCMFetch.mts

@@ -0,0 +1,109 @@
+import { client } from "@gradio/client"
+
+import { generateSeed } from "../../utils/misc/generateSeed.mts"
+import { getValidNumber } from "../../utils/validators/getValidNumber.mts"
+
+// TODO add a system to mark failed instances as "unavailable" for a couple of minutes
+// console.log("process.env:", process.env)
+
+// note: to reduce costs I use the small A10s (not the large)
+// anyway, we will soon not need to use this cloud anymore 
+// since we will be able to leverage the Inference API
+const instance = `${process.env.VC_LCM_SPACE_API_URL || ""}`
+const secretToken = `${process.env.VC_MICROSERVICE_SECRET_TOKEN || ""}`
+
+// console.log("DEBUG:", JSON.stringify({ instances, secretToken }, null, 2))
+
+export async function generateImageLCMAsBase64(options: {
+  positivePrompt: string;
+  negativePrompt?: string;
+  seed?: number;
+  width?: number;
+  height?: number;
+  nbSteps?: number;
+}): Promise<string> {
+
+  // console.log("querying " + instance)
+  const positivePrompt = options?.positivePrompt || ""
+  if (!positivePrompt) {
+    throw new Error("missing prompt")
+  }
+
+  // the negative prompt CAN be missing, since we use a trick
+  // where we make the interface mandatory in the TS doc,
+  // but browsers might send something partial
+  const negativePrompt = options?.negativePrompt || ""
+  
+  // we treat 0 as meaning "random seed"
+  const seed = (options?.seed ? options.seed : 0) || generateSeed()
+
+  const width = getValidNumber(options?.width, 256, 1024, 512)
+  const height = getValidNumber(options?.height, 256, 1024, 512)
+  const nbSteps = getValidNumber(options?.nbSteps, 1, 8, 4)
+  // console.log("SEED:", seed)
+
+  const positive = [
+
+    // oh well.. is it too late to move this to the bottom?
+    "beautiful",
+
+    // too opinionated, so let's remove it
+    // "intricate details",
+
+    positivePrompt,
+
+    "award winning",
+    "high resolution"
+  ].filter(word => word)
+  .join(", ")
+
+  const negative =  [
+    negativePrompt,
+    "watermark",
+    "copyright",
+    "blurry",
+    // "artificial",
+    // "cropped",
+    "low quality",
+    "ugly"
+  ].filter(word => word)
+  .join(", ")
+
+  const res = await fetch(instance + (instance.endsWith("/") ? "" : "/") + "api/predict", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      // Authorization: `Bearer ${token}`,
+    },
+    body: JSON.stringify({
+      fn_index: 1, // <- important!
+      data: [
+        positive, // string  in 'Prompt' Textbox component		
+        negative, // string  in 'Negative prompt' Textbox component		
+        seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
+        width, // number (numeric value between 256 and 1024) in 'Width' Slider component		
+        height, // number (numeric value between 256 and 1024) in 'Height' Slider component		
+        0.0, // can be disabled for LCM-LORA-SSD-1B
+        nbSteps, // number (numeric value between 2 and 8) in 'Number of inference steps for base' Slider component			
+        secretToken
+      ],
+    }),
+    cache: "no-store",
+  })
+
+  const { data } = await res.json()
+
+
+  // Recommendation: handle errors
+  if (res.status !== 200 || !Array.isArray(data)) {
+    // This will activate the closest `error.js` Error Boundary
+    throw new Error(`Failed to fetch data (status: ${res.status})`)
+  }
+  // console.log("data:", data.slice(0, 50))
+
+  if (!data[0]) {
+    throw new Error(`the returned image was empty`)
+  }
+
+  return data[0] as string
+}

src/providers/image-generation/{generateImageLCM.mts → generateImageLCMGradio.mts}

@@ -1,3 +1,4 @@
+
 import { client } from "@gradio/client"
 
 import { generateSeed } from "../../utils/misc/generateSeed.mts"
@@ -21,9 +22,9 @@ export async function generateImageLCMAsBase64(options: {
   width?: number;
   height?: number;
   nbSteps?: number;
-}) {
+}): Promise<string> {
 
-  console.log("querying " + instance)
+  // console.log("querying " + instance)
   const positivePrompt = options?.positivePrompt || ""
   if (!positivePrompt) {
     throw new Error("missing prompt")
@@ -39,7 +40,7 @@ export async function generateImageLCMAsBase64(options: {
 
   const width = getValidNumber(options?.width, 256, 1024, 512)
   const height = getValidNumber(options?.height, 256, 1024, 512)
-  const nbSteps = getValidNumber(options?.nbSteps, 2, 40, 20)
+  const nbSteps = getValidNumber(options?.nbSteps, 1, 8, 4)
   // console.log("SEED:", seed)
 
   const positive = [
@@ -73,16 +74,14 @@ export async function generateImageLCMAsBase64(options: {
     hf_token: `${process.env.VC_HF_API_TOKEN}` as any
   })
 
-  
   const rawResponse = (await api.predict("/run", [		
     positive, // string  in 'Prompt' Textbox component		
     negative, // string  in 'Negative prompt' Textbox component		
-    true, // boolean  in 'Use negative prompt' Checkbox component		
     seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
     width, // number (numeric value between 256 and 1024) in 'Width' Slider component		
     height, // number (numeric value between 256 and 1024) in 'Height' Slider component		
-    8, // number (numeric value between 1 and 20) in 'Guidance scale for base' Slider component		
-    nbSteps, // number (numeric value between 210 and 40) in 'Number of inference steps for base' Slider component			
+    0.0, // can be disabled for LCM-LORA-SSD-1B
+    nbSteps, // number (numeric value between 2 and 8) in 'Number of inference steps for base' Slider component			
     secretToken
   ])) as any
     
@@ -91,4 +90,4 @@ export async function generateImageLCMAsBase64(options: {
     throw new Error(`the returned image was empty`)
   }
   return result
-}
+}

src/providers/image-generation/generateImageSDXL360.mts

@@ -18,7 +18,7 @@ export async function generateImageSDXL360AsBase64(options: {
   width?: number;
   height?: number;
   nbSteps?: number;
-}) {
+}): Promise<string> {
 
   const positivePrompt = options?.positivePrompt || ""
   if (!positivePrompt) {
@@ -32,7 +32,7 @@ export async function generateImageSDXL360AsBase64(options: {
   const width = getValidNumber(options?.width, 256, 1024, 512)
   const height = getValidNumber(options?.height, 256, 1024, 512)
   const nbSteps = getValidNumber(options?.nbSteps, 5, 100, 20)
-  console.log("SEED FOR 360:", seed)
+  // console.log("SEED FOR 360:", seed)
 
   const instance = instances.shift()
   instances.push(instance)

src/providers/image-generation/generateImageSDXLFetch.mts

@@ -0,0 +1,113 @@
+import { client } from "@gradio/client"
+
+import { generateSeed } from "../../utils/misc/generateSeed.mts"
+import { getValidNumber } from "../../utils/validators/getValidNumber.mts"
+
+// TODO add a system to mark failed instances as "unavailable" for a couple of minutes
+// console.log("process.env:", process.env)
+
+// note: to reduce costs I use the small A10s (not the large)
+// anyway, we will soon not need to use this cloud anymore 
+// since we will be able to leverage the Inference API
+const instance = `${process.env.VC_SDXL_SPACE_API_URL || ""}`
+const secretToken = `${process.env.VC_MICROSERVICE_SECRET_TOKEN || ""}`
+
+// console.log("DEBUG:", JSON.stringify({ instances, secretToken }, null, 2))
+
+export async function generateImageSDXLAsBase64(options: {
+  positivePrompt: string;
+  negativePrompt?: string;
+  seed?: number;
+  width?: number;
+  height?: number;
+  nbSteps?: number;
+}): Promise<string> {
+
+  const positivePrompt = options?.positivePrompt || ""
+  if (!positivePrompt) {
+    throw new Error("missing prompt")
+  }
+
+  // the negative prompt CAN be missing, since we use a trick
+  // where we make the interface mandatory in the TS doc,
+  // but browsers might send something partial
+  const negativePrompt = options?.negativePrompt || ""
+  
+  // we treat 0 as meaning "random seed"
+  const seed = (options?.seed ? options.seed : 0) || generateSeed()
+
+  const width = getValidNumber(options?.width, 256, 1024, 512)
+  const height = getValidNumber(options?.height, 256, 1024, 512)
+  const nbSteps = getValidNumber(options?.nbSteps, 5, 100, 20)
+  // console.log("SEED:", seed)
+
+  const positive = [
+
+    // oh well.. is it too late to move this to the bottom?
+    "beautiful",
+    // "intricate details",
+    positivePrompt,
+
+    "award winning",
+    "high resolution"
+  ].filter(word => word)
+  .join(", ")
+
+  const negative =  [
+    negativePrompt,
+    "watermark",
+    "copyright",
+    "blurry",
+    // "artificial",
+    // "cropped",
+    "low quality",
+    "ugly"
+  ].filter(word => word)
+  .join(", ")
+
+  const res = await fetch(instance + (instance.endsWith("/") ? "" : "/") + "api/predict", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      // Authorization: `Bearer ${token}`,
+    },
+    body: JSON.stringify({
+      fn_index: 1, // <- important!
+      data: [
+        positive, // string  in 'Prompt' Textbox component		
+        negative, // string  in 'Negative prompt' Textbox component		
+        positive, // string  in 'Prompt 2' Textbox component		
+        negative, // string  in 'Negative prompt 2' Textbox component		
+        true, // boolean  in 'Use negative prompt' Checkbox component		
+        false, // boolean  in 'Use prompt 2' Checkbox component		
+        false, // boolean  in 'Use negative prompt 2' Checkbox component		
+        seed, // number (numeric value between 0 and 2147483647) in 'Seed' Slider component		
+        width, // number (numeric value between 256 and 1024) in 'Width' Slider component		
+        height, // number (numeric value between 256 and 1024) in 'Height' Slider component		
+        8, // number (numeric value between 1 and 20) in 'Guidance scale for base' Slider component		
+        8, // number (numeric value between 1 and 20) in 'Guidance scale for refiner' Slider component		
+        nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for base' Slider component		
+        nbSteps, // number (numeric value between 10 and 100) in 'Number of inference steps for refiner' Slider component		
+        true, // boolean  in 'Apply refiner' Checkbox component,
+        secretToken
+      ],
+    }),
+    cache: "no-store",
+  })
+
+  const { data } = await res.json()
+
+  // console.log("data:", data)
+  // Recommendation: handle errors
+  if (res.status !== 200 || !Array.isArray(data)) {
+    // This will activate the closest `error.js` Error Boundary
+    throw new Error(`Failed to fetch data (status: ${res.status})`)
+  }
+  // console.log("data:", data.slice(0, 50))
+
+  if (!data[0]) {
+    throw new Error(`the returned image was empty`)
+  }
+
+  return data[0] as string
+}

src/providers/image-generation/{generateImageSDXL.mts → generateImageSDXLGradio.mts}

@@ -21,7 +21,7 @@ export async function generateImageSDXLAsBase64(options: {
   width?: number;
   height?: number;
   nbSteps?: number;
-}) {
+}): Promise<string> {
 
   const positivePrompt = options?.positivePrompt || ""
   if (!positivePrompt) {

src/utils/requests/parseRenderRequest.mts

@@ -5,7 +5,7 @@ import { getValidNumber } from "../validators/getValidNumber.mts"
 
 export function parseRenderRequest(request: RenderRequest) {
 
-  console.log("parseRenderRequest: "+JSON.stringify(request, null, 2))
+  // console.log("parseRenderRequest: "+JSON.stringify(request, null, 2))
   try {
     request.nbFrames = getValidNumber(request.nbFrames, 1, 24, 16)
 
@@ -22,16 +22,16 @@ export function parseRenderRequest(request: RenderRequest) {
 
     request.upscalingFactor = getValidNumber(request.upscalingFactor, 0, 4, 0)
 
-    request.nbSteps = getValidNumber(request.nbSteps, 5, 50, 10)
+    request.nbSteps = getValidNumber(request.nbSteps, 1, 50, 10)
 
     request.analyze = request?.analyze ? true : false
 
     if (isVideo) {
-      request.width = getValidNumber(request.width, 256, 1024, 1024)
-      request.height = getValidNumber(request.height, 256, 1024, 512)
+      request.width = getValidNumber(request.width, 256, 2048, 576)
+      request.height = getValidNumber(request.height, 256, 2048, 320)
     } else {
-      request.width = getValidNumber(request.width, 256, 1280, 576)
-      request.height = getValidNumber(request.height, 256, 720, 320)
+      request.width = getValidNumber(request.width, 256, 2048, 1024)
+      request.height = getValidNumber(request.height, 256, 2048, 1024)
     }
 
     request.turbo = getValidBoolean(request.turbo, false)
@@ -42,6 +42,6 @@ export function parseRenderRequest(request: RenderRequest) {
     console.error(`failed to parse the render request: ${err}`)
   }
 
-  console.log("parsed request: "+JSON.stringify(request, null, 2))
+  // console.log("parsed request: "+JSON.stringify(request, null, 2))
   return request
 }