working on the assistant

package.json

@@ -12,13 +12,16 @@
   "dependencies": {
     "@aitube/clap": "0.0.26",
     "@aitube/engine": "0.0.19",
-    "@aitube/timeline": "0.0.23",
+    "@aitube/timeline": "file:/Users/jbilcke/Projects/Typescript_Libraries/aitube-timeline",
     "@fal-ai/serverless-client": "^0.10.3",
     "@huggingface/hub": "^0.15.1",
     "@huggingface/inference": "^2.7.0",
     "@langchain/anthropic": "^0.2.0",
+    "@langchain/cohere": "^0.0.11",
     "@langchain/core": "^0.2.6",
+    "@langchain/google-vertexai": "^0.0.18",
     "@langchain/groq": "^0.0.12",
+    "@langchain/mistralai": "^0.0.24",
     "@langchain/openai": "^0.1.1",
     "@monaco-editor/react": "^4.6.0",
     "@radix-ui/react-accordion": "^1.1.2",

src/app/api/assistant/askAnyAssistant.ts

@@ -0,0 +1,178 @@
+"use server"
+
+import { ClapSegmentCategory } from "@aitube/clap"
+import { RunnableLike } from "@langchain/core/runnables"
+import { ChatPromptValueInterface } from "@langchain/core/dist/prompt_values"
+import { AIMessageChunk } from "@langchain/core/messages"
+import { ChatPromptTemplate } from "@langchain/core/prompts"
+import { StructuredOutputParser } from "@langchain/core/output_parsers"
+import { ChatOpenAI } from "@langchain/openai"
+import { ChatGroq } from "@langchain/groq"
+import { ChatAnthropic } from "@langchain/anthropic"
+import { ChatCohere } from "@langchain/cohere"
+import { ChatMistralAI } from "@langchain/mistralai"
+import { ChatVertexAI } from "@langchain/google-vertexai"
+// Hugging Face will be supported once the following package becomes available
+// import { ChatHuggingFace } from "@langchain/huggingface"
+
+import { AssistantRequest, AssistantResponse, ComputeProvider } from "@/types"
+
+import { SimplifiedSegmentData, simplifiedSegmentDataZ } from "./types"
+import { examples, humanTemplate, systemTemplate } from "./templates"
+
+const parser = StructuredOutputParser.fromZodSchema(simplifiedSegmentDataZ)
+
+const formatInstructions = parser.getFormatInstructions()
+
+/**
+ * Query the preferred language model on the user prompt + the segments of the current scene
+ *
+ * @param userPrompt 
+ * @param segments 
+ * @returns 
+ */
+export async function askAnyAssistant({
+  settings,
+
+  prompt,
+
+  // the slice to edit
+  segments = [],
+
+  fullScene = "",
+
+  actionLine = "",
+
+  // used to provide more context
+  entities = {},
+
+  // used to provide more context
+  projectInfo = ""
+}: AssistantRequest): Promise<AssistantResponse> {
+
+  const provider = settings.assistantProvider
+
+  if (!provider) { throw new Error(`Missing assistant provider`)}
+
+  let coerceable: undefined | RunnableLike<ChatPromptValueInterface, AIMessageChunk> =
+    provider === ComputeProvider.GROQ
+    ? new ChatGroq({
+      apiKey: settings.groqApiKey,
+      modelName: settings.groqModelForAssistant,
+      // temperature: 0.7,
+    })
+    : provider === ComputeProvider.OPENAI
+    ? new ChatOpenAI({
+        openAIApiKey: settings.openaiApiKey,
+        modelName: settings.openaiModelForAssistant,
+       // temperature: 0.7,
+      })
+    : provider === ComputeProvider.ANTHROPIC
+    ? new ChatAnthropic({
+      anthropicApiKey: settings.anthropicApiKey,
+      modelName: settings.anthropicModelForAssistant,
+      // temperature: 0.7,
+    })
+    : provider === ComputeProvider.COHERE
+    ? new ChatCohere({
+      apiKey: settings.cohereApiKey,
+      model: settings.cohereModelForAssistant,
+      // temperature: 0.7,
+    })
+    : provider === ComputeProvider.MISTRALAI
+    ? new ChatMistralAI({
+      apiKey: settings.mistralAiApiKey,
+      modelName: settings.mistralAiModelForAssistant,
+      // temperature: 0.7,
+    })
+    : provider === ComputeProvider.GOOGLE
+    ? new ChatVertexAI({
+      apiKey: settings.googleApiKey,
+      modelName: settings.googleModelForAssistant,
+      // temperature: 0.7,
+    })
+    : undefined
+
+  if (!coerceable) { throw new Error(`Provider ${provider} is not supported yet. If a LangChain bridge exists for this provider, then you can add it to Clapper.`)}
+
+  const chatPrompt = ChatPromptTemplate.fromMessages(
+    [
+      ["system", systemTemplate],
+      ["human", humanTemplate],
+    ]
+  )
+
+  // we don't give the whole thing to the LLM as to not confuse it,
+  // and also to keep things tight and performant
+  const inputData: SimplifiedSegmentData[] = segments.map((segment) => ({
+    prompt: segment.prompt,
+    category: segment.category,
+  } as SimplifiedSegmentData))
+
+ // console.log("INPUT:", JSON.stringify(inputData, null, 2))
+
+  const chain = chatPrompt.pipe(coerceable).pipe(parser)
+
+  try {
+    const result = await chain.invoke({
+      formatInstructions,
+      examples,
+      projectInfo,
+      fullScene,
+      actionLine,
+      userPrompt: prompt,
+      inputData,
+    })
+
+    console.log("OUTPUT:", JSON.stringify(result, null, 2))
+
+    /*
+    this whole code doesn't work well actually..
+
+    let match: SegmentData | undefined = segments[result.index] || undefined
+
+    // LLM gave an object, but the index is wrong
+    if (!match) {
+      match = segments.find(s => s.category === result.category) || undefined
+    }
+    */
+    
+    // let's create a new segment then!
+    const categoryName: ClapSegmentCategory =
+      result?.category && Object.keys(ClapSegmentCategory).includes(result.category.toUpperCase())
+      ? (result.category as ClapSegmentCategory)
+      : ClapSegmentCategory.GENERIC
+
+    return {
+      prompt: result?.prompt || "",
+      categoryName,
+      llmOutput: "",
+    }
+  } catch (err1) {
+
+    // a common scenario is when the output from the LLM is just not a JSON
+    // this can happen quite often, for instance if the user tried to bypass
+    // our prompt, or if they are just asking generic questions
+    const errObj = err1 as any
+    try {
+      const keys = Object.keys(errObj)
+      // console.log("keys:", keys)
+      if (errObj.llmOutput) {
+        return {
+          prompt: "",
+          categoryName: ClapSegmentCategory.GENERIC,
+          llmOutput: `${errObj.llmOutput || ""}`,
+        }
+      }
+    } catch (err2) {
+      // err2 is just the error for when the LLM failed to reply
+      console.error(`----<${err1}>----`)
+    }
+
+    return {
+      prompt: "",
+      categoryName: ClapSegmentCategory.GENERIC,
+      llmOutput: ""
+    }
+  }
+}

src/app/api/assistant/providers/openai/askAssistant.ts

@@ -9,19 +9,17 @@ import { AssistantRequest, AssistantResponse } from "@/types"
 import { SimplifiedSegmentData, simplifiedSegmentDataZ } from "../../types"
 import { examples, humanTemplate, systemTemplate } from "../../templates"
 
-
 const parser = StructuredOutputParser.fromZodSchema(simplifiedSegmentDataZ)
 
 const formatInstructions = parser.getFormatInstructions()
 
-
 /**
  * Query the preferred language model on the user prompt + the segments of the current scene
  * @param userPrompt 
  * @param segments 
  * @returns 
  */
-export async function queryAssistant({
+export async function askAssistant({
   settings,
 
   prompt,
@@ -54,9 +52,6 @@ export async function queryAssistant({
     ]
   )
 
-
-
-
   // we don't give the whole thing to the LLM as to not confuse it,
   // and also to keep things tight and performant
   const inputData: SimplifiedSegmentData[] = segments.map((segment) => ({

src/app/api/assistant/route.ts

@@ -0,0 +1,17 @@
+import { NextResponse, NextRequest } from "next/server"
+
+import { AssistantRequest, AssistantResponse } from "@/types"
+import { askAnyAssistant } from "./askAnyAssistant"
+
+export async function POST(req: NextRequest) {
+  // do we really need to secure it?
+  // I mean.. in the end, the user is using their own credentials,
+  // so they cannot siphon free OpenAI, HF, Replicate tokens
+  // console.log(`TODO Julian: secure the endpoint`)
+  // await throwIfInvalidToken(req.headers.get("Authorization"))
+  const request = (await req.json()) as AssistantRequest
+
+  const response: AssistantResponse = await askAnyAssistant(request)
+
+  return NextResponse.json(response)
+}

src/app/api/resolve/route.ts

@@ -1,4 +1,5 @@
 import { NextResponse, NextRequest } from "next/server"
+import { ClapSegmentCategory } from "@aitube/clap"
 
 import { resolveSegment as resolveSegmentUsingHuggingFace } from "./providers/huggingface"
 import { resolveSegment as resolveSegmentUsingComfyReplicate } from "./providers/comfy-replicate"
@@ -8,7 +9,6 @@ import { resolveSegment as resolveSegmentUsingFalAi } from "./providers/falai"
 import { resolveSegment as resolveSegmentUsingModelsLab } from "./providers/modelslab"
 
 import { ComputeProvider, ResolveRequest } from "@/types"
-import { ClapSegmentCategory } from "@aitube/clap"
 
 export async function POST(req: NextRequest) {
   // do we really need to secure it?

src/app/main.tsx

@@ -21,6 +21,7 @@ import { useUI } from "@/controllers/ui"
 import { TopBar } from "@/components/toolbars/top-bar"
 import { Timeline } from "@/components/core/timeline"
 import { useIO } from "@/controllers/io/useIO"
+import { ChatView } from "@/components/assistant/ChatView"
 
 type DroppableThing = { files: File[] }
 
@@ -71,23 +72,31 @@ function MainContent() {
           `flex flex-row flex-grow w-full overflow-hidden`, 
           isEmpty ? "opacity-0" : "opacity-100"
         )}>
-          <ReflexContainer orientation="horizontal">
-            <ReflexElement
-               minSize={showTimeline ? 100 : 1}
-              >
-              <Monitor />
+          <ReflexContainer orientation="vertical">
+          
+            <ReflexElement>
+              <ReflexContainer orientation="horizontal">
+                <ReflexElement
+                  minSize={showTimeline ? 100 : 1}
+                  >
+                  <Monitor />
+                </ReflexElement>
+                <ReflexSplitter />
+                <ReflexElement
+                  size={showTimeline ? 400 : 1}
+                  minSize={showTimeline ? 100 : 1}
+                  maxSize={showTimeline ? 1600 : 1}
+                  >
+                  <Timeline />
+                </ReflexElement>
+              </ReflexContainer>
             </ReflexElement>
-            <ReflexSplitter />
-            <ReflexElement
-              size={showTimeline ? 400 : 1}
-              minSize={showTimeline ? 100 : 1}
-              maxSize={showTimeline ? 1600 : 1}
-              >
-              <Timeline />
-            </ReflexElement>
-            {/* showChat && <ReflexSplitter /> */}
-            {/* showChat && <ReflexElement size={300}><ChatView /></ReflexElement> */}
+
+          {showChat && <ReflexSplitter />}
+          {showChat && <ReflexElement size={300}><ChatView /></ReflexElement>}
+
           </ReflexContainer>
+          
         </div>
 
       <SettingsDialog />

src/components/assistant/ChatView.tsx

@@ -1,12 +1,8 @@
 "use client"
 
 import { useState, useTransition } from "react"
-import { ClapOutputType, ClapProject, ClapSegment, ClapSegmentCategory, newSegment } from "@aitube/clap"
-import { DEFAULT_DURATION_IN_MS_PER_STEP, findFreeTrack, useTimeline } from "@aitube/timeline"
 
 import { useAssistant } from "@/controllers/assistant/useAssistant"
-import { queryAssistant } from "@/app/api/assistant/providers/openai/askAssistant"
-import { useSettings } from "@/controllers/settings"
 
 import { ChatBubble } from "./ChatBubble"
 import { Input } from "../ui/input"
@@ -14,22 +10,10 @@ import { Input } from "../ui/input"
 export function ChatView() {
   const [_isPending, startTransition] = useTransition()
 
-  /*
-  const others = useLiveProject((state) => state.liveblocks.others)
-  const userCount = others.length
-  console.log(`TODO: finish this There are ${userCount} other user(s) online`)
-  */
   
   const [draft, setDraft] = useState("")
-  const runCommand = useAssistant((s) => s.runCommand)
   const history = useAssistant((s) => s.history)
-  const addEventToHistory = useAssistant((s) => s.addEventToHistory)
-
-  /*
-  const updateSegment = useApp((state) => state.updateSegment)
-  const addSegment = useApp((state) => state.addSegment)
-  const projectInfo = useApp((state) => state.projectInfo)
-  */
+  const processMessage = useAssistant((s) => s.processMessage)
 
   const handleSubmit = () => {
     const message = draft.trim()
@@ -39,146 +23,7 @@ export function ChatView() {
     }
 
     setDraft("")
-
-    addEventToHistory({
-      senderId: "director",
-      senderName: "Director",
-      message,
-    })
-  
-    // the LLM is behind a server action
-    startTransition(async () => {
-  
-       
-      const basicCommand = await runCommand(message)
-      // LLM analysis can be slow and expensive, so first we try to see if this was a trivial command
-      // like "start", "pause", "stop" etc
-      if (basicCommand) {
-
-        addEventToHistory({
-          senderId: "assistant",
-          senderName: "Assistant",
-          message: `${basicCommand}`,
-        })
-        return // no need to go further
-      }
-    
-
-      const clap: ClapProject = useTimeline.getState().clap
-
-      console.log(`TODO @julian: restore the concept of "addSegment()", "updateSegment()", "active segment" and "cursor position" inside @aitube-timeline`)
-      // const { addSegment, activeSegments, cursorInSteps,  } = useTimeline.getState()
-
-      const activeSegments: ClapSegment[] = []
-      const cursorInSteps = 0
-  
-      const referenceSegment: ClapSegment | undefined = activeSegments.at(0)
-      
-      if (!referenceSegment) {
-        throw new Error(`No segment under the current cursor`)
-      }
-
-      console.log(`TODO @julian: filter entities to only keep the ones in the current active segment? (although.. maybe a bad idea since the LLM need as much context as possible to "fill in the gap" eg. repair/invent missing elements of the story)`)
-
-      const entities = clap.entityIndex
-
-      const projectInfo = clap.meta.description
-
-      const sceneId = referenceSegment.sceneId
-
-      const scene = clap.scenes.find(s => s.id === sceneId)
-      
-      const fullScene: string = scene?.sequenceFullText || ""
-      const actionLine: string = scene?.line || ""
-    
-      const segments: ClapSegment[] = activeSegments
-        .filter(s =>
-          s.category === ClapSegmentCategory.LOCATION ||
-          s.category === ClapSegmentCategory.TIME ||
-          s.category === ClapSegmentCategory.LIGHTING ||
-          s.category === ClapSegmentCategory.ACTION ||
-          s.category === ClapSegmentCategory.DIALOGUE ||
-          s.category === ClapSegmentCategory.WEATHER  
-        )
-  
-      console.log(`TODO @julian: provide both contextual segments and editable ones to the LLM?`)
-
-      const { prompt, categoryName, llmOutput } = await queryAssistant({
-        settings: useSettings.getState().getSettings(),
-        prompt: message,
-        segments,
-        fullScene,
-        actionLine,
-        entities,
-        projectInfo,
-      })
-      if (!prompt.length) {
-        addEventToHistory({
-          senderId: "assistant",
-          senderName: "Assistant",
-          message: llmOutput || "🤔" // or "???" for a "boomer" theme
-        })
-        return
-      }
-  
-      let match = segments.find(s => s.category === categoryName) || undefined
-      if (!match) {
-  
-        const startTimeInMs = cursorInSteps * DEFAULT_DURATION_IN_MS_PER_STEP
-        const durationInSteps = 4
-        const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP
-        const endTimeInMs = startTimeInMs + durationInMs
-  
-        const newSeg = newSegment({
-          startTimeInSteps: cursorInSteps,
-          prompt: [prompt],
-          durationInSteps: 4,
-          trackId: findFreeTrack({
-            segments,
-            startTimeInMs,
-            endTimeInMs,
-          }), 
-          outputType: ClapOutputType.TEXT,
-          categoryName,
-        })
-        console.log("Creating new existing segment:", newSeg)
-  
-        console.log(`TODO Julian: add the segment!!`)
-        // addSegment(newSeg)
-
-        addEventToHistory({
-          senderId: "assistant",
-          senderName: "Assistant",
-          message: `Segment added: ${newSeg.prompt}`,
-        })
-      } else {
-    
-        console.log("Updating an existing segment to:", {
-          ...match,
-          prompt,
-          label: prompt,
-        })
-  
-        console.log(`TODO Julian: update the segment!!`)
-        // addSegment(newSeg)
-
-        /*
-        updateSegment({
-          ...match,
-          prompt,
-          label: prompt,
-        })
-          */
-
-        addEventToHistory({
-          senderId: "assistant",
-          senderName: "Assistant",
-          message: `Segment updated: ${prompt}`,
-        })
-      }
-
-    })
-  
+    processMessage(draft.trim())
   }
 
   return (

src/components/settings/constants.ts

@@ -1,25 +1,33 @@
 import { ComfyIcuAccelerator, ComputeProvider } from "@/types"
 
 export const computeProviderShortNames = {
-  [ComputeProvider.NONE]: "None",
-  [ComputeProvider.CUSTOM]: "Custom API",
-  [ComputeProvider.HUGGINGFACE]: "Hugging Face",
+  [ComputeProvider.ANTHROPIC]: "Anthropic",
+  [ComputeProvider.COHERE]: "Cohere",
+  [ComputeProvider.COMFY_COMFYICU]: "Comfy.icu",
   [ComputeProvider.COMFY_HUGGINGFACE]: "Hugging Face Comfy",
-  [ComputeProvider.REPLICATE]: "Replicate",
   [ComputeProvider.COMFY_REPLICATE]: "Replicate Comfy",
-  [ComputeProvider.COMFY_COMFYICU]: "Comfy.icu",
+  [ComputeProvider.CUSTOM]: "Custom API",
   [ComputeProvider.ELEVENLABS]: "ElevenLabs",
+  [ComputeProvider.FALAI]: "Fal.ai",
+  [ComputeProvider.FIREWORKSAI]: "FireworksAI",
+  [ComputeProvider.GOOGLE]: "Google (VertexAI)",
+  [ComputeProvider.GROQ]: "Groq",
+  [ComputeProvider.HUGGINGFACE]: "Hugging Face",
+  [ComputeProvider.KITSAI]: "Kits.ai",
+  [ComputeProvider.MISTRALAI]: "MistralAI",
+  [ComputeProvider.MODELSLAB]: "ModelsLab",
+  [ComputeProvider.NONE]: "None", // <-- this is the default
   [ComputeProvider.OPENAI]: "OpenAI",
+  [ComputeProvider.REPLICATE]: "Replicate",
   [ComputeProvider.STABILITYAI]: "StabilityAI",
-  [ComputeProvider.GROQ]: "Groq",
-  [ComputeProvider.FALAI]: "Fal.ai",
-  [ComputeProvider.MODELSLAB]: "ModelsLab"
 }
 
 export const availableComputeProvidersForAssistant = [
   ComputeProvider.HUGGINGFACE,
   ComputeProvider.GROQ,
   ComputeProvider.OPENAI,
+  ComputeProvider.ANTHROPIC,
+  ComputeProvider.FIREWORKSAI,
 ]
 
 export const availableComputeProvidersForImages = [
@@ -27,6 +35,8 @@ export const availableComputeProvidersForImages = [
   ComputeProvider.REPLICATE,
   ComputeProvider.COMFY_REPLICATE,
   ComputeProvider.COMFY_COMFYICU,
+  ComputeProvider.STABILITYAI,
+  ComputeProvider.FIREWORKSAI,
   ComputeProvider.FALAI,
   ComputeProvider.MODELSLAB,
 ]
@@ -36,6 +46,8 @@ export const availableComputeProvidersForVideos = [
   ComputeProvider.REPLICATE,
   ComputeProvider.COMFY_REPLICATE,
   ComputeProvider.COMFY_COMFYICU,
+  ComputeProvider.STABILITYAI,
+  // ComputeProvider.FIREWORKSAI,
   ComputeProvider.FALAI,
   ComputeProvider.MODELSLAB,
 ]
@@ -45,6 +57,7 @@ export const availableComputeProvidersForMusic = [
   ComputeProvider.COMFY_REPLICATE,
   ComputeProvider.COMFY_COMFYICU,
   ComputeProvider.STABILITYAI,
+  // ComputeProvider.FIREWORKSAI,
   ComputeProvider.FALAI,
   ComputeProvider.MODELSLAB,
 ]
@@ -54,13 +67,16 @@ export const availableComputeProvidersForSound = [
   ComputeProvider.COMFY_REPLICATE,
   ComputeProvider.COMFY_COMFYICU,
   ComputeProvider.STABILITYAI,
+  // ComputeProvider.FIREWORKSAI,
   ComputeProvider.FALAI,
   ComputeProvider.ELEVENLABS,
 ]
 
 export const availableComputeProvidersForVoice = [
   ComputeProvider.ELEVENLABS,
+  ComputeProvider.KITSAI,
   ComputeProvider.STABILITYAI,
+  // ComputeProvider.FIREWORKSAI,
   ComputeProvider.HUGGINGFACE,
   ComputeProvider.COMFY_REPLICATE,
   ComputeProvider.COMFY_COMFYICU,
@@ -79,7 +95,36 @@ export const availableComfyIcuAccelerators = {
 
 export const availableModelsForAssistant: Partial<Record<ComputeProvider, string[]>> = {
   [ComputeProvider.OPENAI]: [
+    "gpt-4",
+    "gpt-4-turbo",
     "gpt-4o",
+  ],
+  [ComputeProvider.GROQ]: [
+    "Mixtral-8x7b-32768",
+    "Gemma-7b-lt",
+    "Llama3-70b-8192",
+    "Llama3-8b-8192",
+  ],
+  [ComputeProvider.ANTHROPIC]: [
+    "claude-3-opus-20240229",
+    "claude-3-sonnet-20240229",
+    "claude-3-haiku-20240307"
+  ], 
+  [ComputeProvider.GOOGLE]: [
+    "claude-3-opus@20240229",
+    "claude-3-sonnet@20240229",
+    "claude-3-haiku@20240307"
+  ],
+  [ComputeProvider.MISTRALAI]: [
+    "mistral-small-latest",
+    "open-mistral-7b"
+  ],
+  [ComputeProvider.HUGGINGFACE]: [
+    "HuggingFaceH4/zephyr-7b-beta",
+    "mistralai/Mixtral-8x7B-Instruct-v0.1"
+  ],
+  [ComputeProvider.FIREWORKSAI]: [
+    "fireworks/llama-v3-70b-instruct",
   ]
 }
 
@@ -102,6 +147,17 @@ export const availableModelsForImageGeneration: Partial<Record<ComputeProvider,
     // "fal-ai/pulid",
     // "fal-ai/image-to-image",
     // "fal-ai/omni-zero",
+  ],
+  [ComputeProvider.STABILITYAI]: [
+    "stable-image/generate/ultra",
+    "stable-image/generate/core",
+    "stable-image/generate/sd3"
+  ],
+  [ComputeProvider.FIREWORKSAI]: [
+    "stability/sd3",
+    "accounts/stability/models/sd3-turbo",
+    "fireworks/stable-diffusion-xl-1024-v1-0",
+    "accounts/fireworks/models/playground-v2-5-1024px-aesthetic",
   ]
 }
 
@@ -109,12 +165,19 @@ export const availableModelsForImageGeneration: Partial<Record<ComputeProvider,
 export const availableModelsForImageUpscaling: Partial<Record<ComputeProvider, string[]>> = {
   [ComputeProvider.FALAI]: [
     "fal-ai/ccsr",
-  ]
+  ],
+  [ComputeProvider.STABILITYAI]: [
+    "stable-image/upscale/conservative",
+    "stable-image/upscale/creative"
+  ],
 }
 
 export const availableModelsForVideoGeneration: Partial<Record<ComputeProvider, string[]>> = {
   [ComputeProvider.FALAI]: [
     "fal-ai/stable-video",
+  ],
+  [ComputeProvider.STABILITYAI]: [
+    "image-to-video",
   ]
 }
 

src/components/toolbars/top-menu/assistant/index.tsx

@@ -0,0 +1,45 @@
+"use client"
+
+import {
+  MenubarCheckboxItem,
+  MenubarContent,
+  MenubarItem,
+  MenubarMenu,
+  MenubarSeparator,
+  MenubarSub,
+  MenubarSubContent,
+  MenubarSubTrigger,
+  MenubarTrigger
+} from "@/components/ui/menubar"
+import { useSettings } from "@/controllers/settings"
+import { useUI } from "@/controllers/ui"
+
+import { ProviderList } from "../lists/ProviderList"
+import { availableComputeProvidersForAssistant } from "@/components/settings/constants"
+import { SettingsCategory } from "@/types"
+import { AssistantModelList } from "../lists/AssistantModelList"
+
+export function TopMenuAssistant() {
+  const setShowSettings = useUI(s => s.setShowSettings)
+  const assistantProvider = useSettings(s => s.assistantProvider)
+  const setAssistantProvider = useSettings(s => s.setAssistantProvider)
+  const assistantModel = useSettings(s => s.assistantModel)
+  const setAssistantModel = useSettings(s => s.setAssistantModel)
+  return (
+    <MenubarMenu>
+      <MenubarTrigger>Assistant</MenubarTrigger>
+      <MenubarContent>
+        <MenubarSub>
+          <MenubarItem onClick={() => { setShowSettings(SettingsCategory.ASSISTANT) }}>Show advanced settings</MenubarItem>
+          <MenubarSeparator />
+          <AssistantModelList provider={assistantProvider} current={assistantModel} setter={setAssistantModel} />
+          <ProviderList providers={availableComputeProvidersForAssistant} current={assistantProvider} setter={setAssistantProvider} />
+          <MenubarSeparator />
+          <MenubarItem
+             disabled
+            >Usage and costs: not implemented</MenubarItem>
+        </MenubarSub>
+      </MenubarContent>
+    </MenubarMenu>
+  )
+}

src/components/toolbars/top-menu/index.tsx

@@ -7,6 +7,7 @@ import { TopMenuVideo } from "./video"
 import { TopMenuVoice } from "./voice"
 import { TopMenuSound } from "./sound"
 import { TopMenuMusic } from "./music"
+import { TopMenuAssistant } from "./assistant"
 
 import { TopMenuView } from "./view"
 import { cn } from "@aitube/timeline"
@@ -30,6 +31,7 @@ export function TopMenu() {
       <TopMenuVoice />
       <TopMenuSound />
       <TopMenuMusic />
+      <TopMenuAssistant />
       <TopMenuView />
       <div className={cn(`
         flex flex-row flex-grow

src/components/toolbars/top-menu/lists/AssistantModelList.tsx

@@ -0,0 +1,55 @@
+"use client"
+
+import {
+  MenubarCheckboxItem,
+  MenubarContent,
+  MenubarItem,
+  MenubarMenu,
+  MenubarSeparator,
+  MenubarSub,
+  MenubarSubContent,
+  MenubarSubTrigger,
+} from "@/components/ui/menubar"
+
+import { TagColor } from "@/components/tags/types"
+import { Tag } from "@/components/tags/Tag"
+import { ComputeProvider } from "@/types"
+import { availableModelsForAssistant } from "@/components/settings/constants"
+
+export function AssistantModelList({
+  provider,
+  current,
+  setter,
+}: {
+  provider?: ComputeProvider
+  current?: string
+  setter: (model: string) => void
+}) {
+  const models: string[] = provider ? (availableModelsForAssistant[provider] || []) : []
+
+  if (models.length === 0) { return null }
+  
+  return (
+    <MenubarSub>
+      <MenubarSubTrigger>
+        <Tag size="lg" color={TagColor.GREEN}>ai&nbsp;assistant</Tag>
+        {current || "None"}
+      </MenubarSubTrigger>
+      <MenubarSubContent>
+        {models.map(model => (
+          <MenubarCheckboxItem
+             key={model}
+            checked={current === model}
+            onClick={(e) => {
+              setter(model)
+              e.stopPropagation()
+              e.preventDefault()
+              return false
+            }}>
+            {model}
+          </MenubarCheckboxItem>
+        ))}
+      </MenubarSubContent>
+    </MenubarSub>
+  )
+}

src/components/toolbars/top-menu/music/index.tsx

@@ -18,13 +18,17 @@ import { ProviderList } from "../lists/ProviderList"
 import { RenderingStrategyList } from "../lists/RenderingStrategyList"
 import { availableComputeProvidersForMusic } from "@/components/settings/constants"
 import { SettingsCategory } from "@/types"
+import { MusicGenerationModelList } from "../lists/MusicGenerationModelList"
 
 export function TopMenuMusic() {
   const setShowSettings = useUI(s => s.setShowSettings)
   const musicProvider = useSettings(s => s.musicProvider)
   const setMusicProvider = useSettings(s => s.setMusicProvider)
+  const musicGenerationModel = useSettings(s => s.musicGenerationModel)
+  const setMusicGenerationModel = useSettings(s => s.setMusicGenerationModel)
   const musicRenderingStrategy = useSettings((s) => s.musicRenderingStrategy)
   const setMusicRenderingStrategy = useSettings((s) => s.setMusicRenderingStrategy)
+
   return (
     <MenubarMenu>
       <MenubarTrigger>Music</MenubarTrigger>
@@ -32,6 +36,7 @@ export function TopMenuMusic() {
         <MenubarSub>
           <MenubarItem onClick={() => { setShowSettings(SettingsCategory.MUSIC) }}>Show advanced settings</MenubarItem>
           <MenubarSeparator />
+          <MusicGenerationModelList provider={musicProvider} current={musicGenerationModel} setter={setMusicGenerationModel} />
           <ProviderList providers={availableComputeProvidersForMusic} current={musicProvider} setter={setMusicProvider} />
           <RenderingStrategyList current={musicRenderingStrategy} setter={setMusicRenderingStrategy} />
           <MenubarSeparator />

src/components/toolbars/top-menu/sound/index.tsx

@@ -18,11 +18,14 @@ import { ProviderList } from "../lists/ProviderList"
 import { RenderingStrategyList } from "../lists/RenderingStrategyList"
 import { availableComputeProvidersForSound } from "@/components/settings/constants"
 import { SettingsCategory } from "@/types"
+import { SoundGenerationModelList } from "../lists/SoundGenerationModelList"
 
 export function TopMenuSound() {
   const setShowSettings = useUI(s => s.setShowSettings)
   const soundProvider = useSettings(s => s.soundProvider)
   const setSoundProvider = useSettings(s => s.setSoundProvider)
+  const soundGenerationModel = useSettings(s => s.soundGenerationModel)
+  const setSoundGenerationModel = useSettings(s => s.setSoundGenerationModel)
   const soundRenderingStrategy = useSettings((s) => s.soundRenderingStrategy)
   const setSoundRenderingStrategy = useSettings((s) => s.setSoundRenderingStrategy)
   return (
@@ -32,6 +35,7 @@ export function TopMenuSound() {
         <MenubarSub>
           <MenubarItem onClick={() => { setShowSettings(SettingsCategory.SOUND) }}>Show advanced settings</MenubarItem>
           <MenubarSeparator />
+          <SoundGenerationModelList provider={soundProvider} current={soundGenerationModel} setter={setSoundGenerationModel} />
           <ProviderList providers={availableComputeProvidersForSound} current={soundProvider} setter={setSoundProvider} />
           <RenderingStrategyList current={soundRenderingStrategy} setter={setSoundRenderingStrategy} />
           <MenubarSeparator />

src/components/toolbars/top-menu/view/index.tsx

@@ -77,6 +77,7 @@ export function TopMenuView() {
             return false
           }}
           >Show asset explorer</MenubarCheckboxItem>
+      */}
         <MenubarCheckboxItem
           checked={showChat}
           onClick={(e) => {
@@ -86,6 +87,7 @@ export function TopMenuView() {
             return false
           }}
           >Show chat assistant</MenubarCheckboxItem>
+        {/*
         <MenubarCheckboxItem
           checked={showVideoPlayer}
           onClick={(e) => {

src/controllers/assistant/askAssistant.ts

@@ -0,0 +1,14 @@
+import { AssistantRequest, AssistantResponse } from "@/types"
+
+export async function askAssistant(request: AssistantRequest) {
+  const res = await fetch("/api/assistant", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(request)
+  })
+
+  const response = (await res.json()) as AssistantResponse
+  return response
+}

src/controllers/assistant/types.ts

@@ -34,12 +34,13 @@ export type AssistantState = {
   socket?: WebSocket
 }
 export type AssistantControls = {
+
   /**
    * Toggle the speech to text on or off
    * 
    * @returns true if operation succeeded, false otherwise
    */
-  toggleVoice: () => Promise<boolean>
+  // toggleVoice: () => Promise<boolean>
 
   /**
    * Run a prompt command (which can come from a transcript or somewhere else)
@@ -59,6 +60,8 @@ export type AssistantControls = {
   addEventToHistory: (event: Partial<ChatEvent>) => ChatEvent
 
   clearHistory: () => void
+
+  processMessage: (input: string) => void
 }
 
 export type AssistantStore =

src/controllers/assistant/useAssistant.ts

@@ -1,10 +1,14 @@
 "use client"
 
 import { create } from "zustand"
-import { UUID } from "@aitube/clap"
+import { ClapOutputType, ClapProject, ClapSegment, ClapSegmentCategory, newSegment, UUID } from "@aitube/clap"
 
 import { AssistantStore, ChatEvent } from "./types"
 import { getDefaultAssistantState } from "./getDefaultAssistantState"
+import { DEFAULT_DURATION_IN_MS_PER_STEP, findFreeTrack, TimelineStore, useTimeline } from "@aitube/timeline"
+import { useSettings } from "../settings"
+import { AssistantRequest } from "@/types"
+import { askAssistant } from "./askAssistant"
 
 // URL to the speech to text websocket server
 export const STT_API_URL = process.env.NEXT_PUBLIC_SPEECH_TO_TEXT_API_URL || ""
@@ -13,6 +17,7 @@ const enableTextToSpeech = false
 
 export const useAssistant = create<AssistantStore>((set, get) => ({
   ...getDefaultAssistantState(),
+  /*
   toggleVoice: async (): Promise<boolean> => {
    
     if (!navigator?.mediaDevices?.getUserMedia || !MediaRecorder.isTypeSupported("audio/webm")) {
@@ -89,6 +94,7 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
 
     return true
   },
+  */
 
   runCommand: (prompt: string) => {
 
@@ -135,6 +141,154 @@ export const useAssistant = create<AssistantStore>((set, get) => ({
 
   clearHistory: () => {
     set({ history: [] })
-  }
-}))
+  },
+
+  processMessage: async (input: string) => {
+    const message = input.trim()
+    if (!message) {
+      return
+    }
+    const { addEventToHistory, runCommand } = get()
+
+    const timelineState: TimelineStore = useTimeline.getState()
+    const { clap } = timelineState
+    if (!clap) { return }
+
+    // note: settings is not the store (with methods etc)
+    // but a serializable snapshot of the values only
+    const settings = useSettings.getState().getSettings()
+
+    addEventToHistory({
+      senderId: "director",
+      senderName: "Director",
+      message,
+    })
+  
+
+    const basicCommand = await runCommand(message)
+    // LLM analysis can be slow and expensive, so first we try to see if this was a trivial command
+    // like "start", "pause", "stop" etc
+    if (basicCommand) {
+
+      addEventToHistory({
+        senderId: "assistant",
+        senderName: "Assistant",
+        message: `${basicCommand}`,
+      })
+      return // no need to go further
+    }
+    
+    console.log(`TODO @julian: restore the concept of "addSegment()", "updateSegment()", "active segment" and "cursor position" inside @aitube-timeline`)
+    // const { addSegment, activeSegments, cursorInSteps,  } = useTimeline.getState()
+
+    const activeSegments: ClapSegment[] = []
+    const cursorInSteps = 0
+  
+    const referenceSegment: ClapSegment | undefined = activeSegments.at(0)
+    
+    if (!referenceSegment) {
+      throw new Error(`No segment under the current cursor`)
+    }
+
+    console.log(`TODO @julian: filter entities to only keep the ones in the current active segment? (although.. maybe a bad idea since the LLM need as much context as possible to "fill in the gap" eg. repair/invent missing elements of the story)`)
+
+    const entities = clap.entityIndex
 
+    const projectInfo = clap.meta.description
+
+    const sceneId = referenceSegment.sceneId
+
+    const scene = clap.scenes.find(s => s.id === sceneId)
+    
+    const fullScene: string = scene?.sequenceFullText || ""
+    const actionLine: string = scene?.line || ""
+  
+    const segments: ClapSegment[] = activeSegments
+      .filter(s =>
+        s.category === ClapSegmentCategory.LOCATION ||
+        s.category === ClapSegmentCategory.TIME ||
+        s.category === ClapSegmentCategory.LIGHTING ||
+        s.category === ClapSegmentCategory.ACTION ||
+        s.category === ClapSegmentCategory.DIALOGUE ||
+        s.category === ClapSegmentCategory.WEATHER  
+      )
+  
+    console.log(`TODO @julian: provide both contextual segments and editable ones to the LLM?`)
+
+    const request: AssistantRequest = {
+      settings,
+      prompt: message,
+      segments,
+      fullScene,
+      actionLine,
+      entities,
+      projectInfo,
+    }
+  
+    const { prompt, categoryName, llmOutput } = await askAssistant(request)
+    if (!prompt.length) {
+      addEventToHistory({
+        senderId: "assistant",
+        senderName: "Assistant",
+        message: llmOutput || "🤔" // or "???" for a "boomer" theme
+      })
+      return
+    }
+  
+    let match = segments.find(s => s.category === categoryName) || undefined
+    if (!match) {
+  
+      const startTimeInMs = cursorInSteps * DEFAULT_DURATION_IN_MS_PER_STEP
+      const durationInSteps = 4
+      const durationInMs = durationInSteps * DEFAULT_DURATION_IN_MS_PER_STEP
+      const endTimeInMs = startTimeInMs + durationInMs
+  
+      const newSeg = newSegment({
+        startTimeInSteps: cursorInSteps,
+        prompt: [prompt],
+        durationInSteps: 4,
+        trackId: findFreeTrack({
+          segments,
+          startTimeInMs,
+          endTimeInMs,
+        }), 
+        outputType: ClapOutputType.TEXT,
+        categoryName,
+      })
+      console.log("Creating new existing segment:", newSeg)
+  
+      console.log(`TODO Julian: add the segment!!`)
+      // addSegment(newSeg)
+
+      addEventToHistory({
+        senderId: "assistant",
+        senderName: "Assistant",
+        message: `Segment added: ${newSeg.prompt}`,
+      })
+    } else {
+  
+      console.log("Updating an existing segment to:", {
+        ...match,
+        prompt,
+        label: prompt,
+      })
+  
+      console.log(`TODO Julian: update the segment!!`)
+      // addSegment(newSeg)
+
+      /*
+      updateSegment({
+        ...match,
+        prompt,
+        label: prompt,
+      })
+        */
+
+      addEventToHistory({
+        senderId: "assistant",
+        senderName: "Assistant",
+        message: `Segment updated: ${prompt}`,
+      })
+    }
+  }
+}))

src/controllers/metrics/constants.ts

@@ -43,10 +43,28 @@ export const estimatedMetrics: Record<ComputeProvider, Record<string, ProviderMe
   },
   [ComputeProvider.STABILITYAI]: {
     // TODO list the most popular models
+  },
+  [ComputeProvider.FIREWORKSAI]: {
+
   },
   [ComputeProvider.GROQ]: {
     // TODO list the most popular models
   },
+  [ComputeProvider.KITSAI]: {
+    // TODO list the most popular models
+  },
+  [ComputeProvider.ANTHROPIC]: {
+    // TODO list the most popular models
+  },
+  [ComputeProvider.GOOGLE]: {
+    // TODO list the most popular models
+  },
+  [ComputeProvider.MISTRALAI]: {
+    // TODO list the most popular models
+  },
+  [ComputeProvider.COHERE]: {
+    // TODO list the most popular models
+  },
   [ComputeProvider.FALAI]: {
     "fal-ai/fast-sdxl": {
       estimationType: ProviderMetricsEstimationType.MANUAL_MEASUREMENTS,

src/controllers/metrics/getDefaultMetricsPerProvider.ts

@@ -35,6 +35,9 @@ export function getDefaultMetricsPerProvider(): MetricsPerProvider {
     [ComputeProvider.STABILITYAI]: {
       ...getDefaultComputeProviderMetrics(),
     },
+    [ComputeProvider.FIREWORKSAI]: {
+      ...getDefaultComputeProviderMetrics(),
+    },
     [ComputeProvider.GROQ]: {
       ...getDefaultComputeProviderMetrics(),
     },
@@ -44,6 +47,21 @@ export function getDefaultMetricsPerProvider(): MetricsPerProvider {
     [ComputeProvider.MODELSLAB]: {
       ...getDefaultComputeProviderMetrics(),
     },
+    [ComputeProvider.KITSAI]: {
+      ...getDefaultComputeProviderMetrics(),
+    },
+    [ComputeProvider.ANTHROPIC]: {
+      ...getDefaultComputeProviderMetrics(),
+    },
+    [ComputeProvider.GOOGLE]: {
+      ...getDefaultComputeProviderMetrics(),
+    },
+    [ComputeProvider.MISTRALAI]: {
+      ...getDefaultComputeProviderMetrics(),
+    },
+    [ComputeProvider.COHERE]: {
+      ...getDefaultComputeProviderMetrics(),
+    },
   }
   return metricsPerProvider
 }

src/controllers/metrics/useMetrics.ts

@@ -8,5 +8,5 @@ import { getDefaultMetricsState } from "./getDefaultMetricsState"
 export const useMetrics = create<MetricsStore>((set, get) => ({
   ...getDefaultMetricsState(),
   
-
+  // TODO: add a track metric callback
 }))

src/controllers/resolver/useResolver.ts

@@ -91,8 +91,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
     // segments visible on screen are show first,
     // then those nearby, then the hidden ones
     const segments: RuntimeSegment[] = ([...allSegments] as RuntimeSegment[]).sort((segment1, segment2) => {
-      const priority1 = SegmentVisibilityPriority[segment1.visibility || SegmentVisibility.HIDDEN] || 0
-      const priority2 = SegmentVisibilityPriority[segment2.visibility || SegmentVisibility.HIDDEN] || 0
+      const priority1 = (SegmentVisibilityPriority as any)[segment1.visibility || SegmentVisibility.HIDDEN] || 0
+      const priority2 = (SegmentVisibilityPriority as any)[segment2.visibility || SegmentVisibility.HIDDEN] || 0
       
       return priority2 - priority1
     })
@@ -278,7 +278,7 @@ export const useResolver = create<ResolverStore>((set, get) => ({
       // throw new Error(`please call setSegmentRender(...) first`)
     }
 
-    const shotSegments = filterSegments(
+    const shotSegments: ClapSegment[] = filterSegments(
       ClapSegmentFilteringMode.ANY,
       segment,
       allSegments

src/controllers/settings/getDefaultSettingsState.ts

@@ -20,6 +20,10 @@ export function getDefaultSettingsState(): SettingsState {
     groqApiKey: "",
     anthropicApiKey: "",
     elevenLabsApiKey: "",
+    kitsAiApiKey: "",
+    cohereApiKey: "",
+    mistralAiApiKey: "",
+    fireworksAiApiKey: "",
 
     assistantProvider: ComputeProvider.NONE,
     imageProvider: ComputeProvider.NONE,
@@ -93,6 +97,13 @@ export function getDefaultSettingsState(): SettingsState {
     stabilityAiModelForSound: "",
     stabilityAiModelForMusic: "",
   
+    fireworksAiModelForAssistant: "",
+    fireworksAiModelForImage: "",
+    fireworksAiModelForVideo: "",
+    fireworksAiModelForVoice: "",
+    fireworksAiModelForSound: "",
+    fireworksAiModelForMusic: "",
+  
     falAiModelForImage: "fal-ai/fast-sdxl", // "fal-ai/fast-lightning-sdxl",
     falAiModelForVideo: "fal-ai/stable-video",
     falAiModelForVoice: "fal-ai/metavoice-v1",
@@ -122,6 +133,10 @@ export function getDefaultSettingsState(): SettingsState {
 
     elevenLabsModelForVoice: "v1",
     elevenLabsModelForSound: "v1",
+
+    kitsAiModelForVoice: "",
+    cohereModelForAssistant: "",
+    mistralAiModelForAssistant: ""
   }
   return state
 }

src/controllers/settings/types.ts

@@ -15,6 +15,10 @@ export type SettingsState = {
   groqApiKey: string
   anthropicApiKey: string
   elevenLabsApiKey: string
+  kitsAiApiKey: string
+  cohereApiKey: string
+  mistralAiApiKey: string
+  fireworksAiApiKey: string
 
   // ------------- CATEGORY PROVIDERS ---------------
   assistantProvider: ComputeProvider
@@ -92,6 +96,13 @@ export type SettingsState = {
   stabilityAiModelForSound: string
   stabilityAiModelForMusic: string
 
+  fireworksAiModelForAssistant: string
+  fireworksAiModelForImage: string
+  fireworksAiModelForVideo: string
+  fireworksAiModelForVoice: string
+  fireworksAiModelForSound: string
+  fireworksAiModelForMusic: string
+
   falAiModelForImage: string
   falAiModelForVideo: string
   falAiModelForVoice: string
@@ -121,6 +132,12 @@ export type SettingsState = {
 
   elevenLabsModelForVoice: string
   elevenLabsModelForSound: string
+
+  kitsAiModelForVoice: string
+
+  cohereModelForAssistant: string
+
+  mistralAiModelForAssistant: string
 }
 
 export type SettingsControls = {
@@ -131,12 +148,15 @@ export type SettingsControls = {
   setHuggingFaceApiKey: (huggingFaceApiKey?: string) => void
   setModelsLabApiKey: (modelsLabApiKey?: string) => void
   setFalAiApiKey: (falAiApiKey?: string) => void
-  
   setOpenaiApiKey: (openaiApiKey?: string) => void
   setGoogleApiKey: (googleApiKey?: string) => void
   setGroqApiKey: (groqApiKey?: string) => void
+  setFireworksAiApiKey: (fireworksAiApiKey?: string) => void
   setAnthropicApiKey: (anthropicApiKey?: string) => void
   setElevenLabsApiKey: (elevenLabsApiKey?: string) => void
+  setCohereApiKey: (cohereApiKey?: string) => void
+  setMistralAiApiKey: (mistralAiApiKey?: string) => void
+  setKitsAiApiKey: (kitsAiApiKey?: string) => void
 
   setAssistantProvider: (assistantProvider?: ComputeProvider) => void
   setVideoProvider: (videoProvider?: ComputeProvider) => void
@@ -204,6 +224,13 @@ export type SettingsControls = {
   setStabilityAiModelForSound: (stabilityAiModelForSound?: string) => void
   setStabilityAiModelForMusic: (stabilityAiModelForMusic?: string) => void
 
+  setFireworksAiModelForAssistant: (fireworksAiModelForAssistant?: string) => void
+  setFireworksAiModelForImage: (fireworksAiModelForImage?: string) => void
+  setFireworksAiModelForVideo: (fireworksAiModelForVideo?: string) => void
+  setFireworksAiModelForVoice: (fireworksAiModelForVoice?: string) => void
+  setFireworksAiModelForSound: (fireworksAiModelForSound?: string) => void
+  setFireworksAiModelForMusic: (fireworksAiModelForMusic?: string) => void
+
   setFalAiModelForImage: (falAiModelForImage?: string) => void
   setFalAiModelForVideo: (falAiModelForVideo?: string) => void
   setFalAiModelForVoice: (falAiModelForVoice?: string) => void
@@ -234,6 +261,11 @@ export type SettingsControls = {
   setElevenLabsModelForVoice: (elevenLabsModelForVoice?: string) => void
   setElevenLabsModelForSound: (elevenLabsModelForSound?: string) => void
   
+  setCohereModelForAssistant: (cohereModelForAssistant?: string) => void
+  setMistralAiModelForAssistant: (mistralAiModelForAssistant?: string) => void
+  
+  setKitsAiModelForVoice: (kitsAiModelForVoice?: string) => void
+  
   getSettings: () => SettingsState
 }
 

src/controllers/settings/useSettings.ts

@@ -84,6 +84,18 @@ export const useSettings = create<SettingsStore>()(
       setElevenLabsApiKey: (elevenLabsApiKey?: string) => {
         set({ elevenLabsApiKey: getValidString(elevenLabsApiKey, getDefaultSettingsState().elevenLabsApiKey) })
       },
+      setKitsAiApiKey: (kitsAiApiKey?: string) => {
+        set({ kitsAiApiKey: getValidString(kitsAiApiKey, getDefaultSettingsState().kitsAiApiKey) })
+      },
+      setCohereApiKey: (cohereApiKey?: string) => {
+        set({ cohereApiKey: getValidString(cohereApiKey, getDefaultSettingsState().cohereApiKey) })
+      },
+      setMistralAiApiKey: (mistralAiApiKey?: string) => {
+        set({ mistralAiApiKey: getValidString(mistralAiApiKey, getDefaultSettingsState().mistralAiApiKey) })
+      },
+      setFireworksAiApiKey: (fireworksAiApiKey?: string) => {
+        set({ fireworksAiApiKey: getValidString(fireworksAiApiKey, getDefaultSettingsState().fireworksAiApiKey) })
+      },
       setCensorNotForAllAudiencesContent: (censorNotForAllAudiencesContent?: boolean) => {
         set({ censorNotForAllAudiencesContent: getValidBoolean(censorNotForAllAudiencesContent, getDefaultSettingsState().censorNotForAllAudiencesContent) })
       },
@@ -269,6 +281,24 @@ export const useSettings = create<SettingsStore>()(
       setStabilityAiModelForMusic: (stabilityAiModelForMusic?: string) => {
         set({ stabilityAiModelForMusic: getValidString(stabilityAiModelForMusic, getDefaultSettingsState().stabilityAiModelForMusic) })
       },
+      setFireworksAiModelForAssistant: (fireworksAiModelForAssistant?: string) => {
+        set({ fireworksAiModelForAssistant: getValidString(fireworksAiModelForAssistant, getDefaultSettingsState().fireworksAiModelForAssistant) })
+      },
+      setFireworksAiModelForImage: (fireworksAiModelForImage?: string) => {
+        set({ fireworksAiModelForImage: getValidString(fireworksAiModelForImage, getDefaultSettingsState().fireworksAiModelForImage) })
+      },
+      setFireworksAiModelForVideo: (fireworksAiModelForVideo?: string) => {
+        set({ fireworksAiModelForVideo: getValidString(fireworksAiModelForVideo, getDefaultSettingsState().fireworksAiModelForVideo) })
+      },
+      setFireworksAiModelForVoice: (fireworksAiModelForVoice?: string) => {
+        set({ fireworksAiModelForVoice: getValidString(fireworksAiModelForVoice, getDefaultSettingsState().fireworksAiModelForVoice) })
+      },
+      setFireworksAiModelForSound: (fireworksAiModelForSound?: string) => {
+        set({ fireworksAiModelForSound: getValidString(fireworksAiModelForSound, getDefaultSettingsState().fireworksAiModelForSound) })
+      },
+      setFireworksAiModelForMusic: (fireworksAiModelForMusic?: string) => {
+        set({ fireworksAiModelForMusic: getValidString(fireworksAiModelForMusic, getDefaultSettingsState().fireworksAiModelForMusic) })
+      },
       setFalAiModelForImage: (falAiModelForImage?: string) => {
         set({ falAiModelForImage: getValidString(falAiModelForImage, getDefaultSettingsState().falAiModelForImage) })
       },
@@ -338,6 +368,15 @@ export const useSettings = create<SettingsStore>()(
       setElevenLabsModelForSound: (elevenLabsModelForSound?: string) => {
         set({ elevenLabsModelForSound: getValidString(elevenLabsModelForSound, getDefaultSettingsState().elevenLabsModelForSound) })
       },
+      setCohereModelForAssistant: (cohereModelForAssistant?: string) => {
+        set({ cohereModelForAssistant: getValidString(cohereModelForAssistant, getDefaultSettingsState().cohereModelForAssistant) })
+      },
+      setMistralAiModelForAssistant: (mistralAiModelForAssistant?: string) => {
+        set({ mistralAiModelForAssistant: getValidString(mistralAiModelForAssistant, getDefaultSettingsState().mistralAiModelForAssistant) })
+      },
+      setKitsAiModelForVoice: (kitsAiModelForVoice?: string) => {
+        set({ kitsAiModelForVoice: getValidString(kitsAiModelForVoice, getDefaultSettingsState().kitsAiModelForVoice) })
+      },
       getSettings: (): SettingsState => {
         const state = get()
         const defaultSettings = getDefaultSettingsState()
@@ -362,6 +401,10 @@ export const useSettings = create<SettingsStore>()(
           googleApiKey: state.googleApiKey || defaultSettings.googleApiKey,
           anthropicApiKey: state.anthropicApiKey || defaultSettings.anthropicApiKey,
           elevenLabsApiKey: state.elevenLabsApiKey || defaultSettings.elevenLabsApiKey,
+          cohereApiKey: state.cohereApiKey || defaultSettings.cohereApiKey,
+          mistralAiApiKey: state.mistralAiApiKey || defaultSettings.mistralAiApiKey,
+          kitsAiApiKey: state.kitsAiApiKey || defaultSettings.kitsAiApiKey,
+          fireworksAiApiKey: state.fireworksAiApiKey || defaultSettings.fireworksAiApiKey,
           censorNotForAllAudiencesContent: state.censorNotForAllAudiencesContent || defaultSettings.censorNotForAllAudiencesContent,
           imagePromptPrefix: state.imagePromptPrefix || defaultSettings.imagePromptPrefix,
           imagePromptSuffix: state.imagePromptSuffix || defaultSettings.imagePromptSuffix,
@@ -415,6 +458,12 @@ export const useSettings = create<SettingsStore>()(
           stabilityAiModelForVoice: state.stabilityAiModelForVoice || defaultSettings.stabilityAiModelForVoice,
           stabilityAiModelForSound: state.stabilityAiModelForSound || defaultSettings.stabilityAiModelForSound,
           stabilityAiModelForMusic: state.stabilityAiModelForMusic || defaultSettings.stabilityAiModelForMusic,
+          fireworksAiModelForAssistant: state.fireworksAiModelForAssistant || defaultSettings.fireworksAiModelForAssistant,
+          fireworksAiModelForImage: state.fireworksAiModelForImage || defaultSettings.fireworksAiModelForImage,
+          fireworksAiModelForVideo: state.fireworksAiModelForVideo || defaultSettings.fireworksAiModelForVideo,
+          fireworksAiModelForVoice: state.fireworksAiModelForVoice || defaultSettings.fireworksAiModelForVoice,
+          fireworksAiModelForSound: state.fireworksAiModelForSound || defaultSettings.fireworksAiModelForSound,
+          fireworksAiModelForMusic: state.fireworksAiModelForMusic || defaultSettings.fireworksAiModelForMusic,
           falAiModelForImage: state.falAiModelForImage || defaultSettings.falAiModelForImage,
           falAiModelForVideo: state.falAiModelForVideo || defaultSettings.falAiModelForVideo,
           falAiModelForVoice: state.falAiModelForVoice || defaultSettings.falAiModelForVoice,
@@ -438,6 +487,9 @@ export const useSettings = create<SettingsStore>()(
           anthropicModelForAssistant: state.anthropicModelForAssistant || defaultSettings.anthropicModelForAssistant,
           elevenLabsModelForVoice: state.elevenLabsModelForVoice || defaultSettings.elevenLabsModelForVoice,
           elevenLabsModelForSound: state.elevenLabsModelForSound || defaultSettings.elevenLabsModelForSound,
+          cohereModelForAssistant: state.cohereModelForAssistant || defaultSettings.cohereModelForAssistant,
+          mistralAiModelForAssistant: state.mistralAiModelForAssistant || defaultSettings.mistralAiModelForAssistant,
+          kitsAiModelForVoice: state.kitsAiModelForVoice || defaultSettings.kitsAiModelForVoice,
         }
       },
     }),

src/lib/utils/parseComputeProvider.ts

@@ -34,6 +34,9 @@ export function parseComputeProvider(input: any, defaultVendor?: ComputeProvider
   else if (unknownString === "stabilityai") {
     vendor = ComputeProvider.STABILITYAI
   } 
+  else if (unknownString === "fireworksai") {
+    vendor = ComputeProvider.FIREWORKSAI
+  }
   else if (unknownString === "groq") {
     vendor = ComputeProvider.GROQ
   } 

src/types.ts

@@ -21,9 +21,15 @@ export enum ComputeProvider {
   COMFY_REPLICATE = "COMFY_REPLICATE", // https://replicate.com
   COMFY_COMFYICU = "COMFY_COMFYICU", // https://comfy.icu
   ELEVENLABS = "ELEVENLABS", // https://elevenlabs.io
+  KITSAI = "KITSAI", // https://kits.ai
   OPENAI = "OPENAI", // https://openai.com
   STABILITYAI = "STABILITYAI", // https://stability.ai
+  FIREWORKSAI = "FIREWORKSAI", // https://fireworks.ai
   GROQ = "GROQ", // https://groq.com
+  ANTHROPIC = "ANTHROPIC", // https://anthropic.com
+  GOOGLE = "GOOGLE", // https://google.com (in case you didn't know)
+  MISTRALAI = "MISTRALAI", // https://mistral.ai
+  COHERE = "COHERE", // https://cohere.com
   FALAI = "FALAI", // https://fal.ai
   MODELSLAB = "MODELSLAB", // https://modelslab.com
 }