Spaces:
Running
Running
| import { useState, useRef, useEffect, useCallback } from "react"; | |
| import WebcamCapture from "./WebcamCapture"; | |
| import DraggableContainer from "./DraggableContainer"; | |
| import PromptInput from "./PromptInput"; | |
| import LiveCaption from "./LiveCaption"; | |
| import { useVLMContext } from "../context/useVLMContext"; | |
| import { PROMPTS, TIMING } from "../constants"; | |
| interface CaptioningViewProps { | |
| videoRef: React.RefObject<HTMLVideoElement | null>; | |
| } | |
| function useCaptioningLoop( | |
| videoRef: React.RefObject<HTMLVideoElement | null>, | |
| isRunning: boolean, | |
| promptRef: React.RefObject<string>, | |
| onCaptionUpdate: (caption: string) => void, | |
| onError: (error: string) => void, | |
| ) { | |
| const { isLoaded, runInference } = useVLMContext(); | |
| const abortControllerRef = useRef<AbortController | null>(null); | |
| const onCaptionUpdateRef = useRef(onCaptionUpdate); | |
| const onErrorRef = useRef(onError); | |
| useEffect(() => { | |
| onCaptionUpdateRef.current = onCaptionUpdate; | |
| }, [onCaptionUpdate]); | |
| useEffect(() => { | |
| onErrorRef.current = onError; | |
| }, [onError]); | |
| useEffect(() => { | |
| abortControllerRef.current?.abort(); | |
| if (!isRunning || !isLoaded) return; | |
| abortControllerRef.current = new AbortController(); | |
| const signal = abortControllerRef.current.signal; | |
| const video = videoRef.current; | |
| const captureLoop = async () => { | |
| while (!signal.aborted) { | |
| if (video && video.readyState >= 2 && !video.paused && video.videoWidth > 0) { | |
| try { | |
| const currentPrompt = promptRef.current || ""; | |
| const result = await runInference(video, currentPrompt, onCaptionUpdateRef.current); | |
| if (result && !signal.aborted) onCaptionUpdateRef.current(result); | |
| } catch (error) { | |
| if (!signal.aborted) { | |
| const message = error instanceof Error ? error.message : String(error); | |
| onErrorRef.current(message); | |
| console.error("Error processing frame:", error); | |
| } | |
| } | |
| } | |
| if (signal.aborted) break; | |
| await new Promise((resolve) => setTimeout(resolve, TIMING.FRAME_CAPTURE_DELAY)); | |
| } | |
| }; | |
| // NB: Wrap with a setTimeout to ensure abort controller can run before starting the loop | |
| // This is necessary for React's strict mode which calls effects twice in development. | |
| setTimeout(captureLoop, 0); | |
| return () => { | |
| abortControllerRef.current?.abort(); | |
| }; | |
| }, [isRunning, isLoaded, runInference, promptRef, videoRef]); | |
| } | |
| export default function CaptioningView({ videoRef }: CaptioningViewProps) { | |
| const [caption, setCaption] = useState<string>(""); | |
| const [isLoopRunning, setIsLoopRunning] = useState<boolean>(true); | |
| const [currentPrompt, setCurrentPrompt] = useState<string>(PROMPTS.default); | |
| const [error, setError] = useState<string | null>(null); | |
| // Use ref to store current prompt to avoid loop restarts | |
| const promptRef = useRef<string>(currentPrompt); | |
| // Update prompt ref when state changes | |
| useEffect(() => { | |
| promptRef.current = currentPrompt; | |
| }, [currentPrompt]); | |
| const handleCaptionUpdate = useCallback((newCaption: string) => { | |
| setCaption(newCaption); | |
| setError(null); | |
| }, []); | |
| const handleError = useCallback((errorMessage: string) => { | |
| setError(errorMessage); | |
| setCaption(`Error: ${errorMessage}`); | |
| }, []); | |
| useCaptioningLoop(videoRef, isLoopRunning, promptRef, handleCaptionUpdate, handleError); | |
| const handlePromptChange = useCallback((prompt: string) => { | |
| setCurrentPrompt(prompt); | |
| setError(null); | |
| }, []); | |
| const handleToggleLoop = useCallback(() => { | |
| setIsLoopRunning((prev) => !prev); | |
| if (error) setError(null); | |
| }, [error]); | |
| return ( | |
| <div className="absolute inset-0 text-white"> | |
| <div className="relative w-full h-full"> | |
| <WebcamCapture isRunning={isLoopRunning} onToggleRunning={handleToggleLoop} error={error} /> | |
| {/* Draggable Prompt Input - Bottom Left */} | |
| <DraggableContainer initialPosition="bottom-left"> | |
| <PromptInput onPromptChange={handlePromptChange} /> | |
| </DraggableContainer> | |
| {/* Draggable Live Caption - Bottom Right */} | |
| <DraggableContainer initialPosition="bottom-right"> | |
| <LiveCaption caption={caption} isRunning={isLoopRunning} error={error} /> | |
| </DraggableContainer> | |
| </div> | |
| </div> | |
| ); | |
| } | |