import { useState, useRef, useEffect, useCallback } from "react"; import WebcamCapture from "./WebcamCapture"; import DraggableContainer from "./DraggableContainer"; import PromptInput from "./PromptInput"; import LiveCaption from "./LiveCaption"; import { useVLMContext } from "../context/useVLMContext"; import { PROMPTS, TIMING } from "../constants"; interface CaptioningViewProps { videoRef: React.RefObject; } function useCaptioningLoop( videoRef: React.RefObject, isRunning: boolean, promptRef: React.RefObject, onCaptionUpdate: (caption: string) => void, onError: (error: string) => void, ) { const { isLoaded, runInference } = useVLMContext(); const abortControllerRef = useRef(null); const onCaptionUpdateRef = useRef(onCaptionUpdate); const onErrorRef = useRef(onError); useEffect(() => { onCaptionUpdateRef.current = onCaptionUpdate; }, [onCaptionUpdate]); useEffect(() => { onErrorRef.current = onError; }, [onError]); useEffect(() => { abortControllerRef.current?.abort(); if (!isRunning || !isLoaded) return; abortControllerRef.current = new AbortController(); const signal = abortControllerRef.current.signal; const video = videoRef.current; const captureLoop = async () => { while (!signal.aborted) { if (video && video.readyState >= 2 && !video.paused && video.videoWidth > 0) { try { const currentPrompt = promptRef.current || ""; const result = await runInference(video, currentPrompt, onCaptionUpdateRef.current); if (result && !signal.aborted) onCaptionUpdateRef.current(result); } catch (error) { if (!signal.aborted) { const message = error instanceof Error ? error.message : String(error); onErrorRef.current(message); console.error("Error processing frame:", error); } } } if (signal.aborted) break; await new Promise((resolve) => setTimeout(resolve, TIMING.FRAME_CAPTURE_DELAY)); } }; // NB: Wrap with a setTimeout to ensure abort controller can run before starting the loop // This is necessary for React's strict mode which calls effects twice in development. setTimeout(captureLoop, 0); return () => { abortControllerRef.current?.abort(); }; }, [isRunning, isLoaded, runInference, promptRef, videoRef]); } export default function CaptioningView({ videoRef }: CaptioningViewProps) { const [caption, setCaption] = useState(""); const [isLoopRunning, setIsLoopRunning] = useState(true); const [currentPrompt, setCurrentPrompt] = useState(PROMPTS.default); const [error, setError] = useState(null); // Use ref to store current prompt to avoid loop restarts const promptRef = useRef(currentPrompt); // Update prompt ref when state changes useEffect(() => { promptRef.current = currentPrompt; }, [currentPrompt]); const handleCaptionUpdate = useCallback((newCaption: string) => { setCaption(newCaption); setError(null); }, []); const handleError = useCallback((errorMessage: string) => { setError(errorMessage); setCaption(`Error: ${errorMessage}`); }, []); useCaptioningLoop(videoRef, isLoopRunning, promptRef, handleCaptionUpdate, handleError); const handlePromptChange = useCallback((prompt: string) => { setCurrentPrompt(prompt); setError(null); }, []); const handleToggleLoop = useCallback(() => { setIsLoopRunning((prev) => !prev); if (error) setError(null); }, [error]); return (
{/* Draggable Prompt Input - Bottom Left */} {/* Draggable Live Caption - Bottom Right */}
); }