debugging the connect error on deployment

src/components/control-tray/ControlTray.tsx

@@ -84,6 +84,7 @@ function ControlTray({
 
   useEffect(() => {
     if (!connected && connectButtonRef.current) {
+      console.log('🎯 Setting focus on connect button - connection status:', connected);
       connectButtonRef.current.focus();
     }
   }, [connected]);
@@ -91,13 +92,17 @@ function ControlTray({
   // Add iOS volume simulation effect
   useEffect(() => {
     if (isIOSDevice && connected && !muted) {
+      console.log('📱 Starting iOS volume simulation');
       const interval = setInterval(() => {
         // Create a smooth pulsing effect
         const pulse = (Math.sin(Date.now() / 500) + 1) / 2; // Values between 0 and 1
         setSimulatedVolume(0.02 + pulse * 0.03); // Small range for subtle effect
       }, 50);
       
-      return () => clearInterval(interval);
+      return () => {
+        console.log('🛑 Stopping iOS volume simulation');
+        clearInterval(interval);
+      };
     }
   }, [connected, muted, isIOSDevice]);
 
@@ -110,6 +115,7 @@ function ControlTray({
 
   useEffect(() => {
     const onData = (base64: string) => {
+      console.log('🎤 Sending audio data chunk');
       client.sendRealtimeInput([
         {
           mimeType: "audio/pcm;rate=16000",
@@ -119,18 +125,22 @@ function ControlTray({
     };
 
     if (connected && !muted && audioRecorder) {
+      console.log('🎙️ Starting audio recorder');
       audioRecorder.on("data", onData).on("volume", setInVolume).start();
     } else {
+      console.log('⏹️ Stopping audio recorder');
       audioRecorder.stop();
     }
 
     return () => {
+      console.log('🧹 Cleaning up audio recorder listeners');
       audioRecorder.off("data", onData).off("volume", setInVolume);
     };
   }, [connected, client, muted, audioRecorder]);
 
   useEffect(() => {
     if (videoRef.current) {
+      console.log('🎥 Setting video stream:', activeVideoStream ? 'active' : 'null');
       videoRef.current.srcObject = activeVideoStream;
     }
 
@@ -141,6 +151,7 @@ function ControlTray({
       const canvas = renderCanvasRef.current;
 
       if (!video || !canvas) {
+        console.log('⚠️ Missing video or canvas reference');
         return;
       }
 
@@ -151,6 +162,7 @@ function ControlTray({
         ctx.drawImage(videoRef.current, 0, 0, canvas.width, canvas.height);
         const base64 = canvas.toDataURL("image/jpeg", 1.0);
         const data = base64.slice(base64.indexOf(",") + 1, Infinity);
+        console.log('📸 Sending video frame');
         client.sendRealtimeInput([{ mimeType: "image/jpeg", data }]);
       }
       if (connected) {
@@ -158,9 +170,11 @@ function ControlTray({
       }
     }
     if (connected && activeVideoStream !== null) {
+      console.log('🎬 Starting video frame capture');
       requestAnimationFrame(sendVideoFrame);
     }
     return () => {
+      console.log('⏹️ Stopping video frame capture');
       clearTimeout(timeoutId);
     };
   }, [connected, activeVideoStream, client, videoRef]);
@@ -168,20 +182,20 @@ function ControlTray({
   //handler for swapping from one video-stream to the next
   const changeStreams = (next?: UseMediaStreamResult) => async () => {
     if (next) {
+      console.log('🔄 Starting new video stream');
       const mediaStream = await next.start();
-      if (mediaStream) {
-        setActiveVideoStream(mediaStream);
-        onVideoStreamChange(mediaStream);
-      } else {
-        setActiveVideoStream(null);
-        onVideoStreamChange(null);
-      }
+      setActiveVideoStream(mediaStream);
+      onVideoStreamChange(mediaStream);
     } else {
+      console.log('⏹️ Stopping video stream');
       setActiveVideoStream(null);
       onVideoStreamChange(null);
     }
 
-    videoStreams.filter((msr) => msr !== next).forEach((msr) => msr.stop());
+    videoStreams.filter((msr) => msr !== next).forEach((msr) => {
+      console.log('🛑 Stopping other video streams');
+      msr.stop();
+    });
   };
 
   return (

src/contexts/LiveAPIContext.tsx

@@ -28,6 +28,7 @@ export const LiveAPIProvider: FC<LiveAPIProviderProps> = ({
   url = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`,
   children,
 }) => {
+  console.log('🌐 Initializing LiveAPIProvider with URL:', url);
   const liveAPI = useLiveAPI({ url });
 
   return (
@@ -40,7 +41,9 @@ export const LiveAPIProvider: FC<LiveAPIProviderProps> = ({
 export const useLiveAPIContext = () => {
   const context = useContext(LiveAPIContext);
   if (!context) {
+    console.error('❌ LiveAPIContext used outside of LiveAPIProvider');
     throw new Error("useLiveAPIContext must be used within a LiveAPIProvider");
   }
+  console.log('✅ LiveAPIContext successfully retrieved');
   return context;
 };

src/hooks/use-media-stream-mux.ts

@@ -16,7 +16,7 @@
 
 export type UseMediaStreamResult = {
   type: "webcam" | "screen";
-  start: () => Promise<MediaStream | null>;
+  start: () => Promise<MediaStream>;
   stop: () => void;
   isStreaming: boolean;
   stream: MediaStream | null;

src/hooks/use-screen-capture.ts

@@ -41,20 +41,15 @@ export function useScreenCapture(): UseMediaStreamResult {
   }, [stream]);
 
   const start = async () => {
-    try {
-      // const controller = new CaptureController();
-      // controller.setFocusBehavior("no-focus-change");
-      const mediaStream = await navigator.mediaDevices.getDisplayMedia({
-        video: true,
-        // controller
-      });
-      setStream(mediaStream);
-      setIsStreaming(true);
-      return mediaStream;
-    } catch (err) {
-      console.error('Failed to start screen capture:', err);
-      return null;
-    }
+    // const controller = new CaptureController();
+    // controller.setFocusBehavior("no-focus-change");
+    const mediaStream = await navigator.mediaDevices.getDisplayMedia({
+      video: true,
+      // controller
+    });
+    setStream(mediaStream);
+    setIsStreaming(true);
+    return mediaStream;
   };
 
   const stop = () => {

src/hooks/use-webcam.ts

@@ -20,30 +20,6 @@ import { UseMediaStreamResult } from "./use-media-stream-mux";
 export function useWebcam(): UseMediaStreamResult {
   const [stream, setStream] = useState<MediaStream | null>(null);
   const [isStreaming, setIsStreaming] = useState(false);
-  const [availableCameras, setAvailableCameras] = useState<MediaDeviceInfo[]>([]);
-  const [currentCameraIndex, setCurrentCameraIndex] = useState(-1);
-
-  // Get list of available cameras on mount
-  useEffect(() => {
-    async function getCameras() {
-      try {
-        // First request permission to ensure we can enumerate video devices
-        await navigator.mediaDevices.getUserMedia({ video: true })
-          .then(stream => {
-            // Stop the stream immediately, we just needed permission
-            stream.getTracks().forEach(track => track.stop());
-          });
-
-        const devices = await navigator.mediaDevices.enumerateDevices();
-        const videoDevices = devices.filter(device => device.kind === 'videoinput');
-        setAvailableCameras(videoDevices);
-        console.log('Available cameras:', videoDevices);
-      } catch (err) {
-        console.error('Error getting cameras:', err);
-      }
-    }
-    getCameras();
-  }, []);
 
   useEffect(() => {
     const handleStreamEnded = () => {
@@ -65,41 +41,12 @@ export function useWebcam(): UseMediaStreamResult {
   }, [stream]);
 
   const start = async () => {
-    // If we're already streaming, cycle to next camera
-    if (isStreaming) {
-      const nextIndex = (currentCameraIndex + 1) % (availableCameras.length);
-      setCurrentCameraIndex(nextIndex);
-      
-      // Stop current stream
-      if (stream) {
-        stream.getTracks().forEach((track) => track.stop());
-      }
-
-      // If we've cycled through all cameras, stop streaming
-      if (nextIndex === 0) {
-        setStream(null);
-        setIsStreaming(false);
-        return null;
-      }
-
-      const deviceId = availableCameras[nextIndex].deviceId;
-      const mediaStream = await navigator.mediaDevices.getUserMedia({
-        video: { deviceId: { exact: deviceId } }
-      });
-      setStream(mediaStream);
-      setIsStreaming(true);
-      return mediaStream;
-    } else {
-      // Start with first camera
-      setCurrentCameraIndex(0);
-      const deviceId = availableCameras[0]?.deviceId;
-      const mediaStream = await navigator.mediaDevices.getUserMedia({
-        video: deviceId ? { deviceId: { exact: deviceId } } : true
-      });
-      setStream(mediaStream);
-      setIsStreaming(true);
-      return mediaStream;
-    }
+    const mediaStream = await navigator.mediaDevices.getUserMedia({
+      video: true,
+    });
+    setStream(mediaStream);
+    setIsStreaming(true);
+    return mediaStream;
   };
 
   const stop = () => {
@@ -107,7 +54,6 @@ export function useWebcam(): UseMediaStreamResult {
       stream.getTracks().forEach((track) => track.stop());
       setStream(null);
       setIsStreaming(false);
-      setCurrentCameraIndex(-1);
     }
   };
 

src/lib/multimodal-live-client.ts

@@ -72,6 +72,7 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
 
   constructor({ url, apiKey }: MultimodalLiveAPIClientConnection = {}) {
     super();
+    console.log('🔧 Initializing MultimodalLiveClient with URL:', url || `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`);
     this.url = url || `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`;
     this.send = this.send.bind(this);
   }
@@ -86,19 +87,22 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
   }
 
   connect(config: LiveConfig): Promise<boolean> {
+    console.log('🔌 Attempting WebSocket connection to:', this.url);
     this.config = config;
 
     const ws = new WebSocket(this.url);
 
     ws.addEventListener("message", async (evt: MessageEvent) => {
+      console.log('📨 Received WebSocket message:', evt.data instanceof Blob ? 'Blob data' : evt.data);
       if (evt.data instanceof Blob) {
         this.receive(evt.data);
       } else {
-        console.log("non blob message", evt);
+        console.log("📦 Non-blob message received:", evt);
       }
     });
     return new Promise((resolve, reject) => {
       const onError = (ev: Event) => {
+        console.error('❌ WebSocket connection error:', ev);
         this.disconnect(ws);
         const message = `Could not connect to "${this.url}"`;
         this.log(`server.${ev.type}`, message);
@@ -106,7 +110,9 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
       };
       ws.addEventListener("error", onError);
       ws.addEventListener("open", (ev: Event) => {
+        console.log('✅ WebSocket connection opened successfully');
         if (!this.config) {
+          console.error('❌ Invalid config provided to connect()');
           reject("Invalid config sent to `connect(config)`");
           return;
         }
@@ -118,12 +124,13 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
         const setupMessage: SetupMessage = {
           setup: this.config,
         };
+        console.log('📤 Sending setup message:', setupMessage);
         this._sendDirect(setupMessage);
         this.log("client.send", "setup");
 
         ws.removeEventListener("error", onError);
         ws.addEventListener("close", (ev: CloseEvent) => {
-          console.log(ev);
+          console.log('🔒 WebSocket connection closed:', ev);
           this.disconnect(ws);
           let reason = ev.reason || "";
           if (reason.toLowerCase().includes("error")) {
@@ -136,6 +143,7 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
               );
             }
           }
+          console.log('📝 Close reason:', reason || 'No reason provided');
           this.log(
             `server.${ev.type}`,
             `disconnected ${reason ? `with reason: ${reason}` : ``}`,
@@ -148,14 +156,17 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
   }
 
   disconnect(ws?: WebSocket) {
+    console.log('🔌 Attempting to disconnect WebSocket');
     // could be that this is an old websocket and theres already a new instance
     // only close it if its still the correct reference
     if ((!ws || this.ws === ws) && this.ws) {
+      console.log('🔒 Closing WebSocket connection');
       this.ws.close();
       this.ws = null;
       this.log("client.close", `Disconnected`);
       return true;
     }
+    console.log('⚠️ No active WebSocket to disconnect');
     return false;
   }
 
@@ -163,6 +174,7 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
     const response: LiveIncomingMessage = (await blobToJSON(
       blob,
     )) as LiveIncomingMessage;
+    console.log('📥 Received message:', response);
     if (isToolCallMessage(response)) {
       this.log("server.toolCall", response);
       this.emit("toolcall", response.toolCall);