<script lang="ts"> import { createEventDispatcher, onMount } from "svelte"; import { Camera, Circle, Square, DropdownArrow, Spinner } from "@gradio/icons"; import type { I18nFormatter } from "@gradio/utils"; import { StreamingBar } from "@gradio/statustracker"; import { type FileData, type Client, prepare_files } from "@gradio/client"; import WebcamPermissions from "./WebcamPermissions.svelte"; import { fade } from "svelte/transition"; import { get_devices, get_video_stream, set_available_devices } from "./stream_utils"; import type { Base64File } from "./types"; let video_source: HTMLVideoElement; let available_video_devices: MediaDeviceInfo[] = []; let selected_device: MediaDeviceInfo | null = null; let time_limit: number | null = null; let stream_state: "open" | "waiting" | "closed" = "closed"; export const modify_stream: (state: "open" | "closed" | "waiting") => void = ( state: "open" | "closed" | "waiting" ) => { if (state === "closed") { time_limit = null; stream_state = "closed"; value = null; } else if (state === "waiting") { stream_state = "waiting"; } else { stream_state = "open"; } }; export const set_time_limit = (time: number): void => { if (recording) time_limit = time; }; let canvas: HTMLCanvasElement; export let streaming = false; export let pending = false; export let root = ""; export let stream_every = 1; export let mode: "image" | "video" = "image"; export let mirror_webcam: boolean; export let include_audio: boolean; export let i18n: I18nFormatter; export let upload: Client["upload"]; export let value: FileData | null | Base64File = null; const dispatch = createEventDispatcher<{ stream: Blob | string; capture: FileData | Blob | null; error: string; start_recording: undefined; stop_recording: undefined; close_stream: undefined; }>(); onMount(() => { canvas = document.createElement("canvas"); if (streaming && mode === "image") { window.setInterval(() => { if (video_source && !pending) { take_picture(); } }, stream_every * 1000); } }); const handle_device_change = async (event: InputEvent): Promise<void> => { const target = event.target as HTMLInputElement; const device_id = target.value; await get_video_stream(include_audio, video_source, device_id).then( async (local_stream) => { stream = local_stream; selected_device = available_video_devices.find( (device) => device.deviceId === device_id ) || null; options_open = false; } ); }; async function access_webcam(): Promise<void> { try { get_video_stream(include_audio, video_source) .then(async (local_stream) => { webcam_accessed = true; available_video_devices = await get_devices(); stream = local_stream; }) .then(() => set_available_devices(available_video_devices)) .then((devices) => { available_video_devices = devices; const used_devices = stream .getTracks() .map((track) => track.getSettings()?.deviceId)[0]; selected_device = used_devices ? devices.find((device) => device.deviceId === used_devices) || available_video_devices[0] : available_video_devices[0]; }); if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { dispatch("error", i18n("image.no_webcam_support")); } } catch (err) { if (err instanceof DOMException && err.name == "NotAllowedError") { dispatch("error", i18n("image.allow_webcam_access")); } else { throw err; } } } function take_picture(): void { var context = canvas.getContext("2d")!; if ( (!streaming || (streaming && recording)) && video_source.videoWidth && video_source.videoHeight ) { canvas.width = video_source.videoWidth; canvas.height = video_source.videoHeight; context.drawImage( video_source, 0, 0, video_source.videoWidth, video_source.videoHeight ); if (mirror_webcam) { context.scale(-1, 1); context.drawImage(video_source, -video_source.videoWidth, 0); } if (streaming && (!recording || stream_state === "waiting")) { return; } if (streaming) { const image_data = canvas.toDataURL("image/jpeg"); dispatch("stream", image_data); return; } canvas.toBlob( (blob) => { dispatch(streaming ? "stream" : "capture", blob); }, `image/${streaming ? "jpeg" : "png"}`, 0.8 ); } } let recording = false; let recorded_blobs: BlobPart[] = []; let stream: MediaStream; let mimeType: string; let media_recorder: MediaRecorder; function take_recording(): void { if (recording) { media_recorder.stop(); let video_blob = new Blob(recorded_blobs, { type: mimeType }); let ReaderObj = new FileReader(); ReaderObj.onload = async function (e): Promise<void> { if (e.target) { let _video_blob = new File( [video_blob], "sample." + mimeType.substring(6) ); const val = await prepare_files([_video_blob]); let val_ = ( (await upload(val, root))?.filter(Boolean) as FileData[] )[0]; dispatch("capture", val_); dispatch("stop_recording"); } }; ReaderObj.readAsDataURL(video_blob); } else { dispatch("start_recording"); recorded_blobs = []; let validMimeTypes = ["video/webm", "video/mp4"]; for (let validMimeType of validMimeTypes) { if (MediaRecorder.isTypeSupported(validMimeType)) { mimeType = validMimeType; break; } } if (mimeType === null) { console.error("No supported MediaRecorder mimeType"); return; } media_recorder = new MediaRecorder(stream, { mimeType: mimeType }); media_recorder.addEventListener("dataavailable", function (e) { recorded_blobs.push(e.data); }); media_recorder.start(200); } recording = !recording; } let webcam_accessed = false; function record_video_or_photo(): void { if (mode === "image" && streaming) { recording = !recording; } if (mode === "image") { take_picture(); } else { take_recording(); } if (!recording && stream) { dispatch("close_stream"); stream.getTracks().forEach((track) => track.stop()); video_source.srcObject = null; webcam_accessed = false; window.setTimeout(() => { value = null; }, 500); value = null; } } let options_open = false; export function click_outside(node: Node, cb: any): any { const handle_click = (event: MouseEvent): void => { if ( node && !node.contains(event.target as Node) && !event.defaultPrevented ) { cb(event); } }; document.addEventListener("click", handle_click, true); return { destroy() { document.removeEventListener("click", handle_click, true); } }; } function handle_click_outside(event: MouseEvent): void { event.preventDefault(); event.stopPropagation(); options_open = false; } </script> <div class="wrap"> <StreamingBar {time_limit} /> <!-- svelte-ignore a11y-media-has-caption --> <!-- need to suppress for video streaming https://github.com/sveltejs/svelte/issues/5967 --> <video bind:this={video_source} class:flip={mirror_webcam} class:hide={!webcam_accessed || (webcam_accessed && !!value)} /> <!-- svelte-ignore a11y-missing-attribute --> <img src={value?.url} class:hide={!webcam_accessed || (webcam_accessed && !value)} /> {#if !webcam_accessed} <div in:fade={{ delay: 100, duration: 200 }} title="grant webcam access" style="height: 100%" > <WebcamPermissions on:click={async () => access_webcam()} /> </div> {:else} <div class="button-wrap"> <button on:click={record_video_or_photo} aria-label={mode === "image" ? "capture photo" : "start recording"} > {#if mode === "video" || streaming} {#if streaming && stream_state === "waiting"} <div class="icon-with-text" style="width:var(--size-24);"> <div class="icon color-primary" title="spinner"> <Spinner /> </div> {i18n("audio.waiting")} </div> {:else if (streaming && stream_state === "open") || (!streaming && recording)} <div class="icon-with-text"> <div class="icon color-primary" title="stop recording"> <Square /> </div> {i18n("audio.stop")} </div> {:else} <div class="icon-with-text"> <div class="icon color-primary" title="start recording"> <Circle /> </div> {i18n("audio.record")} </div> {/if} {:else} <div class="icon" title="capture photo"> <Camera /> </div> {/if} </button> {#if !recording} <button class="icon" on:click={() => (options_open = true)} aria-label="select input source" > <DropdownArrow /> </button> {/if} </div> {#if options_open && selected_device} <select class="select-wrap" aria-label="select source" use:click_outside={handle_click_outside} on:change={handle_device_change} > <button class="inset-icon" on:click|stopPropagation={() => (options_open = false)} > <DropdownArrow /> </button> {#if available_video_devices.length === 0} <option value="">{i18n("common.no_devices")}</option> {:else} {#each available_video_devices as device} <option value={device.deviceId} selected={selected_device.deviceId === device.deviceId} > {device.label} </option> {/each} {/if} </select> {/if} {/if} </div> <style> .wrap { position: relative; width: var(--size-full); height: var(--size-full); } .hide { display: none; } video { width: var(--size-full); height: var(--size-full); object-fit: cover; } .button-wrap { position: absolute; background-color: var(--block-background-fill); border: 1px solid var(--border-color-primary); border-radius: var(--radius-xl); padding: var(--size-1-5); display: flex; bottom: var(--size-2); left: 50%; transform: translate(-50%, 0); box-shadow: var(--shadow-drop-lg); border-radius: var(--radius-xl); line-height: var(--size-3); color: var(--button-secondary-text-color); } .icon-with-text { width: var(--size-20); align-items: center; margin: 0 var(--spacing-xl); display: flex; justify-content: space-evenly; } @media (--screen-md) { button { bottom: var(--size-4); } } @media (--screen-xl) { button { bottom: var(--size-8); } } .icon { width: 18px; height: 18px; display: flex; justify-content: space-between; align-items: center; } .color-primary { fill: var(--primary-600); stroke: var(--primary-600); color: var(--primary-600); } .flip { transform: scaleX(-1); } .select-wrap { -webkit-appearance: none; -moz-appearance: none; appearance: none; color: var(--button-secondary-text-color); background-color: transparent; width: 95%; font-size: var(--text-md); position: absolute; bottom: var(--size-2); background-color: var(--block-background-fill); box-shadow: var(--shadow-drop-lg); border-radius: var(--radius-xl); z-index: var(--layer-top); border: 1px solid var(--border-color-primary); text-align: left; line-height: var(--size-4); white-space: nowrap; text-overflow: ellipsis; left: 50%; transform: translate(-50%, 0); max-width: var(--size-52); } .select-wrap > option { padding: 0.25rem 0.5rem; border-bottom: 1px solid var(--border-color-accent); padding-right: var(--size-8); text-overflow: ellipsis; overflow: hidden; } .select-wrap > option:hover { background-color: var(--color-accent); } .select-wrap > option:last-child { border: none; } .inset-icon { position: absolute; top: 5px; right: -6.5px; width: var(--size-10); height: var(--size-5); opacity: 0.8; } @media (--screen-md) { .wrap { font-size: var(--text-lg); } } </style>