working on improvements

README.md

@@ -1,6 +1,6 @@
 ---
 title: FacePoke
-emoji: 💬
+emoji: 🙂‍↔️👈
 colorFrom: yellow
 colorTo: red
 sdk: docker
@@ -115,6 +115,14 @@ The project structure is organized as follows:
   - `src/`: TypeScript source files.
   - `public/`: Static assets and built files.
 
+### Increasing the framerate
+
+I am testing various things to increase the framerate.
+
+One project is to only transmit the modified head, instead of the whole image.
+
+Another one is to automatically adapt to the server and network speed.
+
 ## Contributing
 
 Contributions to FacePoke are welcome! Please read our [Contributing Guidelines](CONTRIBUTING.md) for details on how to submit pull requests, report issues, or request features.

app.py

@@ -9,6 +9,8 @@ import sys
 import asyncio
 from aiohttp import web, WSMsgType
 import json
+from json import JSONEncoder
+import numpy as np
 import uuid
 import logging
 import os
@@ -18,16 +20,18 @@ import base64
 import io
 
 from PIL import Image
+
+# by popular demand, let's add support for avif
 import pillow_avif
 
 # Configure logging
-logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
 
 # Set asyncio logger to DEBUG level
-logging.getLogger("asyncio").setLevel(logging.DEBUG)
+#logging.getLogger("asyncio").setLevel(logging.INFO)
 
-logger.debug(f"Python version: {sys.version}")
+#logger.debug(f"Python version: {sys.version}")
 
 # SIGSEGV handler
 def SIGSEGV_signal_arises(signalNum, stack):
@@ -43,89 +47,51 @@ from engine import Engine, base64_data_uri_to_PIL_Image
 DATA_ROOT = os.environ.get('DATA_ROOT', '/tmp/data')
 MODELS_DIR = os.path.join(DATA_ROOT, "models")
 
-image_cache: Dict[str, Image.Image] = {}
+class NumpyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        else:
+            return super(NumpyEncoder, self).default(obj)
 
 async def websocket_handler(request: web.Request) -> web.WebSocketResponse:
-    """
-    Handle WebSocket connections for the FacePoke application.
-
-    Args:
-        request (web.Request): The incoming request object.
-
-    Returns:
-        web.WebSocketResponse: The WebSocket response object.
-    """
     ws = web.WebSocketResponse()
     await ws.prepare(request)
+    engine = request.app['engine']
     try:
         #logger.info("New WebSocket connection established")
-
         while True:
             msg = await ws.receive()
 
-            if msg.type == WSMsgType.TEXT:
-                data = json.loads(msg.data)
-
-                # let's not log user requests, they are heavy
-                #logger.debug(f"Received message: {data}")
-
-                if data['type'] == 'modify_image':
-                    uuid = data.get('uuid')
-                    if not uuid:
-                        logger.warning("Received message without UUID")
+            if msg.type in (WSMsgType.CLOSE, WSMsgType.ERROR):
+                #logger.warning(f"WebSocket connection closed: {msg.type}")
+                break
 
-                    await handle_modify_image(request, ws, data, uuid)
+            try:
+                if msg.type == WSMsgType.BINARY:
+                    res = await engine.load_image(msg.data)
+                    json_res = json.dumps(res, cls=NumpyEncoder)
+                    await ws.send_str(json_res)
 
+                elif msg.type == WSMsgType.TEXT:
+                    data = json.loads(msg.data)
+                    webp_bytes = engine.transform_image(data.get('hash'), data.get('params'))
+                    await ws.send_bytes(webp_bytes)
 
-            elif msg.type in (WSMsgType.CLOSE, WSMsgType.ERROR):
-                #logger.warning(f"WebSocket connection closed: {msg.type}")
-                break
+            except Exception as e:
+                logger.error(f"Error in engine: {str(e)}")
+                logger.exception("Full traceback:")
+                await ws.send_json({"error": str(e)})
 
     except Exception as e:
         logger.error(f"Error in websocket_handler: {str(e)}")
         logger.exception("Full traceback:")
     return ws
 
-async def handle_modify_image(request: web.Request, ws: web.WebSocketResponse, msg: Dict[str, Any], uuid: str):
-    """
-    Handle the 'modify_image' request.
-
-    Args:
-        request (web.Request): The incoming request object.
-        ws (web.WebSocketResponse): The WebSocket response object.
-        msg (Dict[str, Any]): The message containing the image or image_hash and modification parameters.
-        uuid: A unique identifier for the request.
-    """
-    #logger.info("Received modify_image request")
-    try:
-        engine = request.app['engine']
-        image_hash = msg.get('image_hash')
-
-        if image_hash:
-            image_or_hash = image_hash
-        else:
-            image_data = msg['image']
-            image_or_hash = image_data
-
-        modified_image_base64 = await engine.modify_image(image_or_hash, msg['params'])
-
-        await ws.send_json({
-            "type": "modified_image",
-            "image": modified_image_base64,
-            "image_hash": engine.get_image_hash(image_or_hash),
-            "success": True,
-            "uuid": uuid  # Include the UUID in the response
-        })
-        #logger.info("Successfully sent modified image")
-    except Exception as e:
-        #logger.error(f"Error in modify_image: {str(e)}")
-        await ws.send_json({
-            "type": "modified_image",
-            "success": False,
-            "error": str(e),
-            "uuid": uuid  # Include the UUID even in error responses
-        })
-
 async def index(request: web.Request) -> web.Response:
     """Serve the index.html file"""
     content = open(os.path.join(os.path.dirname(__file__), "public", "index.html"), "r").read()

client/src/app.tsx

@@ -4,50 +4,41 @@ import { Download } from 'lucide-react';
 import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';
 import { truncateFileName } from './lib/utils';
 import { useFaceLandmarkDetection } from './hooks/useFaceLandmarkDetection';
-import { PoweredBy } from './components/PoweredBy';
+import { About } from './components/About';
 import { Spinner } from './components/Spinner';
 import { useFacePokeAPI } from './hooks/useFacePokeAPI';
 import { Layout } from './layout';
 import { useMainStore } from './hooks/useMainStore';
-import { convertImageToBase64 } from './lib/convertImageToBase64';
 
 export function App() {
   const error = useMainStore(s => s.error);
   const setError = useMainStore(s => s.setError);
   const imageFile = useMainStore(s => s.imageFile);
   const setImageFile = useMainStore(s => s.setImageFile);
-  const originalImage = useMainStore(s => s.originalImage);
-  const setOriginalImage = useMainStore(s => s.setOriginalImage);
+  const isGazingAtCursor = useMainStore(s => s.isGazingAtCursor);
+  const setIsGazingAtCursor = useMainStore(s => s.setIsGazingAtCursor);
+  const isFollowingCursor = useMainStore(s => s.isFollowingCursor);
+  const setIsFollowingCursor = useMainStore(s => s.setIsFollowingCursor);
+
   const previewImage = useMainStore(s => s.previewImage);
-  const setPreviewImage = useMainStore(s => s.setPreviewImage);
-  const resetImage = useMainStore(s => s.resetImage);
-  const setOriginalImageHash = useMainStore(s => s.setOriginalImageHash);
+  const status = useMainStore(s => s.status);
+  const blendShapes = useMainStore(s => s.blendShapes);
 
   const {
-    status,
-    setStatus,
     isDebugMode,
     setIsDebugMode,
     interruptMessage,
   } = useFacePokeAPI()
 
-  // State for face detection
   const {
-    canvasRef,
     canvasRefCallback,
-    mediaPipeRef,
-    faceLandmarks,
     isMediaPipeReady,
-    blendShapes,
-
-    setFaceLandmarks,
-    setBlendShapes,
-
     handleMouseDown,
     handleMouseUp,
     handleMouseMove,
-    handleMouseEnter,
-    handleMouseLeave,
+    handleTouchStart,
+    handleTouchMove,
+    handleTouchEnd,
     currentOpacity
   } = useFaceLandmarkDetection()
 
@@ -55,37 +46,10 @@ export function App() {
   const videoRef = useRef<HTMLDivElement>(null);
 
   // Handle file change
-  const handleFileChange = useCallback(async (event: React.ChangeEvent<HTMLInputElement>) => {
+  const handleFileChange = useCallback((event: React.ChangeEvent<HTMLInputElement>) => {
     const files = event.target.files;
-    if (files && files[0]) {
-      setImageFile(files[0]);
-      setStatus(`File selected: ${truncateFileName(files[0].name, 16)}`);
-
-      try {
-        const image = await convertImageToBase64(files[0]);
-        setPreviewImage(image);
-        setOriginalImage(image);
-        setOriginalImageHash('');
-      } catch (err) {
-        console.log(`failed to convert the image: `, err);
-        setImageFile(null);
-        setStatus('');
-        setPreviewImage('');
-        setOriginalImage('');
-        setOriginalImageHash('');
-        setFaceLandmarks([]);
-        setBlendShapes([]);
-      }
-    } else {
-      setImageFile(null);
-      setStatus('');
-      setPreviewImage('');
-      setOriginalImage('');
-      setOriginalImageHash('');
-      setFaceLandmarks([]);
-      setBlendShapes([]);
-    }
-  }, [isMediaPipeReady, setImageFile, setPreviewImage, setOriginalImage, setOriginalImageHash, setFaceLandmarks, setBlendShapes, setStatus]);
+    setImageFile(files?.[0] || undefined)
+  }, [setImageFile]);
 
   const handleDownload = useCallback(() => {
     if (previewImage) {
@@ -139,7 +103,7 @@ export function App() {
         <div className="mb-5 relative">
           <div className="flex flex-row items-center justify-between w-full">
             <div className="flex items-center space-x-2">
-              <div className="relative">
+              <div className="flex items-center justify-center">
                 <input
                   id="imageInput"
                   type="file"
@@ -155,7 +119,7 @@ export function App() {
                   } focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-slate-500 shadow-xl`}
                 >
                   <Spinner />
-                  {imageFile ? truncateFileName(imageFile.name, 32) : (isMediaPipeReady ? 'Choose a portrait photo (.jpg, .png, .webp)' : 'Initializing...')}
+                  {imageFile ? truncateFileName(imageFile.name, 32) : (isMediaPipeReady ? 'Choose a portrait photo' : 'Initializing...')}
                 </label>
               </div>
               {previewImage && (
@@ -168,15 +132,38 @@ export function App() {
                 </button>
               )}
             </div>
-            {previewImage && <label className="mt-4 flex items-center">
-              <input
-                type="checkbox"
-                checked={isDebugMode}
-                onChange={(e) => setIsDebugMode(e.target.checked)}
-                className="mr-2"
-              />
-              Show face landmarks on hover
-            </label>}
+            {previewImage && <div className="flex items-center space-x-2">
+              {/* experimental features, not active yet */}
+              {/*
+              <label className="mt-4 flex items-center">
+                <input
+                  type="checkbox"
+                  checked={isGazingAtCursor}
+                  onChange={(e) => setIsGazingAtCursor(!isGazingAtCursor)}
+                  className="mr-2"
+                />
+                Autotrack eyes
+              </label>
+              <label className="mt-4 flex items-center">
+                <input
+                  type="checkbox"
+                  checked={isFollowingCursor}
+                  onChange={(e) => setIsFollowingCursor(!isFollowingCursor)}
+                  className="mr-2"
+                />
+                Autotrack head
+              </label>
+              */}
+              <label className="mt-4 flex items-center">
+                <input
+                  type="checkbox"
+                  checked={isDebugMode}
+                  onChange={(e) => setIsDebugMode(e.target.checked)}
+                  className="mr-2"
+                />
+                Show face markers
+              </label>
+            </div>}
           </div>
           {previewImage && (
             <div className="mt-5 relative shadow-2xl rounded-xl overflow-hidden">
@@ -188,11 +175,12 @@ export function App() {
               <canvas
                 ref={canvasRefCallback}
                 className="absolute top-0 left-0 w-full h-full select-none"
-                onMouseEnter={handleMouseEnter}
-                onMouseLeave={handleMouseLeave}
                 onMouseDown={handleMouseDown}
                 onMouseUp={handleMouseUp}
                 onMouseMove={handleMouseMove}
+                onTouchStart={handleTouchStart}
+                onTouchMove={handleTouchMove}
+                onTouchEnd={handleTouchEnd}
                 style={{
                   position: 'absolute',
                   top: 0,
@@ -207,7 +195,7 @@ export function App() {
           )}
           {canDisplayBlendShapes && displayBlendShapes}
         </div>
-        <PoweredBy />
+        <About />
     </Layout>
   );
 }

client/src/components/{PoweredBy.tsx → About.tsx}

@@ -1,11 +1,11 @@
-export function PoweredBy() {
+export function About() {
   return (
     <div className="flex flex-row items-center justify-center font-sans mt-4 w-full">
-      {/*<span className="text-neutral-900 text-sm"
+      <span className="text-neutral-900 text-sm"
         style={{ textShadow: "rgb(255 255 255 / 80%) 0px 0px 2px" }}>
-        Best hosted on
-      </span>*/}
-      <span className="mr-1">
+        Click and drag on the image.
+      </span>
+      <span className="ml-2 mr-1">
         <img src="/hf-logo.svg" alt="Hugging Face" className="w-5 h-5" />
       </span>
       <span className="text-neutral-900 text-sm font-semibold"

client/src/hooks/useFaceLandmarkDetection.tsx

@@ -6,59 +6,36 @@ import { useMainStore } from './useMainStore';
 import useThrottledCallback from 'beautiful-react-hooks/useThrottledCallback';
 
 import { landmarkGroups, FACEMESH_LIPS, FACEMESH_LEFT_EYE, FACEMESH_LEFT_EYEBROW, FACEMESH_RIGHT_EYE, FACEMESH_RIGHT_EYEBROW, FACEMESH_FACE_OVAL } from './landmarks';
-
-// New types for improved type safety
-export type LandmarkGroup = 'lips' | 'leftEye' | 'leftEyebrow' | 'rightEye' | 'rightEyebrow' | 'faceOval' | 'background';
-export type LandmarkCenter = { x: number; y: number; z: number };
-export type ClosestLandmark = { group: LandmarkGroup; distance: number; vector: { x: number; y: number; z: number } };
-
-export type MediaPipeResources = {
-  faceLandmarker: vision.FaceLandmarker | null;
-  drawingUtils: vision.DrawingUtils | null;
-};
+import type { ActionMode, ClosestLandmark, LandmarkCenter, LandmarkGroup, MediaPipeResources } from '@/types';
 
 export function useFaceLandmarkDetection() {
-  const error = useMainStore(s => s.error);
   const setError = useMainStore(s => s.setError);
-  const imageFile = useMainStore(s => s.imageFile);
-  const setImageFile = useMainStore(s => s.setImageFile);
-  const originalImage = useMainStore(s => s.originalImage);
-  const originalImageHash = useMainStore(s => s.originalImageHash);
-  const setOriginalImageHash = useMainStore(s => s.setOriginalImageHash);
   const previewImage = useMainStore(s => s.previewImage);
-  const setPreviewImage = useMainStore(s => s.setPreviewImage);
-  const resetImage = useMainStore(s => s.resetImage);
+  const handleServerResponse = useMainStore(s => s.handleServerResponse);
+  const faceLandmarks = useMainStore(s => s.faceLandmarks);
 
-  ;(window as any).debugJuju = useMainStore;
   ////////////////////////////////////////////////////////////////////////
-  // ok so apparently I cannot vary the latency, or else there is a bug
-  // const averageLatency = useMainStore(s => s.averageLatency);
-  const averageLatency = 220
+  // if we only send the face/square then we can use 138ms
+  // unfortunately it doesn't work well yet
+  // const throttleInMs = 138ms
+  const throttleInMs = 180
   ////////////////////////////////////////////////////////////////////////
 
   // State for face detection
-  const [faceLandmarks, setFaceLandmarks] = useState<vision.NormalizedLandmark[][]>([]);
   const [isMediaPipeReady, setIsMediaPipeReady] = useState(false);
   const [isDrawingUtilsReady, setIsDrawingUtilsReady] = useState(false);
-  const [blendShapes, setBlendShapes] = useState<vision.Classifications[]>([]);
 
   // State for mouse interaction
   const [dragStart, setDragStart] = useState<{ x: number; y: number } | null>(null);
-  const [dragEnd, setDragEnd] = useState<{ x: number; y: number } | null>(null);
 
   const [isDragging, setIsDragging] = useState(false);
-  const [isWaitingForResponse, setIsWaitingForResponse] = useState(false);
   const dragStartRef = useRef<{ x: number; y: number } | null>(null);
-  const currentMousePosRef = useRef<{ x: number; y: number } | null>(null);
-  const lastModifiedImageHashRef = useRef<string | null>(null);
 
   const [currentLandmark, setCurrentLandmark] = useState<ClosestLandmark | null>(null);
   const [previousLandmark, setPreviousLandmark] = useState<ClosestLandmark | null>(null);
   const [currentOpacity, setCurrentOpacity] = useState(0);
   const [previousOpacity, setPreviousOpacity] = useState(0);
 
-  const [isHovering, setIsHovering] = useState(false);
-
   // Refs
   const canvasRef = useRef<HTMLCanvasElement>(null);
   const mediaPipeRef = useRef<MediaPipeResources>({
@@ -222,6 +199,9 @@ export function useFaceLandmarkDetection() {
 
   // Detect face landmarks
   const detectFaceLandmarks = useCallback(async (imageDataUrl: string) => {
+    const { setFaceLandmarks,setBlendShapes } = useMainStore.getState();
+
+
     // console.log('Attempting to detect face landmarks...');
     if (!isMediaPipeReady) {
       console.log('MediaPipe not ready. Skipping detection.');
@@ -247,6 +227,7 @@ export function useFaceLandmarkDetection() {
     setFaceLandmarks(faceLandmarkerResult.faceLandmarks);
     setBlendShapes(faceLandmarkerResult.faceBlendshapes || []);
 
+
     if (faceLandmarkerResult.faceLandmarks && faceLandmarkerResult.faceLandmarks[0]) {
       computeLandmarkCenters(faceLandmarkerResult.faceLandmarks[0]);
     }
@@ -352,279 +333,147 @@ export function useFaceLandmarkDetection() {
     detectFaceLandmarks(previewImage);
   }, [isMediaPipeReady, isDrawingUtilsReady, previewImage])
 
-
-
-  const modifyImage = useCallback(({ landmark, vector }: {
-      landmark: ClosestLandmark
-      vector: { x: number; y: number; z: number }
-    }) => {
-
-    const {
-      originalImage,
-      originalImageHash,
-      params: previousParams,
-      setParams,
-      setError
-    } = useMainStore.getState()
-
-
-    if (!originalImage) {
-      console.error('Image file or facePoke not available');
-      return;
-    }
-
-    const params = {
-      ...previousParams
-    }
-
-    const minX = -0.50;
-    const maxX = 0.50;
-    const minY = -0.50;
-    const maxY = 0.50;
-
-    // Function to map a value from one range to another
-    const mapRange = (value: number, inMin: number, inMax: number, outMin: number, outMax: number): number => {
-      return Math.min(outMax, Math.max(outMin, ((value - inMin) * (outMax - outMin)) / (inMax - inMin) + outMin));
-    };
-
-    console.log("modifyImage:", {
-      originalImage,
-      originalImageHash,
-      landmark,
-      vector,
-      minX,
-      maxX,
-      minY,
-      maxY,
-    })
-
-    // Map landmarks to ImageModificationParams
-    switch (landmark.group) {
-      case 'leftEye':
-      case 'rightEye':
-         // eyebrow (min: -20, max: 5, default: 0)
-        const eyesMin = -20
-        const eyesMax = 5
-        params.eyes = mapRange(-vector.y, minX, maxX, eyesMin, eyesMax);
-
-        break;
-      case 'leftEyebrow':
-      case 'rightEyebrow':
-        // moving the mouse vertically for the eyebrow
-        // should make them up/down
-        // eyebrow (min: -10, max: 15, default: 0)
-        const eyebrowMin = -10
-        const eyebrowMax = 15
-        params.eyebrow = mapRange(-vector.y, minY, maxY, eyebrowMin, eyebrowMax);
-
-        break;
-      case 'lips':
-        // aaa (min: -30, max: 120, default: 0)
-        //const aaaMin = -30
-        //const aaaMax = 120
-        //params.aaa = mapRange(vector.x, minY, maxY, aaaMin, aaaMax);
-
-        // eee (min: -20, max: 15, default: 0)
-        const eeeMin = -20
-        const eeeMax = 15
-        params.eee = mapRange(-vector.y, minY, maxY, eeeMin, eeeMax);
-
-
-        // woo (min: -20, max: 15, default: 0)
-        const wooMin = -20
-        const wooMax = 15
-        params.woo = mapRange(-vector.x, minX, maxX, wooMin, wooMax);
-
-        break;
-      case 'faceOval':
-          // displacing the face horizontally by moving the mouse on the X axis
-          // should perform a yaw rotation
-          // rotate_roll (min: -20, max: 20, default: 0)
-          const rollMin = -40
-          const rollMax = 40
-
-          // note: we invert the axis here
-          params.rotate_roll = mapRange(vector.x, minX, maxX, rollMin, rollMax);
-          break;
-
-      case 'background':
-        // displacing the face horizontally by moving the mouse on the X axis
-        // should perform a yaw rotation
-        // rotate_yaw (min: -20, max: 20, default: 0)
-        const yawMin = -40
-        const yawMax = 40
-
-        // note: we invert the axis here
-        params.rotate_yaw = mapRange(-vector.x, minX, maxX, yawMin, yawMax);
-
-        // displacing the face vertically by moving the mouse on the Y axis
-        // should perform a pitch rotation
-        // rotate_pitch (min: -20, max: 20, default: 0)
-        const pitchMin = -40
-        const pitchMax = 40
-        params.rotate_pitch = mapRange(vector.y, minY, maxY, pitchMin, pitchMax);
-        break;
-      default:
-        return
-    }
-
-    for (const [key, value] of Object.entries(params)) {
-      if (isNaN(value as any) || !isFinite(value as any)) {
-        console.log(`${key} is NaN, aborting`)
-        return
-      }
-    }
-    console.log(`PITCH=${params.rotate_pitch || 0}, YAW=${params.rotate_yaw || 0}, ROLL=${params.rotate_roll || 0}`);
-
-    setParams(params)
-    try {
-      // For the first request or when the image file changes, send the full image
-      if (!lastModifiedImageHashRef.current || lastModifiedImageHashRef.current !== originalImageHash) {
-        lastModifiedImageHashRef.current = originalImageHash;
-        facePoke.modifyImage(originalImage, null, params);
-      } else {
-        // For subsequent requests, send only the hash
-        facePoke.modifyImage(null, lastModifiedImageHashRef.current, params);
-      }
-    } catch (error) {
-      // console.error('Error modifying image:', error);
-      setError('Failed to modify image');
-    }
-  }, []);
-
-  // this is throttled by our average latency
   const modifyImageWithRateLimit = useThrottledCallback((params: {
     landmark: ClosestLandmark
     vector: { x: number; y: number; z: number }
+    mode: ActionMode
   }) => {
-    modifyImage(params);
-  }, [modifyImage], averageLatency);
-
-  const handleMouseEnter = useCallback(() => {
-    setIsHovering(true);
-  }, []);
+    useMainStore.getState().modifyImage(params);
+  }, [], throttleInMs);
 
-  const handleMouseLeave = useCallback(() => {
-    setIsHovering(false);
-  }, []);
+  useEffect(() => {
+    facePoke.setOnServerResponse(handleServerResponse);
+  }, [handleServerResponse]);
 
-  // Update mouse event handlers
-  const handleMouseDown = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
+  const handleStart = useCallback((x: number, y: number, mode: ActionMode) => {
     if (!canvasRef.current) return;
 
     const rect = canvasRef.current.getBoundingClientRect();
-    const x = (event.clientX - rect.left) / rect.width;
-    const y = (event.clientY - rect.top) / rect.height;
+    const normalizedX = (x - rect.left) / rect.width;
+    const normalizedY = (y - rect.top) / rect.height;
 
-    const landmark = findClosestLandmark(x, y);
-    console.log(`Mouse down on ${landmark.group}`);
+    const landmark = findClosestLandmark(normalizedX, normalizedY);
+    // console.log(`Interaction start on ${landmark.group}`);
     setActiveLandmark(landmark);
-    setDragStart({ x, y });
-    dragStartRef.current = { x, y };
+    setDragStart({ x: normalizedX, y: normalizedY });
+    dragStartRef.current = { x: normalizedX, y: normalizedY };
   }, [findClosestLandmark, setActiveLandmark, setDragStart]);
 
-  const handleMouseMove = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
+  const handleMove = useCallback((x: number, y: number,  mode: ActionMode) => {
     if (!canvasRef.current) return;
 
     const rect = canvasRef.current.getBoundingClientRect();
-    const x = (event.clientX - rect.left) / rect.width;
-    const y = (event.clientY - rect.top) / rect.height;
+    const normalizedX = (x - rect.left) / rect.width;
+    const normalizedY = (y - rect.top) / rect.height;
+
+    const landmark = findClosestLandmark(
+      normalizedX,
+      normalizedY,
+      dragStart && dragStartRef.current ? currentLandmark?.group : undefined
+    );
+
+    const landmarkData = landmarkCenters[landmark?.group]
+    const vector = landmarkData ? {
+      x: normalizedX - landmarkData.x,
+      y: normalizedY - landmarkData.y,
+      z: 0
+    } :  {
+      x: 0.5,
+      y: 0.5,
+      z: 0
+    }
 
-    // only send an API request to modify the image if we are actively dragging
     if (dragStart && dragStartRef.current) {
-
-      const landmark = findClosestLandmark(x, y, currentLandmark?.group);
-
-      console.log(`Dragging mouse (was over ${currentLandmark?.group || 'nothing'}, now over ${landmark.group})`);
-
-      // Compute the vector from the landmark center to the current mouse position
+      setIsDragging(true);
       modifyImageWithRateLimit({
-        landmark: currentLandmark || landmark, // this will still use the initially selected landmark
-        vector: {
-          x: x - landmarkCenters[landmark.group].x,
-          y: y - landmarkCenters[landmark.group].y,
-          z: 0 // Z is 0 as mouse interaction is 2D
-        }
+        landmark: currentLandmark || landmark,
+        vector,
+        mode
       });
-      setIsDragging(true);
     } else {
-      const landmark = findClosestLandmark(x, y);
-
-      //console.log(`Moving mouse over ${landmark.group}`);
-      // console.log(`Simple mouse move over ${landmark.group}`);
-
-      // we need to be careful here, we don't want to change the active
-      // landmark dynamically if we are busy dragging
-
       if (!currentLandmark || (currentLandmark?.group !== landmark?.group)) {
-        // console.log("setting activeLandmark to ", landmark);
         setActiveLandmark(landmark);
       }
-      setIsHovering(true); // Ensure hovering state is maintained during movement
+      modifyImageWithRateLimit({
+        landmark,
+        vector,
+        mode: 'HOVERING'
+      });
     }
-  }, [currentLandmark, dragStart, setIsHovering, setActiveLandmark, setIsDragging, modifyImageWithRateLimit, landmarkCenters]);
+  }, [currentLandmark, dragStart, setActiveLandmark, setIsDragging, modifyImageWithRateLimit, landmarkCenters]);
 
-  const handleMouseUp = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
+  const handleEnd = useCallback((x: number, y: number,  mode: ActionMode) => {
     if (!canvasRef.current) return;
 
     const rect = canvasRef.current.getBoundingClientRect();
-    const x = (event.clientX - rect.left) / rect.width;
-    const y = (event.clientY - rect.top) / rect.height;
+    const normalizedX = (x - rect.left) / rect.width;
+    const normalizedY = (y - rect.top) / rect.height;
 
-    // only send an API request to modify the image if we are actively dragging
     if (dragStart && dragStartRef.current) {
+      const landmark = findClosestLandmark(normalizedX, normalizedY, currentLandmark?.group);
 
-      const landmark = findClosestLandmark(x, y, currentLandmark?.group);
-
-      console.log(`Mouse up (was over ${currentLandmark?.group || 'nothing'}, now over ${landmark.group})`);
-
-      // Compute the vector from the landmark center to the current mouse position
       modifyImageWithRateLimit({
-        landmark: currentLandmark || landmark, // this will still use the initially selected landmark
+        landmark: currentLandmark || landmark,
         vector: {
-          x: x - landmarkCenters[landmark.group].x,
-          y: y - landmarkCenters[landmark.group].y,
-          z: 0 // Z is 0 as mouse interaction is 2D
-        }
+          x: normalizedX - landmarkCenters[landmark.group].x,
+          y: normalizedY - landmarkCenters[landmark.group].y,
+          z: 0
+        },
+        mode
       });
     }
 
     setIsDragging(false);
     dragStartRef.current = null;
     setActiveLandmark(undefined);
-  }, [currentLandmark, isDragging, modifyImageWithRateLimit, findClosestLandmark, setActiveLandmark, landmarkCenters, modifyImageWithRateLimit, setIsDragging]);
+  }, [currentLandmark, isDragging, modifyImageWithRateLimit, findClosestLandmark, setActiveLandmark, landmarkCenters, setIsDragging]);
 
-  useEffect(() => {
-    facePoke.setOnModifiedImage((image: string, image_hash: string) => {
-      if (image) {
-        setPreviewImage(image);
-      }
-      setOriginalImageHash(image_hash);
-      lastModifiedImageHashRef.current = image_hash;
-    });
-  }, [setPreviewImage, setOriginalImageHash]);
+  const handleMouseDown = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
+    const  mode: ActionMode = event.button === 0 ? 'PRIMARY' : 'SECONDARY';
+    handleStart(event.clientX, event.clientY, mode);
+  }, [handleStart]);
+
+  const handleMouseMove = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
+    const  mode: ActionMode =  event.buttons === 1 ? 'PRIMARY' : 'SECONDARY';
+    handleMove(event.clientX, event.clientY, mode);
+  }, [handleMove]);
+
+  const handleMouseUp = useCallback((event: React.MouseEvent<HTMLCanvasElement>) => {
+    const  mode: ActionMode = event.buttons === 1 ? 'PRIMARY' : 'SECONDARY';
+    handleEnd(event.clientX, event.clientY, mode);
+  }, [handleEnd]);
+
+  const handleTouchStart = useCallback((event: React.TouchEvent<HTMLCanvasElement>) => {
+    const  mode: ActionMode = event.touches.length === 1 ? 'PRIMARY' : 'SECONDARY';
+    const touch = event.touches[0];
+    handleStart(touch.clientX, touch.clientY, mode);
+  }, [handleStart]);
+
+  const handleTouchMove = useCallback((event: React.TouchEvent<HTMLCanvasElement>) => {
+    const  mode: ActionMode = event.touches.length === 1 ? 'PRIMARY' : 'SECONDARY';
+    const touch = event.touches[0];
+    handleMove(touch.clientX, touch.clientY, mode);
+  }, [handleMove]);
+
+  const handleTouchEnd = useCallback((event: React.TouchEvent<HTMLCanvasElement>) => {
+    const  mode: ActionMode = event.changedTouches.length === 1 ? 'PRIMARY' : 'SECONDARY';
+    const touch = event.changedTouches[0];
+    handleEnd(touch.clientX, touch.clientY, mode);
+  }, [handleEnd]);
 
   return {
     canvasRef,
     canvasRefCallback,
     mediaPipeRef,
-    faceLandmarks,
     isMediaPipeReady,
     isDrawingUtilsReady,
-    blendShapes,
-
-    //dragStart,
-    //setDragStart,
-    //dragEnd,
-    //setDragEnd,
-    setFaceLandmarks,
-    setBlendShapes,
 
     handleMouseDown,
     handleMouseUp,
     handleMouseMove,
-    handleMouseEnter,
-    handleMouseLeave,
+    handleTouchStart,
+    handleTouchMove,
+    handleTouchEnd,
 
     currentLandmark,
     currentOpacity,

client/src/hooks/useFacePokeAPI.ts

@@ -1,12 +1,10 @@
 import { useEffect, useState } from "react";
 
 import { facePoke } from "../lib/facePoke";
-import { useMainStore } from "./useMainStore";
 
 export function useFacePokeAPI() {
 
    // State for FacePoke
-  const [status, setStatus] = useState('');
   const [isDebugMode, setIsDebugMode] = useState(false);
   const [interruptMessage, setInterruptMessage] = useState<string | null>(null);
 
@@ -33,8 +31,6 @@ export function useFacePokeAPI() {
 
   return {
     facePoke,
-    status,
-    setStatus,
     isDebugMode,
     setIsDebugMode,
     interruptMessage,

client/src/hooks/useMainStore.ts

@@ -1,32 +1,45 @@
 import { create } from 'zustand'
-import type { ClosestLandmark } from './useFaceLandmarkDetection'
-import type { ImageModificationParams } from '@/lib/facePoke'
-
-interface ImageState {
-  error: string
-  imageFile: File | null
-  originalImage: string
-  previewImage: string
-  originalImageHash: string
-  minLatency: number
-  averageLatency: number
-  maxLatency: number
-  activeLandmark?: ClosestLandmark
-  params: Partial<ImageModificationParams>
+
+import * as vision from '@mediapipe/tasks-vision'
+
+import { truncateFileName } from '@/lib/utils'
+import { convertImageToBase64 } from '@/lib/convertImageToBase64'
+import { type Metadata, type ImageModificationParams, type OnServerResponseParams, type ActionMode, type ImageStateValues, type ClosestLandmark } from '@/types'
+import { mapRange } from '@/lib/mapRange'
+import { facePoke } from '@/lib/facePoke'
+
+
+export type ImageState = ImageStateValues & {
+  setStatus: (status?: string) => void
   setError: (error?: string) => void
-  setImageFile: (file: File | null) => void
+  setFaceLandmarks: (faceLandmarks: vision.NormalizedLandmark[][]) => void
+  setBlendShapes: (blendShapes: vision.Classifications[]) => void
+  setImageFile: (file?: File) => Promise<void>
+  setIsFollowingCursor: (isFollowingCursor: boolean) => void
+  setIsGazingAtCursor: (isGazingAtCursor: boolean) => void
   setOriginalImage: (url: string) => void
   setOriginalImageHash: (hash: string) => void
   setPreviewImage: (url: string) => void
   resetImage: () => void
   setAverageLatency: (averageLatency: number) => void
   setActiveLandmark: (activeLandmark?: ClosestLandmark) => void
+  setMetadata: (metadata?: Metadata) => void
   setParams: (params: Partial<ImageModificationParams>) => void
+  handleServerResponse: (params: OnServerResponseParams) => Promise<void>
+  applyModifiedHeadToCanvas: (headImageBlob: Blob) => Promise<string>
+  modifyImage: ({ landmark, vector, mode }: {
+    landmark: ClosestLandmark
+    vector: { x: number; y: number; z: number }
+    mode: ActionMode
+  }) => Promise<void>
 }
 
-export const useMainStore = create<ImageState>((set, get) => ({
+export const getDefaultState = (): ImageStateValues => ({
+  status: '',
   error: '',
   imageFile: null,
+  isFollowingCursor: false,
+  isGazingAtCursor: false,
   originalImage: '',
   originalImageHash: '',
   previewImage: '',
@@ -34,9 +47,56 @@ export const useMainStore = create<ImageState>((set, get) => ({
   averageLatency: 190, // this should be the average for most people
   maxLatency: 4000, // max time between requests
   activeLandmark: undefined,
+  metadata: {
+    center: [0, 0],
+    size: 0,
+    bbox: [[0,0],[0,0],[0,0],[0,0]],
+    angle: 0,
+  },
   params: {},
+  faceLandmarks: [],
+  blendShapes: [],
+})
+
+export const useMainStore = create<ImageState>((set, get) => ({
+  ...getDefaultState(),
+  setStatus: (status: string = '') => set({ status }),
   setError: (error: string = '') => set({ error }),
-  setImageFile: (file) => set({ imageFile: file }),
+  setFaceLandmarks: (faceLandmarks: vision.NormalizedLandmark[][]) => {
+    set({ faceLandmarks })
+  },
+  setBlendShapes: (blendShapes: vision.Classifications[]) => {
+    set({ blendShapes })
+  },
+  setImageFile: async (file?: File) => {
+    if (!file) {
+      set({
+        ...getDefaultState(),
+        status: 'No file selected',
+      })
+      return;
+    }
+
+    try {
+      const image = await convertImageToBase64(file);
+      set({
+        ...getDefaultState(),
+        imageFile: file,
+        status: `File selected: ${truncateFileName(file.name, 16)}`,
+        previewImage: image,
+        originalImage: image,
+      })
+      facePoke.loadImage(image);
+    } catch (err) {
+      console.log(`failed to load the image: `, err);
+      set({
+        ...getDefaultState(),
+        status: 'Failed to load the image',
+      })
+    }
+  },
+  setIsFollowingCursor: (isFollowingCursor: boolean) => set({ isFollowingCursor }),
+  setIsGazingAtCursor: (isGazingAtCursor: boolean) => set({ isGazingAtCursor }),
   setOriginalImage: (url) => set({ originalImage: url }),
   setOriginalImageHash: (originalImageHash) => set({ originalImageHash }),
   setPreviewImage: (url) => set({ previewImage: url }),
@@ -48,6 +108,11 @@ export const useMainStore = create<ImageState>((set, get) => ({
   },
   setAverageLatency: (averageLatency: number) => set({ averageLatency }),
   setActiveLandmark: (activeLandmark?: ClosestLandmark) => set({ activeLandmark }),
+  setMetadata: (metadata?: Metadata) => set(metadata ? {
+    metadata
+  } : {
+    metadata: getDefaultState().metadata,
+  }),
   setParams: (params: Partial<ImageModificationParams>) => {
     const {params: previousParams } = get()
     set({ params: {
@@ -55,4 +120,317 @@ export const useMainStore = create<ImageState>((set, get) => ({
       ...params
     }})
   },
+  handleServerResponse: async (params: OnServerResponseParams) => {
+    const { originalImage, setMetadata, setPreviewImage, setOriginalImageHash, applyModifiedHeadToCanvas, modifyImage } = useMainStore.getState();
+    if (typeof params.error === "string") {
+      console.error(`handleServerResponse: failed to perform the request, resetting the app (${params.error})`)
+      setPreviewImage(originalImage)
+      setOriginalImageHash('')
+    } else if (typeof params.image !== "undefined") {
+
+      // this is where we decide to paste back the image as a whole,
+      // or apply some shenanigans to only past back the head.
+      // the part about the head is not done yet, so we do it all for now.
+
+      //  --- old way: use it whole ---
+      const image = await convertImageToBase64(params.image);
+
+      //  --- future way: try to only apply the head ---
+      // const image = await applyModifiedHeadToCanvas(params.image);
+
+      setPreviewImage(image);
+    } else if (typeof params.loaded !== "undefined") {
+      //console.log(`handleServerResponse: received a json`, params)
+      setOriginalImageHash(params.loaded.h)
+      setMetadata({
+        center: params.loaded.c, // center - 2x1
+        size: params.loaded.s, // size - scalar
+        bbox: params.loaded.b, // bbox - 4x2
+        angle: params.loaded.a,  //angle - rad, counterclockwise
+      })
+
+      // right after we received the hash, we perform a first blank request
+      await modifyImage({
+        landmark: {
+          group: 'background',
+          distance: 0,
+          vector: { x: 0.5, y: 0.5, z: 0 }
+        },
+        vector: {x: 0, y: 0, z: 0 },
+        mode: 'PRIMARY'
+      })
+    } else {
+      console.log(`handleServerResponse: received an unknown json`, params)
+    }
+  },
+
+  applyModifiedHeadToCanvas: async (headImageBlob: Blob): Promise<string> => {
+    return new Promise(async (resolve, reject) => {
+      const originalImg = new Image();
+
+      const { originalImage, metadata } = useMainStore.getState();
+      originalImg.onload = async () => {
+        const canvas = document.createElement('canvas');
+        const ctx = canvas.getContext('2d');
+
+        if (!ctx) {
+          reject(new Error('Failed to get 2D context'));
+          return;
+        }
+
+        // Get device pixel ratio
+        const pixelRatio = window.devicePixelRatio || 1;
+
+        canvas.width = originalImg.width;
+        canvas.height = originalImg.height;
+
+        // Draw original image
+        ctx.drawImage(originalImg, 0, 0);
+
+        const headImageBitmap = await createImageBitmap(headImageBlob, {
+          resizeQuality: 'high'
+        });
+
+        // Create a temporary canvas for the head image with gradient
+        const tempCanvas = document.createElement('canvas');
+        const tempCtx = tempCanvas.getContext('2d');
+
+        if (!tempCtx) {
+          reject(new Error('Failed to get 2D context for temporary canvas'));
+          return;
+        }
+
+        tempCanvas.width = headImageBitmap.width;
+        tempCanvas.height = headImageBitmap.height;
+
+        // Draw the head image on the temporary canvas
+        tempCtx.drawImage(headImageBitmap, 0, 0);
+
+        // Create gradient mask
+        const gradientSize = 20; // Size of the gradient in pixels
+        const gradient = tempCtx.createRadialGradient(
+          tempCanvas.width / 2, tempCanvas.height / 2, Math.min(tempCanvas.width, tempCanvas.height) / 2 - gradientSize,
+          tempCanvas.width / 2, tempCanvas.height / 2, Math.min(tempCanvas.width, tempCanvas.height) / 2
+        );
+
+        gradient.addColorStop(0, 'rgba(0, 0, 0, 1)');
+        gradient.addColorStop(1, 'rgba(0, 0, 0, 0)');
+
+        // Apply gradient mask
+        tempCtx.globalCompositeOperation = 'destination-in';
+        tempCtx.fillStyle = gradient;
+        tempCtx.fillRect(0, 0, tempCanvas.width, tempCanvas.height);
+
+        console.log("metadata:", metadata);
+        ctx.save();
+        ctx.rotate(metadata.angle);
+
+        // TODO: this is where we need to grab the coordinates from the metadata and start drawing our image
+        // now the issue if that there is an angle, so we need to see how this impacts the transformation
+        ctx.restore();
+        /*
+          TODO finish the implementation
+
+          console.log("metadata:", metadata);
+          ctx.save();
+          ctx.rotate(metadata.angle);
+
+          // TODO we need the coordinate inside the final image
+          ctx.drawImage(
+            tempCanvas,
+            topLeftCornerX,
+            topLeftCornerY,
+            width,
+            height,
+          );
+
+          ctx.restore();
+        */
+
+
+        // Our head image already contains compression artifacts,
+        // so let's avoid double-encoding in here and use PNG
+        resolve(canvas.toDataURL('image/png'));
+      };
+
+      originalImg.src = originalImage;
+    })
+  },
+  modifyImage: async ({ landmark, vector, mode }: {
+    landmark: ClosestLandmark
+    vector: { x: number; y: number; z: number }
+    mode: ActionMode
+  }): Promise<void> => {
+
+    const {
+      originalImage,
+      originalImageHash,
+      params: previousParams,
+      setParams,
+      setError,
+      isFollowingCursor,
+      isGazingAtCursor
+    } = get()
+
+
+    if (!originalImage) {
+      console.error('Image file or facePoke not available');
+      return;
+    }
+
+    const params = {
+      ...previousParams
+    }
+
+    const generalControl = {
+      minX: -0.30,
+      maxX: 0.30,
+      minY: -0.30,
+      maxY: 0.30
+    }
+
+    const pupilControl = {
+      minX: -0.50,
+      maxX: 0.50,
+      minY: -0.50,
+      maxY: 0.50
+    }
+
+    const eyeControl = {
+      minX: -0.50,
+      maxX: 0.50,
+      minY: -0.50,
+      maxY: 0.50
+    }
+
+    // for the min and max value, we can look here:
+    // https://github.com/fofr/cog-expression-editor/blob/main/predict.py
+
+    // regardless of the landmark, those rule will apply
+    if (isFollowingCursor) {
+      // displacing the face horizontally by moving the mouse on the X axis
+      // should perform a yaw rotation
+      // rotate_yaw (min: -20, max: 20, default: 0)
+      const yawMin = -40
+      const yawMax = 40
+
+      // note: we invert the axis here
+      params.rotate_yaw = mapRange(-vector.x, generalControl.minX, generalControl.maxX, yawMin, yawMax);
+
+      // displacing the face vertically by moving the mouse on the Y axis
+      // should perform a pitch rotation
+      // rotate_pitch (min: -20, max: 20, default: 0)
+      const pitchMin = -40
+      const pitchMax = 40
+      params.rotate_pitch = mapRange(vector.y, generalControl.minY, generalControl.maxY, pitchMin, pitchMax);
+    }
+
+    if (isGazingAtCursor) {
+      const pupilsXMin = -15
+      const pupilsXMax = 15
+      params.pupil_x = mapRange(vector.x, pupilControl.minX, pupilControl.maxX, pupilsXMin, pupilsXMax);
+      const pupilsYMin = -2 // -15
+      const pupilsYMax = 8 // 15
+      params.pupil_y = mapRange(-vector.y, pupilControl.minY, pupilControl.maxY, pupilsYMin, pupilsYMax);
+    }
+
+    // if the user clicked on the primary or secondary button,
+    // then it triggers some more things
+    if (mode !== 'HOVERING') {
+      switch (landmark.group) {
+        case 'leftEye':
+        case 'rightEye':
+         // if (mode === 'PRIMARY') {
+            const pupilsXMin = -15
+            const pupilsXMax = 15
+            params.pupil_x = mapRange(vector.x, pupilControl.minX, pupilControl.maxX, pupilsXMin, pupilsXMax);
+
+            const eyesMin = -20
+            const eyesMax = 5
+            params.eyes = mapRange(-vector.y, eyeControl.minX, eyeControl.maxX, eyesMin, eyesMax);
+          //}
+
+          break;
+        case 'leftEyebrow':
+        case 'rightEyebrow':
+          // moving the mouse vertically for the eyebrow
+          // should make them up/down
+          // eyebrow (min: -10, max: 15, default: 0)
+          const eyebrowMin = -10
+          const eyebrowMax = 15
+          params.eyebrow = mapRange(-vector.y, eyeControl.minY, eyeControl.maxY, eyebrowMin, eyebrowMax);
+
+          break;
+        case 'lips':
+          // aaa (min: -30, max: 120, default: 0)
+          const aaaMin = -30
+          const aaaMax = 120
+          params.aaa = mapRange(-vector.y, eyeControl.minY, eyeControl.maxY, aaaMin, aaaMax);
+
+          // eee (min: -20, max: 15, default: 0)
+          const eeeMin = -20
+          const eeeMax = 15
+          params.eee = mapRange(vector.x, eyeControl.minX, eyeControl.maxX, eeeMin, eeeMax);
+
+          // woo (min: -20, max: 15, default: 0)
+          //const wooMin = -20
+          //const wooMax = 15
+          //params.woo = mapRange(-vector.x, eyeControl.minX, eyeControl.maxX, wooMin, wooMax);
+
+          break;
+        case 'faceOval':
+            // displacing the face horizontally by moving the mouse on the X axis
+            // should perform a yaw rotation
+            // rotate_roll (min: -20, max: 20, default: 0)
+            const rollMin = -40
+            const rollMax = 40
+
+            // note: we invert the axis here
+            params.rotate_roll = mapRange(vector.x, eyeControl.minX, eyeControl.maxX, rollMin, rollMax);
+            break;
+
+        case 'background':
+          // displacing the face horizontally by moving the mouse on the X axis
+          // should perform a yaw rotation
+          // rotate_yaw (min: -20, max: 20, default: 0)
+          const yawMin = -40
+          const yawMax = 40
+
+          // note: we invert the axis here
+          params.rotate_yaw = mapRange(-vector.x, generalControl.minX, generalControl.maxX, yawMin, yawMax);
+
+          // displacing the face vertically by moving the mouse on the Y axis
+          // should perform a pitch rotation
+          // rotate_pitch (min: -20, max: 20, default: 0)
+          const pitchMin = -40
+          const pitchMax = 40
+          params.rotate_pitch = mapRange(vector.y, eyeControl.minY, eyeControl.maxY, pitchMin, pitchMax);
+          break;
+        default:
+          return
+      }
+    }
+
+    for (const [key, value] of Object.entries(params)) {
+      if (isNaN(value as any) || !isFinite(value as any)) {
+        console.log(`${key} is NaN, aborting`)
+        return
+      }
+    }
+
+    //console.log(`PITCH=${params.rotate_pitch || 0}, YAW=${params.rotate_yaw || 0}, ROLL=${params.rotate_roll || 0}`);
+
+    setParams(params)
+
+    try {
+
+      if (originalImageHash) {
+        facePoke.transformImage(originalImageHash, params);
+      }
+
+    } catch (error) {
+      // console.error('Error modifying image:', error);
+      setError('Failed to modify image');
+    }
+  }
 }))

client/src/layout.tsx

@@ -5,7 +5,7 @@ export function Layout({ children }: { children: ReactNode }) {
     <div className="fixed min-h-screen w-full flex items-center justify-center bg-gradient-to-br from-gray-300 to-stone-300"
       style={{ boxShadow: "inset 0 0 10vh 0 rgb(0 0 0 / 30%)" }}>
       <div className="min-h-screen w-full py-8 flex flex-col justify-center">
-        <div className="relative p-4 sm:max-w-5xl sm:mx-auto">
+        <div className="flex flex-col items-center justify-center p-4 sm:max-w-5xl sm:mx-auto">
           {children}
         </div>
       </div>

client/src/lib/convertImageToBase64.ts

@@ -1,4 +1,4 @@
-export async function convertImageToBase64(imageFile: File): Promise<string> {
+export async function convertImageToBase64(imageFileOrBlob: File | Blob): Promise<string> {
   return new Promise((resolve, reject) => {
     const reader = new FileReader();
 
@@ -11,9 +11,9 @@ export async function convertImageToBase64(imageFile: File): Promise<string> {
     };
 
     reader.onerror = () => {
-      reject(new Error('Error reading file'));
+      reject(new Error('Error reading file or blob'));
     };
 
-    reader.readAsDataURL(imageFile);
+    reader.readAsDataURL(imageFileOrBlob);
   });
 }

client/src/lib/facePoke.ts

@@ -1,153 +1,35 @@
-import { v4 as uuidv4 } from 'uuid';
-import { CircularBuffer } from './circularBuffer';
-import { useMainStore } from '@/hooks/useMainStore';
+import { WebSocketState, type ImageModificationParams, type OnServerResponse } from "@/types";
 
-/**
- * Represents a tracked request with its UUID and timestamp.
- */
-export interface TrackedRequest {
-  uuid: string;
-  timestamp: number;
-}
-
-/**
- * Represents the parameters for image modification.
- */
-export interface ImageModificationParams {
-  eyes: number;
-  eyebrow: number;
-  wink: number;
-  pupil_x: number;
-  pupil_y: number;
-  aaa: number;
-  eee: number;
-  woo: number;
-  smile: number;
-  rotate_pitch: number;
-  rotate_yaw: number;
-  rotate_roll: number;
-}
-
-/**
- * Represents a message to modify an image.
- */
-export interface ModifyImageMessage {
-  type: 'modify_image';
-  image?: string;
-  image_hash?: string;
-  params: Partial<ImageModificationParams>;
-}
-
-
-/**
- * Callback type for handling modified images.
- */
-type OnModifiedImage = (image: string, image_hash: string) => void;
-
-/**
- * Enum representing the different states of a WebSocket connection.
- */
-enum WebSocketState {
-  CONNECTING = 0,
-  OPEN = 1,
-  CLOSING = 2,
-  CLOSED = 3
-}
 
 /**
  * FacePoke class manages the WebSocket connection
  */
 export class FacePoke {
   private ws: WebSocket | null = null;
-  private readonly connectionId: string = uuidv4();
   private isUnloading: boolean = false;
-  private onModifiedImage: OnModifiedImage = () => {};
+  private onServerResponse: OnServerResponse = async () => {};
   private reconnectAttempts: number = 0;
   private readonly maxReconnectAttempts: number = 5;
   private readonly reconnectDelay: number = 5000;
   private readonly eventListeners: Map<string, Set<Function>> = new Map();
 
-  private requestTracker: Map<string, TrackedRequest> = new Map();
-  private responseTimeBuffer: CircularBuffer<number>;
-  private readonly MAX_TRACKED_TIMES = 5; // Number of recent response times to track
-
   /**
    * Creates an instance of FacePoke.
    * Initializes the WebSocket connection.
    */
   constructor() {
-    console.log(`[FacePoke] Initializing FacePoke instance with connection ID: ${this.connectionId}`);
+    console.log(`[FacePoke] Initializing FacePoke instance`);
     this.initializeWebSocket();
     this.setupUnloadHandler();
-
-    this.responseTimeBuffer = new CircularBuffer<number>(this.MAX_TRACKED_TIMES);
-    console.log(`[FacePoke] Initialized response time tracker with capacity: ${this.MAX_TRACKED_TIMES}`);
-  }
-
-
-  /**
-   * Generates a unique UUID for a request and starts tracking it.
-   * @returns The generated UUID for the request.
-   */
-  private trackRequest(): string {
-    const uuid = uuidv4();
-    this.requestTracker.set(uuid, { uuid, timestamp: Date.now() });
-    // console.log(`[FacePoke] Started tracking request with UUID: ${uuid}`);
-    return uuid;
-  }
-
-   /**
-   * Completes tracking for a request and updates response time statistics.
-   * @param uuid - The UUID of the completed request.
-   */
-   private completeRequest(uuid: string): void {
-    const request = this.requestTracker.get(uuid);
-    if (request) {
-      const responseTime = Date.now() - request.timestamp;
-      this.responseTimeBuffer.push(responseTime);
-      this.requestTracker.delete(uuid);
-      this.updateThrottleTime();
-      console.log(`[FacePoke] Completed request ${uuid}. Response time: ${responseTime}ms`);
-    } else {
-      console.warn(`[FacePoke] Attempted to complete unknown request: ${uuid}`);
-    }
-  }
-
-    /**
-   * Calculates the average response time from recent requests.
-   * @returns The average response time in milliseconds.
-   */
-    private calculateAverageResponseTime(): number {
-      const times = this.responseTimeBuffer.getAll();
-
-      const averageLatency = useMainStore.getState().averageLatency;
-
-      if (times.length === 0) return averageLatency;
-      const sum = times.reduce((acc, time) => acc + time, 0);
-      return sum / times.length;
-    }
-
-  /**
-   * Updates the throttle time based on recent response times.
-   */
-  private updateThrottleTime(): void {
-    const { minLatency, maxLatency, averageLatency, setAverageLatency } = useMainStore.getState();
-    const avgResponseTime = this.calculateAverageResponseTime();
-    const newLatency = Math.min(minLatency, Math.max(minLatency, avgResponseTime));
-
-    if (newLatency !== averageLatency) {
-      setAverageLatency(newLatency)
-      console.log(`[FacePoke] Updated throttle time (latency is ${newLatency}ms)`);
-    }
   }
 
   /**
    * Sets the callback function for handling modified images.
    * @param handler - The function to be called when a modified image is received.
    */
-  public setOnModifiedImage(handler: OnModifiedImage): void {
-    this.onModifiedImage = handler;
-    console.log(`[FacePoke] onModifiedImage handler set`);
+  public setOnServerResponse(handler: OnServerResponse): void {
+    this.onServerResponse = handler;
+    console.log(`[FacePoke] onServerResponse handler set`);
   }
 
   /**
@@ -165,53 +47,36 @@ export class FacePoke {
    * Implements exponential backoff for reconnection attempts.
    */
   private async initializeWebSocket(): Promise<void> {
-    console.log(`[FacePoke][${this.connectionId}] Initializing WebSocket connection`);
+    console.log(`[FacePoke] Initializing WebSocket connection`);
 
     const connect = () => {
       this.ws = new WebSocket(`wss://${window.location.host}/ws`);
 
       this.ws.onopen = this.handleWebSocketOpen.bind(this);
-      this.ws.onmessage = this.handleWebSocketMessage.bind(this);
       this.ws.onclose = this.handleWebSocketClose.bind(this);
       this.ws.onerror = this.handleWebSocketError.bind(this);
+      this.ws.onmessage = (this.handleWebSocketMessage.bind(this) as any)
     };
 
-    // const debouncedConnect = debounce(connect, this.reconnectDelay, { leading: true, trailing: false });
-
     connect(); // Initial connection attempt
   }
 
+  private handleWebSocketMessage(msg: MessageEvent) {
+    if (typeof msg.data === "string") {
+      this.onServerResponse({ loaded: JSON.parse(msg.data) as any });
+    } else if (typeof msg.data !== "undefined" ) {
+      this.onServerResponse({ image: msg.data as unknown as Blob });
+    }
+  }
   /**
    * Handles the WebSocket open event.
    */
   private handleWebSocketOpen(): void {
-    console.log(`[FacePoke][${this.connectionId}] WebSocket connection opened`);
+    console.log(`[FacePoke] WebSocket connection opened`);
     this.reconnectAttempts = 0; // Reset reconnect attempts on successful connection
     this.emitEvent('websocketOpen');
   }
 
-  // Update handleWebSocketMessage to complete request tracking
-  private handleWebSocketMessage(event: MessageEvent): void {
-    try {
-      const data = JSON.parse(event.data);
-      // console.log(`[FacePoke][${this.connectionId}] Received JSON data:`, data);
-
-      if (data.uuid) {
-        this.completeRequest(data.uuid);
-      }
-
-      if (data.type === 'modified_image') {
-        if (data?.image) {
-          this.onModifiedImage(data.image, data.image_hash);
-        }
-      }
-
-      this.emitEvent('message', data);
-    } catch (error) {
-      console.error(`[FacePoke][${this.connectionId}] Error parsing WebSocket message:`, error);
-    }
-  }
-
   /**
    * Handles WebSocket close events.
    * Implements reconnection logic with exponential backoff.
@@ -219,9 +84,9 @@ export class FacePoke {
    */
   private handleWebSocketClose(event: CloseEvent): void {
     if (event.wasClean) {
-      console.log(`[FacePoke][${this.connectionId}] WebSocket connection closed cleanly, code=${event.code}, reason=${event.reason}`);
+      console.log(`[FacePoke] WebSocket connection closed cleanly, code=${event.code}, reason=${event.reason}`);
     } else {
-      console.warn(`[FacePoke][${this.connectionId}] WebSocket connection abruptly closed`);
+      console.warn(`[FacePoke] WebSocket connection abruptly closed`);
     }
 
     this.emitEvent('websocketClose', event);
@@ -230,10 +95,10 @@ export class FacePoke {
     if (!this.isUnloading && this.reconnectAttempts < this.maxReconnectAttempts) {
       this.reconnectAttempts++;
       const delay = Math.min(1000 * (2 ** this.reconnectAttempts), 30000); // Exponential backoff, max 30 seconds
-      console.log(`[FacePoke][${this.connectionId}] Attempting to reconnect in ${delay}ms (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
+      console.log(`[FacePoke] Attempting to reconnect in ${delay}ms (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
       setTimeout(() => this.initializeWebSocket(), delay);
     } else if (this.reconnectAttempts >= this.maxReconnectAttempts) {
-      console.error(`[FacePoke][${this.connectionId}] Max reconnect attempts reached. Please refresh the page.`);
+      console.error(`[FacePoke] Max reconnect attempts reached. Please refresh the page.`);
       this.emitEvent('maxReconnectAttemptsReached');
     }
   }
@@ -243,7 +108,7 @@ export class FacePoke {
    * @param error - The error event.
    */
   private handleWebSocketError(error: Event): void {
-    console.error(`[FacePoke][${this.connectionId}] WebSocket error:`, error);
+    console.error(`[FacePoke] WebSocket error:`, error);
     this.emitEvent('websocketError', error);
   }
 
@@ -261,33 +126,32 @@ export class FacePoke {
     this.emitEvent('cleanup');
   }
 
-  /**
-   * Modifies an image based on the provided parameters
-   * @param image - The data-uri base64 image to modify.
-   * @param imageHash - The hash of the image to modify.
-   * @param params - The parameters for image modification.
-   */
-    public modifyImage(image: string | null, imageHash: string | null, params: Partial<ImageModificationParams>): void {
-      try {
-        const message: ModifyImageMessage = {
-          type: 'modify_image',
-          params: params
-        };
+  public async loadImage(image: string): Promise<void> {
+     // Extract the base64 part if it's a data URL
+    const base64Data = image.split(',')[1] || image;
 
-        if (image) {
-          message.image = image;
-        } else if (imageHash) {
-          message.image_hash = imageHash;
-        } else {
-          throw new Error('Either image or imageHash must be provided');
-        }
+    const buffer = new Uint8Array(atob(base64Data).split('').map(char => char.charCodeAt(0)));
+    const blob = new Blob([buffer], { type: 'application/octet-binary' });
+    this.sendBlobMessage(await blob.arrayBuffer());
+  }
 
-        this.sendJsonMessage(message);
-        // console.log(`[FacePoke] Sent modify image request with UUID: ${uuid}`);
-      } catch (err) {
-        console.error(`[FacePoke] Failed to modify the image:`, err);
-      }
+  public transformImage(hash: string, params: Partial<ImageModificationParams>): void {
+    this.sendJsonMessage({ hash, params });
+  }
+
+  private sendBlobMessage(buffer: ArrayBuffer): void {
+    if (!this.ws || this.ws.readyState !== WebSocketState.OPEN) {
+      const error = new Error('WebSocket connection is not open');
+      console.error('[FacePoke] Error sending JSON message:', error);
+      this.emitEvent('sendJsonMessageError', error);
+      throw error;
+    }
+    try {
+      this.ws.send(buffer);
+    } catch (err) {
+      console.error(`failed to send the WebSocket message: ${err}`)
     }
+  }
 
   /**
    * Sends a JSON message through the WebSocket connection with request tracking.
@@ -301,11 +165,11 @@ export class FacePoke {
       this.emitEvent('sendJsonMessageError', error);
       throw error;
     }
-
-    const uuid = this.trackRequest();
-    const messageWithUuid = { ...message, uuid };
-    // console.log(`[FacePoke] Sending JSON message with UUID ${uuid}:`, messageWithUuid);
-    this.ws.send(JSON.stringify(messageWithUuid));
+    try {
+      this.ws.send(JSON.stringify(message));
+    } catch (err) {
+      console.error(`failed to send the WebSocket message: ${err}`)
+    }
   }
 
 /**

client/src/lib/mapRange.ts

@@ -0,0 +1,4 @@
+// Function to map a value from one range to another
+export const mapRange = (value: number, inMin: number, inMax: number, outMin: number, outMax: number): number => {
+  return Math.min(outMax, Math.max(outMin, ((value - inMin) * (outMax - outMin)) / (inMax - inMin) + outMin));
+};

client/src/types.ts

@@ -0,0 +1,92 @@
+import * as vision from '@mediapipe/tasks-vision';
+
+/**
+ * Represents the parameters for image modification.
+ */
+export interface ImageModificationParams {
+  eyes: number;
+  eyebrow: number;
+  wink: number;
+  pupil_x: number;
+  pupil_y: number;
+  aaa: number;
+  eee: number;
+  woo: number;
+  smile: number;
+  rotate_pitch: number;
+  rotate_yaw: number;
+  rotate_roll: number;
+}
+
+export interface Metadata {
+  center: number[] //center - 2x1
+  size: number // size - scalar
+  bbox: number[][] // bbox - 4x2
+  angle: number   //angle - rad, counterclockwise
+}
+
+/**
+ * Represents a message to modify an image.
+ */
+export interface ModifyImageMessage {
+  image?: string;
+  hash?: string;
+  params: Partial<ImageModificationParams>;
+}
+
+export type OnServerResponseParams = {
+  image?: Blob
+  error?: string
+  loaded?: {
+    h: string
+  } & {
+    c: number[] //center - 2x1
+    s: number // size - scalar
+    b: number[][] // bbox - 4x2
+    a: number // angle - rad, counterclockwise
+  }
+}
+
+/**
+ * Callback type for handling modified images.
+ */
+export type OnServerResponse = (params: OnServerResponseParams) => Promise<void>;
+
+/**
+ * Enum representing the different states of a WebSocket connection.
+ */
+export enum WebSocketState {
+  CONNECTING = 0,
+  OPEN = 1,
+  CLOSING = 2,
+  CLOSED = 3
+}
+
+export type ActionMode = 'HOVERING' | 'PRIMARY' | 'SECONDARY'
+export type LandmarkGroup = 'lips' | 'leftEye' | 'leftEyebrow' | 'rightEye' | 'rightEyebrow' | 'faceOval' | 'background';
+export type LandmarkCenter = { x: number; y: number; z: number };
+export type ClosestLandmark = { group: LandmarkGroup; distance: number; vector: { x: number; y: number; z: number } };
+
+export type MediaPipeResources = {
+  faceLandmarker: vision.FaceLandmarker | null;
+  drawingUtils: vision.DrawingUtils | null;
+};
+
+export interface ImageStateValues {
+  status: string
+  error: string
+  imageFile: File | null
+  isFollowingCursor: boolean
+  isGazingAtCursor: boolean
+  originalImage: string
+  previewImage: string
+  originalImageHash: string
+  minLatency: number
+  averageLatency: number
+  maxLatency: number
+  activeLandmark?: ClosestLandmark
+  metadata: Metadata
+  params: Partial<ImageModificationParams>
+  faceLandmarks: vision.NormalizedLandmark[][]
+  blendShapes: vision.Classifications[]
+}

engine.py

@@ -3,6 +3,7 @@ import hashlib
 import os
 import io
 import asyncio
+from async_lru import alru_cache
 import base64
 from queue import Queue
 from typing import Dict, Any, List, Optional, Union
@@ -15,7 +16,7 @@ from PIL import Image
 from liveportrait.config.argument_config import ArgumentConfig
 from liveportrait.utils.camera import get_rotation_matrix
 from liveportrait.utils.io import resize_to_limit
-from liveportrait.utils.crop import prepare_paste_back, paste_back
+from liveportrait.utils.crop import prepare_paste_back, paste_back, parse_bbox_from_landmark
 
 # Configure logging
 logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
@@ -56,12 +57,11 @@ class Engine:
 
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-        self.image_cache = {}  # Stores the original images
         self.processed_cache = {}  # Stores the processed image data
 
         logger.info("✅ FacePoke Engine initialized successfully.")
 
-    def get_image_hash(self, image: Union[Image.Image, str, bytes]) -> str:
+    async def get_image_hash(self, image: Union[Image.Image, str, bytes]) -> str:
         """
         Compute or retrieve the hash for an image.
 
@@ -86,35 +86,23 @@ class Engine:
         else:
             raise ValueError("Unsupported image type")
 
-    @lru_cache(maxsize=256)
-    def _process_image(self, image_hash: str) -> Dict[str, Any]:
-        """
-        Process the input image and cache the results.
-
-        Args:
-            image_hash (str): Hash of the input image.
-
-        Returns:
-            Dict[str, Any]: Processed image data.
-        """
-        # let's hide the logs as there are thousands of message slike this
-        #logger.info(f"Processing image with hash: {image_hash}")
+    @alru_cache(maxsize=512)
+    async def load_image(self, data):
+        image = Image.open(io.BytesIO(data))
 
-        if image_hash not in self.image_cache:
-            raise ValueError(f"Image with hash {image_hash} not found in cache")
+        image_hash = await self.get_image_hash(image)
 
-        image = self.image_cache[image_hash]
         img_rgb = np.array(image)
 
         inference_cfg = self.live_portrait.live_portrait_wrapper.cfg
-        img_rgb = resize_to_limit(img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
-        crop_info = self.live_portrait.cropper.crop_single_image(img_rgb)
+        img_rgb = await asyncio.to_thread(resize_to_limit, img_rgb, inference_cfg.ref_max_shape, inference_cfg.ref_shape_n)
+        crop_info = await asyncio.to_thread(self.live_portrait.cropper.crop_single_image, img_rgb)
         img_crop_256x256 = crop_info['img_crop_256x256']
 
-        I_s = self.live_portrait.live_portrait_wrapper.prepare_source(img_crop_256x256)
-        x_s_info = self.live_portrait.live_portrait_wrapper.get_kp_info(I_s)
-        f_s = self.live_portrait.live_portrait_wrapper.extract_feature_3d(I_s)
-        x_s = self.live_portrait.live_portrait_wrapper.transform_keypoint(x_s_info)
+        I_s = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.prepare_source, img_crop_256x256)
+        x_s_info = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.get_kp_info, I_s)
+        f_s = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.extract_feature_3d, I_s)
+        x_s = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.transform_keypoint, x_s_info)
 
         processed_data = {
             'img_rgb': img_rgb,
@@ -127,50 +115,78 @@ class Engine:
 
         self.processed_cache[image_hash] = processed_data
 
-        return processed_data
+        # Calculate the bounding box
+        bbox_info = parse_bbox_from_landmark(processed_data['crop_info']['lmk_crop'], scale=1.0)
 
-    async def modify_image(self, image_or_hash: Union[Image.Image, str, bytes], params: Dict[str, float]) -> str:
-        """
-        Modify the input image based on the provided parameters, using caching for efficiency
-        and outputting the result as a WebP image.
+        return {
+            'h': image_hash,
 
-        Args:
-            image_or_hash (Union[Image.Image, str, bytes]): Input image as a PIL Image, base64-encoded string,
-                image bytes, or a hash string.
-            params (Dict[str, float]): Parameters for face transformation.
+            # those aren't easy to serialize
+            'c': bbox_info['center'], # 2x1
+            's': bbox_info['size'], # scalar
+            'b': bbox_info['bbox'],  # 4x2
+            'a': bbox_info['angle'],  # rad, counterclockwise
+            # 'bbox_rot': bbox_info['bbox_rot'].toList(),  # 4x2
+        }
 
-        Returns:
-            str: Modified image as a base64-encoded WebP data URI.
+    def transform_image(self, image_hash: str, params: Dict[str, float]) -> bytes:
+        # If we don't have the image in cache yet, add it
+        if image_hash not in self.processed_cache:
+            raise ValueError("cache miss")
 
-        Raises:
-            ValueError: If there's an error modifying the image or WebP is not supported.
-        """
-        # let's disable those logs completely as there are thousands of message slike this
-        #logger.info("Starting image modification")
-        #logger.debug(f"Modification parameters: {params}")
+        processed_data = self.processed_cache[image_hash]
 
         try:
-            image_hash = self.get_image_hash(image_or_hash)
-
-            # If we don't have the image in cache yet, add it
-            if image_hash not in self.image_cache:
-                if isinstance(image_or_hash, (Image.Image, bytes)):
-                    self.image_cache[image_hash] = image_or_hash
-                elif isinstance(image_or_hash, str) and len(image_or_hash) != 32:
-                    # It's a base64 string, not a hash
-                    self.image_cache[image_hash] = base64_data_uri_to_PIL_Image(image_or_hash)
-                else:
-                    raise ValueError("Image not found in cache and no valid image provided")
-
-            # Process the image (this will use the cache if available)
-            if image_hash not in self.processed_cache:
-                processed_data = await asyncio.to_thread(self._process_image, image_hash)
-            else:
-                processed_data = self.processed_cache[image_hash]
-
             # Apply modifications based on params
             x_d_new = processed_data['x_s_info']['kp'].clone()
-            await self._apply_facial_modifications(x_d_new, params)
+
+            modifications = [
+                ('smile', [
+                    (0, 20, 1, -0.01), (0, 14, 1, -0.02), (0, 17, 1, 0.0065), (0, 17, 2, 0.003),
+                    (0, 13, 1, -0.00275), (0, 16, 1, -0.00275), (0, 3, 1, -0.0035), (0, 7, 1, -0.0035)
+                ]),
+                ('aaa', [
+                    (0, 19, 1, 0.001), (0, 19, 2, 0.0001), (0, 17, 1, -0.0001)
+                ]),
+                ('eee', [
+                    (0, 20, 2, -0.001), (0, 20, 1, -0.001), (0, 14, 1, -0.001)
+                ]),
+                ('woo', [
+                    (0, 14, 1, 0.001), (0, 3, 1, -0.0005), (0, 7, 1, -0.0005), (0, 17, 2, -0.0005)
+                ]),
+                ('wink', [
+                    (0, 11, 1, 0.001), (0, 13, 1, -0.0003), (0, 17, 0, 0.0003),
+                    (0, 17, 1, 0.0003), (0, 3, 1, -0.0003)
+                ]),
+                ('pupil_x', [
+                    (0, 11, 0, 0.0007 if params.get('pupil_x', 0) > 0 else 0.001),
+                    (0, 15, 0, 0.001 if params.get('pupil_x', 0) > 0 else 0.0007)
+                ]),
+                ('pupil_y', [
+                    (0, 11, 1, -0.001), (0, 15, 1, -0.001)
+                ]),
+                ('eyes', [
+                    (0, 11, 1, -0.001), (0, 13, 1, 0.0003), (0, 15, 1, -0.001), (0, 16, 1, 0.0003),
+                    (0, 1, 1, -0.00025), (0, 2, 1, 0.00025)
+                ]),
+                ('eyebrow', [
+                    (0, 1, 1, 0.001 if params.get('eyebrow', 0) > 0 else 0.0003),
+                    (0, 2, 1, -0.001 if params.get('eyebrow', 0) > 0 else -0.0003),
+                    (0, 1, 0, -0.001 if params.get('eyebrow', 0) <= 0 else 0),
+                    (0, 2, 0, 0.001 if params.get('eyebrow', 0) <= 0 else 0)
+                ])
+            ]
+
+            for param_name, adjustments in modifications:
+                param_value = params.get(param_name, 0)
+                for i, j, k, factor in adjustments:
+                    x_d_new[i, j, k] += param_value * factor
+
+            # Special case for pupil_y affecting eyes
+            x_d_new[0, 11, 1] -= params.get('pupil_y', 0) * 0.001
+            x_d_new[0, 15, 1] -= params.get('pupil_y', 0) * 0.001
+            params['eyes'] = params.get('eyes', 0) - params.get('pupil_y', 0) / 2.
+
 
             # Apply rotation
             R_new = get_rotation_matrix(
@@ -181,90 +197,40 @@ class Engine:
             x_d_new = processed_data['x_s_info']['scale'] * (x_d_new @ R_new) + processed_data['x_s_info']['t']
 
             # Apply stitching
-            x_d_new = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.stitching, processed_data['x_s'], x_d_new)
+            x_d_new = self.live_portrait.live_portrait_wrapper.stitching(processed_data['x_s'], x_d_new)
 
             # Generate the output
-            out = await asyncio.to_thread(self.live_portrait.live_portrait_wrapper.warp_decode, processed_data['f_s'], processed_data['x_s'], x_d_new)
-            I_p = self.live_portrait.live_portrait_wrapper.parse_output(out['out'])[0]
+            out = self.live_portrait.live_portrait_wrapper.warp_decode(processed_data['f_s'], processed_data['x_s'], x_d_new)
+            I_p = self.live_portrait.live_portrait_wrapper.parse_output(out['out'])
+
+            buffered = io.BytesIO()
 
-            # Paste back to full size
-            mask_ori = await asyncio.to_thread(
-                prepare_paste_back,
+            ####################################################
+            # this part is about stitching the image back into the original.
+            #
+            # this is an expensive operation, not just because of the compute
+            # but because the payload will also be bigger (we send back the whole pic)
+            #
+            # I'm currently running some experiments to do it in the frontend
+            #
+            #  --- old way: we do it in the server-side: ---
+            mask_ori = prepare_paste_back(
                 processed_data['inference_cfg'].mask_crop, processed_data['crop_info']['M_c2o'],
                 dsize=(processed_data['img_rgb'].shape[1], processed_data['img_rgb'].shape[0])
             )
-            I_p_to_ori_blend = await asyncio.to_thread(
-                paste_back,
-                I_p, processed_data['crop_info']['M_c2o'], processed_data['img_rgb'], mask_ori
+            I_p_to_ori_blend = paste_back(
+                I_p[0], processed_data['crop_info']['M_c2o'], processed_data['img_rgb'], mask_ori
             )
-
-            # Convert the result to a PIL Image
             result_image = Image.fromarray(I_p_to_ori_blend)
 
-            # Save as WebP
-            buffered = io.BytesIO()
-            result_image.save(buffered, format="WebP", quality=85)  # Adjust quality as needed
-            modified_image_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')
+            # --- maybe future way: do it in the frontend: ---
+            #result_image = Image.fromarray(I_p[0])
+            ####################################################
 
-            #logger.info("Image modification completed successfully")
-            return f"data:image/webp;base64,{modified_image_base64}"
+            # write it into a webp
+            result_image.save(buffered, format="WebP", quality=82, lossless=False, method=6)
+
+            return buffered.getvalue()
 
         except Exception as e:
-            #logger.error(f"Error in modify_image: {str(e)}")
-            #logger.exception("Full traceback:")
             raise ValueError(f"Failed to modify image: {str(e)}")
-
-    async def _apply_facial_modifications(self, x_d_new: torch.Tensor, params: Dict[str, float]) -> None:
-        """
-        Apply facial modifications to the keypoints based on the provided parameters.
-
-        Args:
-            x_d_new (torch.Tensor): Tensor of facial keypoints to be modified.
-            params (Dict[str, float]): Parameters for face transformation.
-        """
-        modifications = [
-            ('smile', [
-                (0, 20, 1, -0.01), (0, 14, 1, -0.02), (0, 17, 1, 0.0065), (0, 17, 2, 0.003),
-                (0, 13, 1, -0.00275), (0, 16, 1, -0.00275), (0, 3, 1, -0.0035), (0, 7, 1, -0.0035)
-            ]),
-            ('aaa', [
-                (0, 19, 1, 0.001), (0, 19, 2, 0.0001), (0, 17, 1, -0.0001)
-            ]),
-            ('eee', [
-                (0, 20, 2, -0.001), (0, 20, 1, -0.001), (0, 14, 1, -0.001)
-            ]),
-            ('woo', [
-                (0, 14, 1, 0.001), (0, 3, 1, -0.0005), (0, 7, 1, -0.0005), (0, 17, 2, -0.0005)
-            ]),
-            ('wink', [
-                (0, 11, 1, 0.001), (0, 13, 1, -0.0003), (0, 17, 0, 0.0003),
-                (0, 17, 1, 0.0003), (0, 3, 1, -0.0003)
-            ]),
-            ('pupil_x', [
-                (0, 11, 0, 0.0007 if params.get('pupil_x', 0) > 0 else 0.001),
-                (0, 15, 0, 0.001 if params.get('pupil_x', 0) > 0 else 0.0007)
-            ]),
-            ('pupil_y', [
-                (0, 11, 1, -0.001), (0, 15, 1, -0.001)
-            ]),
-            ('eyes', [
-                (0, 11, 1, -0.001), (0, 13, 1, 0.0003), (0, 15, 1, -0.001), (0, 16, 1, 0.0003),
-                (0, 1, 1, -0.00025), (0, 2, 1, 0.00025)
-            ]),
-            ('eyebrow', [
-                (0, 1, 1, 0.001 if params.get('eyebrow', 0) > 0 else 0.0003),
-                (0, 2, 1, -0.001 if params.get('eyebrow', 0) > 0 else -0.0003),
-                (0, 1, 0, -0.001 if params.get('eyebrow', 0) <= 0 else 0),
-                (0, 2, 0, 0.001 if params.get('eyebrow', 0) <= 0 else 0)
-            ])
-        ]
-
-        for param_name, adjustments in modifications:
-            param_value = params.get(param_name, 0)
-            for i, j, k, factor in adjustments:
-                x_d_new[i, j, k] += param_value * factor
-
-        # Special case for pupil_y affecting eyes
-        x_d_new[0, 11, 1] -= params.get('pupil_y', 0) * 0.001
-        x_d_new[0, 15, 1] -= params.get('pupil_y', 0) * 0.001
-        params['eyes'] = params.get('eyes', 0) - params.get('pupil_y', 0) / 2.

liveportrait/utils/crop.py

@@ -409,4 +409,4 @@ def paste_back(image_to_processed, crop_M_c2o, rgb_ori, mask_ori):
     dsize = (rgb_ori.shape[1], rgb_ori.shape[0])
     result = _transform_img(image_to_processed, crop_M_c2o, dsize=dsize)
     result = np.clip(mask_ori * result + (1 - mask_ori) * rgb_ori, 0, 255).astype(np.uint8)
-    return result
+    return result

loader.py

@@ -5,8 +5,6 @@ import asyncio
 import aiohttp
 import requests
 from huggingface_hub import hf_hub_download
-import sentencepiece
-
 
 # Configure logging
 logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -24,6 +22,7 @@ HF_REPO_ID = "jbilcke-hf/model-cocktail"
 MODEL_FILES = [
     "dwpose/dw-ll_ucoco_384.pth",
     "face-detector/s3fd-619a316812.pth",
+
     "liveportrait/spade_generator.pth",
     "liveportrait/warping_module.pth",
     "liveportrait/motion_extractor.pth",
@@ -31,6 +30,16 @@ MODEL_FILES = [
     "liveportrait/appearance_feature_extractor.pth",
     "liveportrait/landmark.onnx",
 
+    # For animal mode 🐶🐱
+    # however they say animal mode doesn't support stitching yet?
+    # https://github.com/KwaiVGI/LivePortrait/blob/main/assets/docs/changelog/2024-08-02.md#updates-on-animals-mode
+    #"liveportrait-animals/warping_module.pth",
+    #"liveportrait-animals/spade_generator.pth",
+    #"liveportrait-animals/motion_extractor.pth",
+    #"liveportrait-animals/appearance_feature_extractor.pth",
+    #"liveportrait-animals/stitching_retargeting_module.pth",
+    #"liveportrait-animals/xpose.pth",
+
     # this is a hack, instead we should probably try to
     # fix liveportrait/utils/dependencies/insightface/utils/storage.py
     "insightface/models/buffalo_l.zip",

public/index.js

@@ -29660,148 +29660,7 @@ Yh.prototype.detectForVideo = Yh.prototype.F, Yh.prototype.detect = Yh.prototype
   return Ka(Yh, t2, e2);
 }, Yh.POSE_CONNECTIONS = Eh;
 
-// node_modules/uuid/dist/esm-browser/stringify.js
-function unsafeStringify(arr, offset = 0) {
-  return (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + "-" + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + "-" + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + "-" + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + "-" + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase();
-}
-var byteToHex = [];
-for (i2 = 0;i2 < 256; ++i2) {
-  byteToHex.push((i2 + 256).toString(16).slice(1));
-}
-var i2;
-
-// node_modules/uuid/dist/esm-browser/rng.js
-var getRandomValues;
-var rnds8 = new Uint8Array(16);
-function rng() {
-  if (!getRandomValues) {
-    getRandomValues = typeof crypto !== "undefined" && crypto.getRandomValues && crypto.getRandomValues.bind(crypto);
-    if (!getRandomValues) {
-      throw new Error("crypto.getRandomValues() not supported. See https://github.com/uuidjs/uuid#getrandomvalues-not-supported");
-    }
-  }
-  return getRandomValues(rnds8);
-}
-
-// node_modules/uuid/dist/esm-browser/native.js
-var randomUUID = typeof crypto !== "undefined" && crypto.randomUUID && crypto.randomUUID.bind(crypto);
-var native_default = {
-  randomUUID
-};
-
-// node_modules/uuid/dist/esm-browser/v4.js
-var v4 = function(options, buf, offset) {
-  if (native_default.randomUUID && !buf && !options) {
-    return native_default.randomUUID();
-  }
-  options = options || {};
-  var rnds = options.random || (options.rng || rng)();
-  rnds[6] = rnds[6] & 15 | 64;
-  rnds[8] = rnds[8] & 63 | 128;
-  if (buf) {
-    offset = offset || 0;
-    for (var i2 = 0;i2 < 16; ++i2) {
-      buf[offset + i2] = rnds[i2];
-    }
-    return buf;
-  }
-  return unsafeStringify(rnds);
-};
-var v4_default = v4;
-// src/lib/circularBuffer.ts
-class CircularBuffer {
-  capacity;
-  buffer;
-  pointer;
-  constructor(capacity) {
-    this.capacity = capacity;
-    this.buffer = new Array(capacity);
-    this.pointer = 0;
-  }
-  push(item) {
-    this.buffer[this.pointer] = item;
-    this.pointer = (this.pointer + 1) % this.capacity;
-  }
-  getAll() {
-    return this.buffer.filter((item) => item !== undefined);
-  }
-}
-
-// node_modules/zustand/esm/vanilla.mjs
-var createStoreImpl = (createState) => {
-  let state;
-  const listeners = new Set;
-  const setState = (partial, replace) => {
-    const nextState = typeof partial === "function" ? partial(state) : partial;
-    if (!Object.is(nextState, state)) {
-      const previousState = state;
-      state = (replace != null ? replace : typeof nextState !== "object" || nextState === null) ? nextState : Object.assign({}, state, nextState);
-      listeners.forEach((listener) => listener(state, previousState));
-    }
-  };
-  const getState = () => state;
-  const getInitialState = () => initialState;
-  const subscribe = (listener) => {
-    listeners.add(listener);
-    return () => listeners.delete(listener);
-  };
-  const api = { setState, getState, getInitialState, subscribe };
-  const initialState = state = createState(setState, getState, api);
-  return api;
-};
-var createStore = (createState) => createState ? createStoreImpl(createState) : createStoreImpl;
-
-// node_modules/zustand/esm/react.mjs
-var import_react3 = __toESM(require_react(), 1);
-var useStore = function(api, selector = identity) {
-  const slice = import_react3.default.useSyncExternalStore(api.subscribe, () => selector(api.getState()), () => selector(api.getInitialState()));
-  import_react3.default.useDebugValue(slice);
-  return slice;
-};
-var identity = (arg) => arg;
-var createImpl = (createState) => {
-  const api = createStore(createState);
-  const useBoundStore = (selector) => useStore(api, selector);
-  Object.assign(useBoundStore, api);
-  return useBoundStore;
-};
-var create = (createState) => createState ? createImpl(createState) : createImpl;
-
-// src/hooks/useMainStore.ts
-var useMainStore = create((set, get) => ({
-  error: "",
-  imageFile: null,
-  originalImage: "",
-  originalImageHash: "",
-  previewImage: "",
-  minLatency: 20,
-  averageLatency: 190,
-  maxLatency: 4000,
-  activeLandmark: undefined,
-  params: {},
-  setError: (error = "") => set({ error }),
-  setImageFile: (file) => set({ imageFile: file }),
-  setOriginalImage: (url) => set({ originalImage: url }),
-  setOriginalImageHash: (originalImageHash) => set({ originalImageHash }),
-  setPreviewImage: (url) => set({ previewImage: url }),
-  resetImage: () => {
-    const { originalImage } = get();
-    if (originalImage) {
-      set({ previewImage: originalImage });
-    }
-  },
-  setAverageLatency: (averageLatency) => set({ averageLatency }),
-  setActiveLandmark: (activeLandmark) => set({ activeLandmark }),
-  setParams: (params) => {
-    const { params: previousParams } = get();
-    set({ params: {
-      ...previousParams,
-      ...params
-    } });
-  }
-}));
-
-// src/lib/facePoke.ts
+// src/types.ts
 var WebSocketState;
 (function(WebSocketState2) {
   WebSocketState2[WebSocketState2["CONNECTING"] = 0] = "CONNECTING";
@@ -29810,63 +29669,24 @@ var WebSocketState;
   WebSocketState2[WebSocketState2["CLOSED"] = 3] = "CLOSED";
 })(WebSocketState || (WebSocketState = {}));
 
+// src/lib/facePoke.ts
 class FacePoke {
   ws = null;
-  connectionId = v4_default();
   isUnloading = false;
-  onModifiedImage = () => {
+  onServerResponse = async () => {
   };
   reconnectAttempts = 0;
   maxReconnectAttempts = 5;
   reconnectDelay = 5000;
   eventListeners = new Map;
-  requestTracker = new Map;
-  responseTimeBuffer;
-  MAX_TRACKED_TIMES = 5;
   constructor() {
-    console.log(`[FacePoke] Initializing FacePoke instance with connection ID: ${this.connectionId}`);
+    console.log(`[FacePoke] Initializing FacePoke instance`);
     this.initializeWebSocket();
     this.setupUnloadHandler();
-    this.responseTimeBuffer = new CircularBuffer(this.MAX_TRACKED_TIMES);
-    console.log(`[FacePoke] Initialized response time tracker with capacity: ${this.MAX_TRACKED_TIMES}`);
-  }
-  trackRequest() {
-    const uuid = v4_default();
-    this.requestTracker.set(uuid, { uuid, timestamp: Date.now() });
-    return uuid;
-  }
-  completeRequest(uuid) {
-    const request = this.requestTracker.get(uuid);
-    if (request) {
-      const responseTime = Date.now() - request.timestamp;
-      this.responseTimeBuffer.push(responseTime);
-      this.requestTracker.delete(uuid);
-      this.updateThrottleTime();
-      console.log(`[FacePoke] Completed request ${uuid}. Response time: ${responseTime}ms`);
-    } else {
-      console.warn(`[FacePoke] Attempted to complete unknown request: ${uuid}`);
-    }
-  }
-  calculateAverageResponseTime() {
-    const times = this.responseTimeBuffer.getAll();
-    const averageLatency = useMainStore.getState().averageLatency;
-    if (times.length === 0)
-      return averageLatency;
-    const sum = times.reduce((acc, time) => acc + time, 0);
-    return sum / times.length;
-  }
-  updateThrottleTime() {
-    const { minLatency, maxLatency, averageLatency, setAverageLatency } = useMainStore.getState();
-    const avgResponseTime = this.calculateAverageResponseTime();
-    const newLatency = Math.min(minLatency, Math.max(minLatency, avgResponseTime));
-    if (newLatency !== averageLatency) {
-      setAverageLatency(newLatency);
-      console.log(`[FacePoke] Updated throttle time (latency is ${newLatency}ms)`);
-    }
   }
-  setOnModifiedImage(handler) {
-    this.onModifiedImage = handler;
-    console.log(`[FacePoke] onModifiedImage handler set`);
+  setOnServerResponse(handler) {
+    this.onServerResponse = handler;
+    console.log(`[FacePoke] onServerResponse handler set`);
   }
   async startWebSocket() {
     console.log(`[FacePoke] Starting WebSocket connection.`);
@@ -29875,56 +29695,47 @@ class FacePoke {
     }
   }
   async initializeWebSocket() {
-    console.log(`[FacePoke][${this.connectionId}] Initializing WebSocket connection`);
+    console.log(`[FacePoke] Initializing WebSocket connection`);
     const connect = () => {
       this.ws = new WebSocket(`wss://${window.location.host}/ws`);
       this.ws.onopen = this.handleWebSocketOpen.bind(this);
-      this.ws.onmessage = this.handleWebSocketMessage.bind(this);
       this.ws.onclose = this.handleWebSocketClose.bind(this);
       this.ws.onerror = this.handleWebSocketError.bind(this);
+      this.ws.onmessage = this.handleWebSocketMessage.bind(this);
     };
     connect();
   }
+  handleWebSocketMessage(msg) {
+    if (typeof msg.data === "string") {
+      this.onServerResponse({ loaded: JSON.parse(msg.data) });
+    } else if (typeof msg.data !== "undefined") {
+      this.onServerResponse({ image: msg.data });
+    }
+  }
   handleWebSocketOpen() {
-    console.log(`[FacePoke][${this.connectionId}] WebSocket connection opened`);
+    console.log(`[FacePoke] WebSocket connection opened`);
     this.reconnectAttempts = 0;
     this.emitEvent("websocketOpen");
   }
-  handleWebSocketMessage(event) {
-    try {
-      const data = JSON.parse(event.data);
-      if (data.uuid) {
-        this.completeRequest(data.uuid);
-      }
-      if (data.type === "modified_image") {
-        if (data?.image) {
-          this.onModifiedImage(data.image, data.image_hash);
-        }
-      }
-      this.emitEvent("message", data);
-    } catch (error) {
-      console.error(`[FacePoke][${this.connectionId}] Error parsing WebSocket message:`, error);
-    }
-  }
   handleWebSocketClose(event) {
     if (event.wasClean) {
-      console.log(`[FacePoke][${this.connectionId}] WebSocket connection closed cleanly, code=${event.code}, reason=${event.reason}`);
+      console.log(`[FacePoke] WebSocket connection closed cleanly, code=${event.code}, reason=${event.reason}`);
     } else {
-      console.warn(`[FacePoke][${this.connectionId}] WebSocket connection abruptly closed`);
+      console.warn(`[FacePoke] WebSocket connection abruptly closed`);
     }
     this.emitEvent("websocketClose", event);
     if (!this.isUnloading && this.reconnectAttempts < this.maxReconnectAttempts) {
       this.reconnectAttempts++;
       const delay = Math.min(1000 * 2 ** this.reconnectAttempts, 30000);
-      console.log(`[FacePoke][${this.connectionId}] Attempting to reconnect in ${delay}ms (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
+      console.log(`[FacePoke] Attempting to reconnect in ${delay}ms (Attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
       setTimeout(() => this.initializeWebSocket(), delay);
     } else if (this.reconnectAttempts >= this.maxReconnectAttempts) {
-      console.error(`[FacePoke][${this.connectionId}] Max reconnect attempts reached. Please refresh the page.`);
+      console.error(`[FacePoke] Max reconnect attempts reached. Please refresh the page.`);
       this.emitEvent("maxReconnectAttemptsReached");
     }
   }
   handleWebSocketError(error) {
-    console.error(`[FacePoke][${this.connectionId}] WebSocket error:`, error);
+    console.error(`[FacePoke] WebSocket error:`, error);
     this.emitEvent("websocketError", error);
   }
   cleanup() {
@@ -29937,22 +29748,26 @@ class FacePoke {
     console.log("[FacePoke] Cleanup completed");
     this.emitEvent("cleanup");
   }
-  modifyImage(image, imageHash, params) {
+  async loadImage(image) {
+    const base64Data = image.split(",")[1] || image;
+    const buffer = new Uint8Array(atob(base64Data).split("").map((char) => char.charCodeAt(0)));
+    const blob = new Blob([buffer], { type: "application/octet-binary" });
+    this.sendBlobMessage(await blob.arrayBuffer());
+  }
+  transformImage(hash, params) {
+    this.sendJsonMessage({ hash, params });
+  }
+  sendBlobMessage(buffer) {
+    if (!this.ws || this.ws.readyState !== WebSocketState.OPEN) {
+      const error = new Error("WebSocket connection is not open");
+      console.error("[FacePoke] Error sending JSON message:", error);
+      this.emitEvent("sendJsonMessageError", error);
+      throw error;
+    }
     try {
-      const message = {
-        type: "modify_image",
-        params
-      };
-      if (image) {
-        message.image = image;
-      } else if (imageHash) {
-        message.image_hash = imageHash;
-      } else {
-        throw new Error("Either image or imageHash must be provided");
-      }
-      this.sendJsonMessage(message);
+      this.ws.send(buffer);
     } catch (err) {
-      console.error(`[FacePoke] Failed to modify the image:`, err);
+      console.error(`failed to send the WebSocket message: ${err}`);
     }
   }
   sendJsonMessage(message) {
@@ -29962,9 +29777,11 @@ class FacePoke {
       this.emitEvent("sendJsonMessageError", error);
       throw error;
     }
-    const uuid = this.trackRequest();
-    const messageWithUuid = { ...message, uuid };
-    this.ws.send(JSON.stringify(messageWithUuid));
+    try {
+      this.ws.send(JSON.stringify(message));
+    } catch (err) {
+      console.error(`failed to send the WebSocket message: ${err}`);
+    }
   }
   setupUnloadHandler() {
     window.addEventListener("beforeunload", () => {
@@ -30000,6 +29817,337 @@ class FacePoke {
 }
 var facePoke = new FacePoke;
 
+// node_modules/zustand/esm/vanilla.mjs
+var createStoreImpl = (createState) => {
+  let state;
+  const listeners = new Set;
+  const setState = (partial, replace) => {
+    const nextState = typeof partial === "function" ? partial(state) : partial;
+    if (!Object.is(nextState, state)) {
+      const previousState = state;
+      state = (replace != null ? replace : typeof nextState !== "object" || nextState === null) ? nextState : Object.assign({}, state, nextState);
+      listeners.forEach((listener) => listener(state, previousState));
+    }
+  };
+  const getState = () => state;
+  const getInitialState = () => initialState;
+  const subscribe = (listener) => {
+    listeners.add(listener);
+    return () => listeners.delete(listener);
+  };
+  const api = { setState, getState, getInitialState, subscribe };
+  const initialState = state = createState(setState, getState, api);
+  return api;
+};
+var createStore = (createState) => createState ? createStoreImpl(createState) : createStoreImpl;
+
+// node_modules/zustand/esm/react.mjs
+var import_react3 = __toESM(require_react(), 1);
+var useStore = function(api, selector = identity) {
+  const slice = import_react3.default.useSyncExternalStore(api.subscribe, () => selector(api.getState()), () => selector(api.getInitialState()));
+  import_react3.default.useDebugValue(slice);
+  return slice;
+};
+var identity = (arg) => arg;
+var createImpl = (createState) => {
+  const api = createStore(createState);
+  const useBoundStore = (selector) => useStore(api, selector);
+  Object.assign(useBoundStore, api);
+  return useBoundStore;
+};
+var create = (createState) => createState ? createImpl(createState) : createImpl;
+
+// src/lib/convertImageToBase64.ts
+async function convertImageToBase64(imageFileOrBlob) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader;
+    reader.onload = () => {
+      if (typeof reader.result === "string") {
+        resolve(reader.result);
+      } else {
+        reject(new Error("Failed to convert image to base64"));
+      }
+    };
+    reader.onerror = () => {
+      reject(new Error("Error reading file or blob"));
+    };
+    reader.readAsDataURL(imageFileOrBlob);
+  });
+}
+
+// src/lib/mapRange.ts
+var mapRange = (value, inMin, inMax, outMin, outMax) => {
+  return Math.min(outMax, Math.max(outMin, (value - inMin) * (outMax - outMin) / (inMax - inMin) + outMin));
+};
+
+// src/hooks/useMainStore.ts
+var getDefaultState = () => ({
+  status: "",
+  error: "",
+  imageFile: null,
+  isFollowingCursor: false,
+  isGazingAtCursor: false,
+  originalImage: "",
+  originalImageHash: "",
+  previewImage: "",
+  minLatency: 20,
+  averageLatency: 190,
+  maxLatency: 4000,
+  activeLandmark: undefined,
+  metadata: {
+    center: [0, 0],
+    size: 0,
+    bbox: [[0, 0], [0, 0], [0, 0], [0, 0]],
+    angle: 0
+  },
+  params: {},
+  faceLandmarks: [],
+  blendShapes: []
+});
+var useMainStore = create((set, get) => ({
+  ...getDefaultState(),
+  setStatus: (status = "") => set({ status }),
+  setError: (error = "") => set({ error }),
+  setFaceLandmarks: (faceLandmarks) => {
+    set({ faceLandmarks });
+  },
+  setBlendShapes: (blendShapes) => {
+    set({ blendShapes });
+  },
+  setImageFile: async (file) => {
+    if (!file) {
+      set({
+        ...getDefaultState(),
+        status: "No file selected"
+      });
+      return;
+    }
+    try {
+      const image = await convertImageToBase64(file);
+      set({
+        ...getDefaultState(),
+        imageFile: file,
+        status: `File selected: ${truncateFileName(file.name, 16)}`,
+        previewImage: image,
+        originalImage: image
+      });
+      facePoke.loadImage(image);
+    } catch (err) {
+      console.log(`failed to load the image: `, err);
+      set({
+        ...getDefaultState(),
+        status: "Failed to load the image"
+      });
+    }
+  },
+  setIsFollowingCursor: (isFollowingCursor) => set({ isFollowingCursor }),
+  setIsGazingAtCursor: (isGazingAtCursor) => set({ isGazingAtCursor }),
+  setOriginalImage: (url) => set({ originalImage: url }),
+  setOriginalImageHash: (originalImageHash) => set({ originalImageHash }),
+  setPreviewImage: (url) => set({ previewImage: url }),
+  resetImage: () => {
+    const { originalImage } = get();
+    if (originalImage) {
+      set({ previewImage: originalImage });
+    }
+  },
+  setAverageLatency: (averageLatency) => set({ averageLatency }),
+  setActiveLandmark: (activeLandmark) => set({ activeLandmark }),
+  setMetadata: (metadata) => set(metadata ? {
+    metadata
+  } : {
+    metadata: getDefaultState().metadata
+  }),
+  setParams: (params) => {
+    const { params: previousParams } = get();
+    set({ params: {
+      ...previousParams,
+      ...params
+    } });
+  },
+  handleServerResponse: async (params) => {
+    const { originalImage, setMetadata, setPreviewImage, setOriginalImageHash, applyModifiedHeadToCanvas, modifyImage } = useMainStore.getState();
+    if (typeof params.error === "string") {
+      console.error(`handleServerResponse: failed to perform the request, resetting the app (${params.error})`);
+      setPreviewImage(originalImage);
+      setOriginalImageHash("");
+    } else if (typeof params.image !== "undefined") {
+      const image = await convertImageToBase64(params.image);
+      setPreviewImage(image);
+    } else if (typeof params.loaded !== "undefined") {
+      setOriginalImageHash(params.loaded.h);
+      setMetadata({
+        center: params.loaded.c,
+        size: params.loaded.s,
+        bbox: params.loaded.b,
+        angle: params.loaded.a
+      });
+      await modifyImage({
+        landmark: {
+          group: "background",
+          distance: 0,
+          vector: { x: 0.5, y: 0.5, z: 0 }
+        },
+        vector: { x: 0, y: 0, z: 0 },
+        mode: "PRIMARY"
+      });
+    } else {
+      console.log(`handleServerResponse: received an unknown json`, params);
+    }
+  },
+  applyModifiedHeadToCanvas: async (headImageBlob) => {
+    return new Promise(async (resolve, reject) => {
+      const originalImg = new Image;
+      const { originalImage, metadata } = useMainStore.getState();
+      originalImg.onload = async () => {
+        const canvas = document.createElement("canvas");
+        const ctx = canvas.getContext("2d");
+        if (!ctx) {
+          reject(new Error("Failed to get 2D context"));
+          return;
+        }
+        const pixelRatio = window.devicePixelRatio || 1;
+        canvas.width = originalImg.width;
+        canvas.height = originalImg.height;
+        ctx.drawImage(originalImg, 0, 0);
+        const headImageBitmap = await createImageBitmap(headImageBlob, {
+          resizeQuality: "high"
+        });
+        const tempCanvas = document.createElement("canvas");
+        const tempCtx = tempCanvas.getContext("2d");
+        if (!tempCtx) {
+          reject(new Error("Failed to get 2D context for temporary canvas"));
+          return;
+        }
+        tempCanvas.width = headImageBitmap.width;
+        tempCanvas.height = headImageBitmap.height;
+        tempCtx.drawImage(headImageBitmap, 0, 0);
+        const gradientSize = 20;
+        const gradient = tempCtx.createRadialGradient(tempCanvas.width / 2, tempCanvas.height / 2, Math.min(tempCanvas.width, tempCanvas.height) / 2 - gradientSize, tempCanvas.width / 2, tempCanvas.height / 2, Math.min(tempCanvas.width, tempCanvas.height) / 2);
+        gradient.addColorStop(0, "rgba(0, 0, 0, 1)");
+        gradient.addColorStop(1, "rgba(0, 0, 0, 0)");
+        tempCtx.globalCompositeOperation = "destination-in";
+        tempCtx.fillStyle = gradient;
+        tempCtx.fillRect(0, 0, tempCanvas.width, tempCanvas.height);
+        console.log("metadata:", metadata);
+        ctx.save();
+        ctx.rotate(metadata.angle);
+        ctx.restore();
+        resolve(canvas.toDataURL("image/png"));
+      };
+      originalImg.src = originalImage;
+    });
+  },
+  modifyImage: async ({ landmark, vector, mode }) => {
+    const {
+      originalImage,
+      originalImageHash,
+      params: previousParams,
+      setParams,
+      setError,
+      isFollowingCursor,
+      isGazingAtCursor
+    } = get();
+    if (!originalImage) {
+      console.error("Image file or facePoke not available");
+      return;
+    }
+    const params = {
+      ...previousParams
+    };
+    const generalControl = {
+      minX: -0.3,
+      maxX: 0.3,
+      minY: -0.3,
+      maxY: 0.3
+    };
+    const pupilControl = {
+      minX: -0.5,
+      maxX: 0.5,
+      minY: -0.5,
+      maxY: 0.5
+    };
+    const eyeControl = {
+      minX: -0.5,
+      maxX: 0.5,
+      minY: -0.5,
+      maxY: 0.5
+    };
+    if (isFollowingCursor) {
+      const yawMin = -40;
+      const yawMax = 40;
+      params.rotate_yaw = mapRange(-vector.x, generalControl.minX, generalControl.maxX, yawMin, yawMax);
+      const pitchMin = -40;
+      const pitchMax = 40;
+      params.rotate_pitch = mapRange(vector.y, generalControl.minY, generalControl.maxY, pitchMin, pitchMax);
+    }
+    if (isGazingAtCursor) {
+      const pupilsXMin = -15;
+      const pupilsXMax = 15;
+      params.pupil_x = mapRange(vector.x, pupilControl.minX, pupilControl.maxX, pupilsXMin, pupilsXMax);
+      const pupilsYMin = -2;
+      const pupilsYMax = 8;
+      params.pupil_y = mapRange(-vector.y, pupilControl.minY, pupilControl.maxY, pupilsYMin, pupilsYMax);
+    }
+    if (mode !== "HOVERING") {
+      switch (landmark.group) {
+        case "leftEye":
+        case "rightEye":
+          const pupilsXMin = -15;
+          const pupilsXMax = 15;
+          params.pupil_x = mapRange(vector.x, pupilControl.minX, pupilControl.maxX, pupilsXMin, pupilsXMax);
+          const eyesMin = -20;
+          const eyesMax = 5;
+          params.eyes = mapRange(-vector.y, eyeControl.minX, eyeControl.maxX, eyesMin, eyesMax);
+          break;
+        case "leftEyebrow":
+        case "rightEyebrow":
+          const eyebrowMin = -10;
+          const eyebrowMax = 15;
+          params.eyebrow = mapRange(-vector.y, eyeControl.minY, eyeControl.maxY, eyebrowMin, eyebrowMax);
+          break;
+        case "lips":
+          const aaaMin = -30;
+          const aaaMax = 120;
+          params.aaa = mapRange(-vector.y, eyeControl.minY, eyeControl.maxY, aaaMin, aaaMax);
+          const eeeMin = -20;
+          const eeeMax = 15;
+          params.eee = mapRange(vector.x, eyeControl.minX, eyeControl.maxX, eeeMin, eeeMax);
+          break;
+        case "faceOval":
+          const rollMin = -40;
+          const rollMax = 40;
+          params.rotate_roll = mapRange(vector.x, eyeControl.minX, eyeControl.maxX, rollMin, rollMax);
+          break;
+        case "background":
+          const yawMin = -40;
+          const yawMax = 40;
+          params.rotate_yaw = mapRange(-vector.x, generalControl.minX, generalControl.maxX, yawMin, yawMax);
+          const pitchMin = -40;
+          const pitchMax = 40;
+          params.rotate_pitch = mapRange(vector.y, eyeControl.minY, eyeControl.maxY, pitchMin, pitchMax);
+          break;
+        default:
+          return;
+      }
+    }
+    for (const [key, value] of Object.entries(params)) {
+      if (isNaN(value) || !isFinite(value)) {
+        console.log(`${key} is NaN, aborting`);
+        return;
+      }
+    }
+    setParams(params);
+    try {
+      if (originalImageHash) {
+        facePoke.transformImage(originalImageHash, params);
+      }
+    } catch (error) {
+      setError("Failed to modify image");
+    }
+  }
+}));
+
 // node_modules/beautiful-react-hooks/esm/useThrottledCallback.js
 var import_react6 = __toESM(require_react(), 1);
 var import_lodash = __toESM(require_lodash(), 1);
@@ -32818,34 +32966,20 @@ var FACEMESH_TESSELATION = Object.freeze(new Set([
 
 // src/hooks/useFaceLandmarkDetection.tsx
 function useFaceLandmarkDetection() {
-  const error = useMainStore((s2) => s2.error);
   const setError = useMainStore((s2) => s2.setError);
-  const imageFile = useMainStore((s2) => s2.imageFile);
-  const setImageFile = useMainStore((s2) => s2.setImageFile);
-  const originalImage = useMainStore((s2) => s2.originalImage);
-  const originalImageHash = useMainStore((s2) => s2.originalImageHash);
-  const setOriginalImageHash = useMainStore((s2) => s2.setOriginalImageHash);
   const previewImage = useMainStore((s2) => s2.previewImage);
-  const setPreviewImage = useMainStore((s2) => s2.setPreviewImage);
-  const resetImage = useMainStore((s2) => s2.resetImage);
-  window.debugJuju = useMainStore;
-  const averageLatency = 220;
-  const [faceLandmarks, setFaceLandmarks] = import_react7.useState([]);
+  const handleServerResponse = useMainStore((s2) => s2.handleServerResponse);
+  const faceLandmarks = useMainStore((s2) => s2.faceLandmarks);
+  const throttleInMs = 180;
   const [isMediaPipeReady, setIsMediaPipeReady] = import_react7.useState(false);
   const [isDrawingUtilsReady, setIsDrawingUtilsReady] = import_react7.useState(false);
-  const [blendShapes, setBlendShapes] = import_react7.useState([]);
   const [dragStart, setDragStart] = import_react7.useState(null);
-  const [dragEnd, setDragEnd] = import_react7.useState(null);
   const [isDragging, setIsDragging] = import_react7.useState(false);
-  const [isWaitingForResponse, setIsWaitingForResponse] = import_react7.useState(false);
   const dragStartRef = import_react7.useRef(null);
-  const currentMousePosRef = import_react7.useRef(null);
-  const lastModifiedImageHashRef = import_react7.useRef(null);
   const [currentLandmark, setCurrentLandmark] = import_react7.useState(null);
   const [previousLandmark, setPreviousLandmark] = import_react7.useState(null);
   const [currentOpacity, setCurrentOpacity] = import_react7.useState(0);
   const [previousOpacity, setPreviousOpacity] = import_react7.useState(0);
-  const [isHovering, setIsHovering] = import_react7.useState(false);
   const canvasRef = import_react7.useRef(null);
   const mediaPipeRef = import_react7.useRef({
     faceLandmarker: null,
@@ -32882,8 +33016,8 @@ function useFaceLandmarkDetection() {
         } else {
           faceLandmarker.close();
         }
-      } catch (error2) {
-        console.error("Error during MediaPipe initialization:", error2);
+      } catch (error) {
+        console.error("Error during MediaPipe initialization:", error);
         setError("Failed to initialize face detection. Please try refreshing the page.");
       }
     };
@@ -32969,6 +33103,7 @@ function useFaceLandmarkDetection() {
     }
   }, [landmarkCenters]);
   const detectFaceLandmarks = import_react7.useCallback(async (imageDataUrl) => {
+    const { setFaceLandmarks, setBlendShapes } = useMainStore.getState();
     if (!isMediaPipeReady) {
       console.log("MediaPipe not ready. Skipping detection.");
       return;
@@ -33068,200 +33203,137 @@ function useFaceLandmarkDetection() {
     }
     detectFaceLandmarks(previewImage);
   }, [isMediaPipeReady, isDrawingUtilsReady, previewImage]);
-  const modifyImage = import_react7.useCallback(({ landmark, vector }) => {
-    const {
-      originalImage: originalImage2,
-      originalImageHash: originalImageHash2,
-      params: previousParams,
-      setParams,
-      setError: setError2
-    } = useMainStore.getState();
-    if (!originalImage2) {
-      console.error("Image file or facePoke not available");
-      return;
-    }
-    const params = {
-      ...previousParams
-    };
-    const minX = -0.5;
-    const maxX = 0.5;
-    const minY = -0.5;
-    const maxY = 0.5;
-    const mapRange = (value, inMin, inMax, outMin, outMax) => {
-      return Math.min(outMax, Math.max(outMin, (value - inMin) * (outMax - outMin) / (inMax - inMin) + outMin));
-    };
-    console.log("modifyImage:", {
-      originalImage: originalImage2,
-      originalImageHash: originalImageHash2,
-      landmark,
-      vector,
-      minX,
-      maxX,
-      minY,
-      maxY
-    });
-    switch (landmark.group) {
-      case "leftEye":
-      case "rightEye":
-        const eyesMin = -20;
-        const eyesMax = 5;
-        params.eyes = mapRange(-vector.y, minX, maxX, eyesMin, eyesMax);
-        break;
-      case "leftEyebrow":
-      case "rightEyebrow":
-        const eyebrowMin = -10;
-        const eyebrowMax = 15;
-        params.eyebrow = mapRange(-vector.y, minY, maxY, eyebrowMin, eyebrowMax);
-        break;
-      case "lips":
-        const eeeMin = -20;
-        const eeeMax = 15;
-        params.eee = mapRange(-vector.y, minY, maxY, eeeMin, eeeMax);
-        const wooMin = -20;
-        const wooMax = 15;
-        params.woo = mapRange(-vector.x, minX, maxX, wooMin, wooMax);
-        break;
-      case "faceOval":
-        const rollMin = -40;
-        const rollMax = 40;
-        params.rotate_roll = mapRange(vector.x, minX, maxX, rollMin, rollMax);
-        break;
-      case "background":
-        const yawMin = -40;
-        const yawMax = 40;
-        params.rotate_yaw = mapRange(-vector.x, minX, maxX, yawMin, yawMax);
-        const pitchMin = -40;
-        const pitchMax = 40;
-        params.rotate_pitch = mapRange(vector.y, minY, maxY, pitchMin, pitchMax);
-        break;
-      default:
-        return;
-    }
-    for (const [key, value] of Object.entries(params)) {
-      if (isNaN(value) || !isFinite(value)) {
-        console.log(`${key} is NaN, aborting`);
-        return;
-      }
-    }
-    console.log(`PITCH=${params.rotate_pitch || 0}, YAW=${params.rotate_yaw || 0}, ROLL=${params.rotate_roll || 0}`);
-    setParams(params);
-    try {
-      if (!lastModifiedImageHashRef.current || lastModifiedImageHashRef.current !== originalImageHash2) {
-        lastModifiedImageHashRef.current = originalImageHash2;
-        facePoke.modifyImage(originalImage2, null, params);
-      } else {
-        facePoke.modifyImage(null, lastModifiedImageHashRef.current, params);
-      }
-    } catch (error2) {
-      setError2("Failed to modify image");
-    }
-  }, []);
   const modifyImageWithRateLimit = useThrottledCallback_default((params) => {
-    modifyImage(params);
-  }, [modifyImage], averageLatency);
-  const handleMouseEnter = import_react7.useCallback(() => {
-    setIsHovering(true);
-  }, []);
-  const handleMouseLeave = import_react7.useCallback(() => {
-    setIsHovering(false);
-  }, []);
-  const handleMouseDown = import_react7.useCallback((event) => {
+    useMainStore.getState().modifyImage(params);
+  }, [], throttleInMs);
+  import_react7.useEffect(() => {
+    facePoke.setOnServerResponse(handleServerResponse);
+  }, [handleServerResponse]);
+  const handleStart = import_react7.useCallback((x2, y2, mode) => {
     if (!canvasRef.current)
       return;
     const rect = canvasRef.current.getBoundingClientRect();
-    const x2 = (event.clientX - rect.left) / rect.width;
-    const y2 = (event.clientY - rect.top) / rect.height;
-    const landmark = findClosestLandmark(x2, y2);
-    console.log(`Mouse down on ${landmark.group}`);
+    const normalizedX = (x2 - rect.left) / rect.width;
+    const normalizedY = (y2 - rect.top) / rect.height;
+    const landmark = findClosestLandmark(normalizedX, normalizedY);
     setActiveLandmark(landmark);
-    setDragStart({ x: x2, y: y2 });
-    dragStartRef.current = { x: x2, y: y2 };
+    setDragStart({ x: normalizedX, y: normalizedY });
+    dragStartRef.current = { x: normalizedX, y: normalizedY };
   }, [findClosestLandmark, setActiveLandmark, setDragStart]);
-  const handleMouseMove = import_react7.useCallback((event) => {
+  const handleMove = import_react7.useCallback((x2, y2, mode) => {
     if (!canvasRef.current)
       return;
     const rect = canvasRef.current.getBoundingClientRect();
-    const x2 = (event.clientX - rect.left) / rect.width;
-    const y2 = (event.clientY - rect.top) / rect.height;
+    const normalizedX = (x2 - rect.left) / rect.width;
+    const normalizedY = (y2 - rect.top) / rect.height;
+    const landmark = findClosestLandmark(normalizedX, normalizedY, dragStart && dragStartRef.current ? currentLandmark?.group : undefined);
+    const landmarkData = landmarkCenters[landmark?.group];
+    const vector = landmarkData ? {
+      x: normalizedX - landmarkData.x,
+      y: normalizedY - landmarkData.y,
+      z: 0
+    } : {
+      x: 0.5,
+      y: 0.5,
+      z: 0
+    };
     if (dragStart && dragStartRef.current) {
-      const landmark = findClosestLandmark(x2, y2, currentLandmark?.group);
-      console.log(`Dragging mouse (was over ${currentLandmark?.group || "nothing"}, now over ${landmark.group})`);
+      setIsDragging(true);
       modifyImageWithRateLimit({
         landmark: currentLandmark || landmark,
-        vector: {
-          x: x2 - landmarkCenters[landmark.group].x,
-          y: y2 - landmarkCenters[landmark.group].y,
-          z: 0
-        }
+        vector,
+        mode
       });
-      setIsDragging(true);
     } else {
-      const landmark = findClosestLandmark(x2, y2);
       if (!currentLandmark || currentLandmark?.group !== landmark?.group) {
         setActiveLandmark(landmark);
       }
-      setIsHovering(true);
+      modifyImageWithRateLimit({
+        landmark,
+        vector,
+        mode: "HOVERING"
+      });
     }
-  }, [currentLandmark, dragStart, setIsHovering, setActiveLandmark, setIsDragging, modifyImageWithRateLimit, landmarkCenters]);
-  const handleMouseUp = import_react7.useCallback((event) => {
+  }, [currentLandmark, dragStart, setActiveLandmark, setIsDragging, modifyImageWithRateLimit, landmarkCenters]);
+  const handleEnd = import_react7.useCallback((x2, y2, mode) => {
     if (!canvasRef.current)
       return;
     const rect = canvasRef.current.getBoundingClientRect();
-    const x2 = (event.clientX - rect.left) / rect.width;
-    const y2 = (event.clientY - rect.top) / rect.height;
+    const normalizedX = (x2 - rect.left) / rect.width;
+    const normalizedY = (y2 - rect.top) / rect.height;
     if (dragStart && dragStartRef.current) {
-      const landmark = findClosestLandmark(x2, y2, currentLandmark?.group);
-      console.log(`Mouse up (was over ${currentLandmark?.group || "nothing"}, now over ${landmark.group})`);
+      const landmark = findClosestLandmark(normalizedX, normalizedY, currentLandmark?.group);
       modifyImageWithRateLimit({
         landmark: currentLandmark || landmark,
         vector: {
-          x: x2 - landmarkCenters[landmark.group].x,
-          y: y2 - landmarkCenters[landmark.group].y,
+          x: normalizedX - landmarkCenters[landmark.group].x,
+          y: normalizedY - landmarkCenters[landmark.group].y,
           z: 0
-        }
+        },
+        mode
       });
     }
     setIsDragging(false);
     dragStartRef.current = null;
     setActiveLandmark(undefined);
-  }, [currentLandmark, isDragging, modifyImageWithRateLimit, findClosestLandmark, setActiveLandmark, landmarkCenters, modifyImageWithRateLimit, setIsDragging]);
-  import_react7.useEffect(() => {
-    facePoke.setOnModifiedImage((image, image_hash) => {
-      if (image) {
-        setPreviewImage(image);
-      }
-      setOriginalImageHash(image_hash);
-      lastModifiedImageHashRef.current = image_hash;
-    });
-  }, [setPreviewImage, setOriginalImageHash]);
+  }, [currentLandmark, isDragging, modifyImageWithRateLimit, findClosestLandmark, setActiveLandmark, landmarkCenters, setIsDragging]);
+  const handleMouseDown = import_react7.useCallback((event) => {
+    const mode = event.button === 0 ? "PRIMARY" : "SECONDARY";
+    handleStart(event.clientX, event.clientY, mode);
+  }, [handleStart]);
+  const handleMouseMove = import_react7.useCallback((event) => {
+    const mode = event.buttons === 1 ? "PRIMARY" : "SECONDARY";
+    handleMove(event.clientX, event.clientY, mode);
+  }, [handleMove]);
+  const handleMouseUp = import_react7.useCallback((event) => {
+    const mode = event.buttons === 1 ? "PRIMARY" : "SECONDARY";
+    handleEnd(event.clientX, event.clientY, mode);
+  }, [handleEnd]);
+  const handleTouchStart = import_react7.useCallback((event) => {
+    const mode = event.touches.length === 1 ? "PRIMARY" : "SECONDARY";
+    const touch = event.touches[0];
+    handleStart(touch.clientX, touch.clientY, mode);
+  }, [handleStart]);
+  const handleTouchMove = import_react7.useCallback((event) => {
+    const mode = event.touches.length === 1 ? "PRIMARY" : "SECONDARY";
+    const touch = event.touches[0];
+    handleMove(touch.clientX, touch.clientY, mode);
+  }, [handleMove]);
+  const handleTouchEnd = import_react7.useCallback((event) => {
+    const mode = event.changedTouches.length === 1 ? "PRIMARY" : "SECONDARY";
+    const touch = event.changedTouches[0];
+    handleEnd(touch.clientX, touch.clientY, mode);
+  }, [handleEnd]);
   return {
     canvasRef,
     canvasRefCallback,
     mediaPipeRef,
-    faceLandmarks,
     isMediaPipeReady,
     isDrawingUtilsReady,
-    blendShapes,
-    setFaceLandmarks,
-    setBlendShapes,
     handleMouseDown,
     handleMouseUp,
     handleMouseMove,
-    handleMouseEnter,
-    handleMouseLeave,
+    handleTouchStart,
+    handleTouchMove,
+    handleTouchEnd,
     currentLandmark,
     currentOpacity
   };
 }
 
-// src/components/PoweredBy.tsx
+// src/components/About.tsx
 var jsx_dev_runtime2 = __toESM(require_jsx_dev_runtime(), 1);
-function PoweredBy() {
+function About() {
   return jsx_dev_runtime2.jsxDEV("div", {
     className: "flex flex-row items-center justify-center font-sans mt-4 w-full",
     children: [
       jsx_dev_runtime2.jsxDEV("span", {
-        className: "mr-1",
+        className: "text-neutral-900 text-sm",
+        style: { textShadow: "rgb(255 255 255 / 80%) 0px 0px 2px" },
+        children: "Click and drag on the image."
+      }, undefined, false, undefined, this),
+      jsx_dev_runtime2.jsxDEV("span", {
+        className: "ml-2 mr-1",
         children: jsx_dev_runtime2.jsxDEV("img", {
           src: "/hf-logo.svg",
           alt: "Hugging Face",
@@ -33297,7 +33369,6 @@ function Spinner() {
 // src/hooks/useFacePokeAPI.ts
 var import_react8 = __toESM(require_react(), 1);
 function useFacePokeAPI() {
-  const [status, setStatus] = import_react8.useState("");
   const [isDebugMode, setIsDebugMode] = import_react8.useState(false);
   const [interruptMessage, setInterruptMessage] = import_react8.useState(null);
   const [isLoading, setIsLoading] = import_react8.useState(false);
@@ -33316,8 +33387,6 @@ function useFacePokeAPI() {
   }, []);
   return {
     facePoke,
-    status,
-    setStatus,
     isDebugMode,
     setIsDebugMode,
     interruptMessage,
@@ -33335,31 +33404,13 @@ function Layout({ children }) {
     children: jsx_dev_runtime4.jsxDEV("div", {
       className: "min-h-screen w-full py-8 flex flex-col justify-center",
       children: jsx_dev_runtime4.jsxDEV("div", {
-        className: "relative p-4 sm:max-w-5xl sm:mx-auto",
+        className: "flex flex-col items-center justify-center p-4 sm:max-w-5xl sm:mx-auto",
         children
       }, undefined, false, undefined, this)
     }, undefined, false, undefined, this)
   }, undefined, false, undefined, this);
 }
 
-// src/lib/convertImageToBase64.ts
-async function convertImageToBase64(imageFile) {
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader;
-    reader.onload = () => {
-      if (typeof reader.result === "string") {
-        resolve(reader.result);
-      } else {
-        reject(new Error("Failed to convert image to base64"));
-      }
-    };
-    reader.onerror = () => {
-      reject(new Error("Error reading file"));
-    };
-    reader.readAsDataURL(imageFile);
-  });
-}
-
 // src/app.tsx
 var jsx_dev_runtime5 = __toESM(require_jsx_dev_runtime(), 1);
 function App() {
@@ -33367,66 +33418,34 @@ function App() {
   const setError = useMainStore((s2) => s2.setError);
   const imageFile = useMainStore((s2) => s2.imageFile);
   const setImageFile = useMainStore((s2) => s2.setImageFile);
-  const originalImage = useMainStore((s2) => s2.originalImage);
-  const setOriginalImage = useMainStore((s2) => s2.setOriginalImage);
+  const isGazingAtCursor = useMainStore((s2) => s2.isGazingAtCursor);
+  const setIsGazingAtCursor = useMainStore((s2) => s2.setIsGazingAtCursor);
+  const isFollowingCursor = useMainStore((s2) => s2.isFollowingCursor);
+  const setIsFollowingCursor = useMainStore((s2) => s2.setIsFollowingCursor);
   const previewImage = useMainStore((s2) => s2.previewImage);
-  const setPreviewImage = useMainStore((s2) => s2.setPreviewImage);
-  const resetImage = useMainStore((s2) => s2.resetImage);
-  const setOriginalImageHash = useMainStore((s2) => s2.setOriginalImageHash);
+  const status = useMainStore((s2) => s2.status);
+  const blendShapes = useMainStore((s2) => s2.blendShapes);
   const {
-    status,
-    setStatus,
     isDebugMode,
     setIsDebugMode,
     interruptMessage
   } = useFacePokeAPI();
   const {
-    canvasRef,
     canvasRefCallback,
-    mediaPipeRef,
-    faceLandmarks,
     isMediaPipeReady,
-    blendShapes,
-    setFaceLandmarks,
-    setBlendShapes,
     handleMouseDown,
     handleMouseUp,
     handleMouseMove,
-    handleMouseEnter,
-    handleMouseLeave,
+    handleTouchStart,
+    handleTouchMove,
+    handleTouchEnd,
     currentOpacity
   } = useFaceLandmarkDetection();
   const videoRef = import_react9.useRef(null);
-  const handleFileChange = import_react9.useCallback(async (event) => {
+  const handleFileChange = import_react9.useCallback((event) => {
     const files = event.target.files;
-    if (files && files[0]) {
-      setImageFile(files[0]);
-      setStatus(`File selected: ${truncateFileName(files[0].name, 16)}`);
-      try {
-        const image = await convertImageToBase64(files[0]);
-        setPreviewImage(image);
-        setOriginalImage(image);
-        setOriginalImageHash("");
-      } catch (err) {
-        console.log(`failed to convert the image: `, err);
-        setImageFile(null);
-        setStatus("");
-        setPreviewImage("");
-        setOriginalImage("");
-        setOriginalImageHash("");
-        setFaceLandmarks([]);
-        setBlendShapes([]);
-      }
-    } else {
-      setImageFile(null);
-      setStatus("");
-      setPreviewImage("");
-      setOriginalImage("");
-      setOriginalImageHash("");
-      setFaceLandmarks([]);
-      setBlendShapes([]);
-    }
-  }, [isMediaPipeReady, setImageFile, setPreviewImage, setOriginalImage, setOriginalImageHash, setFaceLandmarks, setBlendShapes, setStatus]);
+    setImageFile(files?.[0] || undefined);
+  }, [setImageFile]);
   const handleDownload = import_react9.useCallback(() => {
     if (previewImage) {
       const link = document.createElement("a");
@@ -33503,7 +33522,7 @@ function App() {
                 className: "flex items-center space-x-2",
                 children: [
                   jsx_dev_runtime5.jsxDEV("div", {
-                    className: "relative",
+                    className: "flex items-center justify-center",
                     children: [
                       jsx_dev_runtime5.jsxDEV("input", {
                         id: "imageInput",
@@ -33518,7 +33537,7 @@ function App() {
                         className: `cursor-pointer inline-flex items-center px-3 h-10 border border-transparent text-sm font-medium rounded-md text-white ${isMediaPipeReady ? "bg-slate-600 hover:bg-slate-500" : "bg-slate-500 cursor-not-allowed"} focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-slate-500 shadow-xl`,
                         children: [
                           jsx_dev_runtime5.jsxDEV(Spinner, {}, undefined, false, undefined, this),
-                          imageFile ? truncateFileName(imageFile.name, 32) : isMediaPipeReady ? "Choose a portrait photo (.jpg, .png, .webp)" : "Initializing..."
+                          imageFile ? truncateFileName(imageFile.name, 32) : isMediaPipeReady ? "Choose a portrait photo" : "Initializing..."
                         ]
                       }, undefined, true, undefined, this)
                     ]
@@ -33535,18 +33554,21 @@ function App() {
                   }, undefined, true, undefined, this)
                 ]
               }, undefined, true, undefined, this),
-              previewImage && jsx_dev_runtime5.jsxDEV("label", {
-                className: "mt-4 flex items-center",
-                children: [
-                  jsx_dev_runtime5.jsxDEV("input", {
-                    type: "checkbox",
-                    checked: isDebugMode,
-                    onChange: (e2) => setIsDebugMode(e2.target.checked),
-                    className: "mr-2"
-                  }, undefined, false, undefined, this),
-                  "Show face landmarks on hover"
-                ]
-              }, undefined, true, undefined, this)
+              previewImage && jsx_dev_runtime5.jsxDEV("div", {
+                className: "flex items-center space-x-2",
+                children: jsx_dev_runtime5.jsxDEV("label", {
+                  className: "mt-4 flex items-center",
+                  children: [
+                    jsx_dev_runtime5.jsxDEV("input", {
+                      type: "checkbox",
+                      checked: isDebugMode,
+                      onChange: (e2) => setIsDebugMode(e2.target.checked),
+                      className: "mr-2"
+                    }, undefined, false, undefined, this),
+                    "Show face markers"
+                  ]
+                }, undefined, true, undefined, this)
+              }, undefined, false, undefined, this)
             ]
           }, undefined, true, undefined, this),
           previewImage && jsx_dev_runtime5.jsxDEV("div", {
@@ -33560,11 +33582,12 @@ function App() {
               jsx_dev_runtime5.jsxDEV("canvas", {
                 ref: canvasRefCallback,
                 className: "absolute top-0 left-0 w-full h-full select-none",
-                onMouseEnter: handleMouseEnter,
-                onMouseLeave: handleMouseLeave,
                 onMouseDown: handleMouseDown,
                 onMouseUp: handleMouseUp,
                 onMouseMove: handleMouseMove,
+                onTouchStart: handleTouchStart,
+                onTouchMove: handleTouchMove,
+                onTouchEnd: handleTouchEnd,
                 style: {
                   position: "absolute",
                   top: 0,
@@ -33580,7 +33603,7 @@ function App() {
           canDisplayBlendShapes && displayBlendShapes
         ]
       }, undefined, true, undefined, this),
-      jsx_dev_runtime5.jsxDEV(PoweredBy, {}, undefined, false, undefined, this)
+      jsx_dev_runtime5.jsxDEV(About, {}, undefined, false, undefined, this)
     ]
   }, undefined, true, undefined, this);
 }

requirements.txt

@@ -7,12 +7,15 @@
 torch
 torchvision
 torchaudio
-
+torchgeometry
 
 # --------------------------------------------------------------------
 # Common libraries for LivePortrait and all
 # --------------------------------------------------------------------
 
+# LRU cache compatible with asyncio
+async-lru==2.0.4
+
 # note: gradio is only used for the cropping utility
 gradio==5.0.0b4
 
@@ -35,8 +38,8 @@ ffmpeg-python==0.2.0
 onnxruntime-gpu==1.19.2
 onnx==1.16.2
 scikit-image==0.20.0
-albumentations==1.4.10
-matplotlib==3.9.0
+albumentations==1.3.1
+matplotlib==3.7.2
 tyro==0.8.5
 chumpy==0.70
 
@@ -45,16 +48,24 @@ accelerate==0.34.2
 tensorflow==2.12.0
 tensorboard==2.12.0
 transformers==4.39.2
-huggingface-hub==0.25.1
-safetensors==0.4.5
 
 gdown==5.2.0
 requests==2.32.3
 omegaconf==2.3.0
+
 pydantic==2.9.2
-dnspython>=2.0.0
-ifaddr>=0.2.0
+
+# --------------------------------------------------------------------
+# RESERVED FOR FUTURE USAGE
+#
+# (it adds bloat, so you can remove them if you want)
+# --------------------------------------------------------------------
 aiohttp==3.10.5
+av==12.3.0
+einops==0.7.0
+safetensors==0.4.5
+huggingface-hub==0.25.1
+optimum-quanto==0.2.4
 
 # --------------------------------------------------------------------
 # Used for advanced LivePortrait features