import gradio as gr import cv2 import mediapipe as mp mp_drawing = mp.solutions.drawing_utils mp_drawing_styles = mp.solutions.drawing_styles mp_face_mesh = mp.solutions.face_mesh import numpy as np from mediapipe.framework.formats import landmark_pb2 from typing import List, Mapping, Optional, Tuple, Union import pygltflib import struct import tempfile QUADS = [ [300, 334, 333, 298] , [ 1, 12, 303, 268] , [234, 233, 122, 129] , [270, 304, 305, 271] , [246, 129, 115, 189] , [112, 118, 229, 32] , [104, 55, 69, 105] , [228, 35, 128, 235] , [120, 102, 101, 121] , [ 74, 73, 38, 40] , [ 71, 47, 54, 64] , [135, 132, 116, 221] , [335, 294, 299, 334] , [ 73, 12, 1, 38] , [ 42, 43, 81, 82] , [166, 93, 41, 40] , [122, 233, 232, 121] , [215, 213, 217, 208] , [183, 84, 85, 182] , [376, 308, 321, 322] , [ 30, 161, 160, 28] , [ 57, 29, 159, 158] , [ 84, 202, 201, 19] , [117, 144, 35, 228] , [204, 207, 93, 166] , [139, 216, 59, 173] , [276, 282, 6, 5] , [ 25, 145, 164, 111] , [292, 307, 308, 376] , [143, 127, 48, 101] , [419, 422, 429, 263] , [147, 44, 107, 92] , [ 17, 86, 85, 18] , [ 78, 77, 62, 147] , [127, 210, 199, 218] , [397, 378, 401, 370] , [166, 40, 38, 168] , [245, 234, 129, 246] , [ 31, 248, 247, 162] , [ 34, 247, 248, 131] , [175, 218, 199, 237] , [418, 352, 413, 466] , [125, 114, 226, 47] , [225, 224, 53, 54] , [ 99, 65, 103, 130] , [193, 215, 208, 188] , [219, 80, 240, 238] , [134, 156, 113, 244] , [345, 361, 364, 441] , [141, 171, 150, 177] , [400, 413, 352, 420] , [119, 230, 229, 118] , [282, 276, 441, 364] , [ 71, 64, 69, 72] , [315, 314, 407, 406] , [222, 190, 194, 56] , [114, 248, 31, 226] , [106, 53, 66, 67] , [236, 60, 167, 220] , [108, 56, 9, 10] , [ 67, 66, 56, 108] , [ 69, 64, 106, 105] , [120, 119, 51, 102] , [242, 126, 45, 238] , [ 6, 196, 4, 52] , [143, 130, 210, 127] , [ 34, 131, 26, 8] , [323, 271, 410, 411] , [ 33, 195, 205, 212] , [ 37, 102, 51, 206] , [195, 202, 84, 183] , [238, 240, 239, 242] , [ 26, 111, 164, 8] , [225, 54, 47, 226] , [154, 146, 24, 23] , [211, 203, 213, 215] , [246, 194, 190, 245] , [425, 336, 407, 419] , [318, 317, 404, 403] , [ 33, 212, 171, 141] , [ 12, 73, 39, 13] , [208, 217, 207, 206] , [238, 221, 116, 219] , [ 46, 221, 238, 45] , [184, 43, 75, 185] , [209, 202, 195, 33] , [269, 272, 304, 303] , [214, 148, 178, 216] , [235, 94, 138, 228] , [ 67, 108, 109, 70] , [ 7, 352, 418, 169] , [193, 188, 148, 214] , [ 97, 63, 77, 78] , [125, 47, 71, 157] , [317, 16, 17, 316] , [180, 87, 88, 179] , [106, 64, 54, 53] , [119, 118, 124, 51] , [146, 145, 25, 24] , [325, 319, 320, 326] , [123, 189, 175, 197] , [293, 309, 325, 326] , [150, 171, 170, 151] , [178, 138, 94, 133] , [328, 295, 456, 461] , [361, 421, 457, 364] , [336, 274, 376, 322] , [396, 395, 431, 432] , [ 13, 39, 83, 14] , [278, 330, 350, 351] , [191, 57, 158, 174] , [117, 112, 36, 144] , [224, 223, 66, 53] , [140, 72, 22, 163] , [163, 128, 35, 140] , [366, 365, 395, 380] , [219, 116, 49, 220] , [430, 359, 372, 356] , [157, 144, 36, 125] , [377, 353, 281, 412] , [125, 36, 227, 114] , [355, 20, 95, 371] , [120, 231, 230, 119] , [249, 457, 400, 420] , [162, 161, 30, 31] , [ 46, 45, 2, 5] , [141, 172, 209, 33] , [394, 392, 328, 327] , [ 32, 26, 131, 227] , [300, 298, 339, 338] , [395, 396, 379, 380] , [102, 37, 143, 101] , [217, 213, 58, 187] , [327, 3, 165, 394] , [242, 239, 21, 243] , [186, 41, 93, 187] , [269, 303, 12, 13] , [192, 81, 43, 184] , [140, 35, 144, 157] , [223, 222, 56, 66] , [189, 115, 218, 175] , [323, 427, 424, 392] , [ 37, 204, 130, 143] , [280, 430, 421, 361] , [ 2, 275, 276, 5] , [134, 244, 191, 174] , [241, 76, 60, 236] , [108, 10, 152, 109] , [ 27, 155, 154, 23] , [211, 215, 136, 170] , [355, 275, 2, 20] , [ 90, 89, 96, 97] , [321, 320, 404, 405] , [316, 315, 406, 405] , [107, 44, 203, 205] , [201, 422, 314, 19] , [153, 176, 172, 149] , [376, 274, 288, 292] , [292, 288, 411, 410] , [130, 204, 166, 99] , [115, 48, 127, 218] , [327, 328, 461, 329] , [105, 106, 67, 70] , [236, 65, 99, 241] , [200, 201, 202, 209] , [332, 295, 328, 359] , [100, 61, 76, 241] , [243, 142, 126, 242] , [329, 463, 371, 327] , [220, 167, 80, 219] , [233, 27, 23, 232] , [190, 222, 57, 191] , [223, 29, 57, 222] , [244, 113, 234, 245] , [ 32, 229, 111, 26] , [226, 31, 30, 225] , [232, 23, 24, 231] , [225, 30, 28, 224] , [114, 227, 131, 248] , [ 32, 227, 36, 112] , [234, 113, 27, 233] , [230, 25, 111, 229] , [224, 28, 29, 223] , [ 95, 20, 126, 142] , [239, 240, 80, 21] , [243, 21, 61, 100] , [157, 71, 72, 140] , [ 76, 61, 167, 60] , [189, 123, 194, 246] , [231, 24, 25, 230] , [232, 231, 120, 121] , [121, 101, 48, 122] , [208, 206, 51, 188] , [332, 280, 279, 295] , [196, 249, 420, 198] , [199, 210, 50, 132] , [177, 149, 172, 141] , [117, 124, 118, 112] , [ 28, 160, 159, 29] , [245, 190, 191, 244] , [379, 396, 370, 401] , [268, 303, 304, 270] , [351, 453, 454, 358] , [ 75, 74, 40, 41] , [169, 418, 286, 9] , [283, 444, 445, 284] , [397, 176, 153, 378] , [110, 68, 70, 109] , [301, 277, 354, 384] , [186, 62, 77, 185] , [299, 294, 301, 302] , [ 50, 49, 116, 132] , [422, 201, 200, 429] , [304, 272, 273, 305] , [271, 323, 392, 270] , [296, 443, 444, 283] , [427, 437, 428, 426] , [336, 322, 406, 407] , [ 19, 314, 315, 18] , [387, 388, 260, 258] , [255, 374, 375, 254] , [314, 422, 419, 407] , [297, 335, 334, 300] , [313, 312, 272, 269] , [ 55, 22, 72, 69] , [221, 46, 52, 135] , [391, 374, 255, 340] , [315, 316, 17, 18] , [372, 267, 331, 330] , [423, 274, 336, 425] , [ 58, 44, 147, 62] , [ 91, 78, 147, 92] , [182, 85, 86, 181] , [423, 425, 432, 431] , [357, 265, 448, 455] , [268, 270, 392, 394] , [358, 454, 465, 466] , [264, 360, 468, 467] , [264, 250, 256, 360] , [421, 430, 356, 438] , [194, 123, 7, 169] , [449, 450, 348, 347] , [277, 284, 445, 446] , [241, 99, 98, 100] , [281, 331, 267, 426] , [307, 292, 410, 409] , [260, 388, 389, 261] , [364, 457, 249, 282] , [338, 339, 11, 152] , [438, 344, 413, 400] , [349, 451, 452, 350] , [345, 279, 280, 361] , [402, 377, 434, 436] , [367, 324, 455, 448] , [182, 92, 107, 183] , [418, 414, 442, 286] , [360, 256, 262, 447] , [284, 277, 301, 294] , [291, 251, 463, 329] , [344, 358, 466, 413] , [179, 89, 90, 180] , [266, 341, 346, 373] , [429, 397, 370, 263] , [296, 283, 335, 297] , [275, 355, 462, 458] , [ 4, 237, 135, 52] , [359, 424, 267, 372] , [386, 387, 258, 259] , [394, 165, 1, 268] , [207, 217, 187, 93] , [278, 356, 372, 330] , [ 44, 58, 213, 203] , [459, 460, 458, 462] , [381, 382, 257, 253] , [266, 447, 262, 341] , [399, 385, 287, 415] , [437, 433, 435, 428] , [447, 266, 354, 343] , [183, 107, 205, 195] , [ 43, 42, 74, 75] , [302, 301, 384, 369] , [425, 419, 263, 432] , [295, 279, 440, 456] , [ 49, 50, 103, 65] , [ 74, 42, 39, 73] , [433, 423, 431, 435] , [311, 273, 272, 312] , [353, 367, 448, 346] , [252, 302, 369, 390] , [209, 172, 176, 200] , [ 56, 194, 169, 9] , [377, 412, 417, 434] , [ 90, 97, 78, 91] , [330, 331, 349, 350] , [180, 90, 91, 181] , [281, 348, 349, 331] , [265, 373, 346, 448] , [324, 367, 402, 362] , [308, 326, 320, 321] , [ 16, 15, 88, 87] , [266, 373, 384, 354] , [353, 347, 348, 281] , [363, 399, 415, 464] , [318, 15, 16, 317] , [356, 278, 344, 438] , [ 96, 79, 63, 97] , [ 11, 110, 109, 152] , [398, 368, 365, 366] , [ 2, 45, 126, 20] , [313, 269, 13, 14] , [237, 199, 132, 135] , [187, 58, 62, 186] , [152, 10, 337, 338] , [ 42, 82, 83, 39] , [414, 418, 466, 465] , [467, 468, 261, 389] , [ 9, 286, 337, 10] , [446, 343, 354, 277] , [265, 357, 390, 369] , [436, 434, 417, 368] , [170, 136, 137, 151] , [458, 441, 276, 275] , [212, 205, 203, 211] , [347, 353, 346, 341] , [284, 294, 335, 283] , [452, 453, 351, 350] , [ 95, 3, 327, 371] , [450, 451, 349, 348] , [197, 4, 196, 198] , [254, 375, 381, 253] , [345, 441, 458, 439] , [367, 353, 377, 402] , [449, 347, 341, 262] , [360, 447, 343, 468] , [136, 139, 173, 137] , [289, 436, 368, 398] , [281, 426, 428, 412] , [288, 433, 437, 411] , [ 99, 166, 168, 98] , [142, 243, 100, 98] , [175, 237, 4, 197] , [185, 75, 41, 186] , [307, 293, 326, 308] , [396, 432, 263, 370] , [286, 442, 443, 296] , [428, 435, 417, 412] , [411, 437, 427, 323] , [421, 438, 400, 457] , [165, 3, 98, 168] , [279, 345, 439, 440] , [391, 340, 256, 250] , [306, 291, 329, 461] , [373, 265, 369, 384] , [386, 259, 287, 385] , [435, 365, 368, 417] , [251, 459, 462, 463] , [320, 319, 403, 404] , [ 17, 16, 87, 86] , [322, 321, 405, 406] , [ 85, 84, 19, 18] , [433, 288, 274, 423] , [362, 402, 436, 289] , [185, 77, 63, 184] , [293, 307, 409, 408] , [392, 424, 359, 328] , [352, 7, 198, 420] , [228, 138, 124, 117] , [393, 290, 456, 440] , [176, 397, 429, 200] , [220, 49, 65, 236] , [424, 427, 426, 267] , [332, 359, 430, 280] , [365, 435, 431, 395] , [310, 251, 291, 393] , [355, 371, 463, 462] , [ 98, 3, 95, 142] , [255, 254, 451, 450] , [415, 414, 465, 464] , [254, 253, 452, 451] , [261, 468, 343, 446] , [260, 261, 446, 445] , [258, 260, 445, 444] , [454, 342, 464, 465] , [198, 7, 123, 197] , [259, 258, 444, 443] , [287, 442, 414, 415] , [340, 449, 262, 256] , [340, 255, 450, 449] , [257, 342, 454, 453] , [ 61, 21, 80, 167] , [310, 393, 440, 439] , [338, 337, 297, 300] , [310, 460, 459, 251] , [ 51, 124, 148, 188] , [253, 257, 453, 452] , [215, 193, 139, 136] , [351, 358, 344, 278] , [113, 156, 155, 27] , [ 6, 52, 46, 5] , [206, 207, 204, 37] , [249, 196, 6, 282] , [216, 178, 133, 59] , [286, 296, 297, 337] , [382, 383, 342, 257] , [287, 259, 443, 442] , [211, 170, 171, 212] , [306, 461, 456, 290] , [104, 105, 70, 68] , [271, 305, 409, 410] , [460, 310, 439, 458] , [214, 216, 139, 193] , [317, 316, 405, 404] , [181, 91, 92, 182] , [ 1, 165, 168, 38] , [363, 464, 342, 383] , [210, 130, 103, 50] , [305, 273, 408, 409] , [311, 416, 408, 273] , [309, 293, 408, 416] , [184, 63, 79, 192] , [115, 129, 122, 48] , [148, 124, 138, 178] , [181, 86, 87, 180] , [290, 393, 291, 306] , [252, 285, 299, 302] , [285, 333, 334, 299] ] class face_image_to_face_mesh: def demo(self): demo = gr.Blocks() with demo: gr.Markdown( """ # Face Image to Face Quad Mesh Uses MediaPipe to detect a face in an image and convert it to a quad mesh. Currently saves to OBJ, hopefully glb at some point with color data. The 3d viewer has Y pointing the opposite direction from Blender, so ya hafta spin it. The initial workflow I was imagining was: 1. sculpt high poly mesh in blender 2. snapshot the face 3. generate the mesh using the mediapipe stuff 4. import the low poly mediapipe face 5. snap the mesh to the high poly model 6. model the rest of the low poly model 7. bake the normal / etc maps to the low poly face model 8. it's just that easy 😛 Ideally it would be a plugin... """) with gr.Row(): with gr.Column(): upload_image = gr.Image(label="Input image", type="numpy", source="upload") gr.Examples( examples=[ 'examples/blonde-00019-1421846474.png', 'examples/dude-00110-1227390728.png', 'examples/granny-00056-1867315302.png', 'examples/tuffie-00039-499759385.png', ], inputs=[upload_image] ) upload_image_btn = gr.Button(value="Detect faces") with gr.Group(): min_detection_confidence = gr.Slider(label="Min detection confidence", value=0.5, minimum=0.0, maximum=1.0, step=0.01) gr.Textbox(show_label=False, value="Minimum confidence value ([0.0, 1.0]) from the face detection model for the detection to be considered successful.") with gr.Column(): with gr.Group(): num_faces_detected = gr.Number(label="Number of faces detected", value=0) output_mesh = gr.Model3D(clear_color=[0.0, 0.0, 0.0, 0.0], label="3D Model") output_image = gr.Image(label="Output image") outputs = [output_mesh, output_image, num_faces_detected] upload_image_btn.click( fn=self.detect, inputs=[upload_image, min_detection_confidence], outputs=outputs ) demo.launch() def detect(self, image, min_detection_confidence): width = image.shape[1] height = image.shape[0] ratio = width / height mesh = "examples/jackiechan.obj" drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1) with mp_face_mesh.FaceMesh( static_image_mode=True, max_num_faces=1, min_detection_confidence=min_detection_confidence) as face_mesh: results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) if not results.multi_face_landmarks: return mesh, image, 0 annotated_image = image.copy() for face_landmarks in results.multi_face_landmarks: mesh = self.toObj(ratio=ratio, landmark_list=face_landmarks) mp_drawing.draw_landmarks( image=annotated_image, landmark_list=face_landmarks, connections=mp_face_mesh.FACEMESH_TESSELATION, landmark_drawing_spec=None, connection_drawing_spec=mp_drawing_styles .get_default_face_mesh_tesselation_style()) mp_drawing.draw_landmarks( image=annotated_image, landmark_list=face_landmarks, connections=mp_face_mesh.FACEMESH_CONTOURS, landmark_drawing_spec=None, connection_drawing_spec=mp_drawing_styles .get_default_face_mesh_contours_style()) return mesh, annotated_image,1 def toObj( self, ratio: float, landmark_list: landmark_pb2.NormalizedLandmarkList): print( f'you have such pretty hair' ) lines = [] points = self.landmarksToPoints( ratio, landmark_list ) for point in points: vertex = "v " + " ".join([str(value) for value in point]) lines.append( vertex ) for quad in QUADS: face = "f " + " ".join([str(vertex) for vertex in quad]) lines.append( face ) normal = self.totallyNormal( points[ quad[ 0 ] -1 ], points[ quad[ 1 ] -1 ], points[ quad[ 2 ] -1 ] ) lines.append( "vn " + " ".join([str(value) for value in normal]) ) obj_file = tempfile.NamedTemporaryFile(suffix='.obj', delete=False) output_file = obj_file.name out = open( output_file, 'w' ) out.write( '\n'.join( lines ) ) out.close() print( f'I know it is special to you so I saved it to {output_file} since we are friends' ) return output_file def landmarksToPoints( self, ratio: float, landmark_list: landmark_pb2.NormalizedLandmarkList ): points = [] mins = [+np.inf] * 3 maxs = [-np.inf] * 3 for idx, landmark in enumerate(landmark_list.landmark): if ((landmark.HasField('visibility') and landmark.visibility < _VISIBILITY_THRESHOLD) or (landmark.HasField('presence') and landmark.presence < _PRESENCE_THRESHOLD)): idk_what_to_do_for_this = True point = [landmark.x * ratio, -landmark.y, -landmark.z]; for pidx,value in enumerate( point ): mins[pidx] = min(mins[pidx],value) maxs[pidx] = max(maxs[pidx],value) points.append( point ) mids = [(min_val + max_val) / 2 for min_val, max_val in zip(mins, maxs)] for idx,point in enumerate( points ): points[idx] = [(val-mid) for val, mid in zip(point,mids)] print( f'mins: {mins}' ) print( f'mids: {mids}' ) print( f'maxs: {maxs}' ) return points def totallyNormal(self, p0, p1, p2): v1 = np.array(p1) - np.array(p0) v2 = np.array(p2) - np.array(p0) normal = np.cross(v1, v2) normal = normal / np.linalg.norm(normal) return normal.tolist() face_image_to_face_mesh().demo()