Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import cv2 | |
| import mediapipe as mp | |
| mp_drawing = mp.solutions.drawing_utils | |
| mp_drawing_styles = mp.solutions.drawing_styles | |
| mp_face_mesh = mp.solutions.face_mesh | |
| import numpy as np | |
| from mediapipe.framework.formats import landmark_pb2 | |
| from typing import List, Mapping, Optional, Tuple, Union | |
| import pygltflib | |
| import struct | |
| import tempfile | |
| QUADS = [ | |
| [300, 334, 333, 298] , [ 1, 12, 303, 268] , [234, 233, 122, 129] , [270, 304, 305, 271] , [246, 129, 115, 189] , | |
| [112, 118, 229, 32] , [104, 55, 69, 105] , [228, 35, 128, 235] , [120, 102, 101, 121] , [ 74, 73, 38, 40] , | |
| [ 71, 47, 54, 64] , [135, 132, 116, 221] , [335, 294, 299, 334] , [ 73, 12, 1, 38] , [ 42, 43, 81, 82] , | |
| [166, 93, 41, 40] , [122, 233, 232, 121] , [215, 213, 217, 208] , [183, 84, 85, 182] , [376, 308, 321, 322] , | |
| [ 30, 161, 160, 28] , [ 57, 29, 159, 158] , [ 84, 202, 201, 19] , [117, 144, 35, 228] , [204, 207, 93, 166] , | |
| [139, 216, 59, 173] , [276, 282, 6, 5] , [ 25, 145, 164, 111] , [292, 307, 308, 376] , [143, 127, 48, 101] , | |
| [419, 422, 429, 263] , [147, 44, 107, 92] , [ 17, 86, 85, 18] , [ 78, 77, 62, 147] , [127, 210, 199, 218] , | |
| [397, 378, 401, 370] , [166, 40, 38, 168] , [245, 234, 129, 246] , [ 31, 248, 247, 162] , [ 34, 247, 248, 131] , | |
| [175, 218, 199, 237] , [418, 352, 413, 466] , [125, 114, 226, 47] , [225, 224, 53, 54] , [ 99, 65, 103, 130] , | |
| [193, 215, 208, 188] , [219, 80, 240, 238] , [134, 156, 113, 244] , [345, 361, 364, 441] , [141, 171, 150, 177] , | |
| [400, 413, 352, 420] , [119, 230, 229, 118] , [282, 276, 441, 364] , [ 71, 64, 69, 72] , [315, 314, 407, 406] , | |
| [222, 190, 194, 56] , [114, 248, 31, 226] , [106, 53, 66, 67] , [236, 60, 167, 220] , [108, 56, 9, 10] , | |
| [ 67, 66, 56, 108] , [ 69, 64, 106, 105] , [120, 119, 51, 102] , [242, 126, 45, 238] , [ 6, 196, 4, 52] , | |
| [143, 130, 210, 127] , [ 34, 131, 26, 8] , [323, 271, 410, 411] , [ 33, 195, 205, 212] , [ 37, 102, 51, 206] , | |
| [195, 202, 84, 183] , [238, 240, 239, 242] , [ 26, 111, 164, 8] , [225, 54, 47, 226] , [154, 146, 24, 23] , | |
| [211, 203, 213, 215] , [246, 194, 190, 245] , [425, 336, 407, 419] , [318, 317, 404, 403] , [ 33, 212, 171, 141] , | |
| [ 12, 73, 39, 13] , [208, 217, 207, 206] , [238, 221, 116, 219] , [ 46, 221, 238, 45] , [184, 43, 75, 185] , | |
| [209, 202, 195, 33] , [269, 272, 304, 303] , [214, 148, 178, 216] , [235, 94, 138, 228] , [ 67, 108, 109, 70] , | |
| [ 7, 352, 418, 169] , [193, 188, 148, 214] , [ 97, 63, 77, 78] , [125, 47, 71, 157] , [317, 16, 17, 316] , | |
| [180, 87, 88, 179] , [106, 64, 54, 53] , [119, 118, 124, 51] , [146, 145, 25, 24] , [325, 319, 320, 326] , | |
| [123, 189, 175, 197] , [293, 309, 325, 326] , [150, 171, 170, 151] , [178, 138, 94, 133] , [328, 295, 456, 461] , | |
| [361, 421, 457, 364] , [336, 274, 376, 322] , [396, 395, 431, 432] , [ 13, 39, 83, 14] , [278, 330, 350, 351] , | |
| [191, 57, 158, 174] , [117, 112, 36, 144] , [224, 223, 66, 53] , [140, 72, 22, 163] , [163, 128, 35, 140] , | |
| [366, 365, 395, 380] , [219, 116, 49, 220] , [430, 359, 372, 356] , [157, 144, 36, 125] , [377, 353, 281, 412] , | |
| [125, 36, 227, 114] , [355, 20, 95, 371] , [120, 231, 230, 119] , [249, 457, 400, 420] , [162, 161, 30, 31] , | |
| [ 46, 45, 2, 5] , [141, 172, 209, 33] , [394, 392, 328, 327] , [ 32, 26, 131, 227] , [300, 298, 339, 338] , | |
| [395, 396, 379, 380] , [102, 37, 143, 101] , [217, 213, 58, 187] , [327, 3, 165, 394] , [242, 239, 21, 243] , | |
| [186, 41, 93, 187] , [269, 303, 12, 13] , [192, 81, 43, 184] , [140, 35, 144, 157] , [223, 222, 56, 66] , | |
| [189, 115, 218, 175] , [323, 427, 424, 392] , [ 37, 204, 130, 143] , [280, 430, 421, 361] , [ 2, 275, 276, 5] , | |
| [134, 244, 191, 174] , [241, 76, 60, 236] , [108, 10, 152, 109] , [ 27, 155, 154, 23] , [211, 215, 136, 170] , | |
| [355, 275, 2, 20] , [ 90, 89, 96, 97] , [321, 320, 404, 405] , [316, 315, 406, 405] , [107, 44, 203, 205] , | |
| [201, 422, 314, 19] , [153, 176, 172, 149] , [376, 274, 288, 292] , [292, 288, 411, 410] , [130, 204, 166, 99] , | |
| [115, 48, 127, 218] , [327, 328, 461, 329] , [105, 106, 67, 70] , [236, 65, 99, 241] , [200, 201, 202, 209] , | |
| [332, 295, 328, 359] , [100, 61, 76, 241] , [243, 142, 126, 242] , [329, 463, 371, 327] , [220, 167, 80, 219] , | |
| [233, 27, 23, 232] , [190, 222, 57, 191] , [223, 29, 57, 222] , [244, 113, 234, 245] , [ 32, 229, 111, 26] , | |
| [226, 31, 30, 225] , [232, 23, 24, 231] , [225, 30, 28, 224] , [114, 227, 131, 248] , [ 32, 227, 36, 112] , | |
| [234, 113, 27, 233] , [230, 25, 111, 229] , [224, 28, 29, 223] , [ 95, 20, 126, 142] , [239, 240, 80, 21] , | |
| [243, 21, 61, 100] , [157, 71, 72, 140] , [ 76, 61, 167, 60] , [189, 123, 194, 246] , [231, 24, 25, 230] , | |
| [232, 231, 120, 121] , [121, 101, 48, 122] , [208, 206, 51, 188] , [332, 280, 279, 295] , [196, 249, 420, 198] , | |
| [199, 210, 50, 132] , [177, 149, 172, 141] , [117, 124, 118, 112] , [ 28, 160, 159, 29] , [245, 190, 191, 244] , | |
| [379, 396, 370, 401] , [268, 303, 304, 270] , [351, 453, 454, 358] , [ 75, 74, 40, 41] , [169, 418, 286, 9] , | |
| [283, 444, 445, 284] , [397, 176, 153, 378] , [110, 68, 70, 109] , [301, 277, 354, 384] , [186, 62, 77, 185] , | |
| [299, 294, 301, 302] , [ 50, 49, 116, 132] , [422, 201, 200, 429] , [304, 272, 273, 305] , [271, 323, 392, 270] , | |
| [296, 443, 444, 283] , [427, 437, 428, 426] , [336, 322, 406, 407] , [ 19, 314, 315, 18] , [387, 388, 260, 258] , | |
| [255, 374, 375, 254] , [314, 422, 419, 407] , [297, 335, 334, 300] , [313, 312, 272, 269] , [ 55, 22, 72, 69] , | |
| [221, 46, 52, 135] , [391, 374, 255, 340] , [315, 316, 17, 18] , [372, 267, 331, 330] , [423, 274, 336, 425] , | |
| [ 58, 44, 147, 62] , [ 91, 78, 147, 92] , [182, 85, 86, 181] , [423, 425, 432, 431] , [357, 265, 448, 455] , | |
| [268, 270, 392, 394] , [358, 454, 465, 466] , [264, 360, 468, 467] , [264, 250, 256, 360] , [421, 430, 356, 438] , | |
| [194, 123, 7, 169] , [449, 450, 348, 347] , [277, 284, 445, 446] , [241, 99, 98, 100] , [281, 331, 267, 426] , | |
| [307, 292, 410, 409] , [260, 388, 389, 261] , [364, 457, 249, 282] , [338, 339, 11, 152] , [438, 344, 413, 400] , | |
| [349, 451, 452, 350] , [345, 279, 280, 361] , [402, 377, 434, 436] , [367, 324, 455, 448] , [182, 92, 107, 183] , | |
| [418, 414, 442, 286] , [360, 256, 262, 447] , [284, 277, 301, 294] , [291, 251, 463, 329] , [344, 358, 466, 413] , | |
| [179, 89, 90, 180] , [266, 341, 346, 373] , [429, 397, 370, 263] , [296, 283, 335, 297] , [275, 355, 462, 458] , | |
| [ 4, 237, 135, 52] , [359, 424, 267, 372] , [386, 387, 258, 259] , [394, 165, 1, 268] , [207, 217, 187, 93] , | |
| [278, 356, 372, 330] , [ 44, 58, 213, 203] , [459, 460, 458, 462] , [381, 382, 257, 253] , [266, 447, 262, 341] , | |
| [399, 385, 287, 415] , [437, 433, 435, 428] , [447, 266, 354, 343] , [183, 107, 205, 195] , [ 43, 42, 74, 75] , | |
| [302, 301, 384, 369] , [425, 419, 263, 432] , [295, 279, 440, 456] , [ 49, 50, 103, 65] , [ 74, 42, 39, 73] , | |
| [433, 423, 431, 435] , [311, 273, 272, 312] , [353, 367, 448, 346] , [252, 302, 369, 390] , [209, 172, 176, 200] , | |
| [ 56, 194, 169, 9] , [377, 412, 417, 434] , [ 90, 97, 78, 91] , [330, 331, 349, 350] , [180, 90, 91, 181] , | |
| [281, 348, 349, 331] , [265, 373, 346, 448] , [324, 367, 402, 362] , [308, 326, 320, 321] , [ 16, 15, 88, 87] , | |
| [266, 373, 384, 354] , [353, 347, 348, 281] , [363, 399, 415, 464] , [318, 15, 16, 317] , [356, 278, 344, 438] , | |
| [ 96, 79, 63, 97] , [ 11, 110, 109, 152] , [398, 368, 365, 366] , [ 2, 45, 126, 20] , [313, 269, 13, 14] , | |
| [237, 199, 132, 135] , [187, 58, 62, 186] , [152, 10, 337, 338] , [ 42, 82, 83, 39] , [414, 418, 466, 465] , | |
| [467, 468, 261, 389] , [ 9, 286, 337, 10] , [446, 343, 354, 277] , [265, 357, 390, 369] , [436, 434, 417, 368] , | |
| [170, 136, 137, 151] , [458, 441, 276, 275] , [212, 205, 203, 211] , [347, 353, 346, 341] , [284, 294, 335, 283] , | |
| [452, 453, 351, 350] , [ 95, 3, 327, 371] , [450, 451, 349, 348] , [197, 4, 196, 198] , [254, 375, 381, 253] , | |
| [345, 441, 458, 439] , [367, 353, 377, 402] , [449, 347, 341, 262] , [360, 447, 343, 468] , [136, 139, 173, 137] , | |
| [289, 436, 368, 398] , [281, 426, 428, 412] , [288, 433, 437, 411] , [ 99, 166, 168, 98] , [142, 243, 100, 98] , | |
| [175, 237, 4, 197] , [185, 75, 41, 186] , [307, 293, 326, 308] , [396, 432, 263, 370] , [286, 442, 443, 296] , | |
| [428, 435, 417, 412] , [411, 437, 427, 323] , [421, 438, 400, 457] , [165, 3, 98, 168] , [279, 345, 439, 440] , | |
| [391, 340, 256, 250] , [306, 291, 329, 461] , [373, 265, 369, 384] , [386, 259, 287, 385] , [435, 365, 368, 417] , | |
| [251, 459, 462, 463] , [320, 319, 403, 404] , [ 17, 16, 87, 86] , [322, 321, 405, 406] , [ 85, 84, 19, 18] , | |
| [433, 288, 274, 423] , [362, 402, 436, 289] , [185, 77, 63, 184] , [293, 307, 409, 408] , [392, 424, 359, 328] , | |
| [352, 7, 198, 420] , [228, 138, 124, 117] , [393, 290, 456, 440] , [176, 397, 429, 200] , [220, 49, 65, 236] , | |
| [424, 427, 426, 267] , [332, 359, 430, 280] , [365, 435, 431, 395] , [310, 251, 291, 393] , [355, 371, 463, 462] , | |
| [ 98, 3, 95, 142] , [255, 254, 451, 450] , [415, 414, 465, 464] , [254, 253, 452, 451] , [261, 468, 343, 446] , | |
| [260, 261, 446, 445] , [258, 260, 445, 444] , [454, 342, 464, 465] , [198, 7, 123, 197] , [259, 258, 444, 443] , | |
| [287, 442, 414, 415] , [340, 449, 262, 256] , [340, 255, 450, 449] , [257, 342, 454, 453] , [ 61, 21, 80, 167] , | |
| [310, 393, 440, 439] , [338, 337, 297, 300] , [310, 460, 459, 251] , [ 51, 124, 148, 188] , [253, 257, 453, 452] , | |
| [215, 193, 139, 136] , [351, 358, 344, 278] , [113, 156, 155, 27] , [ 6, 52, 46, 5] , [206, 207, 204, 37] , | |
| [249, 196, 6, 282] , [216, 178, 133, 59] , [286, 296, 297, 337] , [382, 383, 342, 257] , [287, 259, 443, 442] , | |
| [211, 170, 171, 212] , [306, 461, 456, 290] , [104, 105, 70, 68] , [271, 305, 409, 410] , [460, 310, 439, 458] , | |
| [214, 216, 139, 193] , [317, 316, 405, 404] , [181, 91, 92, 182] , [ 1, 165, 168, 38] , [363, 464, 342, 383] , | |
| [210, 130, 103, 50] , [305, 273, 408, 409] , [311, 416, 408, 273] , [309, 293, 408, 416] , [184, 63, 79, 192] , | |
| [115, 129, 122, 48] , [148, 124, 138, 178] , [181, 86, 87, 180] , [290, 393, 291, 306] , [252, 285, 299, 302] , | |
| [285, 333, 334, 299] | |
| ] | |
| class face_image_to_face_mesh: | |
| def demo(self): | |
| demo = gr.Blocks() | |
| with demo: | |
| gr.Markdown( | |
| """ | |
| # Face Image to Face Quad Mesh | |
| Uses MediaPipe to detect a face in an image and convert it to a quad mesh. | |
| Currently saves to OBJ, hopefully glb at some point with color data. | |
| The 3d viewer has Y pointing the opposite direction from Blender, so ya hafta spin it. | |
| The initial workflow I was imagining was: | |
| 1. sculpt high poly mesh in blender | |
| 2. snapshot the face | |
| 3. generate the mesh using the mediapipe stuff | |
| 4. import the low poly mediapipe face | |
| 5. snap the mesh to the high poly model | |
| 6. model the rest of the low poly model | |
| 7. bake the normal / etc maps to the low poly face model | |
| 8. it's just that easy 😛 | |
| Ideally it would be a plugin... | |
| """) | |
| with gr.Row(): | |
| with gr.Column(): | |
| upload_image = gr.Image(label="Input image", type="numpy", source="upload") | |
| gr.Examples( examples=[ | |
| 'examples/blonde-00019-1421846474.png', | |
| 'examples/dude-00110-1227390728.png', | |
| 'examples/granny-00056-1867315302.png', | |
| 'examples/tuffie-00039-499759385.png', | |
| ], inputs=[upload_image] ) | |
| upload_image_btn = gr.Button(value="Detect faces") | |
| with gr.Group(): | |
| min_detection_confidence = gr.Slider(label="Min detection confidence", value=0.5, minimum=0.0, maximum=1.0, step=0.01) | |
| gr.Textbox(show_label=False, value="Minimum confidence value ([0.0, 1.0]) from the face detection model for the detection to be considered successful.") | |
| with gr.Column(): | |
| with gr.Group(): | |
| num_faces_detected = gr.Number(label="Number of faces detected", value=0) | |
| output_mesh = gr.Model3D(clear_color=[0.0, 0.0, 0.0, 0.0], label="3D Model") | |
| output_image = gr.Image(label="Output image") | |
| outputs = [output_mesh, output_image, num_faces_detected] | |
| upload_image_btn.click( | |
| fn=self.detect, | |
| inputs=[upload_image, min_detection_confidence], | |
| outputs=outputs | |
| ) | |
| demo.launch() | |
| def detect(self, image, min_detection_confidence): | |
| width = image.shape[1] | |
| height = image.shape[0] | |
| ratio = width / height | |
| mesh = "examples/jackiechan.obj" | |
| drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1) | |
| with mp_face_mesh.FaceMesh( | |
| static_image_mode=True, | |
| max_num_faces=1, | |
| min_detection_confidence=min_detection_confidence) as face_mesh: | |
| results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) | |
| if not results.multi_face_landmarks: | |
| return mesh, image, 0 | |
| annotated_image = image.copy() | |
| for face_landmarks in results.multi_face_landmarks: | |
| mesh = self.toObj(ratio=ratio, landmark_list=face_landmarks) | |
| mp_drawing.draw_landmarks( | |
| image=annotated_image, | |
| landmark_list=face_landmarks, | |
| connections=mp_face_mesh.FACEMESH_TESSELATION, | |
| landmark_drawing_spec=None, | |
| connection_drawing_spec=mp_drawing_styles | |
| .get_default_face_mesh_tesselation_style()) | |
| mp_drawing.draw_landmarks( | |
| image=annotated_image, | |
| landmark_list=face_landmarks, | |
| connections=mp_face_mesh.FACEMESH_CONTOURS, | |
| landmark_drawing_spec=None, | |
| connection_drawing_spec=mp_drawing_styles | |
| .get_default_face_mesh_contours_style()) | |
| return mesh, annotated_image,1 | |
| def toObj( self, ratio: float, landmark_list: landmark_pb2.NormalizedLandmarkList): | |
| print( f'you have such pretty hair' ) | |
| lines = [] | |
| points = self.landmarksToPoints( ratio, landmark_list ) | |
| for point in points: | |
| vertex = "v " + " ".join([str(value) for value in point]) | |
| lines.append( vertex ) | |
| for quad in QUADS: | |
| face = "f " + " ".join([str(vertex) for vertex in quad]) | |
| lines.append( face ) | |
| normal = self.totallyNormal( points[ quad[ 0 ] -1 ], points[ quad[ 1 ] -1 ], points[ quad[ 2 ] -1 ] ) | |
| lines.append( "vn " + " ".join([str(value) for value in normal]) ) | |
| obj_file = tempfile.NamedTemporaryFile(suffix='.obj', delete=False) | |
| output_file = obj_file.name | |
| out = open( output_file, 'w' ) | |
| out.write( '\n'.join( lines ) ) | |
| out.close() | |
| print( f'I know it is special to you so I saved it to {output_file} since we are friends' ) | |
| return output_file | |
| def landmarksToPoints( self, ratio: float, landmark_list: landmark_pb2.NormalizedLandmarkList ): | |
| points = [] | |
| mins = [+np.inf] * 3 | |
| maxs = [-np.inf] * 3 | |
| for idx, landmark in enumerate(landmark_list.landmark): | |
| if ((landmark.HasField('visibility') and | |
| landmark.visibility < _VISIBILITY_THRESHOLD) or | |
| (landmark.HasField('presence') and | |
| landmark.presence < _PRESENCE_THRESHOLD)): | |
| idk_what_to_do_for_this = True | |
| point = [landmark.x * ratio, -landmark.y, -landmark.z]; | |
| for pidx,value in enumerate( point ): | |
| mins[pidx] = min(mins[pidx],value) | |
| maxs[pidx] = max(maxs[pidx],value) | |
| points.append( point ) | |
| mids = [(min_val + max_val) / 2 for min_val, max_val in zip(mins, maxs)] | |
| for idx,point in enumerate( points ): | |
| points[idx] = [(val-mid) for val, mid in zip(point,mids)] | |
| print( f'mins: {mins}' ) | |
| print( f'mids: {mids}' ) | |
| print( f'maxs: {maxs}' ) | |
| return points | |
| def totallyNormal(self, p0, p1, p2): | |
| v1 = np.array(p1) - np.array(p0) | |
| v2 = np.array(p2) - np.array(p0) | |
| normal = np.cross(v1, v2) | |
| normal = normal / np.linalg.norm(normal) | |
| return normal.tolist() | |
| face_image_to_face_mesh().demo() | |