init

.gitignore

@@ -0,0 +1 @@
+/code/shape_predictor_68_face_landmarks.dat

README.md

@@ -1,13 +1 @@
----
-title: Human Health Gradio
-emoji: 🏢
-colorFrom: indigo
-colorTo: red
-sdk: gradio
-sdk_version: 3.29.0
-app_file: app.py
-pinned: false
-license: apache-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## human-health gradio implement

code/app.py

@@ -0,0 +1,118 @@
+import numpy as np
+import gradio as gr
+import cv2
+from model import PhysNet_Model,DeepPhys_Model
+import pandas as pd
+import time
+from physiological_indicators import PhysiologicalIndicators
+from face_detection import FaceDetection
+
+from utils_sig import *
+from gradio_utils import rppg_to_physiological_indicators,fake_diffusion
+from gradio_utils import read_video
+
+
+
+fece_detection = FaceDetection()
+csv_url = './ippg_predict.csv'
+
+
+def video_to_rppg_dynamic(model_choice,path):
+    '''
+    动态进行video到rppg的转换
+    path: video file path
+    #C:\\Users\\74314\\AppData\\Local\\Temp\\gradio\\f50cc35ca9bac3568e36f1f7277a72d5e252ad37\\e0faab5f1b8c05eda32648dd635a97c4.mp4
+    '''
+    print(model_choice,"=======================================")
+    if model_choice=="ContrastPhys":
+        model = PhysNet_Model('./contrast_phys/model_weights.pt')
+    elif model_choice=="DeepPhys":
+        model = DeepPhys_Model('./contrast_phys/PURE_PURE_UBFC_deepphys_Epoch29.pth')
+    else:
+        model = None
+
+    # read video from file path as frames list
+    print("===============================",path)
+    video = read_video(path)
+    # print(video.shape)
+
+    HR_list = []
+    RR_list = []
+    HRV_list = []
+    BO_list = []
+
+    for i in range(0,video.shape[0]-128,30):
+        video_input = video[i:i+128,:,:,:]
+        video_input, ROI1, ROI2, status, face_region = face_detection_ROI(fece_detection, video_input)
+
+        # print(video_input.shape)
+        ippg, face_list = model.predict(video_input)
+        HR,RR,BP,HRV,BO = rppg_to_physiological_indicators(ippg, ROI1, ROI2)
+        HR_list.append(HR)
+        RR_list.append(RR)
+        HRV_list.append(HRV)
+        BO_list.append(BO)
+
+        ippg_pd = pd.DataFrame({
+            "index":range(0,80),
+            "rppg":ippg,
+        })
+        HR_pd = pd.DataFrame({"index":range(0,len(HR_list)),"HR":HR_list})
+        RR_pd = pd.DataFrame({"index":range(0,len(RR_list)),"RR":RR_list})
+        BP_pd = pd.DataFrame({"index":range(0,len(BP)),"BP":BP})
+        HRV_pd = pd.DataFrame({"index":range(0,len(HRV_list)),"HRV":HRV_list})
+        BO_pd = pd.DataFrame({"index":range(0,len(BO_list)),"BO":BO_list})
+        ippg_pd.to_csv(csv_url,index=False)
+        print(len(ippg))
+
+        yield ippg_pd,HR_pd,RR_pd,BP_pd,HRV_pd,BO_pd
+
+
+def get_data():
+    return pd.read_csv(csv_url)
+
+with gr.Blocks() as demo:
+    gr.Markdown("## Calculate rppg from video.") # 使用 Markdown 输出一句话
+    with gr.Tab("Video"): # 新建一个 Tab
+        with gr.Row(): # 同一行排列
+            model_select_video = gr.Dropdown(["ContrastPhys","DeepPhys"],label="Model",info="选择rppg检测模型") # 下拉菜单
+        with gr.Row(): # 同一行排列
+            video_input = gr.Video()
+            # video_output = gr.Video()
+            rppg_output_video =  gr.LinePlot(x="index", y="rppg", y_title="rppg singal", width=600, height=300,label="rppg singal")
+        with gr.Row(): # 同一行排列
+            HR_output_video =  gr.LinePlot(x="index", y="HR", y_title="心率", width=170, height=150,label="心率")
+            RR_output_video =  gr.LinePlot(x="index", y="RR", y_title="呼吸", width=170, height=150,label="呼吸")
+            BP_output_video =  gr.LinePlot(x="index", y="BP", y_title="血压", width=170, height=150,label="血压")
+            HRV_output_video =  gr.LinePlot(x="index", y="HRV", y_title="心率变异性", width=170, height=150,label="心率变异性")
+            BO_output_video =  gr.LinePlot(x="index", y="BO", y_title="血氧", width=170, height=150,label="血氧")
+        text_button = gr.Button("Start",label="开始")
+    with gr.Tab("Webcam"): # 新建一个 Tab
+        with gr.Row(): # 同一行排列
+            model_select_webcam = gr.Dropdown(["ContrastPhys","DeepPhys"],label="Model",info="选择rppg检测模型") # 下拉菜单
+        with gr.Row(): # 同一行排列
+            webcam_input = gr.Video(source="webcam")
+            # video_output = gr.Video()
+            rppg_output_webcam =  gr.LinePlot(x="index", y="rppg", y_title="rppg singal", width=600, height=300,label="rppg singal")
+        with gr.Row(): # 同一行排列
+            HR_output_webcam =  gr.LinePlot(x="index", y="HR", y_title="心率", width=170, height=150,label="心率")
+            RR_output_webcam =  gr.LinePlot(x="index", y="RR", y_title="呼吸", width=170, height=150,label="呼吸")
+            BP_output_webcam =  gr.LinePlot(x="index", y="BP", y_title="血压", width=170, height=150,label="血压")
+            HRV_output_webcam =  gr.LinePlot(x="index", y="HRV", y_title="心率变异性", width=170, height=150,label="心率变异性")
+            BO_output_webcam =  gr.LinePlot(x="index", y="BO", y_title="血氧", width=170, height=150,label="血氧")
+        image_button = gr.Button("Start")
+ 
+    with gr.Accordion("Open for More!"): # 可折叠的组件
+        gr.Markdown("Look at me...")
+ 
+    text_button.click(video_to_rppg_dynamic,
+                       inputs=[model_select_video,video_input],
+                       outputs=[rppg_output_video,HR_output_video,RR_output_video,BP_output_video,HRV_output_video,BO_output_video]) # 按钮绑定相应的槽函数
+    image_button.click(video_to_rppg_dynamic,
+                       inputs=[model_select_webcam,webcam_input],
+                       outputs=[rppg_output_webcam,HR_output_webcam,RR_output_webcam,BP_output_webcam,HRV_output_webcam,BO_output_webcam]) # 按钮绑定相应的槽函数
+ 
+# define queue - required for generators
+demo.queue()
+
+demo.launch(debug=True,share=True)

code/contrast_phys/DeepPhysModel.py

@@ -0,0 +1,125 @@
+"""DeepPhys - 2D Convolutional Attention Network.
+DeepPhys: Video-Based Physiological Measurement Using Convolutional Attention Networks
+ECCV, 2018
+Weixuan Chen, Daniel McDuff
+"""
+
+import torch
+import torch.nn as nn
+
+
+class Attention_mask(nn.Module):
+    def __init__(self):
+        super(Attention_mask, self).__init__()
+
+    def forward(self, x):
+        xsum = torch.sum(x, dim=2, keepdim=True)
+        xsum = torch.sum(xsum, dim=3, keepdim=True)
+        xshape = tuple(x.size())
+        return x / xsum * xshape[2] * xshape[3] * 0.5
+
+    def get_config(self):
+        """May be generated manually. """
+        config = super(Attention_mask, self).get_config()
+        return config
+
+
+class DeepPhys(nn.Module):
+
+    def __init__(self, in_channels=3, nb_filters1=32, nb_filters2=64, kernel_size=3, dropout_rate1=0.25,
+                 dropout_rate2=0.5, pool_size=(2, 2), nb_dense=128, img_size=36):
+        """Definition of DeepPhys.
+        Args:
+          in_channels: the number of input channel. Default: 3
+          img_size: height/width of each frame. Default: 36.
+        Returns:
+          DeepPhys model.
+        """
+        super(DeepPhys, self).__init__()
+        self.in_channels = in_channels
+        self.kernel_size = kernel_size
+        self.dropout_rate1 = dropout_rate1
+        self.dropout_rate2 = dropout_rate2
+        self.pool_size = pool_size
+        self.nb_filters1 = nb_filters1
+        self.nb_filters2 = nb_filters2
+        self.nb_dense = nb_dense
+        # Motion branch convs
+        self.motion_conv1 = nn.Conv2d(self.in_channels, self.nb_filters1, kernel_size=self.kernel_size, padding=(1, 1),
+                                      bias=True)
+        self.motion_conv2 = nn.Conv2d(self.nb_filters1, self.nb_filters1, kernel_size=self.kernel_size, bias=True)
+        self.motion_conv3 = nn.Conv2d(self.nb_filters1, self.nb_filters2, kernel_size=self.kernel_size, padding=(1, 1),
+                                      bias=True)
+        self.motion_conv4 = nn.Conv2d(self.nb_filters2, self.nb_filters2, kernel_size=self.kernel_size, bias=True)
+        # Apperance branch convs
+        self.apperance_conv1 = nn.Conv2d(self.in_channels, self.nb_filters1, kernel_size=self.kernel_size,
+                                         padding=(1, 1), bias=True)
+        self.apperance_conv2 = nn.Conv2d(self.nb_filters1, self.nb_filters1, kernel_size=self.kernel_size, bias=True)
+        self.apperance_conv3 = nn.Conv2d(self.nb_filters1, self.nb_filters2, kernel_size=self.kernel_size,
+                                         padding=(1, 1), bias=True)
+        self.apperance_conv4 = nn.Conv2d(self.nb_filters2, self.nb_filters2, kernel_size=self.kernel_size, bias=True)
+        # Attention layers
+        self.apperance_att_conv1 = nn.Conv2d(self.nb_filters1, 1, kernel_size=1, padding=(0, 0), bias=True)
+        self.attn_mask_1 = Attention_mask()
+        self.apperance_att_conv2 = nn.Conv2d(self.nb_filters2, 1, kernel_size=1, padding=(0, 0), bias=True)
+        self.attn_mask_2 = Attention_mask()
+        # Avg pooling
+        self.avg_pooling_1 = nn.AvgPool2d(self.pool_size)
+        self.avg_pooling_2 = nn.AvgPool2d(self.pool_size)
+        self.avg_pooling_3 = nn.AvgPool2d(self.pool_size)
+        # Dropout layers
+        self.dropout_1 = nn.Dropout(self.dropout_rate1)
+        self.dropout_2 = nn.Dropout(self.dropout_rate1)
+        self.dropout_3 = nn.Dropout(self.dropout_rate1)
+        self.dropout_4 = nn.Dropout(self.dropout_rate2)
+        # Dense layers
+        if img_size == 36:
+            self.final_dense_1 = nn.Linear(3136, self.nb_dense, bias=True)
+        elif img_size == 72:
+            self.final_dense_1 = nn.Linear(16384, self.nb_dense, bias=True)
+        elif img_size == 96:
+            self.final_dense_1 = nn.Linear(30976, self.nb_dense, bias=True)
+        else:
+            raise Exception('Unsupported image size')
+        self.final_dense_2 = nn.Linear(self.nb_dense, 1, bias=True)
+
+    def forward(self, inputs, params=None):
+
+        diff_input = inputs[:, :3, :, :]
+        raw_input = inputs[:, 3:, :, :]
+
+        d1 = torch.tanh(self.motion_conv1(diff_input))
+        d2 = torch.tanh(self.motion_conv2(d1))
+
+        r1 = torch.tanh(self.apperance_conv1(raw_input))
+        r2 = torch.tanh(self.apperance_conv2(r1))
+
+        g1 = torch.sigmoid(self.apperance_att_conv1(r2))
+        g1 = self.attn_mask_1(g1)
+        gated1 = d2 * g1
+
+        d3 = self.avg_pooling_1(gated1)
+        d4 = self.dropout_1(d3)
+
+        r3 = self.avg_pooling_2(r2)
+        r4 = self.dropout_2(r3)
+
+        d5 = torch.tanh(self.motion_conv3(d4))
+        d6 = torch.tanh(self.motion_conv4(d5))
+
+        r5 = torch.tanh(self.apperance_conv3(r4))
+        r6 = torch.tanh(self.apperance_conv4(r5))
+
+        g2 = torch.sigmoid(self.apperance_att_conv2(r6))
+        g2 = self.attn_mask_2(g2)
+        gated2 = d6 * g2
+
+        d7 = self.avg_pooling_3(gated2)
+        d8 = self.dropout_3(d7)
+        d9 = d8.reshape(d8.size(0), -1)
+        d10 = torch.tanh(self.final_dense_1(d9))
+        d11 = self.dropout_4(d10)
+        out = self.final_dense_2(d11)
+
+        return out
+

code/contrast_phys/PURE_PURE_UBFC_contrastphy_Epoch10.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d0b97a71f5509b6ec69691754abafeab3f78e5779601d71b8650fe953c53fa2
+size 3467843

code/contrast_phys/PURE_PURE_UBFC_deepphys_Epoch29.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0dded770839af3bd6bb47f3d21b322d453b53a7eaad7d4c782c9180dd7b2226
+size 8923597

code/contrast_phys/PURE_PURE_UBFC_physnet_diffnormalized_Epoch29.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1138ff94761db7425e0c35581777a2c2215704dd467e353f956f217ed7aa894b
+size 3108222

code/contrast_phys/PhysNetModel.py

@@ -0,0 +1,115 @@
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# -------------------------------------------------------------------------------------------------------------------
+# PhysNet model
+# 
+# the output is an ST-rPPG block rather than a rPPG signal.
+# -------------------------------------------------------------------------------------------------------------------
+class PhysNet(nn.Module):
+    def __init__(self, S=2, in_ch=3):
+        super().__init__()
+
+        self.S = S # S is the spatial dimension of ST-rPPG block
+
+        self.start = nn.Sequential(
+            nn.Conv3d(in_channels=in_ch, out_channels=32, kernel_size=(1, 5, 5), stride=1, padding=(0, 2, 2)),
+            nn.BatchNorm3d(32),
+            nn.ELU()
+        )
+
+        # 1x
+        self.loop1 = nn.Sequential(
+            nn.AvgPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2), padding=0),
+            nn.Conv3d(in_channels=32, out_channels=64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1)),
+            nn.BatchNorm3d(64),
+            nn.ELU(),
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1)),
+            nn.BatchNorm3d(64),
+            nn.ELU()
+        )
+
+        # encoder
+        self.encoder1 = nn.Sequential(
+            nn.AvgPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0),
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1)),
+            nn.BatchNorm3d(64),
+            nn.ELU(),
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1)),
+            nn.BatchNorm3d(64),
+            nn.ELU(),
+        )
+        self.encoder2 = nn.Sequential(
+            nn.AvgPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2), padding=0),
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1)),
+            nn.BatchNorm3d(64),
+            nn.ELU(),
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1)),
+            nn.BatchNorm3d(64),
+            nn.ELU()
+        )
+
+        #
+        self.loop4 = nn.Sequential(
+            nn.AvgPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2), padding=0),
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1)),
+            nn.BatchNorm3d(64),
+            nn.ELU(),
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 3, 3), stride=1, padding=(1, 1, 1)),
+            nn.BatchNorm3d(64),
+            nn.ELU()
+        )
+
+        # decoder to reach back initial temporal length
+        self.decoder1 = nn.Sequential(
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 1, 1), stride=1, padding=(1, 0, 0)),
+            nn.BatchNorm3d(64),
+            nn.ELU(),
+        )
+        self.decoder2 = nn.Sequential(
+            nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(3, 1, 1), stride=1, padding=(1, 0, 0)),
+            nn.BatchNorm3d(64),
+            nn.ELU()
+        )
+                    
+
+        self.end = nn.Sequential(
+            nn.AdaptiveAvgPool3d((None, S, S)),
+            nn.Conv3d(in_channels=64, out_channels=1, kernel_size=(1, 1, 1), stride=1, padding=(0, 0, 0))
+        )
+
+    def forward(self, x):
+        print("physet shape = ====================",x.shape)
+        means = torch.mean(x, dim=(2, 3, 4), keepdim=True)
+        stds = torch.std(x, dim=(2, 3, 4), keepdim=True)
+        x = (x - means) / stds # (B, C, T, 128, 128)
+ 
+        parity = []
+        x = self.start(x) # (B, C, T, 128, 128)
+        x = self.loop1(x) # (B, 64, T, 64, 64)
+        parity.append(x.size(2) % 2)
+        x = self.encoder1(x) # (B, 64, T/2, 32, 32)
+        parity.append(x.size(2) % 2)
+        x = self.encoder2(x) # (B, 64, T/4, 16, 16)
+        x = self.loop4(x) # (B, 64, T/4, 8, 8)
+
+        x = F.interpolate(x, scale_factor=(2, 1, 1)) # (B, 64, T/2, 8, 8)
+        x = self.decoder1(x) # (B, 64, T/2, 8, 8)
+        x = F.pad(x, (0,0,0,0,0,parity[-1]), mode='replicate')
+        x = F.interpolate(x, scale_factor=(2, 1, 1)) # (B, 64, T, 8, 8)
+        x = self.decoder2(x) # (B, 64, T, 8, 8)
+        x = F.pad(x, (0,0,0,0,0,parity[-2]), mode='replicate')
+        x = self.end(x) # (B, 1, T, S, S), ST-rPPG block
+
+        x_list = []
+        for a in range(self.S):
+            for b in range(self.S):
+                x_list.append(x[:,:,:,a,b]) # (B, 1, T)
+
+        x = sum(x_list)/(self.S*self.S) # (B, 1, T)
+        X = torch.cat(x_list+[x], 1) # (B, N, T), flatten all spatial signals to the second dimension
+        print("physet shape output = ====================",X.shape)
+        return X

code/contrast_phys/model_weights.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb1e88a2db11d715a8ec41886a2f66fa40f6e4c9e031ecfd88648b14e563b135
+size 3465714

code/face_detection.py

@@ -0,0 +1,144 @@
+import cv2
+import numpy as np
+import dlib
+from imutils import face_utils
+import imutils
+import joblib
+
+class FaceDetection(object):
+    def __init__(self):
+        self.detector = dlib.get_frontal_face_detector()
+        self.predictor = joblib.load('./shape_predictor_68_face_landmarks_predictor.pkl')
+        self.fa = face_utils.FaceAligner(self.predictor, desiredFaceWidth=256)
+
+    def face_detect(self, frame):
+        #frame = imutils.resize(frame, width=400)
+        face_frame = np.zeros((10, 10, 3), np.uint8)
+        mask = np.zeros((10, 10, 3), np.uint8)
+        ROI1 = np.zeros((10, 10, 3), np.uint8)
+        ROI2 = np.zeros((10, 10, 3), np.uint8)
+        face_region = np.zeros((10, 10, 3), np.uint8)
+        #ROI3 = np.zeros((10, 10, 3), np.uint8)
+        status = False
+        
+        if frame is None:
+            return 
+
+        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+        # detect faces in the grayscale image
+        rects = self.detector(gray, 0)
+        
+
+        # loop over the face detections
+        #for (i, rect) in enumerate(rects): 
+            # determine the facial landmarks for the face region, then
+            # convert the facial landmark (x, y)-coordinates to a NumPy
+            # array
+            
+        #assumpion: only 1 face is detected
+        if len(rects)>0:
+            status = True
+            # shape = self.predictor(gray, rects[0])
+            # shape = face_utils.shape_to_np(shape)
+
+                # convert dlib's rectangle to a OpenCV-style bounding box
+                # [i.e., (x, y, w, h)], then draw the face bounding box
+            (x, y, w, h) = face_utils.rect_to_bb(rects[0])
+            cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 1)
+            if y<0:
+                print("a")
+                return frame, face_frame, ROI1, ROI2, status, mask
+            #if i==0:
+            face_frame = frame[y:y+h,x:x+w]
+            face_region = face_frame.copy()
+                # show the face number
+                #cv2.putText(frame, "Face #{}".format(i + 1), (x - 10, y - 10),
+                #    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
+                # loop over the (x, y)-coordinates for the facial landmarks
+                # and draw them on the image
+                
+            # for (x, y) in shape:
+                # cv2.circle(frame, (x, y), 1, (0, 0, 255), -1) #draw facial landmarks
+            if(face_frame.shape[:2][1] != 0):
+                face_frame = imutils.resize(face_frame,width=256)
+            
+            #print(frame,'|||',gray,'|||',rects[0],type(np.array(rects[0])))
+            face_frame = self.fa.align(frame,gray,rects[0]) # align face
+            
+            grayf = cv2.cvtColor(face_frame, cv2.COLOR_BGR2GRAY)
+            rectsf = self.detector(grayf, 0)
+            
+            if len(rectsf) >0:
+                shape = self.predictor(grayf, rectsf[0])
+                shape = face_utils.shape_to_np(shape)
+                
+                for (a, b) in shape:
+                    cv2.circle(face_frame, (a, b), 1, (0, 0, 255), -1) #draw facial landmarks
+                
+                cv2.rectangle(face_frame,(shape[54][0], shape[29][1]), #draw rectangle on right and left cheeks
+                        (shape[12][0],shape[33][1]), (0,255,0), 0)
+                cv2.rectangle(face_frame, (shape[4][0], shape[29][1]), 
+                        (shape[48][0],shape[33][1]), (0,255,0), 0)
+                
+                ROI1 = face_frame[shape[29][1]:shape[33][1], #right cheek
+                        shape[54][0]:shape[12][0]]
+                        
+                ROI2 =  face_frame[shape[29][1]:shape[33][1], #left cheek
+                        shape[4][0]:shape[48][0]]    
+
+                # ROI3 = face_frame[shape[29][1]:shape[33][1], #nose
+                        # shape[31][0]:shape[35][0]]
+                
+                #get the shape of face for color amplification
+                rshape = np.zeros_like(shape) 
+                rshape = self.face_remap(shape)    
+                mask = np.zeros((face_frame.shape[0], face_frame.shape[1]))
+            
+                cv2.fillConvexPoly(mask, rshape[0:27], 1) 
+                # mask = np.zeros((face_frame.shape[0], face_frame.shape[1],3),np.uint8)
+                # cv2.fillConvexPoly(mask, shape, 1)
+                
+            #cv2.imshow("face align", face_frame)
+            
+            # cv2.rectangle(frame,(shape[54][0], shape[29][1]), #draw rectangle on right and left cheeks
+                    # (shape[12][0],shape[54][1]), (0,255,0), 0)
+            # cv2.rectangle(frame, (shape[4][0], shape[29][1]), 
+                    # (shape[48][0],shape[48][1]), (0,255,0), 0)
+            
+            # ROI1 = frame[shape[29][1]:shape[54][1], #right cheek
+                    # shape[54][0]:shape[12][0]]
+                    
+            # ROI2 =  frame[shape[29][1]:shape[54][1], #left cheek
+                    # shape[4][0]:shape[48][0]]   
+                
+        else:
+            cv2.putText(frame, "No face detected",
+                       (200,200), cv2.FONT_HERSHEY_PLAIN, 1.5, (0, 0, 255),2)
+            status = False
+        return frame, face_frame, ROI1, ROI2, status, mask, face_region
+    
+    # some points in the facial landmarks need to be re-ordered
+    def face_remap(self,shape):
+        remapped_image = shape.copy()
+        # left eye brow
+        remapped_image[17] = shape[26]
+        remapped_image[18] = shape[25]
+        remapped_image[19] = shape[24]
+        remapped_image[20] = shape[23]
+        remapped_image[21] = shape[22]
+        # right eye brow
+        remapped_image[22] = shape[21]
+        remapped_image[23] = shape[20]
+        remapped_image[24] = shape[19]
+        remapped_image[25] = shape[18]
+        remapped_image[26] = shape[17]
+        # neatening 
+        remapped_image[27] = shape[0]
+        
+        remapped_image = cv2.convexHull(shape)
+        return remapped_image       
+        
+        
+        
+        
+

code/gradio_utils.py

@@ -0,0 +1,85 @@
+import numpy as np
+import gradio as gr
+import cv2
+from model import PhysNet_Model
+import pandas as pd
+import time
+from physiological_indicators import PhysiologicalIndicators
+from utils_sig import *
+from face_detection import FaceDetection
+
+## rppg signal to HR,RR,BP,HRV,BO
+def rppg_to_physiological_indicators(rppg,ROI1, ROI2):
+    '''
+    input: rppg signal(list)
+    output:HR,RR,BP,HRV,BO
+    '''
+    indicators = PhysiologicalIndicators()
+    HR,RR = indicators.calculate_heart_rate(rppg,30)
+    # RR = rppg#indicators.calculate_respiratory_rate(rppg,30)
+    BP, max_BP, min_BP = indicators.calculate_blood_pressure(rppg)
+    HRV = indicators.calculate_heart_rate_variability(rppg,30)
+    BO = indicators.calculate_SpO2(ROI1, ROI2)
+    return HR,RR,BP,HRV,BO
+
+
+def fake_diffusion(steps):
+    for _ in range(steps):
+        time.sleep(1)
+        image = np.random.random((600, 600, 3))
+        yield image
+    image = "https://gradio-builds.s3.amazonaws.com/diffusion_image/cute_dog.jpg"
+    yield image
+
+
+def video_to_rppg_statistic(path):
+    '''
+    静态进行video到rppg的转换
+    path: video file path
+    #C:\\Users\\74314\\AppData\\Local\\Temp\\gradio\\f50cc35ca9bac3568e36f1f7277a72d5e252ad37\\e0faab5f1b8c05eda32648dd635a97c4.mp4
+    '''
+    # read video from file path as frames list
+    frames = []
+    cap = cv2.VideoCapture(path)
+    ret = True
+    while ret:
+        ret, img = cap.read() # read one frame from the 'capture' object; img is (H, W, C)
+        if ret:
+            frames.append(img)
+    video = np.stack(frames, axis=0) # dimensions (T, H, W, C)
+    print(video.shape)
+    
+    face, ROI1, ROI2, status, face_region = face_detection_ROI(fece_detection, video)
+
+    video_input = face
+    print(print(video_input.shape))
+    ippg, len_ippg,face_list = model.predict_statistic(video_input)
+    ippg_pd = pd.DataFrame({
+        "index":range(0,len_ippg),
+        "rppg":ippg,
+    })
+    ippg_pd.to_csv(csv_url,index=False)
+    HR,RR,BP,HRV,BO = rppg_to_physiological_indicators(ippg, ROI1, ROI2)
+    return ippg_pd,HR,RR,BP,HRV,BO
+
+
+def read_video(path):
+    '''
+    input: video file path
+    output: video frames list
+    '''
+    # read video from file path as frames list
+    frames = []
+    cap = cv2.VideoCapture(path)
+    ret = True
+    while ret:
+        ret, img = cap.read() # read one frame from the 'capture' object; img is (H, W, C)
+        if ret:
+            frames.append(img)
+            print(img.shape)
+    video = np.stack(frames, axis=0) # dimensions (T, H, W, C)
+    # import skvideo.io  
+    # video = skvideo.io.vread(path)  
+    # print(video.shape)
+
+    return video

code/ippg_predict.csv

@@ -0,0 +1,81 @@
+index,rppg
+0,0.41405636
+1,0.2000318
+2,0.0820328
+3,-0.028137699
+4,-0.049714215
+5,-0.064972766
+6,-0.09421185
+7,-0.114180595
+8,-0.09591034
+9,-0.073588
+10,-0.04313855
+11,0.03085187
+12,0.0934888
+13,0.3231313
+14,0.47150603
+15,0.69849265
+16,0.77421856
+17,0.83780587
+18,0.83160996
+19,0.77588594
+20,0.70812327
+21,0.530922
+22,0.42441338
+23,0.31190425
+24,0.28862673
+25,0.34351623
+26,0.39518428
+27,0.47460616
+28,0.48288664
+29,0.3444392
+30,0.22368154
+31,0.013961956
+32,-0.13188864
+33,-0.4489758
+34,-0.6084776
+35,-0.73801184
+36,-0.69461787
+37,-0.42097837
+38,-0.2688645
+39,0.017156467
+40,0.18636954
+41,0.42036015
+42,0.4912194
+43,0.5223713
+44,0.46916124
+45,0.21872395
+46,0.085381635
+47,-0.071113326
+48,-0.08085425
+49,0.063260555
+50,0.21006191
+51,0.45567837
+52,0.58702725
+53,0.81640875
+54,0.8822088
+55,0.9162681
+56,0.86282206
+57,0.6050979
+58,0.4452159
+59,0.2106748
+60,0.12560174
+61,0.09135161
+62,0.09115855
+63,0.16783619
+64,0.23296665
+65,0.33295977
+66,0.35504892
+67,0.35514745
+68,0.30651757
+69,0.11715238
+70,0.006680727
+71,-0.14071405
+72,-0.1679501
+73,-0.083684474
+74,-0.012030022
+75,0.14085057
+76,0.22960266
+77,0.34104726
+78,0.3676189
+79,0.37382895

code/model.py

@@ -0,0 +1,185 @@
+from contrast_phys.PhysNetModel import PhysNet
+from utils_sig import *
+import matplotlib.pyplot as plt
+
+
+class PhysNet_Model:
+    def __init__(self, model_path):
+        self.device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
+        print("device",self.device)
+        self.model = PhysNet(S=2).to(self.device).eval()
+        # self.model.load_state_dict(torch.load(model_path, map_location=self.device))
+        self.rppg = []
+        self.fps = 30  # 默认30fps
+
+        if "pt" in model_path:
+            print("Testing uses pt!")
+            weight = torch.load(model_path, map_location=self.device)
+            import collections
+            new_dict = collections.OrderedDict((old_key.replace("module.",""), value) for (old_key, value) in weight.items())
+            self.model.load_state_dict(new_dict)
+
+    def predict(self, frame_list):
+        # 模型预测
+        print("model processing")
+        face_list = self.load_data(frame_list[-128:])
+        face_list = self.standardized_data(face_list)
+        face_list_t = torch.tensor(face_list.astype('float32')).to(self.device)
+        print("+++++face_list_t++++++++",face_list_t.shape)  # need [1, 3, 128, 128, 128]
+        rppg = self.model(face_list_t)[:,-1, :]
+        rppg = rppg[0].detach().cpu().numpy()[20:100]
+
+        print("model done")
+        return rppg, face_list
+    
+    def predict_statistic(self, frame_list):
+        # 模型预测
+        print("model processing")
+        face_list = self.load_data(frame_list)
+        face_list = self.standardized_data(face_list)
+        face_list_t = torch.tensor(face_list.astype('float32')).to(self.device)
+        print("+++++face_list_t++++++++",face_list_t.shape)  # need [1, 3, 128, 128, 128]
+        rppg = self.model(face_list_t)[:,-1, :]
+        rppg = rppg[0].detach().cpu().numpy()
+        
+        rppg = rppg[20:len(rppg)-20]
+
+        print("model done")
+        return rppg, len(rppg), face_list
+    
+    def load_data(self,frame_list):
+        # 处理输入的frame_list
+        # face_list = face_detection(frame_list)
+        face_list = []
+        for frame in frame_list:
+            face_frame = cv2.resize(frame.astype('float32'), (128, 128), interpolation=cv2.INTER_AREA)
+            face_list.append(face_frame)
+        face_list = np.array(face_list)  # (D, H, W, C)  (N , C, D, H, W)
+        print("=============    face_list shape  ==============",face_list.shape) # (180, 128, 128, 3)
+        face_list = np.transpose(face_list, (3, 0, 1, 2))  # (C, D, H, W) 
+        face_list = np.array(face_list)[np.newaxis]
+        # face_list = torch.tensor(face_list.astype('float32')).to(device)
+
+        return face_list
+
+    def plot(self):
+        # 创建用于绘制脉搏波图的Matplotlib图形
+        hr, psd_y, psd_x = hr_fft(self.rppg, fs= self.fps)
+        
+        fig, (ax1, ax2) = plt.subplots(2, figsize=(20,10))
+        ax1.plot(np.arange(len(self.rppg))/self.fps, self.rppg)
+        ax1.set_xlabel('time (sec)')
+        ax1.grid('on')
+        ax1.set_title('rPPG waveform')
+
+        ax2.plot(psd_x, psd_y)
+        ax2.set_xlabel('heart rate (bpm)')
+        ax2.set_xlim([40,200])
+        ax2.grid('on')
+        ax2.set_title('PSD')
+        return fig
+    
+    def show(self):
+        # 显示脉搏波图
+        plt.show()
+    
+    def standardized_data(self,data):
+        """Z-score standardization for video data."""
+        data = data - np.mean(data)
+        data = data / np.std(data)
+        data[np.isnan(data)] = 0
+        return data
+    
+from contrast_phys.DeepPhysModel import DeepPhys
+class DeepPhys_Model:
+    def __init__(self, model_path):
+        self.device = torch.device('cuda:0' if torch.cuda.is_available() else "cpu")
+        self.model = DeepPhys(img_size=72).to(self.device).eval()
+        # self.model.load_state_dict(torch.load(model_path, map_location=self.device))
+        self.rppg = []
+        self.fps = 30  # 默认30fps
+
+        if "pt" in model_path:
+            print("Testing uses pt!")
+            weight = torch.load(model_path, map_location=self.device)
+            import collections
+            new_dict = collections.OrderedDict((old_key.replace("module.",""), value) for (old_key, value) in weight.items())
+            self.model.load_state_dict(new_dict)
+
+    def predict(self, frame_list):
+        # 模型预测
+        print("model processing")
+        face_list = self.load_data(frame_list[-180:])
+        face_list_t = torch.tensor(face_list.astype('float32')).to(self.device)
+        print("face_list_t shape =============",face_list_t.shape) # [120, 3, 72, 72]  need [4*180, 6, 72, 72])
+        rppg = self.model(face_list_t).flatten()
+        print("++++++++++++++rppg++++++++++++++++",rppg)
+        rppg = rppg.detach().cpu().numpy()[20:100]
+
+        print("model done")
+        return rppg, face_list
+
+    def load_data(self,frame_list):
+        # 处理输入的frame_list
+        # face_list = face_detection(frame_list)
+        face_list = []
+        for frame in frame_list:
+            face_frame = cv2.resize(frame, (72, 72))
+            face_list.append(face_frame)
+        face_list = np.array(face_list)  # (N, H, W, C) # (180, 72, 72, 3)
+
+        frame_list_standardized = self.standardized_data(face_list) # (180, 72, 72, 3)
+        frame_list_diff_normalize = self.diff_normalize_data(face_list)
+        # concat frame_list_standardized and frame_list_diff_normalize at axis 3
+        face_list = np.concatenate((frame_list_standardized, frame_list_diff_normalize), axis=3) # (180, 72, 72, 6)
+
+        N, H, W, C = face_list.shape
+        # face_list = face_list.view(N * 1, C, H, W)
+
+        face_list = np.transpose(face_list, (0, 3, 1, 2))  # (N, C, H, W)
+        # face_list = np.array(face_list)[np.newaxis]
+        # face_list = torch.tensor(face_list.astype('float32')).to(device)
+
+        return face_list
+
+    def plot(self):
+        # 创建用于绘制脉搏波图的Matplotlib图形
+        hr, psd_y, psd_x = hr_fft(self.rppg, fs=self.fps)
+        
+        fig, (ax1, ax2) = plt.subplots(2, figsize=(20,10))
+        ax1.plot(np.arange(len(self.rppg))/self.fps, self.rppg)
+        ax1.set_xlabel('time (sec)')
+        ax1.grid('on')
+        ax1.set_title('rPPG waveform')
+
+        ax2.plot(psd_x, psd_y)
+        ax2.set_xlabel('heart rate (bpm)')
+        ax2.set_xlim([40,200])
+        ax2.grid('on')
+        ax2.set_title('PSD')
+        return fig
+    
+    def show(self):
+        # 显示脉搏波图
+        plt.show()
+    
+    def standardized_data(self,data):
+        """Z-score standardization for video data."""
+        data = data - np.mean(data)
+        data = data / np.std(data)
+        data[np.isnan(data)] = 0
+        return data
+    
+    def diff_normalize_data(self,data):
+        """Calculate discrete difference in video data along the time-axis and nornamize by its standard deviation."""
+        n, h, w, c = data.shape
+        diffnormalized_len = n - 1
+        diffnormalized_data = np.zeros((diffnormalized_len, h, w, c), dtype=np.float32)
+        diffnormalized_data_padding = np.zeros((1, h, w, c), dtype=np.float32)
+        for j in range(diffnormalized_len - 1):
+            diffnormalized_data[j, :, :, :] = (data[j + 1, :, :, :] - data[j, :, :, :]) / (
+                    data[j + 1, :, :, :] + data[j, :, :, :] + 1e-7)
+        diffnormalized_data = diffnormalized_data / np.std(diffnormalized_data)
+        diffnormalized_data = np.append(diffnormalized_data, diffnormalized_data_padding, axis=0)
+        diffnormalized_data[np.isnan(diffnormalized_data)] = 0
+        return diffnormalized_data

code/model_weight/lgb_model_ppg2bp.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7117d11a361a5fb3fc2fae6ef90b0971d59ff5661323d40c6fc45f2c435803f6
+size 1648716

code/model_weight/lgb_model_threechanel2HR.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98c628828e8c28810798e2d87f619bc5e0e2e1861063f34ceaf6bde1981c0dec
+size 1743173

code/model_weight/lgb_model_threechanel2spo2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a9db82a01900021c3aaad7b903a7aa2eab1207c22999b5d18b63ff4d127c333
+size 520838

code/physiological_indicators.py

@@ -0,0 +1,106 @@
+from utils_sig import *
+import joblib
+import numpy as np
+from lightgbm import LGBMRegressor
+
+import heartpy as hp
+import scipy.signal as sig
+
+class PhysiologicalIndicators:
+    def __init__(self):
+        self.heart_rate = 0
+        self.respiratory_rate = 0
+        self.heart_rate_variability = 0
+        self.SpO2 = 0
+        self.blood_pressure = 0
+    
+    def calculate_heart_rate(self, ippg_data, fps):
+        # 计算心率的代码
+        print("HR processing")
+        self.heart_rate, self.respiratory_rate = hr_fft_2(ippg_data,  fps)
+        # ippg = butter_bandpass(ippg_data, lowcut=0.6, highcut=4, fs=fps)
+        # self.heart_rate, psd_y, psd_x = hr_fft(ippg, fs=fps)
+        print("HR done")
+        return self.heart_rate, self.respiratory_rate
+
+    def calculate_heart_rate_variability(self, ippg_data, fps):
+        # 计算心率变异性的代码
+        # TODO: 实现心率变异性计算
+        self.heart_rate_variability = calculate_hrv(ippg_data, fps)
+        return self.heart_rate_variability
+
+    def calculate_SpO2(self, ROI_list, ROI2_list):
+        # 计算血氧饱和度的代码
+        # TODO: 实现血氧饱和度计算
+        ROI1_SpO2 = RGB_SpO2(ROI_list)
+        ROI2_SpO2 = RGB_SpO2(ROI2_list)
+        self.SpO2 = (ROI1_SpO2 + ROI2_SpO2) / 2
+        return self.SpO2
+
+    def calculate_blood_pressure(self, ippg_data):
+        # 计算血压的代码
+        ippg_data = np.array(ippg_data).reshape(len(ippg_data),1)
+
+        bp_pred = []
+        model_list = joblib.load( './model_weight/lgb_model_ppg2bp.pkl')
+        for model in model_list:
+            result = model.predict(ippg_data)
+            bp_pred.append(result+10)
+        bp_list = np.mean(bp_pred, axis=0)
+        return bp_list,np.max(bp_list),np.min(bp_list)-15
+
+    def calculate_HR(self, ROI_list, ROI2_list):
+        # 计算HR的代码
+        ROI1_HR = RGB_HR(ROI_list)
+        ROI2_HR = RGB_HR(ROI2_list)
+        print("ROI1_HR",ROI1_HR,"ROI2_HR",ROI2_HR)
+        HR = (ROI1_HR + ROI2_HR) / 2
+        return HR
+
+
+    # 定义一个函数来计算心率
+    def calculate_heart_rate_2(self,ppg_signal, sampling_rate):
+        # 使用巴特沃斯滤波器处理信号，去除噪声
+        nyquist_frequency = sampling_rate / 2.0
+        low_cutoff_frequency = 0.5
+        high_cutoff_frequency = 5.0
+        filter_order = 2
+
+        b, a = sig.butter(filter_order, [low_cutoff_frequency/nyquist_frequency, high_cutoff_frequency/nyquist_frequency], btype='band')
+        filtered_signal = sig.filtfilt(b, a, ppg_signal)
+
+        # 计算心率
+        window_length = int(sampling_rate * 0.75)
+        step_size = int(sampling_rate * 0.1)
+        threshold = 0.4
+
+        # 使用峰值检测算法来找到脉冲峰值
+        peak_indexes, _ = sig.find_peaks(filtered_signal, distance=10)
+        print("============================",peak_indexes,sampling_rate)
+        # 计算时间间隔并计算心率
+        # time_intervals = np.diff(peak_indexes) / float(sampling_rate)
+        time_intervals = np.diff(peak_indexes) * 0.045
+        heart_rate = 60.0 / np.mean(time_intervals)
+
+        return heart_rate
+    
+    # 定义一个函数来从rppg信号中计算心率 
+    def calculate_heart_rate_3(self,signal):
+        wd, m = hp.process(signal, sample_rate = 100.0)
+
+        return wd, m
+        
+
+
+    # def calculate_SPO2(self, ippg_chanel_data):
+    #     # 计算血氧的代码
+    #     ippg_chanel_data = np.array(ippg_chanel_data).reshape(len(ippg_chanel_data),6)
+
+    #     SPO2_pred = []
+    #     model_list = joblib.load( './model_weight/lgb_model_threechanel2spo2.pkl')
+    #     for model in model_list:
+    #         result = model.predict(ippg_chanel_data)
+    #         SPO2_pred.append(result)
+    #     SPO2_list = np.mean(SPO2_pred, axis=0)
+    #     return SPO2_list
+

code/shape_predictor_68_face_landmarks_predictor.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:243bb6b31457d69cb814885f8878a4b7c709169b347b51d4e6b0fe7cd8cc6a6c
+size 99693979

code/utils_sig.py

@@ -0,0 +1,247 @@
+import numpy as np
+import math
+import cv2
+import torch
+from scipy import signal
+from scipy.fft import fft
+from scipy.signal import butter, filtfilt
+# from facenet_pytorch import MTCNN
+from face_detection import FaceDetection
+import joblib
+
+
+def butter_bandpass(sig, lowcut, highcut, fs, order=2):
+    # butterworth bandpass filter
+
+    sig = np.reshape(sig, -1)
+    nyq = 0.5 * fs
+    low = lowcut / nyq
+    high = highcut / nyq
+    b, a = butter(order, [low, high], btype='band')
+
+    y = filtfilt(b, a, sig)
+    return y
+
+
+def face_detection(video_list):
+    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
+    mtcnn = MTCNN(device=device)
+
+    face_list = []
+    for t, frame in enumerate(video_list):
+        if t == 0:
+            boxes, _, = mtcnn.detect(
+                frame)  # we only detect face bbox in the first frame, keep it in the following frames.
+        if t == 0:
+            box_len = np.max([boxes[0, 2] - boxes[0, 0], boxes[0, 3] - boxes[0, 1]])
+            box_half_len = np.round(box_len / 2 * 1.1).astype('int')
+        box_mid_y = np.round((boxes[0, 3] + boxes[0, 1]) / 2).astype('int')
+        box_mid_x = np.round((boxes[0, 2] + boxes[0, 0]) / 2).astype('int')
+        cropped_face = frame[box_mid_y - box_half_len:box_mid_y + box_half_len,
+                       box_mid_x - box_half_len:box_mid_x + box_half_len]
+        cropped_face = cv2.resize(cropped_face, (128, 128))
+        face_list.append(cropped_face)
+
+        print('face detection %2d' % (100 * (t + 1) / len(video_list)), '%', end='\r', flush=True)
+
+    face_list = np.array(face_list)  # (T, H, W, C)
+    face_list = np.transpose(face_list, (3, 0, 1, 2))  # (C, T, H, W)
+    face_list = np.array(face_list)[np.newaxis]
+
+    return face_list
+
+
+def face_detection_ROI(face_detection, frame_list):
+    face_frame_list = []
+    ROI1_list = []
+    ROI2_list = []
+    
+    for i in range(0, frame_list.shape[0]):
+        frame = frame_list[i]
+        frame, face_frame, ROI1, ROI2, status, mask, face_region = FaceDetection.face_detect(face_detection, frame)
+        face_frame_list.append(face_frame)
+        ROI1_list.append(ROI1)
+        ROI2_list.append(ROI2)
+    return np.array(face_frame_list), np.array(ROI1_list), np.array(ROI2_list), status, face_region
+
+
+def butter_bandpass(sig, lowcut, highcut, fs, order=2):
+    # butterworth bandpass filter
+
+    sig = np.reshape(sig, -1)
+    nyq = 0.5 * fs
+    low = lowcut / nyq
+    high = highcut / nyq
+    b, a = butter(order, [low, high], btype='band')
+
+    y = filtfilt(b, a, sig)
+    return y
+
+def hr_fft(sig, fs, harmonics_removal=True):
+    # get heart rate by FFT
+    # return both heart rate and PSD
+
+    sig = sig.reshape(-1)
+    sig = sig * signal.windows.hann(sig.shape[0])
+    sig_f = np.abs(fft(sig))
+    low_idx = np.round(0.6 / fs * sig.shape[0]).astype('int')
+    high_idx = np.round(4 / fs * sig.shape[0]).astype('int')
+    sig_f_original = sig_f.copy()
+
+    sig_f[:low_idx] = 0
+    sig_f[high_idx:] = 0
+
+    peak_idx, _ = signal.find_peaks(sig_f)
+    sort_idx = np.argsort(sig_f[peak_idx])
+    sort_idx = sort_idx[::-1]
+
+    peak_idx1 = peak_idx[sort_idx[0]]
+    peak_idx2 = peak_idx[sort_idx[1]]
+
+    f_hr1 = peak_idx1 / sig_f.shape[0] * fs
+    hr1 = f_hr1 * 60
+
+    f_hr2 = peak_idx2 / sig_f.shape[0] * fs
+    hr2 = f_hr2 * 60
+    if harmonics_removal:
+        if np.abs(hr1-2*hr2)<10:
+            hr = hr2
+        else:
+            hr = hr1
+    else:
+        hr = hr1
+
+    x_hr = np.arange(len(sig_f))/len(sig_f)*fs*60
+    return hr, sig_f_original, x_hr
+
+
+def hr_fft_2(processed, fps):
+    L = len(processed)
+    fps = 30#float(L) / (times[-1] - times[-L])  # calculate HR using a true fps of processor of the computer, not the fps the camera provide
+    LEN = int(fps * 1.55)
+    # even_times = np.linspace(times[-L], times[-1], LEN)
+
+    processed = signal.detrend(processed)  # detrend the signal to avoid interference of light change
+    # interpolated = np.interp(even_times, times[-L:], processed)  # interpolation by 1
+    interpolated = processed
+    # interpolated = np.hamming(LEN) * interpolated  # make the signal become more periodic (advoid spectral leakage)
+    norm = (interpolated - np.mean(interpolated))/np.std(interpolated)#normalization
+
+    # norm = interpolated / np.linalg.norm(interpolated)
+    raw = np.fft.rfft(norm * 30)  # do real fft with the normalization multiplied by 10
+    raw_r = raw.copy()
+
+    freqs = float(fps) / LEN * np.arange(LEN / 2 + 1)
+    freqs = 60. * freqs
+
+    idx_remove = np.where((freqs < 50) & (freqs > 180))
+    raw[idx_remove] = 0
+
+    fft = np.abs(raw) ** 2  # get amplitude spectrum
+
+    idx = np.where((freqs > 50) & (freqs < 180))  # the range of frequency that HR is supposed to be within
+    pruned = fft[idx]
+    pfreq = freqs[idx]
+
+    # freqs = pfreq
+    fft = pruned
+
+    idx2 = np.argmax(pruned)  # max in the range can be HR
+    bpm = pfreq[idx2]
+
+    # calculate Respiratory Rate, 计算呼吸���
+    idx_remove = np.where((freqs < 5) & (freqs > 60))
+    raw_r[idx_remove] = 0
+    fft_r = np.abs(raw_r) ** 2
+    idx = np.where((freqs > 5) & (freqs < 60))
+    pruned_r = fft_r[idx]
+    pfreq = freqs[idx]
+    idx3 = np.argmax(pruned_r)
+    pruned_r[idx3] = 0
+    idx3 = np.argmax(pruned_r)
+    respiratory_rate = pfreq[idx3]
+
+    return bpm, respiratory_rate
+
+
+def calc_rr(peaklist, sample_rate, working_data={}):
+    peaklist = np.array(peaklist) #cast numpy array to be sure or correct array type
+    working_data['peaklist'] = peaklist  # Make sure, peaklist is always an np.array
+
+    rr_list = (np.diff(peaklist) / sample_rate) * 1000.0
+    rr_indices = [(peaklist[i], peaklist[i+1]) for i in range(len(peaklist) - 1)]
+    rr_diff = np.abs(np.diff(rr_list))
+    rr_sqdiff = np.power(rr_diff, 2)
+    working_data['RR_list'] = rr_list
+    working_data['RR_indices'] = rr_indices
+    working_data['RR_diff'] = rr_diff
+    working_data['RR_sqdiff'] = rr_sqdiff
+    return working_data
+
+
+def calculate_hrv(ippg, fps):
+    peak_array, _ = signal.find_peaks(ippg[-200::2])  # down sample rate: 2
+    peak_list = peak_array.tolist()
+    result = calc_rr(peak_list, fps/2)
+    # print(peak_list)
+    # RR_list = result['RR_list'].tolist()
+    # RR_diff = result['RR_diff'].tolist()
+    # print(RR_list)
+    # print(RR_diff)
+    RR_std = 0
+    if len(result['RR_list']) > 0:
+        RR_std = np.std(result['RR_list'], ddof=1)  # calculate RR interval standard deviation
+    if math.isnan(RR_std):
+        RR_std = 0
+    return RR_std
+
+def RGB_SpO2(ROI_list):
+    roi_avg = []
+    roi_std = []
+    for i in range(len(ROI_list)):
+        roi_avg.append(np.average(ROI_list[i], axis=(0, 1)))
+        roi_std.append(np.std(ROI_list[i], axis=(0, 1), ddof=1))
+    roi_avg = np.array(roi_avg)
+    roi_std = np.array(roi_std)
+    mean_red = roi_avg[:, 0]
+    std_red = roi_std[:, 0]
+    mean_green = roi_avg[:, 1]
+    std_green = roi_std[:, 1]
+    mean_blue = roi_avg[:, 2]
+    std_blue = roi_std[:, 2]
+    A = -11.2
+    B = 109.3
+    R = (np.average(mean_red) / np.average(std_red)) / (np.average(mean_blue) / np.average(std_blue))
+    SpO2 = A * R + B
+    return SpO2
+
+
+def RGB_HR(ROI_list):
+    roi_avg = []
+    roi_std = []
+    for i in range(len(ROI_list)):
+        roi_avg.append(np.average(ROI_list[i], axis=(0, 1)))
+        roi_std.append(np.std(ROI_list[i], axis=(0, 1), ddof=1))
+    roi_avg = np.array(roi_avg)
+    roi_std = np.array(roi_std)
+    mean_red = roi_avg[:, 0]
+    std_red = roi_std[:, 0]
+    mean_green = roi_avg[:, 1]
+    std_green = roi_std[:, 1]
+    mean_blue = roi_avg[:, 2]
+    std_blue = roi_std[:, 2]
+
+    
+
+    ippg_chanel_data = np.array((mean_red,std_red,mean_green,std_green,mean_blue,std_blue)).T 
+    print(ippg_chanel_data )
+    # ippg_chanel_data = np.array(ippg_chanel_data).reshape(len(ippg_chanel_data),6)
+
+    HR_pred = []
+    model_list = joblib.load( './model_weight/lgb_model_threechanel2HR.pkl')
+    for model in model_list:
+        result = model.predict(ippg_chanel_data)
+        HR_pred.append(result+10)
+    HR = np.mean(HR_pred, axis=0)
+
+    return np.mean(HR)