add deepsort

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+deep_sort/deep_sort/deep/checkpoint/ckpt.t7 filter=lfs diff=lfs merge=lfs -text

deep_sort/configs/deep_sort.yaml

@@ -0,0 +1,10 @@
+DEEPSORT:
+  REID_CKPT: "deep_sort/deep_sort/deep/checkpoint/ckpt.t7"
+  MAX_DIST: 0.2
+  MIN_CONFIDENCE: 0.3
+  NMS_MAX_OVERLAP: 0.5
+  MAX_IOU_DISTANCE: 0.7
+  MAX_AGE: 70
+  N_INIT: 3
+  NN_BUDGET: 100
+  

deep_sort/deep_sort/README.md

@@ -0,0 +1,3 @@
+# Deep Sort 
+
+This is the implemention of deep sort with pytorch.

deep_sort/deep_sort/__init__.py

@@ -0,0 +1,21 @@
+from .deep_sort import DeepSort
+
+
+__all__ = ['DeepSort', 'build_tracker']
+
+
+def build_tracker(cfg, use_cuda):
+    return DeepSort(cfg.DEEPSORT.REID_CKPT, 
+                max_dist=cfg.DEEPSORT.MAX_DIST, min_confidence=cfg.DEEPSORT.MIN_CONFIDENCE, 
+                nms_max_overlap=cfg.DEEPSORT.NMS_MAX_OVERLAP, max_iou_distance=cfg.DEEPSORT.MAX_IOU_DISTANCE, 
+                max_age=cfg.DEEPSORT.MAX_AGE, n_init=cfg.DEEPSORT.N_INIT, nn_budget=cfg.DEEPSORT.NN_BUDGET, use_cuda=use_cuda)
+    
+
+
+
+
+
+
+
+
+

deep_sort/deep_sort/deep/checkpoint/ckpt.t7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22628596f112dc7eb1fe7adfbfaf95bbc6ce8eb024205beafdc705232a646c29
+size 46061055

deep_sort/deep_sort/deep/evaluate.py

@@ -0,0 +1,15 @@
+import torch
+
+features = torch.load("features.pth")
+qf = features["qf"]
+ql = features["ql"]
+gf = features["gf"]
+gl = features["gl"]
+
+scores = qf.mm(gf.t())
+res = scores.topk(5, dim=1)[1][:,0]
+top1correct = gl[res].eq(ql).sum().item()
+
+print("Acc top1:{:.3f}".format(top1correct/ql.size(0)))
+
+

deep_sort/deep_sort/deep/feature_extractor.py

@@ -0,0 +1,65 @@
+import torch
+import torchvision.transforms as transforms
+import numpy as np
+import cv2
+import logging
+
+from .model import Net
+
+'''
+特征提取器：
+提取对应bounding box中的特征, 得到一个固定维度的embedding作为该bounding box的代表，
+供计算相似度时使用。
+
+模型训练是按照传统ReID的方法进行，使用Extractor类的时候输入为一个list的图片，得到图片对应的特征。
+'''
+
+class Extractor(object):
+    def __init__(self, model_path, use_cuda=True):
+        self.net = Net(reid=True)
+        self.device = "cuda" if torch.cuda.is_available() and use_cuda else "cpu"
+        state_dict = torch.load(model_path, map_location=lambda storage, loc: storage)['net_dict']
+        self.net.load_state_dict(state_dict)
+        logger = logging.getLogger("root.tracker")
+        logger.info("Loading weights from {}... Done!".format(model_path))
+        self.net.to(self.device)
+        self.size = (64, 128)
+        self.norm = transforms.Compose([
+            # RGB图片数据范围是[0-255]，需要先经过ToTensor除以255归一化到[0,1]之后，
+            # 再通过Normalize计算(x - mean)/std后，将数据归一化到[-1,1]。
+            transforms.ToTensor(),
+            # mean=[0.485, 0.456, 0.406] and std=[0.229, 0.224, 0.225]是从imagenet训练集中算出来的
+            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+        ])
+        
+    def _preprocess(self, im_crops):
+        """
+        TODO:
+            1. to float with scale from 0 to 1
+            2. resize to (64, 128) as Market1501 dataset did
+            3. concatenate to a numpy array
+            3. to torch Tensor
+            4. normalize
+        """
+        def _resize(im, size):
+            return cv2.resize(im.astype(np.float32)/255., size)
+
+        im_batch = torch.cat([self.norm(_resize(im, self.size)).unsqueeze(0) for im in im_crops], dim=0).float()
+        return im_batch
+
+# __call__()是一个非常特殊的实例方法。该方法的功能类似于在类中重载 () 运算符，
+# 使得类实例对象可以像调用普通函数那样，以“对象名()”的形式使用。
+    def __call__(self, im_crops):
+        im_batch = self._preprocess(im_crops)
+        with torch.no_grad():
+            im_batch = im_batch.to(self.device)
+            features = self.net(im_batch)
+        return features.cpu().numpy()
+
+
+if __name__ == '__main__':
+    img = cv2.imread("demo.jpg")[:,:,(2,1,0)]
+    extr = Extractor("checkpoint/ckpt.t7")
+    feature = extr(img)
+    print(feature.shape)
+

deep_sort/deep_sort/deep/model.py

@@ -0,0 +1,105 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class BasicBlock(nn.Module):
+    def __init__(self, c_in, c_out,is_downsample=False):
+        super(BasicBlock,self).__init__()
+        self.is_downsample = is_downsample
+        if is_downsample:
+            self.conv1 = nn.Conv2d(c_in, c_out, 3, stride=2, padding=1, bias=False)
+        else:
+            self.conv1 = nn.Conv2d(c_in, c_out, 3, stride=1, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(c_out)
+        self.relu = nn.ReLU(True)
+        self.conv2 = nn.Conv2d(c_out,c_out,3,stride=1,padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(c_out)
+        if is_downsample:
+            self.downsample = nn.Sequential(
+                nn.Conv2d(c_in, c_out, 1, stride=2, bias=False),
+                nn.BatchNorm2d(c_out)
+            )
+        elif c_in != c_out:
+            self.downsample = nn.Sequential(
+                nn.Conv2d(c_in, c_out, 1, stride=1, bias=False),
+                nn.BatchNorm2d(c_out)
+            )
+            self.is_downsample = True
+
+    def forward(self,x):
+        y = self.conv1(x)
+        y = self.bn1(y)
+        y = self.relu(y)
+        y = self.conv2(y)
+        y = self.bn2(y)
+        if self.is_downsample:
+            x = self.downsample(x)
+        return F.relu(x.add(y),True)
+
+def make_layers(c_in,c_out,repeat_times, is_downsample=False):
+    blocks = []
+    for i in range(repeat_times):
+        if i ==0:
+            blocks += [BasicBlock(c_in,c_out, is_downsample=is_downsample),]
+        else:
+            blocks += [BasicBlock(c_out,c_out),]
+    return nn.Sequential(*blocks)
+
+class Net(nn.Module):
+    def __init__(self, num_classes=751, reid=False):
+        super(Net,self).__init__()
+        # 3 128 64
+        self.conv = nn.Sequential(
+            nn.Conv2d(3,64,3,stride=1,padding=1),
+            nn.BatchNorm2d(64),
+            nn.ReLU(inplace=True),
+            # nn.Conv2d(32,32,3,stride=1,padding=1),
+            # nn.BatchNorm2d(32),
+            # nn.ReLU(inplace=True),
+            nn.MaxPool2d(3,2,padding=1),
+        )
+        # 32 64 32
+        self.layer1 = make_layers(64,64,2,False)
+        # 32 64 32
+        self.layer2 = make_layers(64,128,2,True)
+        # 64 32 16
+        self.layer3 = make_layers(128,256,2,True)
+        # 128 16 8
+        self.layer4 = make_layers(256,512,2,True)
+        # 256 8 4
+        self.avgpool = nn.AvgPool2d((8,4),1)
+        # 256 1 1 
+        self.reid = reid
+
+        self.classifier = nn.Sequential(
+            nn.Linear(512, 256),
+            nn.BatchNorm1d(256),
+            nn.ReLU(inplace=True),
+            nn.Dropout(),
+            nn.Linear(256, num_classes),
+        )
+    
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        x = self.avgpool(x)
+        x = x.view(x.size(0),-1)
+        # B x 128
+        if self.reid:
+            x = x.div(x.norm(p=2,dim=1,keepdim=True))
+            return x
+        # classifier
+        x = self.classifier(x)
+        return x
+
+
+if __name__ == '__main__':
+    net = Net()
+    x = torch.randn(4,3,128,64)
+    y = net(x)
+    import ipdb; ipdb.set_trace()
+
+

deep_sort/deep_sort/deep/original_model.py

@@ -0,0 +1,106 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class BasicBlock(nn.Module):
+    def __init__(self, c_in, c_out,is_downsample=False):
+        super(BasicBlock,self).__init__()
+        self.is_downsample = is_downsample
+        if is_downsample:
+            self.conv1 = nn.Conv2d(c_in, c_out, 3, stride=2, padding=1, bias=False)
+        else:
+            self.conv1 = nn.Conv2d(c_in, c_out, 3, stride=1, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(c_out)
+        self.relu = nn.ReLU(True)
+        self.conv2 = nn.Conv2d(c_out,c_out,3,stride=1,padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(c_out)
+        if is_downsample:
+            self.downsample = nn.Sequential(
+                nn.Conv2d(c_in, c_out, 1, stride=2, bias=False),
+                nn.BatchNorm2d(c_out)
+            )
+        elif c_in != c_out:
+            self.downsample = nn.Sequential(
+                nn.Conv2d(c_in, c_out, 1, stride=1, bias=False),
+                nn.BatchNorm2d(c_out)
+            )
+            self.is_downsample = True
+
+    def forward(self,x):
+        y = self.conv1(x)
+        y = self.bn1(y)
+        y = self.relu(y)
+        y = self.conv2(y)
+        y = self.bn2(y)
+        if self.is_downsample:
+            x = self.downsample(x)
+        return F.relu(x.add(y),True)
+
+def make_layers(c_in,c_out,repeat_times, is_downsample=False):
+    blocks = []
+    for i in range(repeat_times):
+        if i ==0:
+            blocks += [BasicBlock(c_in,c_out, is_downsample=is_downsample),]
+        else:
+            blocks += [BasicBlock(c_out,c_out),]
+    return nn.Sequential(*blocks)
+
+class Net(nn.Module):
+    def __init__(self, num_classes=625 ,reid=False):
+        super(Net,self).__init__()
+        # 3 128 64
+        self.conv = nn.Sequential(
+            nn.Conv2d(3,32,3,stride=1,padding=1),
+            nn.BatchNorm2d(32),
+            nn.ELU(inplace=True),
+            nn.Conv2d(32,32,3,stride=1,padding=1),
+            nn.BatchNorm2d(32),
+            nn.ELU(inplace=True),
+            nn.MaxPool2d(3,2,padding=1),
+        )
+        # 32 64 32
+        self.layer1 = make_layers(32,32,2,False)
+        # 32 64 32
+        self.layer2 = make_layers(32,64,2,True)
+        # 64 32 16
+        self.layer3 = make_layers(64,128,2,True)
+        # 128 16 8
+        self.dense = nn.Sequential(
+            nn.Dropout(p=0.6),
+            nn.Linear(128*16*8, 128),
+            nn.BatchNorm1d(128),
+            nn.ELU(inplace=True)
+        )
+        # 256 1 1 
+        self.reid = reid
+        self.batch_norm = nn.BatchNorm1d(128)
+        self.classifier = nn.Sequential(
+            nn.Linear(128, num_classes),
+        )
+    
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+
+        x = x.view(x.size(0),-1)
+        if self.reid:
+            x = self.dense[0](x)
+            x = self.dense[1](x)
+            x = x.div(x.norm(p=2,dim=1,keepdim=True))
+            return x
+        x = self.dense(x)
+        # B x 128
+        # classifier
+        x = self.classifier(x)
+        return x
+
+
+if __name__ == '__main__':
+    net = Net(reid=True)
+    x = torch.randn(4,3,128,64)
+    y = net(x)
+    import ipdb; ipdb.set_trace()
+
+

deep_sort/deep_sort/deep/prepare_car.py

@@ -0,0 +1,129 @@
+# -*- coding:utf8 -*-
+
+import os
+from PIL import Image
+from shutil import copyfile, copytree, rmtree, move
+
+PATH_DATASET = './car-dataset' # 需要处理的文件夹
+PATH_NEW_DATASET = './car-reid-dataset' # 处理后的文件夹
+PATH_ALL_IMAGES = PATH_NEW_DATASET + '/all_images'
+PATH_TRAIN = PATH_NEW_DATASET + '/train'
+PATH_TEST = PATH_NEW_DATASET + '/test'
+
+# 定义创建目录函数
+def mymkdir(path):
+    path = path.strip() # 去除首位空格
+    path = path.rstrip("\\") # 去除尾部 \ 符号
+    isExists = os.path.exists(path) # 判断路径是否存在
+    if not isExists:
+        os.makedirs(path) # 如果不存在则创建目录
+        print(path + ' 创建成功')
+        return True
+    else:
+        # 如果目录存在则不创建，并提示目录已存在
+        print(path + ' 目录已存在')
+        return False
+
+class BatchRename():
+    '''
+    批量重命名文件夹中的图片文件
+    '''
+
+    def __init__(self):
+        self.path = PATH_DATASET # 表示需要命名处理的文件夹
+
+    # 修改图像尺寸
+    def resize(self):
+        for aroot, dirs, files in os.walk(self.path):
+            # aroot是self.path目录下的所有子目录（含self.path）,dir是self.path下所有的文件夹的列表.
+            filelist = files  # 注意此处仅是该路径下的其中一个列表
+            # print('list', list)
+
+            # filelist = os.listdir(self.path) #获取文件路径
+            total_num = len(filelist)  # 获取文件长度（个数）
+
+            for item in filelist:
+                if item.endswith('.jpg'):  # 初始的图片的格式为jpg格式的（或者源文件是png格式及其他格式，后面的转换格式就可以调整为自己需要的格式即可）
+                    src = os.path.join(os.path.abspath(aroot), item)
+
+                    # 修改图片尺寸到128宽*256高
+                    im = Image.open(src)
+                    out = im.resize((128, 256), Image.ANTIALIAS)  # resize image with high-quality
+                    out.save(src)  # 原路径保存
+
+    def rename(self):
+
+        for aroot, dirs, files in os.walk(self.path):
+            # aroot是self.path目录下的所有子目录（含self.path）,dir是self.path下所有的文件夹的列表.
+            filelist = files  # 注意此处仅是该路径下的其中一个列表
+            # print('list', list)
+
+            # filelist = os.listdir(self.path) #获取文件路径
+            total_num = len(filelist)  # 获取文件长度（个数）
+
+            i = 1  # 表示文件的命名是从1开始的
+            for item in filelist:
+                if item.endswith('.jpg'):  # 初始的图片的格式为jpg格式的（或者源文件是png格式及其他格式，后面的转换格式就可以调整为自己需要的格式即可）
+                    src = os.path.join(os.path.abspath(aroot), item)
+
+                    # 根据图片名创建图片目录
+                    dirname = str(item.split('_')[0])
+                    # 为相同车辆创建目录
+                    #new_dir = os.path.join(self.path, '..', 'bbox_all', dirname)
+                    new_dir = os.path.join(PATH_ALL_IMAGES, dirname)
+                    if not os.path.isdir(new_dir):
+                        mymkdir(new_dir)
+
+                    # 获得new_dir中的图片数
+                    num_pic = len(os.listdir(new_dir))
+
+                    dst = os.path.join(os.path.abspath(new_dir),
+                                       dirname + 'C1T0001F' + str(num_pic + 1) + '.jpg')
+                    # 处理后的格式也为jpg格式的，当然这里可以改成png格式    C1T0001F见mars.py filenames 相机ID，跟踪指数
+                    # dst = os.path.join(os.path.abspath(self.path), '0000' + format(str(i), '0>3s') + '.jpg')    这种情况下的命名格式为0000000.jpg形式，可以自主定义想要的格式
+                    try:
+                        copyfile(src, dst) #os.rename(src, dst)
+                        print ('converting %s to %s ...' % (src, dst))
+                        i = i + 1
+                    except:
+                        continue
+            print ('total %d to rename & converted %d jpgs' % (total_num, i))
+            
+    def split(self):
+        #---------------------------------------
+        #train_test
+        images_path = PATH_ALL_IMAGES
+        train_save_path = PATH_TRAIN
+        test_save_path = PATH_TEST
+        if not os.path.isdir(train_save_path):
+            os.mkdir(train_save_path)
+            os.mkdir(test_save_path)
+        
+        for _, dirs, _ in os.walk(images_path, topdown=True):
+            for i, dir in enumerate(dirs):
+                for root, _, files in os.walk(images_path + '/' + dir, topdown=True):
+                    for j, file in enumerate(files):
+                        if(j==0): # test dataset；每个车辆的第一幅图片
+                            print("序号：%s  文件夹： %s  图片：%s ��为测试集" % (i + 1, root, file))
+                            src_path = root + '/' + file
+                            dst_dir = test_save_path + '/' + dir
+                            if not os.path.isdir(dst_dir):
+                                os.mkdir(dst_dir)
+                            dst_path = dst_dir + '/' + file
+                            move(src_path, dst_path)
+                        else:
+                            src_path = root + '/' + file
+                            dst_dir = train_save_path + '/' + dir
+                            if not os.path.isdir(dst_dir):
+                                os.mkdir(dst_dir)
+                            dst_path = dst_dir + '/' + file
+                            move(src_path, dst_path)
+        rmtree(PATH_ALL_IMAGES)
+        
+if __name__ == '__main__':
+    demo = BatchRename()
+    demo.resize()
+    demo.rename()
+    demo.split()
+
+

deep_sort/deep_sort/deep/prepare_person.py

@@ -0,0 +1,108 @@
+import os
+from shutil import copyfile
+
+# You only need to change this line to your dataset download path
+download_path = './Market-1501-v15.09.15'
+
+if not os.path.isdir(download_path):
+    print('please change the download_path')
+
+save_path = download_path + '/pytorch'
+if not os.path.isdir(save_path):
+    os.mkdir(save_path)
+#-----------------------------------------
+#query
+query_path = download_path + '/query'
+query_save_path = download_path + '/pytorch/query'
+if not os.path.isdir(query_save_path):
+    os.mkdir(query_save_path)
+
+for root, dirs, files in os.walk(query_path, topdown=True):
+    for name in files:
+        if not name[-3:]=='jpg':
+            continue
+        ID  = name.split('_')
+        src_path = query_path + '/' + name
+        dst_path = query_save_path + '/' + ID[0] 
+        if not os.path.isdir(dst_path):
+            os.mkdir(dst_path)
+        copyfile(src_path, dst_path + '/' + name)
+
+#-----------------------------------------
+#multi-query
+query_path = download_path + '/gt_bbox'
+# for dukemtmc-reid, we do not need multi-query
+if os.path.isdir(query_path):
+    query_save_path = download_path + '/pytorch/multi-query'
+    if not os.path.isdir(query_save_path):
+        os.mkdir(query_save_path)
+
+    for root, dirs, files in os.walk(query_path, topdown=True):
+        for name in files:
+            if not name[-3:]=='jpg':
+                continue
+            ID  = name.split('_')
+            src_path = query_path + '/' + name
+            dst_path = query_save_path + '/' + ID[0]
+            if not os.path.isdir(dst_path):
+                os.mkdir(dst_path)
+            copyfile(src_path, dst_path + '/' + name)
+
+#-----------------------------------------
+#gallery
+gallery_path = download_path + '/bounding_box_test'
+gallery_save_path = download_path + '/pytorch/gallery'
+if not os.path.isdir(gallery_save_path):
+    os.mkdir(gallery_save_path)
+
+for root, dirs, files in os.walk(gallery_path, topdown=True):
+    for name in files:
+        if not name[-3:]=='jpg':
+            continue
+        ID  = name.split('_')
+        src_path = gallery_path + '/' + name
+        dst_path = gallery_save_path + '/' + ID[0]
+        if not os.path.isdir(dst_path):
+            os.mkdir(dst_path)
+        copyfile(src_path, dst_path + '/' + name)
+
+#---------------------------------------
+#train_all
+train_path = download_path + '/bounding_box_train'
+train_save_path = download_path + '/pytorch/train_all'
+if not os.path.isdir(train_save_path):
+    os.mkdir(train_save_path)
+
+for root, dirs, files in os.walk(train_path, topdown=True):
+    for name in files:
+        if not name[-3:]=='jpg':
+            continue
+        ID  = name.split('_')
+        src_path = train_path + '/' + name
+        dst_path = train_save_path + '/' + ID[0]
+        if not os.path.isdir(dst_path):
+            os.mkdir(dst_path)
+        copyfile(src_path, dst_path + '/' + name)
+
+
+#---------------------------------------
+#train_val
+train_path = download_path + '/bounding_box_train'
+train_save_path = download_path + '/pytorch/train'
+val_save_path = download_path + '/pytorch/test'
+if not os.path.isdir(train_save_path):
+    os.mkdir(train_save_path)
+    os.mkdir(val_save_path)
+
+for root, dirs, files in os.walk(train_path, topdown=True):
+    for name in files:
+        if not name[-3:]=='jpg':
+            continue
+        ID  = name.split('_')
+        src_path = train_path + '/' + name
+        dst_path = train_save_path + '/' + ID[0]
+        if not os.path.isdir(dst_path):
+            os.mkdir(dst_path)
+            dst_path = val_save_path + '/' + ID[0]  #first image is used as val image
+            os.mkdir(dst_path)
+        copyfile(src_path, dst_path + '/' + name)

deep_sort/deep_sort/deep/test.py

@@ -0,0 +1,77 @@
+import torch
+import torch.backends.cudnn as cudnn
+import torchvision
+
+import argparse
+import os
+
+from model import Net
+
+parser = argparse.ArgumentParser(description="Train on market1501")
+parser.add_argument("--data-dir",default='data',type=str)
+parser.add_argument("--no-cuda",action="store_true")
+parser.add_argument("--gpu-id",default=0,type=int)
+args = parser.parse_args()
+
+# device
+device = "cuda:{}".format(args.gpu_id) if torch.cuda.is_available() and not args.no_cuda else "cpu"
+if torch.cuda.is_available() and not args.no_cuda:
+    cudnn.benchmark = True
+
+# data loader
+root = args.data_dir
+query_dir = os.path.join(root,"query")
+gallery_dir = os.path.join(root,"gallery")
+transform = torchvision.transforms.Compose([
+    torchvision.transforms.Resize((128,64)),
+    torchvision.transforms.ToTensor(),
+    torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+])
+queryloader = torch.utils.data.DataLoader(
+    torchvision.datasets.ImageFolder(query_dir, transform=transform),
+    batch_size=64, shuffle=False
+)
+galleryloader = torch.utils.data.DataLoader(
+    torchvision.datasets.ImageFolder(gallery_dir, transform=transform),
+    batch_size=64, shuffle=False
+)
+
+# net definition
+net = Net(reid=True)
+assert os.path.isfile("./checkpoint/ckpt.t7"), "Error: no checkpoint file found!"
+print('Loading from checkpoint/ckpt.t7')
+checkpoint = torch.load("./checkpoint/ckpt.t7")
+net_dict = checkpoint['net_dict']
+net.load_state_dict(net_dict, strict=False)
+net.eval()
+net.to(device)
+
+# compute features
+query_features = torch.tensor([]).float()
+query_labels = torch.tensor([]).long()
+gallery_features = torch.tensor([]).float()
+gallery_labels = torch.tensor([]).long()
+
+with torch.no_grad():
+    for idx,(inputs,labels) in enumerate(queryloader):
+        inputs = inputs.to(device)
+        features = net(inputs).cpu()
+        query_features = torch.cat((query_features, features), dim=0)
+        query_labels = torch.cat((query_labels, labels))
+
+    for idx,(inputs,labels) in enumerate(galleryloader):
+        inputs = inputs.to(device)
+        features = net(inputs).cpu()
+        gallery_features = torch.cat((gallery_features, features), dim=0)
+        gallery_labels = torch.cat((gallery_labels, labels))
+
+gallery_labels -= 2
+
+# save features
+features = {
+    "qf": query_features,
+    "ql": query_labels,
+    "gf": gallery_features,
+    "gl": gallery_labels
+}
+torch.save(features,"features.pth")

deep_sort/deep_sort/deep/train.py

@@ -0,0 +1,192 @@
+import argparse
+import os
+import time
+
+import numpy as np
+import matplotlib.pyplot as plt
+import torch
+import torch.backends.cudnn as cudnn
+import torchvision
+
+from model import Net
+
+parser = argparse.ArgumentParser(description="Train on market1501")
+parser.add_argument("--data-dir",default='data',type=str)
+parser.add_argument("--no-cuda",action="store_true")
+parser.add_argument("--gpu-id",default=0,type=int)
+parser.add_argument("--lr",default=0.1, type=float)
+parser.add_argument("--interval",'-i',default=20,type=int)
+parser.add_argument('--resume', '-r',action='store_true')
+args = parser.parse_args()
+
+# device
+device = "cuda:{}".format(args.gpu_id) if torch.cuda.is_available() and not args.no_cuda else "cpu"
+if torch.cuda.is_available() and not args.no_cuda:
+    cudnn.benchmark = True
+
+# data loading
+root = args.data_dir
+train_dir = os.path.join(root,"train")
+test_dir = os.path.join(root,"test")
+
+transform_train = torchvision.transforms.Compose([
+    torchvision.transforms.RandomCrop((128,64),padding=4),
+    torchvision.transforms.RandomHorizontalFlip(),
+    torchvision.transforms.ToTensor(),
+    torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+])
+transform_test = torchvision.transforms.Compose([
+    torchvision.transforms.Resize((128,64)),
+    torchvision.transforms.ToTensor(),
+    torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+])
+trainloader = torch.utils.data.DataLoader(
+    torchvision.datasets.ImageFolder(train_dir, transform=transform_train),
+    batch_size=64,shuffle=True
+)
+testloader = torch.utils.data.DataLoader(
+    torchvision.datasets.ImageFolder(test_dir, transform=transform_test),
+    batch_size=64,shuffle=True
+)
+num_classes = max(len(trainloader.dataset.classes), len(testloader.dataset.classes))
+print("num_classes = %s" %num_classes)
+
+# net definition
+start_epoch = 0
+net = Net(num_classes=num_classes)
+if args.resume:
+    assert os.path.isfile("./checkpoint/ckpt.t7"), "Error: no checkpoint file found!"
+    print('Loading from checkpoint/ckpt.t7')
+    checkpoint = torch.load("./checkpoint/ckpt.t7")
+    # import ipdb; ipdb.set_trace()
+    net_dict = checkpoint['net_dict']
+    net.load_state_dict(net_dict)
+    best_acc = checkpoint['acc']
+    start_epoch = checkpoint['epoch']
+net.to(device)
+
+# loss and optimizer
+criterion = torch.nn.CrossEntropyLoss()
+optimizer = torch.optim.SGD(net.parameters(), args.lr, momentum=0.9, weight_decay=5e-4)
+best_acc = 0.
+
+# train function for each epoch
+def train(epoch):
+    print("\nEpoch : %d"%(epoch+1))
+    net.train()
+    training_loss = 0.
+    train_loss = 0.
+    correct = 0
+    total = 0
+    interval = args.interval
+    start = time.time()
+    for idx, (inputs, labels) in enumerate(trainloader):
+        # forward
+        inputs,labels = inputs.to(device),labels.to(device)
+        outputs = net(inputs)
+        loss = criterion(outputs, labels)
+
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        # accumurating
+        training_loss += loss.item()
+        train_loss += loss.item()
+        correct += outputs.max(dim=1)[1].eq(labels).sum().item()
+        total += labels.size(0)
+
+        # print 
+        if (idx+1)%interval == 0:
+            end = time.time()
+            print("[progress:{:.1f}%]time:{:.2f}s Loss:{:.5f} Correct:{}/{} Acc:{:.3f}%".format(
+                100.*(idx+1)/len(trainloader), end-start, training_loss/interval, correct, total, 100.*correct/total
+            ))
+            training_loss = 0.
+            start = time.time()
+    
+    return train_loss/len(trainloader), 1.- correct/total
+
+def test(epoch):
+    global best_acc
+    net.eval()
+    test_loss = 0.
+    correct = 0
+    total = 0
+    start = time.time()
+    with torch.no_grad():
+        for idx, (inputs, labels) in enumerate(testloader):
+            inputs, labels = inputs.to(device), labels.to(device)
+            outputs = net(inputs)
+            loss = criterion(outputs, labels)
+
+            test_loss += loss.item()
+            correct += outputs.max(dim=1)[1].eq(labels).sum().item()
+            total += labels.size(0)
+        
+        print("Testing ...")
+        end = time.time()
+        print("[progress:{:.1f}%]time:{:.2f}s Loss:{:.5f} Correct:{}/{} Acc:{:.3f}%".format(
+                100.*(idx+1)/len(testloader), end-start, test_loss/len(testloader), correct, total, 100.*correct/total
+            ))
+
+    # saving checkpoint
+    acc = 100.*correct/total
+    if acc > best_acc:
+        best_acc = acc
+        print("Saving parameters to checkpoint/ckpt.t7")
+        checkpoint = {
+            'net_dict':net.state_dict(),
+            'acc':acc,
+            'epoch':epoch,
+        }
+        if not os.path.isdir('checkpoint'):
+            os.mkdir('checkpoint')
+        torch.save(checkpoint, './checkpoint/ckpt.t7')
+
+    return test_loss/len(testloader), 1.- correct/total
+
+# plot figure
+x_epoch = []
+record = {'train_loss':[], 'train_err':[], 'test_loss':[], 'test_err':[]}
+fig = plt.figure()
+ax0 = fig.add_subplot(121, title="loss")
+ax1 = fig.add_subplot(122, title="top1err")
+def draw_curve(epoch, train_loss, train_err, test_loss, test_err):
+    global record
+    record['train_loss'].append(train_loss)
+    record['train_err'].append(train_err)
+    record['test_loss'].append(test_loss)
+    record['test_err'].append(test_err)
+
+    x_epoch.append(epoch)
+    ax0.plot(x_epoch, record['train_loss'], 'bo-', label='train')
+    ax0.plot(x_epoch, record['test_loss'], 'ro-', label='val')
+    ax1.plot(x_epoch, record['train_err'], 'bo-', label='train')
+    ax1.plot(x_epoch, record['test_err'], 'ro-', label='val')
+    if epoch == 0:
+        ax0.legend()
+        ax1.legend()
+    fig.savefig("train.jpg")
+
+# lr decay
+def lr_decay():
+    global optimizer
+    for params in optimizer.param_groups:
+        params['lr'] *= 0.1
+        lr = params['lr']
+        print("Learning rate adjusted to {}".format(lr))
+
+def main():
+    total_epoches = 40
+    for epoch in range(start_epoch, start_epoch+total_epoches):
+        train_loss, train_err = train(epoch)
+        test_loss, test_err = test(epoch)
+        draw_curve(epoch, train_loss, train_err, test_loss, test_err)
+        if (epoch+1)%(total_epoches//2)==0:
+            lr_decay()
+
+
+if __name__ == '__main__':
+    main()

deep_sort/deep_sort/deep_sort.py

@@ -0,0 +1,125 @@
+import numpy as np
+import torch
+
+from .deep.feature_extractor import Extractor
+from .sort.nn_matching import NearestNeighborDistanceMetric
+from .sort.preprocessing import non_max_suppression
+from .sort.detection import Detection
+from .sort.tracker import Tracker
+
+
+__all__ = ['DeepSort'] # __all__ 提供了暴露接口用的”白名单“
+
+
+class DeepSort(object):
+    def __init__(self, model_path, max_dist=0.2, min_confidence=0.3, nms_max_overlap=1.0, max_iou_distance=0.7, max_age=70, n_init=3, nn_budget=100, use_cuda=True):
+        self.min_confidence = min_confidence # 检测结果置信度阈值 
+        self.nms_max_overlap = nms_max_overlap # 非极大抑制阈值，设置为1代表不进行抑制
+
+        self.extractor = Extractor(model_path, use_cuda=use_cuda) # 用于提取一个batch图片对应的特征
+
+        max_cosine_distance = max_dist # 最大余弦距离，用于级联匹配，如果大于该阈值，则忽略
+        nn_budget = 100 # 每个类别gallery最多的外观描述子的个数，如果超过，删除旧的
+        # NearestNeighborDistanceMetric 最近邻距离度量
+        # 对于每个目标，返回到目前为止已观察到的任何样本的最近距离（欧式或余弦）。
+        # 由距离度量方法构造一个 Tracker。
+        # 第一个参数可选'cosine' or 'euclidean'
+        metric = NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
+        self.tracker = Tracker(metric, max_iou_distance=max_iou_distance, max_age=max_age, n_init=n_init)
+
+    def update(self, bbox_xywh, confidences, ori_img):
+        self.height, self.width = ori_img.shape[:2]
+        # generate detections
+        # 从原图中抠取bbox对应图片并计算得到相应的特征
+        features = self._get_features(bbox_xywh, ori_img)
+        bbox_tlwh = self._xywh_to_tlwh(bbox_xywh)        
+        # 筛选掉小于min_confidence的目标，并构造一个Detection对象构成的列表
+        detections = [Detection(bbox_tlwh[i], conf, features[i]) for i,conf in enumerate(confidences) if conf>self.min_confidence]
+
+        # run on non-maximum supression
+        boxes = np.array([d.tlwh for d in detections])
+        scores = np.array([d.confidence for d in detections])
+        indices = non_max_suppression(boxes, self.nms_max_overlap, scores)
+        detections = [detections[i] for i in indices]
+
+        # update tracker
+        self.tracker.predict() # 将跟踪状态分布向前传播一步
+        self.tracker.update(detections) # 执行测量更新和跟踪管理
+
+        # output bbox identities
+        outputs = []
+        for track in self.tracker.tracks:
+            if not track.is_confirmed() or track.time_since_update > 1:
+                continue
+            box = track.to_tlwh()
+            x1,y1,x2,y2 = self._tlwh_to_xyxy(box)
+            track_id = track.track_id
+            outputs.append(np.array([x1,y1,x2,y2,track_id], dtype=np.int16))
+        if len(outputs) > 0:
+            outputs = np.stack(outputs,axis=0)
+        return outputs
+
+
+    """
+    TODO:
+        Convert bbox from xc_yc_w_h to xtl_ytl_w_h
+    Thanks JieChen91@github.com for reporting this bug!
+    """
+    #将bbox的[x,y,w,h] 转换成[t,l,w,h]
+    @staticmethod
+    def _xywh_to_tlwh(bbox_xywh):
+        if isinstance(bbox_xywh, np.ndarray):
+            bbox_tlwh = bbox_xywh.copy()
+        elif isinstance(bbox_xywh, torch.Tensor):
+            bbox_tlwh = bbox_xywh.clone()
+        bbox_tlwh[:,0] = bbox_xywh[:,0] - bbox_xywh[:,2]/2.
+        bbox_tlwh[:,1] = bbox_xywh[:,1] - bbox_xywh[:,3]/2.
+        return bbox_tlwh
+
+    #将bbox的[x,y,w,h] 转换成[x1,y1,x2,y2]
+    #某些数据集例如 pascal_voc 的标注方式是采用[x，y，w，h]
+    """Convert [x y w h] box format to [x1 y1 x2 y2] format."""
+    def _xywh_to_xyxy(self, bbox_xywh):
+        x,y,w,h = bbox_xywh
+        x1 = max(int(x-w/2),0)
+        x2 = min(int(x+w/2),self.width-1)
+        y1 = max(int(y-h/2),0)
+        y2 = min(int(y+h/2),self.height-1)
+        return x1,y1,x2,y2
+
+    def _tlwh_to_xyxy(self, bbox_tlwh):
+        """
+        TODO:
+            Convert bbox from xtl_ytl_w_h to xc_yc_w_h
+        Thanks JieChen91@github.com for reporting this bug!
+        """
+        x,y,w,h = bbox_tlwh
+        x1 = max(int(x),0)
+        x2 = min(int(x+w),self.width-1)
+        y1 = max(int(y),0)
+        y2 = min(int(y+h),self.height-1)
+        return x1,y1,x2,y2
+
+    def _xyxy_to_tlwh(self, bbox_xyxy):
+        x1,y1,x2,y2 = bbox_xyxy
+
+        t = x1
+        l = y1
+        w = int(x2-x1)
+        h = int(y2-y1)
+        return t,l,w,h
+    
+    # 获取抠图部分的特征
+    def _get_features(self, bbox_xywh, ori_img):
+        im_crops = []
+        for box in bbox_xywh:
+            x1,y1,x2,y2 = self._xywh_to_xyxy(box)
+            im = ori_img[y1:y2,x1:x2] # 抠图部分
+            im_crops.append(im)
+        if im_crops:
+            features = self.extractor(im_crops) # 对抠图部分提取特征
+        else:
+            features = np.array([])
+        return features
+
+

deep_sort/deep_sort/sort/detection.py

@@ -0,0 +1,49 @@
+# vim: expandtab:ts=4:sw=4
+import numpy as np
+
+
+class Detection(object):
+    """
+    This class represents a bounding box detection in a single image.
+
+    Parameters
+    ----------
+    tlwh : array_like
+        Bounding box in format `(top left x, top left y, width, height)`.
+    confidence : float
+        Detector confidence score.
+    feature : array_like
+        A feature vector that describes the object contained in this image.
+
+    Attributes
+    ----------
+    tlwh : ndarray
+        Bounding box in format `(top left x, top left y, width, height)`.
+    confidence : ndarray
+        Detector confidence score.
+    feature : ndarray | NoneType
+        A feature vector that describes the object contained in this image.
+
+    """
+
+    def __init__(self, tlwh, confidence, feature):
+        self.tlwh = np.asarray(tlwh, dtype=np.float32)
+        self.confidence = float(confidence)
+        self.feature = np.asarray(feature, dtype=np.float32)
+
+    def to_tlbr(self):
+        """Convert bounding box to format `(min x, min y, max x, max y)`, i.e.,
+        `(top left, bottom right)`.
+        """
+        ret = self.tlwh.copy()
+        ret[2:] += ret[:2]
+        return ret
+
+    def to_xyah(self):
+        """Convert bounding box to format `(center x, center y, aspect ratio,
+        height)`, where the aspect ratio is `width / height`.
+        """
+        ret = self.tlwh.copy()
+        ret[:2] += ret[2:] / 2
+        ret[2] /= ret[3]
+        return ret

deep_sort/deep_sort/sort/iou_matching.py

@@ -0,0 +1,84 @@
+# vim: expandtab:ts=4:sw=4
+from __future__ import absolute_import
+import numpy as np
+from . import linear_assignment
+
+#计算两个框的IOU
+def iou(bbox, candidates):
+    """Computer intersection over union.
+
+    Parameters
+    ----------
+    bbox : ndarray
+        A bounding box in format `(top left x, top left y, width, height)`.
+    candidates : ndarray
+        A matrix of candidate bounding boxes (one per row) in the same format
+        as `bbox`.
+
+    Returns
+    -------
+    ndarray
+        The intersection over union in [0, 1] between the `bbox` and each
+        candidate. A higher score means a larger fraction of the `bbox` is
+        occluded by the candidate.
+
+    """
+    bbox_tl, bbox_br = bbox[:2], bbox[:2] + bbox[2:]
+    candidates_tl = candidates[:, :2]
+    candidates_br = candidates[:, :2] + candidates[:, 2:]
+
+    # np.c_ Translates slice objects to concatenation along the second axis.
+    tl = np.c_[np.maximum(bbox_tl[0], candidates_tl[:, 0])[:, np.newaxis],
+               np.maximum(bbox_tl[1], candidates_tl[:, 1])[:, np.newaxis]]
+    br = np.c_[np.minimum(bbox_br[0], candidates_br[:, 0])[:, np.newaxis],
+               np.minimum(bbox_br[1], candidates_br[:, 1])[:, np.newaxis]]
+    wh = np.maximum(0., br - tl)
+
+    area_intersection = wh.prod(axis=1)
+    area_bbox = bbox[2:].prod()
+    area_candidates = candidates[:, 2:].prod(axis=1)
+    return area_intersection / (area_bbox + area_candidates - area_intersection)
+
+# 计算tracks和detections之间的IOU距离成本矩阵
+def iou_cost(tracks, detections, track_indices=None,
+             detection_indices=None):
+    """An intersection over union distance metric.
+
+    用于计算tracks和detections之间的iou距离矩阵
+
+    Parameters
+    ----------
+    tracks : List[deep_sort.track.Track]
+        A list of tracks.
+    detections : List[deep_sort.detection.Detection]
+        A list of detections.
+    track_indices : Optional[List[int]]
+        A list of indices to tracks that should be matched. Defaults to
+        all `tracks`.
+    detection_indices : Optional[List[int]]
+        A list of indices to detections that should be matched. Defaults
+        to all `detections`.
+
+    Returns
+    -------
+    ndarray
+        Returns a cost matrix of shape
+        len(track_indices), len(detection_indices) where entry (i, j) is
+        `1 - iou(tracks[track_indices[i]], detections[detection_indices[j]])`.
+
+    """
+    if track_indices is None:
+        track_indices = np.arange(len(tracks))
+    if detection_indices is None:
+        detection_indices = np.arange(len(detections))
+
+    cost_matrix = np.zeros((len(track_indices), len(detection_indices)))
+    for row, track_idx in enumerate(track_indices):
+        if tracks[track_idx].time_since_update > 1:
+            cost_matrix[row, :] = linear_assignment.INFTY_COST
+            continue
+
+        bbox = tracks[track_idx].to_tlwh()
+        candidates = np.asarray([detections[i].tlwh for i in detection_indices])
+        cost_matrix[row, :] = 1. - iou(bbox, candidates)
+    return cost_matrix

deep_sort/deep_sort/sort/kalman_filter.py

@@ -0,0 +1,286 @@
+# vim: expandtab:ts=4:sw=4
+import numpy as np
+import scipy.linalg
+
+
+"""
+Table for the 0.95 quantile of the chi-square distribution with N degrees of
+freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv
+function and used as Mahalanobis gating threshold.
+"""
+chi2inv95 = {
+    1: 3.8415,
+    2: 5.9915,
+    3: 7.8147,
+    4: 9.4877,
+    5: 11.070,
+    6: 12.592,
+    7: 14.067,
+    8: 15.507,
+    9: 16.919}
+
+'''
+卡尔曼滤波分为两个阶段：
+(1) 预测track在下一时刻的位置，
+(2) 基于detection来更新预测的位置。
+'''
+class KalmanFilter(object):
+    """
+    A simple Kalman filter for tracking bounding boxes in image space.
+
+    The 8-dimensional state space
+
+        x, y, a, h, vx, vy, va, vh
+
+    contains the bounding box center position (x, y), aspect ratio a, height h,
+    and their respective velocities.
+
+    Object motion follows a constant velocity model. The bounding box location
+    (x, y, a, h) is taken as direct observation of the state space (linear
+    observation model).
+
+    对于每个轨迹，由一个 KalmanFilter 预测状态分布。每个轨迹记录自己的均值和方差作为滤波器输入。
+
+    8维状态空间[x, y, a, h, vx, vy, va, vh]包含边界框中心位置(x, y)，纵横比a，高度h和它们各自的速度。
+    物体运动遵循恒速模型。 边界框位置(x, y, a, h)被视为状态空间的直接观察（线性观察模型）
+
+    """
+
+    def __init__(self):
+        ndim, dt = 4, 1.
+
+        # Create Kalman filter model matrices.
+        self._motion_mat = np.eye(2 * ndim, 2 * ndim)
+        for i in range(ndim):
+            self._motion_mat[i, ndim + i] = dt
+        self._update_mat = np.eye(ndim, 2 * ndim)
+
+        # Motion and observation uncertainty are chosen relative to the current
+        # state estimate. These weights control the amount of uncertainty in
+        # the model. This is a bit hacky.
+        # 依据当前状态估计（高度）选择运动和观测不确定性。这些权重控制模型中的不确定性。
+        self._std_weight_position = 1. / 20
+        self._std_weight_velocity = 1. / 160
+
+    def initiate(self, measurement):
+        """Create track from unassociated measurement.
+
+        Parameters
+        ----------
+        measurement : ndarray
+            Bounding box coordinates (x, y, a, h) with center position (x, y),
+            aspect ratio a, and height h.
+
+        Returns
+        -------
+        (ndarray, ndarray)
+            Returns the mean vector (8 dimensional) and covariance matrix (8x8
+            dimensional) of the new track. Unobserved velocities are initialized
+            to 0 mean.
+
+        """
+        
+
+        mean_pos = measurement
+        mean_vel = np.zeros_like(mean_pos)
+        # Translates slice objects to concatenation along the first axis
+        mean = np.r_[mean_pos, mean_vel]
+
+        # 由测量初始化均值向量（8维）和协方差矩阵（8x8维）
+        std = [
+            2 * self._std_weight_position * measurement[3],
+            2 * self._std_weight_position * measurement[3],
+            1e-2,
+            2 * self._std_weight_position * measurement[3],
+            10 * self._std_weight_velocity * measurement[3],
+            10 * self._std_weight_velocity * measurement[3],
+            1e-5,
+            10 * self._std_weight_velocity * measurement[3]]
+        covariance = np.diag(np.square(std))
+        return mean, covariance
+
+    def predict(self, mean, covariance):
+        """Run Kalman filter prediction step.
+
+        Parameters
+        ----------
+        mean : ndarray
+            The 8 dimensional mean vector of the object state at the previous
+            time step.
+        covariance : ndarray
+            The 8x8 dimensional covariance matrix of the object state at the
+            previous time step.
+
+        Returns
+        -------
+        (ndarray, ndarray)
+            Returns the mean vector and covariance matrix of the predicted
+            state. Unobserved velocities are initialized to 0 mean.
+
+        """
+        #卡尔曼滤波器由目标上一时刻的均值和协方差进行预测。
+        std_pos = [
+            self._std_weight_position * mean[3],
+            self._std_weight_position * mean[3],
+            1e-2,
+            self._std_weight_position * mean[3]]
+        std_vel = [
+            self._std_weight_velocity * mean[3],
+            self._std_weight_velocity * mean[3],
+            1e-5,
+            self._std_weight_velocity * mean[3]]
+       
+        # 初始化噪声矩阵Q；np.r_ 按列连接两个矩阵
+        # motion_cov是过程噪声 W_k的 协方差矩阵Qk 
+        motion_cov = np.diag(np.square(np.r_[std_pos, std_vel]))
+
+        # Update time state x' = Fx (1)
+        # x为track在t-1时刻的均值，F称为状态转移矩阵，该公式预测t时刻的x'
+        # self._motion_mat为F_k是作用在 x_{k-1}上的状态变换模型
+        mean = np.dot(self._motion_mat, mean)
+        # Calculate error covariance P' = FPF^T+Q (2)
+        # P为track在t-1时刻的协方差，Q为系统的噪声矩阵，代表整个系统的可靠程度，一般初始化为很小的值，
+        # 该公式预测t时刻的P'
+        # covariance为P_{k|k} ，后验估计误差协方差矩阵，度量估计值的精确程度
+        covariance = np.linalg.multi_dot((
+            self._motion_mat, covariance, self._motion_mat.T)) + motion_cov
+
+        return mean, covariance
+
+    def project(self, mean, covariance):
+        """Project state distribution to measurement space.
+        投影状态分布到测量空间
+
+        Parameters
+        ----------
+        mean : ndarray
+            The state's mean vector (8 dimensional array).
+        covariance : ndarray
+            The state's covariance matrix (8x8 dimensional).
+
+       mean：ndarray，状态的平均向量（8维数组）。
+       covariance：ndarray，状态的协方差矩阵（8x8维）。
+
+        Returns
+        -------
+        (ndarray, ndarray)
+            Returns the projected mean and covariance matrix of the given state
+            estimate.
+
+       返回（ndarray，ndarray），返回给定状态估计的投影平均值和协方差矩阵
+
+        """
+        # 在公式4中，R为检测器的噪声矩阵，它是一个4x4的对角矩阵，
+        # 对角线上的值分别为中心点两个坐标以及宽高的噪声，
+        # 以任意值初始化，一般设置宽高的噪声大于中心点的噪声，
+        # 该公式先将协方差矩阵P'映射到检测空间，然后再加上噪声矩阵R；
+        std = [
+            self._std_weight_position * mean[3],
+            self._std_weight_position * mean[3],
+            1e-1,
+            self._std_weight_position * mean[3]]
+            
+        # R为测量过程中噪声的协方差；初始化噪声矩阵R
+        innovation_cov = np.diag(np.square(std))
+
+        # 将均值向量映射到检测空间，即 Hx'
+        mean = np.dot(self._update_mat, mean)
+        # 将协方差矩阵映射到检测空间，即 HP'H^T
+        covariance = np.linalg.multi_dot((
+            self._update_mat, covariance, self._update_mat.T))
+        return mean, covariance + innovation_cov # 公式(4)
+
+    def update(self, mean, covariance, measurement):
+        """Run Kalman filter correction step.
+        通过估计值和观测值估计最新结果
+
+        Parameters
+        ----------
+        mean : ndarray
+            The predicted state's mean vector (8 dimensional).
+        covariance : ndarray
+            The state's covariance matrix (8x8 dimensional).
+        measurement : ndarray
+            The 4 dimensional measurement vector (x, y, a, h), where (x, y)
+            is the center position, a the aspect ratio, and h the height of the
+            bounding box.
+
+        Returns
+        -------
+        (ndarray, ndarray)
+            Returns the measurement-corrected state distribution.
+
+        """
+        # 将均值和协方差映射到检测空间，得到 Hx'和S
+        projected_mean, projected_cov = self.project(mean, covariance)
+
+        # 矩阵分解
+        chol_factor, lower = scipy.linalg.cho_factor(
+            projected_cov, lower=True, check_finite=False)
+        # 计算卡尔曼增益K；相当于求解公式(5)
+        # 公式5计算卡尔曼增益K，卡尔曼增益用于估计误差的重要程度
+        # 求解卡尔曼滤波增益K 用到了cholesky矩阵分解加快求解；
+        # 公式5的右边有一个S的逆，如果S矩阵很大，S的逆求解消耗时间太大，
+        # 所以代码中把公式两边同时乘上S，右边的S*S的逆变成了单位矩阵，转化成AX=B形式求解。
+        kalman_gain = scipy.linalg.cho_solve(
+            (chol_factor, lower), np.dot(covariance, self._update_mat.T).T,
+            check_finite=False).T
+        # y = z - Hx' (3)
+        # 在公式3中，z为detection的均值向量，不包含速度变化值，即z=[cx, cy, r, h]，
+        # H称为测量矩阵，它将track的均值向量x'映射到检测空间，该公式计算detection和track的均值误差
+        innovation = measurement - projected_mean
+
+        # 更新后的均值向量 x = x' + Ky (6)
+        new_mean = mean + np.dot(innovation, kalman_gain.T)
+        # 更新后的协方差矩阵 P = (I - KH)P' (7)
+        new_covariance = covariance - np.linalg.multi_dot((
+            kalman_gain, projected_cov, kalman_gain.T))
+        return new_mean, new_covariance
+
+    def gating_distance(self, mean, covariance, measurements,
+                        only_position=False):
+        """Compute gating distance between state distribution and measurements.
+
+        A suitable distance threshold can be obtained from `chi2inv95`. If
+        `only_position` is False, the chi-square distribution has 4 degrees of
+        freedom, otherwise 2.
+
+        Parameters
+        ----------
+        mean : ndarray
+            Mean vector over the state distribution (8 dimensional).
+            状态分布上的平均向量（8维）
+        covariance : ndarray
+            Covariance of the state distribution (8x8 dimensional).
+            状态分布的协方差（8x8维）
+        measurements : ndarray
+            An Nx4 dimensional matrix of N measurements, each in
+            format (x, y, a, h) where (x, y) is the bounding box center
+            position, a the aspect ratio, and h the height.
+            N 个测量的 N×4维矩阵，每个矩阵的格式为（x，y，a，h），其中（x，y）是边界框中心位置，宽高比和h高度。
+        only_position : Optional[bool]
+            If True, distance computation is done with respect to the bounding
+            box center position only.
+             如果为True，则只计算盒子中心位置
+
+        Returns
+        -------
+        ndarray
+            Returns an array of length N, where the i-th element contains the
+            squared Mahalanobis distance between (mean, covariance) and
+            `measurements[i]`.
+       返回一个长度为N的数组，其中第i个元素包含（mean，covariance）和measurements [i]之间的平方Mahalanobis距离
+
+        """
+        mean, covariance = self.project(mean, covariance)
+        if only_position:
+            mean, covariance = mean[:2], covariance[:2, :2]
+            measurements = measurements[:, :2]
+
+        cholesky_factor = np.linalg.cholesky(covariance)
+        d = measurements - mean
+        z = scipy.linalg.solve_triangular(
+            cholesky_factor, d.T, lower=True, check_finite=False,
+            overwrite_b=True)
+        squared_maha = np.sum(z * z, axis=0)
+        return squared_maha

deep_sort/deep_sort/sort/linear_assignment.py

@@ -0,0 +1,240 @@
+# vim: expandtab:ts=4:sw=4
+from __future__ import absolute_import
+import numpy as np
+# The linear sum assignment problem is also known as minimum weight matching in bipartite graphs.
+from scipy.optimize import linear_sum_assignment as linear_assignment
+from . import kalman_filter
+
+
+INFTY_COST = 1e+5
+
+# min_cost_matching 使用匈牙利算法解决线性分配问题。
+# 传入 门控余弦距离成本 或 iou cost
+def min_cost_matching(
+        distance_metric, max_distance, tracks, detections, track_indices=None,
+        detection_indices=None):
+    """Solve linear assignment problem.
+
+    Parameters
+    ----------
+    distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray
+        The distance metric is given a list of tracks and detections as well as
+        a list of N track indices and M detection indices. The metric should
+        return the NxM dimensional cost matrix, where element (i, j) is the
+        association cost between the i-th track in the given track indices and
+        the j-th detection in the given detection_indices.
+    max_distance : float
+        Gating threshold. Associations with cost larger than this value are
+        disregarded.
+    tracks : List[track.Track]
+        A list of predicted tracks at the current time step.
+    detections : List[detection.Detection]
+        A list of detections at the current time step.
+    track_indices : List[int]
+        List of track indices that maps rows in `cost_matrix` to tracks in
+        `tracks` (see description above).
+    detection_indices : List[int]
+        List of detection indices that maps columns in `cost_matrix` to
+        detections in `detections` (see description above).
+
+    Returns
+    -------
+    (List[(int, int)], List[int], List[int])
+        Returns a tuple with the following three entries:
+        * A list of matched track and detection indices.
+        * A list of unmatched track indices.
+        * A list of unmatched detection indices.
+
+    """
+    if track_indices is None:
+        track_indices = np.arange(len(tracks))
+    if detection_indices is None:
+        detection_indices = np.arange(len(detections))
+
+    if len(detection_indices) == 0 or len(track_indices) == 0:
+        return [], track_indices, detection_indices  # Nothing to match.
+
+    # 计算成本矩阵
+    cost_matrix = distance_metric(
+        tracks, detections, track_indices, detection_indices)
+    cost_matrix[cost_matrix > max_distance] = max_distance + 1e-5
+
+    # 执行匈牙利算法，得到指派成功的索引对，行索引为tracks的索引，列索引为detections的索引
+    row_indices, col_indices = linear_assignment(cost_matrix)
+
+    matches, unmatched_tracks, unmatched_detections = [], [], []
+    # 找出未匹配的detections
+    for col, detection_idx in enumerate(detection_indices):
+        if col not in col_indices:
+            unmatched_detections.append(detection_idx)
+    # 找出未匹配的tracks
+    for row, track_idx in enumerate(track_indices):
+        if row not in row_indices:
+            unmatched_tracks.append(track_idx)
+    # 遍历匹配的(track, detection)索引对
+    for row, col in zip(row_indices, col_indices):
+        track_idx = track_indices[row]
+        detection_idx = detection_indices[col]
+        # 如果相应的cost大于阈值max_distance，也视为未匹配成功
+        if cost_matrix[row, col] > max_distance:
+            unmatched_tracks.append(track_idx)
+            unmatched_detections.append(detection_idx)
+        else:
+            matches.append((track_idx, detection_idx))
+    return matches, unmatched_tracks, unmatched_detections
+
+
+def matching_cascade(
+        distance_metric, max_distance, cascade_depth, tracks, detections,
+        track_indices=None, detection_indices=None):
+    """Run matching cascade.
+
+    Parameters
+    ----------
+    distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray
+        The distance metric is given a list of tracks and detections as well as
+        a list of N track indices and M detection indices. The metric should
+        return the NxM dimensional cost matrix, where element (i, j) is the
+        association cost between the i-th track in the given track indices and
+        the j-th detection in the given detection indices.
+        距离度量：
+        输入：一个轨迹和检测列表，以及一个N个轨迹索引和M个检测索引的列表。 
+        返回：NxM维的代价矩阵，其中元素(i，j)是给定轨迹索引中第i个轨迹与
+        给定检测索引中第j个检测之间的关联成本。
+    max_distance : float
+        Gating threshold. Associations with cost larger than this value are
+        disregarded.
+        门控阈值。成本大于此值的关联将被忽略。
+    cascade_depth: int
+        The cascade depth, should be se to the maximum track age.
+        级联深度应设置为最大轨迹寿命。
+    tracks : List[track.Track]
+        A list of predicted tracks at the current time step.
+        当前时间步的预测轨迹列表。
+    detections : List[detection.Detection]
+        A list of detections at the current time step.
+        当前时间步的检测列表。
+    track_indices : Optional[List[int]]
+        List of track indices that maps rows in `cost_matrix` to tracks in
+        `tracks` (see description above). Defaults to all tracks.
+        轨迹索引列表，用于将 cost_matrix中的行映射到tracks的
+         轨迹（请参见上面的说明）。 默认为所有轨迹。
+    detection_indices : Optional[List[int]]
+        List of detection indices that maps columns in `cost_matrix` to
+        detections in `detections` (see description above). Defaults to all
+        detections.
+        将 cost_matrix中的列映射到的检测索引列表
+         detections中的检测（请参见上面的说明）。 默认为全部检测。
+
+    Returns
+    -------
+    (List[(int, int)], List[int], List[int])
+        Returns a tuple with the following three entries:
+        * A list of matched track and detection indices.
+        * A list of unmatched track indices.
+        * A list of unmatched detection indices.
+
+    返回包含以下三个条目的元组：
+    
+    匹配的跟踪和检测的索引列表，
+    不匹配的轨迹索引的列表，
+    未匹配的检测索引的列表。
+
+    """
+    
+    # 分配track_indices和detection_indices两个列表
+    if track_indices is None:
+        track_indices = list(range(len(tracks)))
+    if detection_indices is None:
+        detection_indices = list(range(len(detections)))
+
+    # 初始化匹配集matches M ← ∅ 
+    # 未匹配检测集unmatched_detections U ← D 
+    unmatched_detections = detection_indices
+    matches = []
+    # 由小到大依次对每个level的tracks做匹配
+    for level in range(cascade_depth):
+        # 如果没有detections，退出循环
+        if len(unmatched_detections) == 0:  # No detections left
+            break
+
+        # 当前level的所有tracks索引
+        # 步骤6：Select tracks by age
+        track_indices_l = [
+            k for k in track_indices
+            if tracks[k].time_since_update == 1 + level
+        ]
+        # 如果当前level没有track，继续
+        if len(track_indices_l) == 0:  # Nothing to match at this level
+            continue
+            
+        # 步骤7：调用min_cost_matching函数进行匹配 
+        matches_l, _, unmatched_detections = \
+            min_cost_matching(
+                distance_metric, max_distance, tracks, detections,
+                track_indices_l, unmatched_detections)
+        matches += matches_l # 步骤8
+    unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches))  # 步骤9
+    return matches, unmatched_tracks, unmatched_detections
+
+'''
+门控成本矩阵：通过计算卡尔曼滤波的状态分布和测量值之间的距离对成本矩阵进行限制，
+成本矩阵中的距离是track和detection之间的外观相似度。
+如果一个轨迹要去匹配两个外观特征非常相似的 detection，很容易出错；
+分别让两个detection计算与这个轨迹的马氏距离，并使用一个阈值gating_threshold进行限制，
+就可以将马氏距离较远的那个detection区分开，从而减少错误的匹配。
+'''
+def gate_cost_matrix(
+        kf, cost_matrix, tracks, detections, track_indices, detection_indices,
+        gated_cost=INFTY_COST, only_position=False):
+    """Invalidate infeasible entries in cost matrix based on the state
+    distributions obtained by Kalman filtering.
+
+    Parameters
+    ----------
+    kf : The Kalman filter.
+    cost_matrix : ndarray
+        The NxM dimensional cost matrix, where N is the number of track indices
+        and M is the number of detection indices, such that entry (i, j) is the
+        association cost between `tracks[track_indices[i]]` and
+        `detections[detection_indices[j]]`.
+    tracks : List[track.Track]
+        A list of predicted tracks at the current time step.
+    detections : List[detection.Detection]
+        A list of detections at the current time step.
+    track_indices : List[int]
+        List of track indices that maps rows in `cost_matrix` to tracks in
+        `tracks` (see description above).
+    detection_indices : List[int]
+        List of detection indices that maps columns in `cost_matrix` to
+        detections in `detections` (see description above).
+    gated_cost : Optional[float]
+        Entries in the cost matrix corresponding to infeasible associations are
+        set this value. Defaults to a very large value.
+        代价矩阵中与不可行关联相对应的条目设置此值。 默认为一个很大的值。
+    only_position : Optional[bool]
+        If True, only the x, y position of the state distribution is considered
+        during gating. Defaults to False.
+        如果为True，则在门控期间仅考虑状态分布的x，y位置。默认为False。
+
+    Returns
+    -------
+    ndarray
+        Returns the modified cost matrix.
+
+    """
+    # 根据通过卡尔曼滤波获得的状态分布，使成本矩阵中的不可行条目无效。
+    gating_dim = 2 if only_position else 4 # 测量空间维度 
+    # 马氏距离通过测算检测与平均轨迹位置的距离超过多少标准差来考虑状态估计的不确定性。
+    # 通过从逆chi^2分布计算95%置信区间的阈值，排除可能性小的关联。
+    # 四维测量空间对应的马氏阈值为9.4877
+    gating_threshold = kalman_filter.chi2inv95[gating_dim]
+    measurements = np.asarray(
+        [detections[i].to_xyah() for i in detection_indices])
+    for row, track_idx in enumerate(track_indices):
+        track = tracks[track_idx]
+        #KalmanFilter.gating_distance 计算状态分布和测量之间的选通距离
+        gating_distance = kf.gating_distance(
+            track.mean, track.covariance, measurements, only_position)
+        cost_matrix[row, gating_distance > gating_threshold] = gated_cost
+    return cost_matrix

deep_sort/deep_sort/sort/nn_matching.py

@@ -0,0 +1,207 @@
+# vim: expandtab:ts=4:sw=4
+import numpy as np
+
+
+def _pdist(a, b):
+    """Compute pair-wise squared distance between points in `a` and `b`.
+
+    Parameters
+    ----------
+    a : array_like
+        An NxM matrix of N samples of dimensionality M.
+    b : array_like
+        An LxM matrix of L samples of dimensionality M.
+
+    Returns
+    -------
+    ndarray
+        Returns a matrix of size len(a), len(b) such that element (i, j)
+        contains the squared distance between `a[i]` and `b[j]`.
+
+    
+    用于计算成对点之间的平方距离
+    a ：NxM 矩阵，代表 N 个样本，每个样本 M 个数值 
+    b ：LxM 矩阵，代表 L 个样本，每个样本有 M 个数值 
+    返回的是 NxL 的矩阵，比如 dist[i][j] 代表 a[i] 和 b[j] 之间的平方和距离
+    参考：https://blog.csdn.net/frankzd/article/details/80251042
+
+    """
+    a, b = np.asarray(a), np.asarray(b)
+    if len(a) == 0 or len(b) == 0:
+        return np.zeros((len(a), len(b)))
+    a2, b2 = np.square(a).sum(axis=1), np.square(b).sum(axis=1)
+    r2 = -2. * np.dot(a, b.T) + a2[:, None] + b2[None, :]
+    r2 = np.clip(r2, 0., float(np.inf))
+    return r2
+
+
+def _cosine_distance(a, b, data_is_normalized=False):
+    """Compute pair-wise cosine distance between points in `a` and `b`.
+
+    Parameters
+    ----------
+    a : array_like
+        An NxM matrix of N samples of dimensionality M.
+    b : array_like
+        An LxM matrix of L samples of dimensionality M.
+    data_is_normalized : Optional[bool]
+        If True, assumes rows in a and b are unit length vectors.
+        Otherwise, a and b are explicitly normalized to lenght 1.
+
+    Returns
+    -------
+    ndarray
+        Returns a matrix of size len(a), len(b) such that eleement (i, j)
+        contains the squared distance between `a[i]` and `b[j]`.
+
+    用于计算成对点之间的余弦距离
+    a ：NxM 矩阵，代表 N 个样本，每个样本 M 个数值 
+    b ：LxM 矩阵，代表 L 个样本，每个样本有 M 个数值 
+    返回的是 NxL 的矩阵，比如 c[i][j] 代表 a[i] 和 b[j] 之间的余弦距离
+    参考：
+    https://blog.csdn.net/u013749540/article/details/51813922
+    
+
+    """
+    if not data_is_normalized:
+        # np.linalg.norm 求向量的范式，默认是 L2 范式 
+        a = np.asarray(a) / np.linalg.norm(a, axis=1, keepdims=True)
+        b = np.asarray(b) / np.linalg.norm(b, axis=1, keepdims=True)
+    return 1. - np.dot(a, b.T) # 余弦距离 = 1 - 余弦相似度
+
+
+def _nn_euclidean_distance(x, y):
+    """ Helper function for nearest neighbor distance metric (Euclidean).
+
+    Parameters
+    ----------
+    x : ndarray
+        A matrix of N row-vectors (sample points).
+    y : ndarray
+        A matrix of M row-vectors (query points).
+
+    Returns
+    -------
+    ndarray
+        A vector of length M that contains for each entry in `y` the
+        smallest Euclidean distance to a sample in `x`.
+
+    """
+    distances = _pdist(x, y)
+    return np.maximum(0.0, distances.min(axis=0))
+
+
+def _nn_cosine_distance(x, y):
+    """ Helper function for nearest neighbor distance metric (cosine).
+
+    Parameters
+    ----------
+    x : ndarray
+        A matrix of N row-vectors (sample points).
+    y : ndarray
+        A matrix of M row-vectors (query points).
+
+    Returns
+    -------
+    ndarray
+        A vector of length M that contains for each entry in `y` the
+        smallest cosine distance to a sample in `x`.
+
+    """
+    distances = _cosine_distance(x, y)
+    return distances.min(axis=0)
+
+
+class NearestNeighborDistanceMetric(object):
+    """
+    A nearest neighbor distance metric that, for each target, returns
+    the closest distance to any sample that has been observed so far.
+
+    对于每个目标，返回最近邻居的距离度量, 即与到目前为止已观察到的任何样本的最接近距离。
+
+    Parameters
+    ----------
+    metric : str
+        Either "euclidean" or "cosine".
+    matching_threshold: float
+        The matching threshold. Samples with larger distance are considered an
+        invalid match.
+        匹配阈值。 距离较大的样本对被认为是无效的匹配。
+    budget : Optional[int]
+        If not None, fix samples per class to at most this number. Removes
+        the oldest samples when the budget is reached.
+        如果不是None，则将每个类别的样本最多固定为该数字。 
+        删除达到budget时最古老的样本。
+
+    Attributes
+    ----------
+    samples : Dict[int -> List[ndarray]]
+        A dictionary that maps from target identities to the list of samples
+        that have been observed so far.
+        一个从目标ID映射到到目前为止已经观察到的样本列表的字典
+
+    """
+
+    def __init__(self, metric, matching_threshold, budget=None):
+
+
+        if metric == "euclidean":
+            self._metric = _nn_euclidean_distance # 欧式距离
+        elif metric == "cosine":
+            self._metric = _nn_cosine_distance # 余弦距离
+        else:
+            raise ValueError(
+                "Invalid metric; must be either 'euclidean' or 'cosine'")
+        self.matching_threshold = matching_threshold
+        self.budget = budget # budge用于控制 feature 的数目
+        self.samples = {}
+
+    def partial_fit(self, features, targets, active_targets):
+        """Update the distance metric with new data.
+        用新的数据更新测量距离
+
+        Parameters
+        ----------
+        features : ndarray
+            An NxM matrix of N features of dimensionality M.
+        targets : ndarray
+            An integer array of associated target identities.
+        active_targets : List[int]
+            A list of targets that are currently present in the scene.
+        传入特征列表及其对应id，partial_fit构造一个活跃目标的特征字典。
+
+        """
+        for feature, target in zip(features, targets):
+            # 对应目标下添加新的feature，更新feature集合
+            # samples字典    d: feature list}
+            self.samples.setdefault(target, []).append(feature)
+            if self.budget is not None:
+                # 只考虑budget个目标，超过直接忽略
+                self.samples[target] = self.samples[target][-self.budget:]
+        
+        # 筛选激活的目标；samples是一个字典{id->feature list}
+        self.samples = {k: self.samples[k] for k in active_targets}
+
+    def distance(self, features, targets):
+        """Compute distance between features and targets.
+
+        Parameters
+        ----------
+        features : ndarray
+            An NxM matrix of N features of dimensionality M.
+        targets : List[int]
+            A list of targets to match the given `features` against.
+
+        Returns
+        -------
+        ndarray
+            Returns a cost matrix of shape len(targets), len(features), where
+            element (i, j) contains the closest squared distance between
+            `targets[i]` and `features[j]`.
+        
+        计算features和targets之间的距离，返回一个成本矩阵（代价矩阵）
+        """
+        cost_matrix = np.zeros((len(targets), len(features)))
+        for i, target in enumerate(targets):
+            cost_matrix[i, :] = self._metric(self.samples[target], features)
+        return cost_matrix