Upload 8 files

.env

@@ -0,0 +1,3 @@
+DET_MODEL_PATH = 'person_detection_v3.pt'
+ACTIVITY_DET_MODEL_PATH = 'final_activity_detection.pt'
+IMG_DIR_PATH = 'images/valid'

app.py

@@ -0,0 +1,120 @@
+import streamlit as st
+import cv2
+import os
+import io
+import numpy as np    
+from PIL import Image
+from inferance import pipline
+import pandas as pd
+
+
+code = """
+<style>
+  .block-container{
+    max-width: 100%;
+    padding: 50px;
+  }
+  # [data-testid="stImage"], .e115fcil2, [data-testid="StyledFullScreenButton"], [data-testid="stFullScreenFrame"].e1vs0wn30, [data-testid="element-container"].e1f1d6gn4.element-container{
+  #   width: fit-content !important;
+  # }
+  # [data-testid="stVerticalBlock"].e1f1d6gn2{
+  #   flex-direction: row;
+  #   flex-wrap: wrap;
+  # }
+  [data-testid="StyledFullScreenButton"]{
+    display: none;
+  }
+  [data-testid="stVerticalBlockBorderWrapper"], [data-testid="stVerticalBlock"]{
+    width: 100%;
+  }
+  .e115fcil2{
+    justify-content: center;
+    margin-top: 20px;
+  }
+</style>
+"""
+st.html(code)
+
+st.title("Automated Surveillance System")
+
+col1, col2 = st.columns([5, 5])
+container = col2.container(height=800)
+col3, col4= container.columns([1,1])
+
+with col1:
+  
+  image = st.file_uploader("File upload", label_visibility="hidden")
+  
+  if image is not None:
+    image = Image.open(io.BytesIO(image.getvalue()))
+    image = np.asarray(image)
+    
+    cv2.imwrite("image.jpg", image)
+    image = cv2.imread("image.jpg")
+    
+    results = pipline(image)
+    
+    for result in results:
+      image = cv2.rectangle(image, result['updated_boxes']['top_left'], result['updated_boxes']['bottom_right'], (255, 0, 0), 1)
+
+    st.image(image)
+  
+  else:
+    image = cv2.imread("default_img.jpg")
+
+    results = pipline(image)
+    
+    for result in results:
+      image = cv2.rectangle(image, result['updated_boxes']['top_left'], result['updated_boxes']['bottom_right'], (255, 0, 0), 1)
+
+    st.image(image)
+
+if image is not None:
+  with col2:
+      results_1 = results[:len(results)//2]
+      results_2 = results[len(results)//2:]
+      with col4:
+        for result in results_1:
+          img = result['zoomed_img']
+          
+          df = pd.DataFrame(columns=['Object Type', 'Distance', 'Activity'])
+          actual_width, actual_height = result['updated_boxes']['bottom_right'][0] - result['updated_boxes']['top_left'][0], result['updated_boxes']['bottom_right'][1] - result['updated_boxes']['top_left'][1]
+
+          for box in result['actual_boxes']:
+            top_left = (box['top_left'][0] - result['updated_boxes']['top_left'][0], (box['top_left'][1] - result['updated_boxes']['top_left'][1]))
+            bottom_right = (box['bottom_right'][0] - result['updated_boxes']['top_left'][0], (box['bottom_right'][1] - result['updated_boxes']['top_left'][1]))
+
+            print(img.shape, actual_height, actual_width)
+            bottom_right = (bottom_right[0]*img.shape[0]//(actual_height), bottom_right[1]*img.shape[1]//(actual_width))
+            top_left = (top_left[0]*img.shape[0]//(actual_height), top_left[1]*img.shape[1]//(actual_width))
+            
+            print(box['top_left'], result['updated_boxes']['top_left'], box['bottom_right'], result['updated_boxes']['bottom_right'], top_left, bottom_right)
+            img = cv2.rectangle(img, top_left, bottom_right, (255, 0, 0), 1)
+            img = cv2.putText(img, "ID: "+str(len(df)), top_left, 1, 1, (255, 255, 255))
+            df.loc[len(df)] = [box['class'], box['distance'], box['activity']]
+          
+          st.image(img)
+          st.table(df)
+
+      with col3:
+        for result in results_2:
+          img = result['zoomed_img']
+          
+          df = pd.DataFrame(columns=['Object Type', 'Distance', 'Activity'])
+          actual_width, actual_height = result['updated_boxes']['bottom_right'][0] - result['updated_boxes']['top_left'][0], result['updated_boxes']['bottom_right'][1] - result['updated_boxes']['top_left'][1]
+
+          for box in result['actual_boxes']:
+            top_left = (box['top_left'][0] - result['updated_boxes']['top_left'][0], (box['top_left'][1] - result['updated_boxes']['top_left'][1]))
+            bottom_right = (box['bottom_right'][0] - result['updated_boxes']['top_left'][0], (box['bottom_right'][1] - result['updated_boxes']['top_left'][1]))
+
+            print(img.shape, actual_height, actual_width)
+            bottom_right = (bottom_right[0]*img.shape[0]//(actual_height), bottom_right[1]*img.shape[1]//(actual_width))
+            top_left = (top_left[0]*img.shape[0]//(actual_height), top_left[1]*img.shape[1]//(actual_width))
+            
+            print(box['top_left'], result['updated_boxes']['top_left'], box['bottom_right'], result['updated_boxes']['bottom_right'], top_left, bottom_right)
+            img = cv2.rectangle(img, top_left, bottom_right, (255, 0, 0), 1)
+            img = cv2.putText(img, "ID: "+str(len(df)), top_left, 1, 1, (255, 255, 255))
+            df.loc[len(df)] = [box['class'], box['distance'], box['activity']]
+          
+          st.image(img)
+          st.table(df)

final_activity_detection.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70759328a3222fb40001170759f7ed6577acd54f3283d087f9fbe63974989ee6
+size 2968321

inferance.py

@@ -0,0 +1,43 @@
+import cv2
+from pipline_functions import croped_images,object_detection,image_enhancements,detect_activity,get_distances,get_json_data
+import os
+
+def pipline(image):
+    """_summary_
+
+    Args:
+        image (numpy array): get numpy array of image which has 3 channels
+
+    Returns:
+        final_results: JSON Array which has below object
+        {
+            'zoomed_img':np.array([]) ,
+            'actual_boxes':[],
+            'updated_boxes':{},
+        }
+    """
+    # detect object of given image using YOLO and get json_data of each object
+    json_data = object_detection(image)
+
+    # get croped_images list which has overlapping boundry box and also get croped single object images
+    croped_images_list,single_object_images=  croped_images(image,json_data)
+
+    # enhance images of both croped images and single object images
+    enhanced_images,single_object_images = image_enhancements(croped_images_list,single_object_images)
+
+    # detect activity of person object using image classification
+    detected_activity = detect_activity(single_object_images)
+
+    # Calculate distances of all objects
+    distances_list = get_distances(json_data)
+
+    # get final json array 
+    final_results = get_json_data(json_data,enhanced_images,detected_activity,distances_list)
+
+    # print(distances_list)
+    # print(detected_activity)
+    # print(final_results)
+
+    return final_results
+
+pipline(cv2.imread('distance_test\distance_test\images\car_99-94168281555176_Mon-Dec-13-16-37-40-2021_jpg.rf.a8c56aba60dd3a19f2c2f159a2c9062d.jpg'))

person_detection_v3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64b1203bd0c8e4fb317eb0b11816a9e2a95ab887f708d9ede3c4ef40f1daf94c
+size 52026625

pipline_functions.py

@@ -0,0 +1,345 @@
+import cv2
+from ultralytics import YOLO
+import os
+from dotenv import load_dotenv
+from pathlib import Path
+import math
+import json
+import numpy as np
+
+env_path = Path('.') / '.env'
+load_dotenv(dotenv_path=env_path)
+
+path = {
+    'DET_MODEL_PATH': str(os.getenv('DET_MODEL_PATH')),
+    'IMG_DIR_PATH': str(os.getenv('IMG_DIR_PATH')),
+    'ACTIVITY_DET_MODEL_PATH':str(os.getenv('ACTIVITY_DET_MODEL_PATH')),
+}
+#constants
+PERSON_HEIGHT = 1.5
+VEHICAL_HEIGHT = 1.35
+ANIMAL_HEIGHT = 0.6
+FOCAL_LENGTH = 6400
+# CONF = 0.0
+
+#Load models
+det_model = YOLO(path['DET_MODEL_PATH'])
+activity_det_model = YOLO(path['ACTIVITY_DET_MODEL_PATH'])
+
+activity_classes = ['Standing','Running','Sitting']
+
+def object_detection(image):
+    
+    """
+    Args:
+        image (numpy array): get numpy array of image which has 3 channels
+
+    Returns:
+        new_boxes: returns json object which has below format
+        [
+            {
+                "actual_boundries": [
+                    {
+                        "top_left": [48, 215],
+                        "bottom_right": [62, 245],
+                        "class": "person"
+                    }
+                ],
+                "updated_boundries": {
+                    "top_left": [41, 199],
+                    "bottom_right": [73, 269],
+                    "person_count": 1,
+                    "vehical_count": 0,
+                    "animal_count": 0
+                }
+            }
+        ]
+    """
+
+    #detect object using yolo model
+    results = det_model(image)
+
+    boxes = results[0].boxes.xyxy.tolist()
+    classes = results[0].boxes.cls.tolist()
+    names = results[0].names
+    confidences = results[0].boxes.conf.tolist()
+    ctr = 0
+    my_boxes = []  # ((x1, y1), (x2,y2), person_count, vehical_count, animal_count)
+
+    for box, cls, conf in zip(boxes, classes, confidences):
+        x1, y1, x2, y2 = box
+        name = names[int(cls)]
+        my_obj = {"actual_boundries": [{"top_left": (int(x1), int(y1)),
+                                        "bottom_right": (int(x2), int(y2)),
+                                        "class": name}]}
+        # img = cv2.imread(img_path)
+        x1 = max(0, x1 - (x2-x1)/2)
+        y1 = max(0, y1 - (y2-y1)/2)
+        x2 = min(len(image[0])-1, x2 + (x2-x1)/2)
+        y2 = min(len(image)-1, y2 + (y2-y1)/2)
+        x1, y1, x2, y2 = math.floor(x1), math.floor(y1), math.ceil(x2), math.ceil(y2)
+        # image = cv2.rectangle(image, (x1, y1), (x2, y2), (255, 0, 0), 2)
+        my_obj["updated_boundries"] = {"top_left": (x1, y1),
+                                        "bottom_right": (x2, y2),
+                                        "person_count": 1 if name == 'person' else 0,
+                                        "vehical_count": 1 if name == 'vehical' else 0,
+                                        "animal_count": 1 if name == 'animal' else 0}
+        my_boxes.append(my_obj)
+        ctr += 1
+    my_boxes.sort(key=lambda x: (x['updated_boundries']['top_left'], x['updated_boundries']['bottom_right']))
+
+    new_boxes = []
+    if len(my_boxes) > 0:
+        new_boxes.append(my_boxes[0])
+
+    for indx, box in enumerate(my_boxes):
+        if indx != 0:
+            top_left_last = new_boxes[-1]['updated_boundries']['top_left']
+            bottom_right_last = new_boxes[-1]['updated_boundries']['bottom_right']
+            top_left_curr = box['updated_boundries']['top_left']
+            bottom_right_curr = box['updated_boundries']['bottom_right']
+
+            if bottom_right_last[0] >= top_left_curr[0] and bottom_right_last[1] >= top_left_curr[1]:
+                new_x1 = min(top_left_last[0], top_left_curr[0])
+                new_y1 = min(top_left_last[1], top_left_curr[1])
+                new_x2 = max(bottom_right_last[0], bottom_right_curr[0])
+                new_y2 = max(bottom_right_last[1], bottom_right_curr[1])
+                
+                new_boxes[-1]['actual_boundries'] += box['actual_boundries']
+                new_boxes[-1]['updated_boundries'] = {"top_left": (new_x1, new_y1), 
+                                                    "bottom_right": (new_x2, new_y2), 
+                                                    "person_count": new_boxes[-1]['updated_boundries']['person_count'] + box['updated_boundries']['person_count'], 
+                                                    "vehical_count": new_boxes[-1]['updated_boundries']['vehical_count'] + box['updated_boundries']['vehical_count'], 
+                                                    "animal_count": new_boxes[-1]['updated_boundries']['animal_count'] + box['updated_boundries']['animal_count']}
+            else:
+                new_boxes.append(box)
+            
+    return new_boxes
+
+def croped_images(image,new_boxes):
+    """_summary_
+
+    Args:
+        image (numpy array): get numpy array of image which has 3 channels
+        new_boxes (json array): get json array 
+
+    Returns:
+        croped_images_list(list of numpy array): returns list which has croped images
+        single_object_images(list of numpy array): returns list which has single object images
+    """
+    croped_images_list = []
+    single_object_images = []
+
+    for data in new_boxes:
+        print(data['updated_boundries'])
+        crop_image = image[data['updated_boundries']['top_left'][1]:data['updated_boundries']['bottom_right'][1],data['updated_boundries']['top_left'][0]:data['updated_boundries']['bottom_right'][0]]
+        croped_images_list.append(crop_image)
+
+        for object in data['actual_boundries']:
+            if object['class']=='person':
+                crop_object= image[object['top_left'][1]:object['bottom_right'][1],object['top_left'][0]:object['bottom_right'][0]]
+                single_object_images.append(crop_object)
+
+
+    return croped_images_list,single_object_images
+
+def image_enhancements(croped_images_list,single_object_images):
+    """_summary_
+
+    Args:
+        croped_images_list (list numpy array): croped images list 
+        single_object_images (list numpy array): single object images list
+
+    Returns:
+        enhanced croped images: returns enhanced images
+        enhanced single_object_images: returns enhanced images
+    """
+    enhanced_images = []
+    enhanced_single_object_images = []
+
+    for image in croped_images_list:
+
+        # resize the image
+        res = cv2.resize(image,(500*image.shape[1]//image.shape[0],500), interpolation = cv2.INTER_CUBIC)
+       
+        # brightness and contrast
+        brightness = 16
+        contrast = 0.95
+        res2 = cv2.addWeighted(res, contrast, np.zeros(res.shape, res.dtype), 0, brightness) 
+
+        # Sharpen the image 
+        kernel = np.array([[0, -1, 0], [-1, 5, -1], [0, -1, 0]]) 
+        sharpened_image = cv2.filter2D(res2, -1, kernel) 
+
+        #append in the list
+        enhanced_images.append(sharpened_image)
+
+    
+    for image in single_object_images:
+
+        # resize the image
+        res = cv2.resize(image,(500*image.shape[1]//image.shape[0],500), interpolation = cv2.INTER_CUBIC)   
+
+        # brightness and contrast
+        brightness = 16
+        contrast = 0.95 
+        res2 = cv2.addWeighted(res, contrast, np.zeros(res.shape, res.dtype), 0, brightness) 
+        
+        # Sharpen the image 
+        kernel = np.array([[0, -1, 0], [-1, 5, -1], [0, -1, 0]]) 
+        sharpened_image = cv2.filter2D(res2, -1, kernel) 
+
+        #append enhnaced single object image
+        enhanced_single_object_images.append(sharpened_image)
+    
+    return enhanced_images,enhanced_single_object_images
+
+
+def detect_activity(single_object_images):
+    """_summary_
+
+    Args:
+        single_object_images (list of numpy array): list of single object images
+
+    Returns:
+        activities(list of strings): returns list of activities perform by person
+    """
+    activities = []
+
+    for img in single_object_images:
+
+        predictions =activity_det_model.predict(img)
+
+        for result in predictions:
+
+            probs = result.probs
+            class_index = probs.top1
+
+        activities.append(activity_classes[class_index])
+
+    return activities
+
+
+def get_distances(new_boxes):
+    """_summary_
+
+    Args:
+        new_boxes (json array): takes json array of detected image's data
+
+    Returns:
+        distance_list: list of distances of each object
+    """
+  
+    distance_list = []
+    for box in new_boxes:
+        for actual_box in box['actual_boundries']:
+            height = actual_box['bottom_right'][1] - actual_box['top_left'][1]
+
+            if actual_box['class'] == "person":
+                distance = FOCAL_LENGTH*PERSON_HEIGHT/height
+                
+            elif actual_box['class'] == "vehical":
+                distance = FOCAL_LENGTH*PERSON_HEIGHT/height
+
+            else:
+                distance = FOCAL_LENGTH*PERSON_HEIGHT/height
+
+            distance_list.append(str(round(distance)) + "m")
+
+    return distance_list
+
+
+def get_json_data(json_data,enhanced_images,detected_activity,distances_list):
+    """_summary_
+
+    Args:
+        json_data (json Array): get json data of image
+        enhanced_images (list of numpy array): list of enhanced images 
+        detected_activity (list of strings): list of activities of person
+        distances_list (lsit of integers): list of distances of each object
+
+    Returns:
+        results(json Array): contains all informations needed for frontend 
+                            {'zoomed_img':np.array([]) ,
+                             'actual_boxes':[],
+                             'updated_boxes':{},
+                            }
+    """
+    results = []
+    object_count = 0
+    activity_count = 0
+    for idx,box in enumerate(json_data):
+        final_json_output = {'zoomed_img':np.array([]) ,
+                            'actual_boxes':[],
+                            'updated_boxes':{},
+                            }
+
+        final_json_output['zoomed_img'] = enhanced_images[idx]
+        final_json_output['updated_boxes'] = { "top_left": box['updated_boundries']['top_left'],
+                                                "bottom_right": box['updated_boundries']['bottom_right']}
+        
+        for actual_box in box['actual_boundries']:
+            
+            temp  = {"top_left": [],
+                "bottom_right": [],
+                "class": "",
+                "distance":0,
+                "activity":'none'}
+            temp['top_left'] = actual_box['top_left']
+            temp['bottom_right'] = actual_box['bottom_right']
+            temp['class'] = actual_box['class']
+            temp['distance'] = distances_list[object_count]
+            object_count+=1
+            
+            if temp['class'] == 'person':
+                temp['activity'] = detected_activity[activity_count]
+                activity_count+=1
+            
+            final_json_output['actual_boxes'].append(temp)
+            final_json_output = fix_distance(final_json_output)
+
+        results.append(final_json_output)
+    
+    return results
+
+
+def fix_distance(final_json_output):
+    """_summary_
+
+    Args:
+        final_json_output (json Array): array of json object 
+
+    Returns:
+       final_json_output (json Array): array of json object
+    """
+    distances = []
+    DIFF  = 90
+
+    for idx,box in enumerate(final_json_output['actual_boxes']):
+        distances.append({'idx':idx,'distance':int(box['distance'][:-1])})
+
+    sorted_dist = sorted(distances, key=lambda d: d['distance'])
+    sum_dist = []
+    idx= 0
+    sum_dist.append({'sum':sorted_dist[0]['distance'],'idxes':[sorted_dist[0]['idx']]})
+
+    for i in range(1,len(sorted_dist)):
+        print(sorted_dist[i]['distance'],sorted_dist[i-1]['distance'])
+        if abs(sorted_dist[i]['distance']-sorted_dist[i-1]['distance']) <=DIFF:
+            sum_dist[idx]['sum']+= sorted_dist[i]['distance']
+            sum_dist[idx]['idxes'].append(sorted_dist[i]['idx'])
+        
+        else:
+            sum_dist.append({'sum':sorted_dist[i]['distance'],'idxes':[sorted_dist[i]['idx']]})
+            idx+=1
+
+    #change values in distance array 
+    for data in sum_dist:
+        count  = len(data['idxes'])
+        mean = data['sum']//count
+        for i in data['idxes']:
+            final_json_output['actual_boxes'][i]['distance'] = str(mean)+'m'
+        
+    return final_json_output
+    
+    

requirements.txt

@@ -0,0 +1,127 @@
+absl-py==2.1.0
+altair==5.3.0
+asttokens==2.4.1
+astunparse==1.6.3
+attrs==23.2.0
+backcall==0.2.0
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blinker==1.8.1
+cachetools==5.3.3
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.1
+cycler==0.12.1
+decorator==5.1.1
+defusedxml==0.7.1
+docopt==0.6.2
+executing==2.0.1
+fastjsonschema==2.19.1
+filelock==3.14.0
+flatbuffers==24.3.25
+fonttools==4.51.0
+fsspec==2024.3.1
+gast==0.5.4
+gitdb==4.0.11
+GitPython==3.1.43
+google-pasta==0.2.0
+grpcio==1.63.0
+h5py==3.11.0
+idna==3.7
+importlib_metadata==7.1.0
+importlib_resources==6.4.0
+intel-openmp==2021.4.0
+ipython==8.12.3
+jedi==0.19.1
+Jinja2==3.1.3
+jsonschema==4.22.0
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.1
+jupyter_core==5.7.2
+jupyterlab_pygments==0.3.0
+keras==3.3.3
+kiwisolver==1.4.5
+libclang==18.1.1
+Markdown==3.6
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.8.4
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mistune==3.0.2
+mkl==2021.4.0
+ml-dtypes==0.3.2
+mpmath==1.3.0
+namex==0.0.8
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+networkx==3.2.1
+numpy==1.26.4
+opencv-python==4.9.0.80
+opt-einsum==3.3.0
+optree==0.11.0
+packaging==24.0
+pandas==2.2.2
+pandocfilters==1.5.1
+parso==0.8.4
+pickleshare==0.7.5
+pillow==10.3.0
+platformdirs==4.2.1
+prompt-toolkit==3.0.43
+protobuf==4.25.3
+psutil==5.9.8
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==16.0.0
+pydeck==0.9.0
+Pygments==2.17.2
+pyparsing==3.1.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+pytz==2024.1
+pywin32==306
+PyYAML==6.0.1
+pyzmq==26.0.3
+referencing==0.35.1
+requests==2.31.0
+rich==13.7.1
+rpds-py==0.18.0
+scipy==1.13.0
+seaborn==0.13.2
+six==1.16.0
+smmap==5.0.1
+soupsieve==2.5
+stack-data==0.6.3
+streamlit==1.34.0
+sympy==1.12
+tbb==2021.12.0
+tenacity==8.2.3
+tensorboard==2.16.2
+tensorboard-data-server==0.7.2
+tensorflow==2.16.1
+tensorflow-intel==2.16.1
+tensorflow-io-gcs-filesystem==0.31.0
+termcolor==2.4.0
+thop==0.1.1.post2209072238
+tinycss2==1.3.0
+toml==0.10.2
+toolz==0.12.1
+torch==2.3.0
+torchvision==0.18.0
+tornado==6.4
+tqdm==4.66.2
+traitlets==5.14.3
+typing_extensions==4.11.0
+tzdata==2024.1
+ultralytics==8.2.6
+urllib3==2.2.1
+watchdog==4.0.0
+wcwidth==0.2.13
+webencodings==0.5.1
+Werkzeug==3.0.2
+wrapt==1.16.0
+yarg==0.1.9
+zipp==3.18.1