change detection to use rolling mode, add json output

app.py

@@ -31,8 +31,8 @@ def get_auto_comparison(url, target, smoothing_window_size=10, method="CUSUM"):
     # fig = plot_comparison(lims, D, I, hash_vectors, MIN_DISTANCE = distance)
     df = get_videomatch_df(url, target, min_distance=MIN_DISTANCE, vanilla_df=False)
     change_points = get_change_points(df, smoothing_window_size=smoothing_window_size, method=method)
-    fig = plot_segment_comparison(df, change_points)
-    return fig
+    fig, segment_decisions = plot_segment_comparison(df, change_points)
+    return fig, segment_decisions
 
 def get_auto_edit_decision(url, target, smoothing_window_size=10):
     """ Function for Gradio to combine all helper functions"""
@@ -74,8 +74,8 @@ compare_iface = gr.Interface(fn=get_comparison,
                      examples=[[x, video_urls[-1]] for x in video_urls[:-1]])
 
 auto_compare_iface = gr.Interface(fn=get_auto_comparison,
-                     inputs=["text", "text", gr.Slider(1, 50, 10, step=1), gr.Dropdown(choices=["CUSUM", "Robust"], value="Robust")], 
-                     outputs="plot", 
+                     inputs=["text", "text", gr.Slider(2, 50, 10, step=1), gr.Dropdown(choices=["CUSUM", "Robust"], value="Robust")], 
+                     outputs=["plot", "json"], 
                      examples=[[x, video_urls[-1]] for x in video_urls[:-1]])
 
 iface = gr.TabbedInterface([auto_compare_iface, compare_iface, index_iface,], ["AutoCompare", "Compare", "Index"])

config.py

@@ -5,3 +5,4 @@ VIDEO_DIRECTORY = tempfile.gettempdir()
 FPS = 5
 MIN_DISTANCE = 4
 MAX_DISTANCE = 30
+ROLLING_WINDOW_SIZE = 10

plot.py

@@ -71,32 +71,48 @@ def add_seconds_to_datetime64(datetime64, seconds, subtract=False):
 
 def plot_segment_comparison(df, change_points):
     """ From the dataframe plot the current set of plots, where the bottom right is most indicative """
-    fig, ax_arr = plt.subplots(2, 1, figsize=(16, 6), dpi=100, sharex=True)
-    sns.scatterplot(data = df, x='time', y='SOURCE_S', ax=ax_arr[0])
+    fig, ax_arr = plt.subplots(3, 1, figsize=(16, 6), dpi=100, sharex=True)
+    # sns.scatterplot(data = df, x='time', y='SOURCE_S', ax=ax_arr[0])
+    sns.scatterplot(data = df, x='time', y='OFFSET', ax=ax_arr[0], label="OFFSET", alpha=0.5)
     # sns.lineplot(data = df, x='time', y='SOURCE_LIP_S', ax=ax_arr[0,1])
 
-    # Plot change point as lines 
+    # Get rolling average offset
+    # window_size = 30
+    # df['ROLL_OFFSET'] = df['OFFSET_LIP'].rolling(window_size, center=False, min_periods=1).median()
+    # df['ROLL_OFFSET'] = df['OFFSET_LIP'].rolling(window_size, center=False, min_periods=1).apply(lambda x: st.mode(x)[0])
+    metric = 'ROLL_OFFSET_MODE' #'OFFSET'
+    sns.scatterplot(data = df, x='time', y=metric, ax=ax_arr[1], label=metric, alpha=0.5)
+
+    # Plot linearly interpolated values
+    sns.lineplot(data = df, x='time', y='OFFSET_LIP', ax=ax_arr[1], label="OFFSET_LIP")
+
+    # Plot change point as lines
+    
     # sns.lineplot(data = df, x='time', y='OFFSET_LIP', ax=ax_arr[1,0])
-    sns.lineplot(data = df, x='time', y='OFFSET_LIP', ax=ax_arr[1])
+    sns.scatterplot(data = df, x='time', y=metric, ax=ax_arr[2], label=metric, s=20)
     timestamps = change_points_to_segments(df, change_points) 
 
+    segment_decisions = {}
+    seg_i = 0
+
     # To plot the detected segment lines 
     for x in timestamps:
-        plt.vlines(x=x, ymin=np.min(df['OFFSET_LIP']), ymax=np.max(df['OFFSET_LIP']), colors='black', lw=2, alpha=0.5)
-        rand_y_pos = np.random.uniform(low=np.min(df['OFFSET_LIP']), high=np.max(df['OFFSET_LIP']), size=None)
+        plt.vlines(x=x, ymin=np.min(df[metric]), ymax=np.max(df[metric]), colors='black', lw=2, alpha=0.5)
 
     # To get each detected segment and their mean?
     threshold_diff = 1.5 # Average diff threshold 
     # threshold = 3.0 # s diff threshold
     for start_time, end_time in zip(timestamps[:-1], timestamps[1:]):
 
+        # add_offset = df.iloc[0]['SOURCE_S'] # np.min(df['SOURCE_S'])
         add_offset = np.min(df['SOURCE_S'])
         
-        # Cut out the segment between the segment lines 
+        # Cut out the segment between the segment lines
         segment = df[(df['time'] > start_time) & (df['time'] < end_time)] # Not offset LIP
-        segment_no_nan = segment[~np.isnan(segment['OFFSET'])] # Remove NaNs
-        segment_offsets = segment_no_nan['OFFSET'] # np.round(segment_no_nan['OFFSET'], 1)
+        segment_no_nan = segment[~np.isnan(segment[metric])] # Remove NaNs
+        segment_offsets = segment_no_nan[metric] # np.round(segment_no_nan['OFFSET'], 1)
         # segment_offsets = np.round(segment_no_nan['OFFSET'], 0)
+        # print(segment_offsets)
         
         # Calculate mean/median/mode
         # seg_sum_stat = np.mean(segment_offsets)
@@ -104,27 +120,40 @@ def plot_segment_comparison(df, change_points):
         seg_sum_stat = st.mode(segment_offsets)[0][0]
 
         # Get average difference from mean/median/mode of the segment to see if it is a "straight line" or not 
-        average_diff = np.mean(np.abs(segment_offsets - seg_sum_stat))
+        average_diff = np.median(np.abs(segment_offsets - seg_sum_stat))
         
         # If the time where the segment comes from (origin time) is close to the start_time, it's a "good match", so no editing
         noisy = False if average_diff < threshold_diff else True
-        origin_time = add_seconds_to_datetime64(start_time, seg_sum_stat + add_offset)
+        origin_start_time = add_seconds_to_datetime64(start_time, seg_sum_stat + add_offset)
+        origin_end_time  = add_seconds_to_datetime64(end_time, seg_sum_stat + add_offset)
 
         # Plot green for a confident prediction (straight line), red otherwise
         if not noisy:
             # Plot estimated straight line
-            plt.hlines(y=seg_sum_stat, xmin=start_time, xmax=end_time, color='green', lw=3, alpha=0.5)
+            plt.hlines(y=seg_sum_stat, xmin=start_time, xmax=end_time, color='green', lw=5, alpha=0.5)
             plt.text(x=start_time, y=seg_sum_stat, s=str(np.round(average_diff, 1)), color='green', rotation=-0.0, fontsize=14)   
         else:
             # Plot estimated straight line
-            plt.hlines(y=seg_sum_stat, xmin=start_time, xmax=end_time, color='red', lw=3, alpha=0.5)
+            plt.hlines(y=seg_sum_stat, xmin=start_time, xmax=end_time, color='red', lw=5, alpha=0.5)
             plt.text(x=start_time, y=seg_sum_stat, s=str(np.round(average_diff, 1)), color='red', rotation=-0.0, fontsize=14)
-
         
-        
-        # print(f"DIFF={average_diff:.1f} SUMSTAT={seg_sum_stat:.1f} {start_time} -> {end_time} comes from video X, from {origin_time}")
-
+        # Decisions about segments
+        start_time_str = pd.to_datetime(start_time).strftime('%H:%M:%S')
+        end_time_str = pd.to_datetime(end_time).strftime('%H:%M:%S')
+        origin_start_time_str = pd.to_datetime(origin_start_time).strftime('%H:%M:%S')
+        origin_end_time_str = pd.to_datetime(origin_end_time).strftime('%H:%M:%S')
+        video_id = "placeholder_video_id"
+
+        decision = {"Target Start Time" : start_time_str,
+                    "Target End Time" : end_time_str,
+                    "Source Start Time" : origin_start_time_str,
+                    "Source End Time" : origin_end_time_str,
+                    "Video ID" : video_id,
+                    "Explanation" : f"{start_time_str} -> {end_time_str} comes from video {video_id} from {origin_start_time_str} -> {origin_end_time_str}"}
+        segment_decisions[f'Segment {seg_i}'] = decision 
+        seg_i += 1
+        print(decision)
 
     # Return figure
     plt.xticks(rotation=90)
-    return fig
+    return fig, segment_decisions

videomatch.py

@@ -7,11 +7,13 @@ from kats.detectors.cusum_detection import CUSUMDetector
 from kats.detectors.robust_stat_detection import RobustStatDetector
 from kats.consts import TimeSeriesData
 
+from scipy import stats as st
+
 import numpy as np
 import pandas as pd
 
 from videohash import compute_hashes, filepath_from_url
-from config import FPS, MIN_DISTANCE, MAX_DISTANCE
+from config import FPS, MIN_DISTANCE, MAX_DISTANCE, ROLLING_WINDOW_SIZE
 
 def index_hashes_for_video(url: str) -> faiss.IndexBinaryIVF:
     """ Compute hashes of a video and index the video using faiss indices and return the index. """
@@ -79,6 +81,7 @@ def get_decent_distance(filepath, target, MIN_DISTANCE, MAX_DISTANCE):
     
 def get_change_points(df, smoothing_window_size=10, method='CUSUM'):
     tsd = TimeSeriesData(df.loc[:,['time','OFFSET_LIP']])
+    # tsd = TimeSeriesData(df.loc[:,['time','ROLL_OFFSET_MODE']])
     if method.upper() == "CUSUM":
         detector = CUSUMDetector(tsd)
     elif method.upper() == "ROBUST":
@@ -93,7 +96,7 @@ def get_change_points(df, smoothing_window_size=10, method='CUSUM'):
         print(f"Video jumps {jump_s:.1f}s in time at {mean_offset_prechange:.1f} seconds")
     return change_points
 
-def get_videomatch_df(url, target, min_distance=MIN_DISTANCE, vanilla_df=False):
+def get_videomatch_df(url, target, min_distance=MIN_DISTANCE, window_size=ROLLING_WINDOW_SIZE, vanilla_df=False):
     distance = get_decent_distance(url, target, MIN_DISTANCE, MAX_DISTANCE)
     _, hash_vectors = get_video_index(url)
     target_index, _ = get_video_index(target)
@@ -147,6 +150,9 @@ def get_videomatch_df(url, target, min_distance=MIN_DISTANCE, vanilla_df=False):
     df['OFFSET'] = df['SOURCE_S'] - df['TARGET_S'] - np.min(df['SOURCE_S'])
     df['OFFSET_LIP'] = df['SOURCE_LIP_S'] - df['TARGET_S'] - np.min(df['SOURCE_LIP_S'])
     
+    # Add rolling window mode
+    df['ROLL_OFFSET_MODE'] = np.round(df['OFFSET_LIP'], 0).rolling(window_size, center=True, min_periods=1).apply(lambda x: st.mode(x)[0])
+
     # Add time column for plotting
     df['time'] = pd.to_datetime(df["TARGET_S"], unit='s') # Needs a datetime as input
     return df