ijru2025 updates

app.py

@@ -73,6 +73,7 @@ def preprocess_image(img, img_size):
 
 
 def run_inference(batch_X):
+    global ort_sess
     batch_X = torch.cat(batch_X)
     return ort_sess.run(None, {'video': batch_X.numpy()})
 
@@ -114,6 +115,7 @@ def detect_beeps(video_path, target_event_length=30, beep_height=0.8):
     
     # Extract audio from video
     audio_convert_command = f'ffmpeg -i {video_path} -vn -acodec pcm_s16le -ar {fs} -ac 2 temp.wav'
+    print(audio_convert_command)
     subprocess.call(audio_convert_command, shell=True)
     
     # Read the extracted audio
@@ -307,17 +309,45 @@ def upload_video(out_text, in_video):
         )
 
 
+def count_phases(phase_sin, phase_cos, threshold=0.5):
+    """
+    Count the number of phase transitions in the sine and cosine phases.
+    
+    Args:
+        phase_sin: Numpy array of sine phase values
+        phase_cos: Numpy array of cosine phase values
+        threshold: Threshold to consider a transition
+    Returns:
+        count: Number of phase transitions
+        phase_indices: Indices where transitions occur
+    """
+    phase_indices = []
+    count = 0
+    for i in range(1, len(phase_sin)):
+        # Check if the sine and cosine phases cross each other
+        if (phase_sin[i-1] < threshold and phase_sin[i] >= threshold) or \
+           (phase_sin[i-1] >= threshold and phase_sin[i] < threshold):
+            # Check if the cosine phase crosses the threshold
+            if (phase_cos[i-1] < threshold and phase_cos[i] >= threshold) or \
+            (phase_cos[i-1] >= threshold and phase_cos[i] < threshold):
+                phase_indices.append(i)
+                count += 1
+    return count, phase_indices
+
+
+
 def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choice, 
               beep_detection_on, event_length, relay_detection_on, relay_length, switch_delay,
-              count_only_api, api_key, seq_len=64, stride_length=32, stride_pad=3, batch_size=4,
-              miss_threshold=0.8, marks_threshold=0.5, median_pred_filter=True, both_feet=True, 
+              count_only_api, api_key, seq_len=64, stride_length=32, stride_pad=3, batch_size=2,
+              miss_threshold=0.5, marks_threshold=0.5, median_pred_filter=True, both_feet=True, 
               api_call=False,
               progress=gr.Progress()):
-    global current_model
+    global current_model, ort_sess
+    print(in_video)
     if model_choice != current_model:
         current_model = model_choice
         onnx_file = hf_hub_download(repo_id="lumos-motion/nextjump", filename=f"{current_model}.onnx", repo_type="model", token=os.environ['DATASET_SECRET'])
-
+        #onnx_file = f'{current_model}.onnx'
 
         if torch.cuda.is_available():
             print("Using CUDA")
@@ -333,8 +363,10 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
         # warmup inference
         ort_sess.run(None, {'video': np.zeros((4, 64, 3, IMG_SIZE, IMG_SIZE), dtype=np.float32)})
     if in_video is None:
+        print("No video input provided.")
         in_video = download_clips(stream_url, os.path.join(os.getcwd(), 'clips'), start_time, end_time, use_60fps=use_60fps)
     else:  # local uploaded video (still resize with ffmpeg)
+        print("Using uploaded video input.")
         in_video = download_clips(in_video, os.path.join(os.getcwd(), 'clips'), start_time, end_time, use_60fps=use_60fps)
     progress(0, desc="Running inference...")
     has_access = False
@@ -401,7 +433,7 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
     batch_list = []
     idx_list = []
     inference_futures = []
-    with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
         for i in progress.tqdm(range(0, length + stride_length - stride_pad, stride_length)):
             batch = all_frames[i:i + seq_len]
             Xlist = []
@@ -499,17 +531,20 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
     full_marks_mask = np.zeros(len(full_marks))
     full_marks_mask[pred_marks_peaks] = 1
     periodicity_mask = np.int32(periodicity > miss_threshold)
+    phase_count, phase_indices = count_phases(phase_sin, phase_cos, threshold=-0.5)
     numofReps = 0
     count = []
     miss_detected = True
-    num_misses = 0
+    num_misses = -1  # end of event is not counted as a miss
+    miss_frames = []
     for i in range(len(periodLength)):
         if periodLength[i] < 2 or periodicity_mask[i] == 0:
             numofReps += 0
             if not miss_detected:
                 miss_detected = True
                 num_misses += 1
-                numofReps -= 2
+                miss_frames.append(i)
+                #numofReps -= 2
         elif full_marks_mask[i]:  # high confidence mark detected
             if math.modf(numofReps)[0] < 0.2:  # probably false positive/late detection
                 numofReps = float(int(numofReps))
@@ -526,6 +561,7 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
         # if a jump was counted, and periodicity is high, and the next frame was not counted (to avoid double counting)
         if full_marks_mask[i] > 0 and periodicity_mask[i] > 0 and full_marks_mask[i + 1] == 0:
             marks_count_pred += 1
+    
     if not both_feet:
         count_pred = count_pred / 2
         marks_count_pred = marks_count_pred / 2
@@ -545,11 +581,84 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
         self_pct_err = 0
     total_confidence = confidence * (1 - self_pct_err)
 
+    # find the fastest second (30 frames if 30fp and 60 frames if 60fps) based on the period_length
+    scan_window = 60 if use_60fps else 30
+    fastest_frames_start = 0
+    fastest_period = float('inf')
+    for i in range(0, len(periodLength) - scan_window, scan_window // 2):
+        #if np.sum(periodicity_mask[i:i + scan_window]) > 0:
+        avg_period = np.mean(periodLength[i:i + scan_window])
+        if avg_period < fastest_period:
+            fastest_period = avg_period
+            fastest_frames_start = i
+    fastest_frames_end = fastest_frames_start + scan_window
+    fastest_jumps_per_second = np.clip(1 / ((fastest_period / fps) + 0.0001), 0, 10)
+    print(f"Fastest jumps per second: {fastest_jumps_per_second:.2f} (from frames {fastest_frames_start} to {fastest_frames_end})")
+
+    # measure the reaction time to the beep (if beep detection is on) as the time to reach average speed 
+    time_to_speed = 0
+    if beep_detection_on:
+        avg_speed = np.mean(periodLength[periodicity_mask])
+        reaction_frame = np.argmax((periodLength < avg_speed) & (periodicity_mask))
+        print(f"Reaction frame: {reaction_frame}, Avg Speed: {avg_speed}")
+        time_to_speed = (reaction_frame - event_start) / fps
+
+    # get peak speed and lowest speed
+    peak_speed = np.quantile(periodLength[periodicity_mask], 0.01) if np.any(periodicity_mask) else 0
+    lowest_speed = np.quantile(periodLength[periodicity_mask], 0.99) if np.any(periodicity_mask) else 0
+    peak_jps = np.clip(1 / ((peak_speed / fps) + 0.0001), 0, 10)
+    lowest_jps = np.clip(1 / ((lowest_speed / fps) + 0.0001), 0, 10)
+    slowdown = (lowest_jps - peak_jps)
+    slowdown_percent = (slowdown / peak_jps) * 100 if peak_jps > 0 else 0
+
+    print('slowdown', slowdown)
+    print('percent', slowdown_percent)
+
+    # estimate the score assuming no misses and fill in the gaps
+    estimated_score = 0
+    filled_periodLength = np.zeros(len(periodLength))
+    started = False
+    for i in range(len(periodLength)):
+        if beep_detection_on and i < event_start:
+            filled_periodLength[i] = 0
+        elif beep_detection_on and i >= event_end:
+            filled_periodLength[i] = 0
+        elif periodicity_mask[i] > 0:
+            started = True
+            filled_periodLength[i] = periodLength[i]
+        elif not started:
+            filled_periodLength[i] = 0
+        else:
+            # fill in the gaps with the previous value
+            filled_periodLength[i] = filled_periodLength[i - 1]
+    estimated_score = 0
+    for i in range(len(filled_periodLength)):
+        if filled_periodLength[i] < 2:
+            estimated_score += 0
+        else:
+            estimated_score += max(0, periodicity_mask[i] / (filled_periodLength[i]))
+    print(f"Estimated score: {estimated_score:.2f}")
+    
+    # find the recovery times after each miss
+    recovery_times = []
+    if len(miss_frames) > 0:
+        avg_speed = np.mean(periodLength[periodicity_mask])
+        for miss_frame in miss_frames:
+            # find the next frame where the speed is above avg_speed
+            recovery_frame = np.argmax((periodLength[miss_frame:] > avg_speed) & (periodicity_mask[miss_frame:])) + miss_frame
+            if recovery_frame > miss_frame:
+                recovery_time = (recovery_frame - miss_frame) / fps
+                recovery_times.append(recovery_time)
+            else: # end of event
+                pass 
+    print(f"Recovery times: {recovery_times}")
+            
+
     if LOCAL:
         if both_feet:
-            count_msg = f"## Reps Count (both feet): {count_pred:.1f}, Marks Count (both feet): {marks_count_pred:.1f}, Confidence: {total_confidence:.2f}"
+            count_msg = f"## Reps Count (both feet): {count_pred:.1f}, Marks: {marks_count_pred:.1f}, Phase: {phase_count:.1f}, Confidence: {total_confidence:.2f}, Time to Speed: {time_to_speed:.2f} seconds"
         else:
-            count_msg = f"## Reps Count (one foot): {count_pred:.1f}, Marks Count (one foot): {marks_count_pred:.1f}, Confidence: {total_confidence:.2f}"
+            count_msg = f"## Reps Count (one foot): {count_pred:.1f}, Marks: {marks_count_pred:.1f}, Phase: {phase_count:.1f}, Confidence: {total_confidence:.2f}, Time to Speed: {time_to_speed:.2f} seconds"
     else:
         if both_feet:
             count_msg = f"## Reps Count (both feet): {count_pred:.1f}, Confidence: {total_confidence:.2f}"
@@ -575,7 +684,21 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
                 "cum_count": np.array2string(np.array(count), formatter={'float_kind':lambda x: "%.2f" % x}, threshold=np.inf).replace('\n', ''),
                 "count": f"{count_pred:.2f}",
                 "marks": f"{marks_count_pred:.1f}",
+                "phase_count": f"{phase_count:.1f}",
                 "confidence": f"{total_confidence:.2f}",
+                "fastest_frames_start": fastest_frames_start,
+                "fastest_frames_end": fastest_frames_end,
+                "fastest_jumps_per_second": f"{fastest_jumps_per_second:.2f}",
+                "lowest_jumps_per_second": f"{lowest_jps:.2f}",
+                "fastest_period_length": f"{fastest_period:.2f}",
+                "lowest_period_length": f"{lowest_speed:.2f}",
+                "time_to_speed": f"{time_to_speed:.2f}" if beep_detection_on else 0,
+                "slowdown": f"{slowdown:.2f}",
+                "slowdown_percent": f"{slowdown_percent:.2f}",
+                "num_misses": num_misses,
+                "miss_frames": np.array2string(np.array(miss_frames[:num_misses]), formatter={'int':lambda x: str(x)}, threshold=np.inf).replace('\n', ''),
+                "recovery_times": np.array2string(np.array(recovery_times), formatter={'float_kind':lambda x: "%.2f" % x}, threshold=np.inf).replace('\n', ''),
+                "no_miss_score": f"{estimated_score:.2f}" if num_misses > 0 else f"{count_pred:.2f}",
                 "single_rope_speed": f"{event_type_probs[0]:.3f}",
                 "double_dutch": f"{event_type_probs[1]:.3f}",
                 "double_unders": f"{event_type_probs[2]:.3f}",
@@ -710,8 +833,10 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
     # phase_cos = np.cos(np.arctan2(phase_sin, phase_cos) - np.pi / 2)
     
     # plot phase spiral using plotly
+    phase_jumps = np.zeros(len(phase_sin))
+    phase_jumps[phase_indices] = 1
     fig_phase_spiral = px.scatter(x=phase_cos, y=phase_sin,
-                                color=jumps_per_second,
+                                color=phase_jumps,
                                 color_continuous_scale='plasma',
                                 title="Phase Spiral (speed)",
                                 template="plotly_dark")
@@ -725,7 +850,7 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
     )
     # label colorbar as time
     fig_phase_spiral.update_coloraxes(colorbar=dict(
-        title="Jumps per second"))
+        title="Phase Jumps",))
     # make axes equal
     fig_phase_spiral.update_layout(
         xaxis=dict(scaleanchor="y"),
@@ -792,7 +917,7 @@ def inference(in_video, stream_url, start_time, end_time, use_60fps, model_choic
     except FileNotFoundError:
         pass
     
-    return in_video, count_msg, fig, fig_phase_spiral, fig_phase_spiral_marks, hist, bar
+    return count_msg, fig, fig_phase_spiral, fig_phase_spiral_marks, hist, bar
         
 #css = '#phase-spiral {transform: rotate(0.25turn);}\n#phase-spiral-marks {transform: rotate(0.25turn);}'
 with gr.Blocks() as demo:
@@ -823,6 +948,7 @@ with gr.Blocks() as demo:
             use_60fps = gr.Checkbox(label="Use 60 FPS", elem_id='use-60fps', visible=True)
             model_choice =  gr.Dropdown(
                 ["nextjump_speed", "nextjump_all", "nextjump_both_feet"], label="Model Choice", info="For now just speed-only or general model",
+                value="nextjump_speed", elem_id='model-choice'
             )
         with gr.Column():
             gr.Markdown(
@@ -837,8 +963,6 @@ with gr.Blocks() as demo:
             relay_detection_on = gr.Checkbox(label="Relay Event", elem_id='relay-beeps', visible=True)
             relay_length = gr.Textbox(label="Relay Length (s)", elem_id='relay-length', visible=True, value='30')
             switch_delay = gr.Textbox(label="Expected Switch Delay (s)", elem_id='event-length', visible=True, value='0.2')
-        with gr.Column(min_width=480):
-            out_video = gr.PlayableVideo(label="Video Clip", elem_id='output-video', format='mp4', width=400, height=400)
             
     with gr.Row():
         run_button = gr.Button(value="Run", elem_id='run-button', scale=1)
@@ -869,7 +993,7 @@ with gr.Blocks() as demo:
     demo_inference = partial(inference, count_only_api=False, api_key=None)
     
     run_button.click(demo_inference, [in_video, in_stream_url, in_stream_start, in_stream_end, use_60fps, model_choice, beep_detection_on, event_length, relay_detection_on, relay_length, switch_delay], 
-                     outputs=[out_video, out_text, out_plot, out_phase_spiral, out_phase, out_hist, out_event_type_dist])
+                     outputs=[out_text, out_plot, out_phase_spiral, out_phase, out_hist, out_event_type_dist])
     api_inference = partial(inference, api_call=True)
     api_dummy_button.click(api_inference, [in_video, in_stream_url, in_stream_start, in_stream_end, use_60fps, model_choice, beep_detection_on, event_length, relay_detection_on, relay_length, switch_delay, count_only, api_token], 
                            outputs=[period_length], api_name='inference')
@@ -881,7 +1005,7 @@ with gr.Blocks() as demo:
     ]
     gr.Examples(examples, 
                 inputs=[in_video, in_stream_url, in_stream_start, in_stream_end, use_60fps, model_choice, beep_detection_on, event_length, relay_detection_on, relay_length, switch_delay], 
-                outputs=[out_video, out_text, out_plot, out_phase_spiral, out_phase, out_hist, out_event_type_dist],
+                outputs=[out_text, out_plot, out_phase_spiral, out_phase, out_hist, out_event_type_dist],
                 fn=demo_inference, cache_examples=False)
 
 
@@ -891,6 +1015,6 @@ if __name__ == "__main__":
                                                     server_port=7860, 
                                                     debug=False,
                                                     ssl_verify=False,
-                                                    share=False)
+                                                    share=True)
     else:
         demo.queue(api_open=True, max_size=15).launch(share=False)

hls_download.py

@@ -11,6 +11,7 @@ def download_clips(stream_url, out_dir, start_time, end_time, resize=True, use_6
     if resize:  # resize and convert to 30 fps
         ffmpeg_cmd = [
                 'ffmpeg',
+                #'-hwaccel', 'cuda',
                 '-i', stream_url,
                 '-c:v', 'libx264',
                 '-crf', '23',
@@ -30,7 +31,7 @@ def download_clips(stream_url, out_dir, start_time, end_time, resize=True, use_6
         except subprocess.CalledProcessError as e:
             print(f"Error occurred: {e}")
             print(f"ffmpeg output: {e.output}")
-            return None
+            return output_file
     # else:
     #     os.rename(tmp_file, output_file)
     print(f"Converted {output_file}")