MogensR commited on
Commit
7f55866
Β·
1 Parent(s): 940a560

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +195 -338
app.py CHANGED
@@ -1,10 +1,8 @@
1
  #!/usr/bin/env python3
2
  # ========================= PRE-IMPORT ENV GUARDS =========================
3
- # Must run BEFORE importing numpy/cv2/torch to avoid libgomp errors.
4
  import os
5
-
6
- # Remove invalid OMP setting or tame thread counts
7
- os.environ.pop("OMP_NUM_THREADS", None) # or set to e.g. "1"
8
  os.environ.setdefault("MKL_NUM_THREADS", "1")
9
  os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
10
  os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
@@ -16,7 +14,7 @@
16
  """
17
  High-Quality Video Background Replacement - MAIN APPLICATION
18
  Upload video β†’ Choose professional background β†’ Replace with cinema quality
19
- Features: SAM2 + MatAnyone with multi-fallback loading, professional backgrounds,
20
  cinema-quality processing, lazy loading, and enhanced stability
21
  """
22
 
@@ -39,16 +37,13 @@
39
  from typing import Optional, Tuple, Dict, Any
40
  import logging
41
  import warnings
 
 
42
 
43
- # Import all utilities (must provide: PROFESSIONAL_BACKGROUNDS,
44
- # create_professional_background, create_procedural_background,
45
- # segment_person_hq, refine_mask_hq, replace_background_hq, get_model_status)
46
- from utilities import *
47
 
48
- # Suppress warnings
49
  warnings.filterwarnings("ignore")
50
-
51
- # Setup logging
52
  logging.basicConfig(level=logging.INFO)
53
  logger = logging.getLogger(__name__)
54
 
@@ -58,14 +53,12 @@
58
  try:
59
  import gradio_client.utils as gc_utils
60
  original_get_type = gc_utils.get_type
61
-
62
  def patched_get_type(schema):
63
  if not isinstance(schema, dict):
64
  if isinstance(schema, bool):
65
  return "boolean"
66
  return "string"
67
  return original_get_type(schema)
68
-
69
  gc_utils.get_type = patched_get_type
70
  logger.info("βœ… Applied Gradio schema validation monkey patch.")
71
  except (ImportError, AttributeError) as e:
@@ -75,10 +68,7 @@ def patched_get_type(schema):
75
  # SAM2 LOADER (Hydra search path; pass STRING config name to build_sam2)
76
  # ============================================================================ #
77
  def load_sam2_predictor(device: str = "cuda", progress: Optional[gr.Progress] = None):
78
- """
79
- Loads the SAM2 model and returns a SAM2ImagePredictor instance.
80
- Uses Hydra only to set the config search path; passes STRING config name to build_sam2.
81
- """
82
  import hydra
83
 
84
  sam_logger = logging.getLogger("SAM2Loader")
@@ -86,20 +76,12 @@ def load_sam2_predictor(device: str = "cuda", progress: Optional[gr.Progress] =
86
  sam_logger.info(f"Looking for SAM2 configs in absolute path: {configs_dir}")
87
 
88
  if not os.path.isdir(configs_dir):
89
- sam_logger.error(
90
- f"FATAL: Configs directory not found at '{configs_dir}'. "
91
- f"Please ensure the 'Configs' folder exists at repository root."
92
- )
93
- raise gr.Error("FATAL: SAM2 Configs directory not found.")
94
-
95
- tried = []
96
 
97
  def _maybe_progress(pct: float, desc: str):
98
  if progress is not None:
99
- try:
100
- progress(pct, desc=desc)
101
- except Exception:
102
- pass
103
 
104
  def try_load(config_name_with_yaml: str, checkpoint_name: str):
105
  try:
@@ -110,7 +92,7 @@ def try_load(config_name_with_yaml: str, checkpoint_name: str):
110
  sam_logger.info(f"Downloading {checkpoint_name} from Hugging Face Hub...")
111
  _maybe_progress(0.1, f"Downloading {checkpoint_name}...")
112
  from huggingface_hub import hf_hub_download
113
- repo = f"facebook/{config_name_with_yaml.replace('.yaml','')}" # e.g. facebook/sam2_hiera_large
114
  checkpoint_path = hf_hub_download(
115
  repo_id=repo,
116
  filename=checkpoint_name,
@@ -128,7 +110,7 @@ def try_load(config_name_with_yaml: str, checkpoint_name: str):
128
  job_name=f"sam2_load_{int(time.time())}"
129
  )
130
 
131
- # IMPORTANT: pass STRING config name (no ".yaml") to build_sam2
132
  config_name = config_name_with_yaml.replace(".yaml", "")
133
 
134
  from sam2.build_sam import build_sam2
@@ -142,42 +124,65 @@ def try_load(config_name_with_yaml: str, checkpoint_name: str):
142
  predictor = SAM2ImagePredictor(sam2_model)
143
  sam_logger.info(f"βœ… Loaded {config_name_with_yaml} successfully on {device}")
144
  return predictor
 
145
  except Exception as e:
146
- error_msg = f"Failed to load {config_name_with_yaml}: {e}\nTraceback: {traceback.format_exc()}"
147
- tried.append(error_msg)
148
- sam_logger.warning(error_msg)
149
  return None
150
 
151
  predictor = try_load("sam2_hiera_large.yaml", "sam2_hiera_large.pt")
152
-
153
  if predictor is None:
154
- error_message = "SAM2 loading failed for large model. Reasons:\n" + "\n".join(tried)
155
- sam_logger.error(f"❌ {error_message}")
156
- raise gr.Error(error_message)
157
-
158
  return predictor
159
 
160
  # ============================================================================ #
161
- # MatAnyOne LOADER (robust cfg + net + core)
162
  # ============================================================================ #
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
163
  def load_matanyone(device: str):
164
  """
165
- Robust MatAnyOne loader:
166
- - Loads an OmegaConf cfg from common paths or creates a minimal default
167
- - Builds the network and wraps with InferenceCore
168
- - Tries multiple import layouts to accommodate different repos
169
  """
170
  from omegaconf import OmegaConf
171
-
172
  ma_logger = logging.getLogger("MatAnyOneLoader")
173
 
174
- # Try to locate a config file; otherwise minimal default
175
- cfg_path_candidates = [
176
- "Configs/matanyone.yaml",
177
- "configs/matanyone.yaml",
178
  ]
 
 
 
 
 
179
  cfg = None
180
- for p in cfg_path_candidates:
181
  if os.path.exists(p):
182
  ma_logger.info(f"Loading MatAnyOne cfg: {p}")
183
  cfg = OmegaConf.load(p)
@@ -190,34 +195,78 @@ def load_matanyone(device: str):
190
  "device": device,
191
  })
192
 
193
- last_err = None
194
-
195
- # Layout A (common in forks): separate model + inference modules
196
- try:
197
- from matanyone.model.matanyone import MatAnyOne
198
- from matanyone.inference.inference_core import InferenceCore
199
- net = MatAnyOne(cfg)
200
  net.to(device)
201
- core = InferenceCore(net, cfg)
202
- ma_logger.info("βœ… MatAnyOne loaded (layout A)")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
203
  return core
204
- except Exception as e:
205
- last_err = e
206
- ma_logger.warning(f"Layout A failed: {e}")
207
 
208
- # Layout B (single package exposing classes)
 
 
 
 
 
 
 
 
 
 
 
 
 
209
  try:
210
- from matanyone import MatAnyOne, InferenceCore
211
- net = MatAnyOne(cfg)
212
- net.to(device)
213
- core = InferenceCore(net, cfg)
214
- ma_logger.info("βœ… MatAnyOne loaded (layout B)")
215
- return core
 
 
 
 
 
 
 
 
 
 
 
 
216
  except Exception as e:
217
  last_err = e
218
- ma_logger.warning(f"Layout B failed: {e}")
219
 
220
- raise RuntimeError(f"Failed to initialize MatAnyOne/InferenceCore. Last error: {last_err}")
 
221
 
222
  # ============================================================================ #
223
  # GLOBALS & MODEL SETUP
@@ -228,34 +277,27 @@ def load_matanyone(device: str):
228
  loading_lock = threading.Lock()
229
 
230
  def download_and_setup_models(progress: Optional[gr.Progress] = None):
231
- """
232
- Download and setup models (SAM2 and MatAnyOne), robust to HF Spaces and local dev.
233
- """
234
  global sam2_predictor, matanyone_model, models_loaded
235
 
236
  with loading_lock:
237
  if models_loaded:
238
- return "βœ… High-quality models already loaded"
239
- try:
240
- logger.info("πŸ”„ Starting ENHANCED model loading with fallback...")
241
 
 
 
242
  device = "cuda" if torch.cuda.is_available() else "cpu"
243
 
244
- # --- Load SAM2 ---
245
  local_sam2 = load_sam2_predictor(device=device, progress=progress)
246
  sam2_predictor = local_sam2
247
 
248
- # --- Load MatAnyOne ---
249
- try:
250
- local_matanyone = load_matanyone(device)
251
- matanyone_model = local_matanyone
252
- logger.info("βœ… MatAnyOne loaded")
253
- except Exception as e:
254
- logger.warning(f"❌ MatAnyOne load failed: {e}")
255
- raise RuntimeError("MatAnyone model could not be loaded.")
256
 
257
  models_loaded = True
258
- logger.info("--- βœ… All models loaded successfully ---")
259
  return "βœ… SAM2 + MatAnyOne loaded successfully!"
260
  except Exception as e:
261
  logger.error(f"❌ Enhanced loading failed: {str(e)}")
@@ -263,32 +305,23 @@ def download_and_setup_models(progress: Optional[gr.Progress] = None):
263
  return f"❌ Enhanced loading failed: {str(e)}"
264
 
265
  # ============================================================================ #
266
- # TWO-STAGE PROCESSING PIPELINE
267
  # ============================================================================ #
268
- def process_video_hq(
269
- video_path,
270
- background_choice,
271
- custom_background_path,
272
- progress: Optional[gr.Progress] = None
273
- ):
274
  """TWO-STAGE High-quality video processing: Original β†’ Green Screen β†’ Final Background"""
275
  if not models_loaded:
276
  return None, "❌ Models not loaded. Click 'Load Models' first."
277
-
278
  if not video_path:
279
  return None, "❌ No video file provided."
280
 
281
  def _prog(pct: float, desc: str):
282
  if progress is not None:
283
- try:
284
- progress(pct, desc=desc)
285
- except Exception:
286
- pass
287
 
288
  try:
289
  _prog(0.0, "🎬 Initializing TWO-STAGE processing...")
290
 
291
- # Validate and read video
292
  if not os.path.exists(video_path):
293
  return None, f"❌ Video file not found: {video_path}"
294
 
@@ -296,161 +329,119 @@ def _prog(pct: float, desc: str):
296
  if not cap.isOpened():
297
  return None, "❌ Could not open video file. Please check the format."
298
 
299
- # Get video properties
300
  fps = cap.get(cv2.CAP_PROP_FPS)
301
  total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
302
  frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
303
  frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
304
-
305
  logger.info(f"Video properties: {frame_width}x{frame_height}, {fps}fps, {total_frames} frames")
306
 
307
  if total_frames == 0:
308
  return None, "❌ Video appears to be empty or corrupted."
309
 
310
- # Prepare final background for Stage 2
311
  background = None
312
  background_name = ""
313
 
314
  if background_choice == "custom" and custom_background_path:
315
- try:
316
- background = cv2.imread(custom_background_path)
317
- if background is None:
318
- return None, "❌ Could not read custom background image. Please check the file format."
319
- background_name = "Custom Image"
320
- logger.info("Using custom background image")
321
- except Exception as e:
322
- return None, f"❌ Error loading custom background: {str(e)}"
323
  else:
324
  if background_choice in PROFESSIONAL_BACKGROUNDS:
325
- try:
326
- bg_config = PROFESSIONAL_BACKGROUNDS[background_choice]
327
- background = create_professional_background(bg_config, frame_width, frame_height)
328
- background_name = bg_config["name"]
329
- logger.info(f"Using professional background: {background_name}")
330
- except Exception as e:
331
- logger.error(f"Error creating professional background: {e}")
332
- return None, f"❌ Error creating background: {str(e)}"
333
  else:
334
  return None, f"❌ Invalid background selection: {background_choice}"
335
 
336
  if background is None:
337
  return None, "❌ Failed to create background."
338
 
339
- # Setup codec and timestamp
340
  timestamp = int(time.time())
341
  fourcc = cv2.VideoWriter_fourcc(*'mp4v')
342
 
343
- # STAGE 1: Create green screen video (Original β†’ Green Screen)
344
  _prog(0.1, "🟒 STAGE 1: Creating green screen version...")
345
  greenscreen_path = f"/tmp/greenscreen_{timestamp}.mp4"
346
  greenscreen_writer = cv2.VideoWriter(greenscreen_path, fourcc, fps, (frame_width, frame_height))
347
-
348
  if not greenscreen_writer.isOpened():
349
  return None, "❌ Could not create green screen video file."
350
 
351
  frame_count = 0
352
-
353
- # Process original video to green screen
354
  while True:
355
  ret, frame = cap.read()
356
  if not ret:
357
  break
358
-
359
  try:
360
- progress_pct = 0.1 + (frame_count / total_frames) * 0.4
361
- _prog(progress_pct, f"🟒 Green screen frame {frame_count + 1}/{total_frames}")
362
-
363
- # Segment person and create green screen frame
364
- mask = segment_person_hq(frame)
365
- refined_mask = refine_mask_hq(frame, mask)
366
- green_screen = create_green_screen_background(frame)
367
- green_screen_frame = replace_background_hq(frame, refined_mask, green_screen)
368
-
369
  greenscreen_writer.write(green_screen_frame)
370
- frame_count += 1
371
-
372
- if frame_count % 100 == 0:
373
- gc.collect()
374
- if torch.cuda.is_available():
375
- torch.cuda.empty_cache()
376
-
377
  except Exception as e:
378
  logger.warning(f"Error in Stage 1 frame {frame_count}: {e}")
379
  greenscreen_writer.write(frame)
380
- frame_count += 1
 
 
 
 
381
 
382
  greenscreen_writer.release()
383
  cap.release()
384
 
385
- # STAGE 2: Replace green screen with final background (Green Screen β†’ Final)
386
  _prog(0.5, f"🎨 STAGE 2: Replacing green screen with {background_name}...")
387
-
388
  final_path = f"/tmp/final_output_{timestamp}.mp4"
389
  final_writer = cv2.VideoWriter(final_path, fourcc, fps, (frame_width, frame_height))
390
-
391
  if not final_writer.isOpened():
392
  return None, "❌ Could not create final output video file."
393
 
394
- # Open green screen video
395
  greenscreen_cap = cv2.VideoCapture(greenscreen_path)
396
  if not greenscreen_cap.isOpened():
397
  return None, "❌ Could not open green screen video."
398
 
399
  frame_count = 0
400
-
401
- # Process green screen video to final background with enhanced green detection
402
  while True:
403
  ret, green_frame = greenscreen_cap.read()
404
  if not ret:
405
  break
406
-
407
  try:
408
- progress_pct = 0.5 + (frame_count / total_frames) * 0.4
409
- _prog(progress_pct, f"🎬 Final compositing frame {frame_count + 1}/{total_frames}")
410
-
411
- # Detect green screen with wider detection range
412
  hsv = cv2.cvtColor(green_frame, cv2.COLOR_BGR2HSV)
413
  lower_green = np.array([25, 30, 30])
414
  upper_green = np.array([100, 255, 255])
415
  green_mask = cv2.inRange(hsv, lower_green, upper_green)
416
-
417
- # Additional mask processing for cleaner edges
418
  kernel = np.ones((3, 3), np.uint8)
419
  green_mask = cv2.morphologyEx(green_mask, cv2.MORPH_OPEN, kernel)
420
  green_mask = cv2.morphologyEx(green_mask, cv2.MORPH_CLOSE, kernel)
421
- green_mask = 255 - green_mask # Invert mask (person = white, green = black)
422
-
423
- result_frame = replace_background_hq(green_frame, green_mask, background)
424
  final_writer.write(result_frame)
425
- frame_count += 1
426
-
427
- if frame_count % 100 == 0:
428
- gc.collect()
429
- if torch.cuda.is_available():
430
- torch.cuda.empty_cache()
431
-
432
  except Exception as e:
433
  logger.warning(f"Error in Stage 2 frame {frame_count}: {e}")
434
  final_writer.write(green_frame)
435
- frame_count += 1
 
 
 
 
436
 
437
  greenscreen_cap.release()
438
  final_writer.release()
439
 
440
- # Cleanup intermediate green screen file
441
- try:
442
- os.remove(greenscreen_path)
443
- except Exception:
444
- pass
445
 
446
  if frame_count == 0:
447
  return None, "❌ No frames were processed successfully."
448
 
449
  _prog(0.9, "🎡 Adding high-quality audio...")
450
-
451
- # Add audio back with high quality settings
452
  final_output = f"/tmp/final_output_hq_{timestamp}.mp4"
453
-
454
  try:
455
  audio_cmd = (
456
  f'ffmpeg -y -i "{final_path}" -i "{video_path}" '
@@ -459,15 +450,12 @@ def _prog(pct: float, desc: str):
459
  f'-map 0:v:0 -map 1:a:0? -shortest "{final_output}"'
460
  )
461
  result = os.system(audio_cmd)
462
-
463
  if result != 0 or not os.path.exists(final_output):
464
  logger.warning("Audio merging failed, using video without audio")
465
  shutil.copy2(final_path, final_output)
466
-
467
  except Exception as e:
468
  logger.warning(f"Audio processing error: {e}, using video without audio")
469
- try:
470
- shutil.copy2(final_path, final_output)
471
  except Exception as e2:
472
  logger.error(f"Failed to copy video file: {e2}")
473
  return None, f"❌ Failed to finalize video: {str(e2)}"
@@ -476,17 +464,14 @@ def _prog(pct: float, desc: str):
476
  try:
477
  myavatar_path = "/tmp/MyAvatar/My_Videos/"
478
  os.makedirs(myavatar_path, exist_ok=True)
479
-
480
  saved_filename = f"two_stage_bg_replaced_{timestamp}.mp4"
481
  saved_path = os.path.join(myavatar_path, saved_filename)
482
  shutil.copy2(final_output, saved_path)
483
-
484
  logger.info(f"Video saved to: {saved_path}")
485
  except Exception as e:
486
  logger.warning(f"Could not save to MyAvatar directory: {e}")
487
  saved_filename = os.path.basename(final_output)
488
 
489
- # Cleanup temporary files
490
  try:
491
  if os.path.exists(final_path):
492
  os.remove(final_path)
@@ -494,7 +479,6 @@ def _prog(pct: float, desc: str):
494
  pass
495
 
496
  _prog(1.0, "βœ… TWO-STAGE processing complete!")
497
-
498
  success_message = (
499
  f"βœ… TWO-STAGE Success!\n"
500
  f"🟒 Stage 1: Original β†’ Green Screen\n"
@@ -504,22 +488,17 @@ def _prog(pct: float, desc: str):
504
  f"🎯 Quality: Cinema-grade with SAM2 + MatAnyOne\n"
505
  f"πŸš€ Method: Professional two-stage compositing"
506
  )
507
-
508
  return final_output, success_message
509
 
510
  except Exception as e:
511
- error_msg = f"❌ TWO-STAGE Processing Error: {str(e)}"
512
  logger.error(f"Video processing error: {traceback.format_exc()}")
513
- return None, error_msg
514
 
515
  # ============================================================================ #
516
  # GRADIO UI
517
  # ============================================================================ #
518
  def create_interface():
519
- """Create enhanced Gradio interface with comprehensive features and 4-method background system"""
520
-
521
  def extract_video_path(v):
522
- # Robustly extract file path from input (tuple, list, or string)
523
  if isinstance(v, (tuple, list)) and len(v) > 0:
524
  return v[0]
525
  return v
@@ -532,46 +511,27 @@ def extract_video_path(v):
532
  .progress-bar { background: linear-gradient(90deg, #3498db, #2ecc71) !important; }
533
  """
534
  ) as demo:
535
-
536
- # Header
537
  gr.Markdown("# 🎬 Cinema-Quality Video Background Replacement")
538
  gr.Markdown("**Upload a video β†’ Choose a background β†’ Get professional results with AI**")
539
  gr.Markdown("*Powered by SAM2 + MatAnyOne with multi-fallback loading for maximum reliability*")
540
  gr.Markdown("---")
541
 
542
  with gr.Row():
543
- # Left column - Input and controls
544
  with gr.Column(scale=1):
545
  gr.Markdown("### πŸ“₯ Step 1: Upload Your Video")
546
  gr.Markdown("*Supports MP4, MOV, AVI, and other common formats*")
 
547
 
548
- video_input = gr.Video(
549
- label="πŸŽ₯ Drop your video here",
550
- height=300
551
- )
552
-
553
- # Video preview
554
- video_preview = gr.Video(
555
- label="πŸ“Ί Preview of Uploaded Video",
556
- height=200,
557
- interactive=False
558
- )
559
- video_input.change(
560
- fn=extract_video_path,
561
- inputs=video_input,
562
- outputs=video_preview
563
- )
564
 
565
  gr.Markdown("### 🎨 Step 2: Choose Background Method")
566
  gr.Markdown("*Select your preferred background creation method*")
567
-
568
- # FIXED Radio (flat choices)
569
  background_method = gr.Radio(
570
  choices=["upload", "professional", "colors", "ai"],
571
  value="professional",
572
  label="Background Method"
573
  )
574
- # Labels hint
575
  gr.Markdown(
576
  "- **upload** = πŸ“· Upload Image \n"
577
  "- **professional** = 🎨 Professional Presets \n"
@@ -579,15 +539,10 @@ def extract_video_path(v):
579
  "- **ai** = πŸ€– AI Generated"
580
  )
581
 
582
- # Method A: Upload Image
583
  with gr.Group(visible=False) as upload_group:
584
  gr.Markdown("**πŸ“· Upload Your Background Image**")
585
- custom_background = gr.Image(
586
- label="Drop your background image here",
587
- type="filepath"
588
- )
589
 
590
- # Method B: Professional Presets
591
  with gr.Group(visible=True) as professional_group:
592
  gr.Markdown("**🎨 Professional Background Presets**")
593
  professional_choice = gr.Dropdown(
@@ -596,53 +551,43 @@ def extract_video_path(v):
596
  label="Select Professional Background"
597
  )
598
 
599
- # Method C: Colors/Gradients
600
  with gr.Group(visible=False) as colors_group:
601
  gr.Markdown("**🌈 Custom Colors & Gradients**")
602
-
603
  gradient_type = gr.Dropdown(
604
  choices=["solid", "vertical", "horizontal", "diagonal", "radial", "soft_radial"],
605
  value="vertical",
606
  label="Gradient Type"
607
  )
608
-
609
  with gr.Row():
610
  color1 = gr.ColorPicker(label="🎨 Color 1", value="#3498db")
611
  color2 = gr.ColorPicker(label="🎨 Color 2", value="#2ecc71")
612
-
613
  with gr.Row():
614
  color3 = gr.ColorPicker(label="🎨 Color 3", value="#e74c3c")
615
  use_third_color = gr.Checkbox(label="Use 3rd color", value=False)
616
 
617
- # Method D: AI Generated
618
  with gr.Group(visible=False) as ai_group:
619
  gr.Markdown("**πŸ€– AI Generated Background**")
620
-
621
  ai_prompt = gr.Textbox(
622
  label="Describe your background",
623
  placeholder="e.g., 'modern office with plants', 'sunset over mountains', 'abstract tech pattern'",
624
  lines=2
625
  )
626
-
627
  ai_style = gr.Dropdown(
628
  choices=["photorealistic", "artistic", "abstract", "minimalist", "corporate", "nature"],
629
  value="photorealistic",
630
  label="Style"
631
  )
632
-
633
  with gr.Row():
634
  generate_ai_btn = gr.Button("🎨 Generate Background", variant="secondary")
635
  ai_generated_image = gr.Image(label="Generated Background", type="filepath", visible=False)
636
 
637
- # Background method switching function
638
  def switch_background_method(method):
639
  return (
640
- gr.update(visible=(method == "upload")), # upload_group
641
- gr.update(visible=(method == "professional")), # professional_group
642
- gr.update(visible=(method == "colors")), # colors_group
643
- gr.update(visible=(method == "ai")) # ai_group
644
  )
645
-
646
  background_method.change(
647
  fn=switch_background_method,
648
  inputs=background_method,
@@ -651,35 +596,16 @@ def switch_background_method(method):
651
 
652
  gr.Markdown("### 🎬 Processing Controls")
653
  gr.Markdown("*First load the AI models, then process your video*")
654
-
655
  with gr.Row():
656
- load_models_btn = gr.Button(
657
- "πŸš€ Step 1: Load AI Models",
658
- variant="secondary",
659
- )
660
- process_btn = gr.Button(
661
- "✨ Step 2: Process Video",
662
- variant="primary",
663
- )
664
 
665
- # System status
666
- status_text = gr.Textbox(
667
- label="πŸ”§ System Status",
668
- value=get_model_status(),
669
- interactive=False,
670
- lines=3
671
- )
672
 
673
- # Right column - Results and preview
674
  with gr.Column(scale=1):
675
  gr.Markdown("### πŸ“€ Your Results")
676
  gr.Markdown("*Processed video will appear here after Step 2*")
677
-
678
- video_output = gr.Video(
679
- label="🎬 Your Processed Video",
680
- height=400
681
- )
682
-
683
  result_text = gr.Textbox(
684
  label="πŸ“Š Processing Results",
685
  interactive=False,
@@ -688,29 +614,15 @@ def switch_background_method(method):
688
  )
689
 
690
  gr.Markdown("### 🎨 Professional Backgrounds Available")
691
-
692
- # Create background preview grid
693
  bg_preview_html = """
694
  <div style='display: grid; grid-template-columns: repeat(3, 1fr); gap: 8px; padding: 10px; max-height: 400px; overflow-y: auto; border: 1px solid #ddd; border-radius: 8px;'>
695
  """
696
  for key, config in PROFESSIONAL_BACKGROUNDS.items():
697
  colors = config["colors"]
698
- if len(colors) >= 2:
699
- gradient = f"linear-gradient(45deg, {colors[0]}, {colors[-1]})"
700
- else:
701
- gradient = colors[0]
702
  bg_preview_html += f"""
703
- <div style='
704
- padding: 12px 8px;
705
- border: 1px solid #ddd;
706
- border-radius: 6px;
707
- text-align: center;
708
- background: {gradient};
709
- min-height: 60px;
710
- display: flex;
711
- align-items: center;
712
- justify-content: center;
713
- '>
714
  <div>
715
  <strong style='color: white; text-shadow: 1px 1px 2px rgba(0,0,0,0.8); font-size: 12px; display: block;'>{config["name"]}</strong>
716
  <small style='color: rgba(255,255,255,0.9); text-shadow: 1px 1px 1px rgba(0,0,0,0.6); font-size: 10px;'>{config.get("description", "")[:30]}...</small>
@@ -720,126 +632,79 @@ def switch_background_method(method):
720
  bg_preview_html += "</div>"
721
  gr.HTML(bg_preview_html)
722
 
723
- # AI Background Generation Function
724
  def generate_ai_background(prompt, style):
725
- """Generate AI background using procedural methods"""
726
  if not prompt or not prompt.strip():
727
  return None, "❌ Please enter a prompt"
728
-
729
  try:
730
- # Create procedural background based on prompt
731
  bg_image = create_procedural_background(prompt, style, 1920, 1080)
732
-
733
  if bg_image is not None:
734
  with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as tmp:
735
  cv2.imwrite(tmp.name, bg_image)
736
  return tmp.name, f"βœ… Background generated: {prompt[:50]}..."
737
- else:
738
- return None, "❌ Generation failed, try different prompt"
739
  except Exception as e:
740
  logger.error(f"AI generation error: {e}")
741
  return None, f"❌ Generation error: {str(e)}"
742
 
743
- # Enhanced video processing function that handles all 4 methods
744
  def process_video_enhanced(
745
- video_path,
746
- bg_method,
747
- custom_img,
748
- prof_choice,
749
- grad_type,
750
- color1, color2, color3, use_third,
751
- ai_prompt, ai_style, ai_img,
752
  progress: Optional[gr.Progress] = None
753
  ):
754
- """Process video with any of the 4 background methods using TWO-STAGE approach"""
755
-
756
  if not models_loaded:
757
  return None, "❌ Models not loaded. Click 'Load Models' first."
758
-
759
  if not video_path:
760
  return None, "❌ No video file provided."
761
-
762
  try:
763
  if bg_method == "upload":
764
  if custom_img and os.path.exists(custom_img):
765
  return process_video_hq(video_path, "custom", custom_img, progress)
766
- else:
767
- return None, "❌ No image uploaded. Please upload a background image."
768
-
769
  elif bg_method == "professional":
770
  if prof_choice and prof_choice in PROFESSIONAL_BACKGROUNDS:
771
  return process_video_hq(video_path, prof_choice, None, progress)
772
- else:
773
- return None, f"❌ Invalid professional background: {prof_choice}"
774
-
775
  elif bg_method == "colors":
776
  try:
777
  colors = [color1 or "#3498db", color2 or "#2ecc71"]
778
  if use_third and color3:
779
  colors.append(color3)
780
-
781
  bg_config = {
782
  "type": "gradient" if grad_type != "solid" else "color",
783
  "colors": colors if grad_type != "solid" else [colors[0]],
784
  "direction": grad_type if grad_type != "solid" else "vertical"
785
  }
786
-
787
  gradient_bg = create_professional_background(bg_config, 1920, 1080)
788
  temp_path = f"/tmp/gradient_{int(time.time())}.png"
789
  cv2.imwrite(temp_path, gradient_bg)
790
-
791
  return process_video_hq(video_path, "custom", temp_path, progress)
792
  except Exception as e:
793
  return None, f"❌ Error creating gradient: {str(e)}"
794
-
795
  elif bg_method == "ai":
796
  if ai_img and os.path.exists(ai_img):
797
  return process_video_hq(video_path, "custom", ai_img, progress)
798
- else:
799
- return None, "❌ No AI background generated. Click 'Generate Background' first."
800
-
801
  else:
802
  return None, f"❌ Unknown background method: {bg_method}"
803
-
804
  except Exception as e:
805
  logger.error(f"Enhanced processing error: {e}")
806
  return None, f"❌ Processing error: {str(e)}"
807
 
808
- # Wire up callbacks
809
- load_models_btn.click(
810
- fn=download_and_setup_models,
811
- outputs=status_text
812
- )
813
-
814
- generate_ai_btn.click(
815
- fn=generate_ai_background,
816
- inputs=[ai_prompt, ai_style],
817
- outputs=[ai_generated_image, status_text]
818
- )
819
-
820
  process_btn.click(
821
  fn=process_video_enhanced,
822
- inputs=[
823
- video_input, # video_path
824
- background_method, # bg_method
825
- custom_background, # custom_img
826
- professional_choice, # prof_choice
827
- gradient_type, # grad_type
828
- color1, color2, color3, use_third_color, # colors
829
- ai_prompt, ai_style, ai_generated_image # AI
830
- ],
831
  outputs=[video_output, result_text]
832
  )
833
 
834
- # Info
835
  with gr.Accordion("ℹ️ ENHANCED Quality & Features", open=False):
836
  gr.Markdown("""
837
  ### πŸ† TWO-STAGE Cinema-Quality Features:
838
  **Stage 1**: Original β†’ Green Screen (SAM2 + MatAnyOne)
839
  **Stage 2**: Green Screen β†’ Final Background (professional chroma key)
840
-
841
- **Background Methods**: Upload image / Professional presets / Gradients / AI generated
842
-
843
  **Quality**: Edge feathering, gamma correction, mask cleanup, H.264 CRF 18, AAC 192kbps.
844
  """)
845
 
@@ -852,12 +717,10 @@ def process_video_enhanced(
852
  # MAIN
853
  # ============================================================================ #
854
  def main():
855
- """Main application entry point"""
856
  try:
 
857
  print("🎬 Cinema-Quality Video Background Replacement")
858
  print("=" * 50)
859
-
860
- # Initialize application paths
861
  os.makedirs("/tmp/MyAvatar/My_Videos/", exist_ok=True)
862
  os.makedirs(os.path.expanduser("~/.cache/sam2"), exist_ok=True)
863
 
@@ -870,17 +733,11 @@ def main():
870
  print(" β€’ Enhanced stability & error handling")
871
  print("=" * 50)
872
 
873
- # Create and launch interface
874
  logger.info("🌐 Creating Gradio interface...")
875
  demo = create_interface()
876
 
877
  logger.info("πŸš€ Launching application...")
878
- demo.launch(
879
- server_name="0.0.0.0",
880
- server_port=7860,
881
- share=True,
882
- show_error=True
883
- )
884
 
885
  except KeyboardInterrupt:
886
  logger.info("πŸ›‘ Application stopped by user")
 
1
  #!/usr/bin/env python3
2
  # ========================= PRE-IMPORT ENV GUARDS =========================
 
3
  import os
4
+ # Remove invalid OMP setting or tame thread counts BEFORE importing numpy/cv2/torch
5
+ os.environ.pop("OMP_NUM_THREADS", None) # or set "1"
 
6
  os.environ.setdefault("MKL_NUM_THREADS", "1")
7
  os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
8
  os.environ.setdefault("NUMEXPR_NUM_THREADS", "1")
 
14
  """
15
  High-Quality Video Background Replacement - MAIN APPLICATION
16
  Upload video β†’ Choose professional background β†’ Replace with cinema quality
17
+ Features: SAM2 + MatAnyOne with multi-fallback loading, professional backgrounds,
18
  cinema-quality processing, lazy loading, and enhanced stability
19
  """
20
 
 
37
  from typing import Optional, Tuple, Dict, Any
38
  import logging
39
  import warnings
40
+ import subprocess
41
+ import importlib
42
 
43
+ # Import your utilities
44
+ from utilities import * # must provide required helpers & PROFESSIONAL_BACKGROUNDS
 
 
45
 
 
46
  warnings.filterwarnings("ignore")
 
 
47
  logging.basicConfig(level=logging.INFO)
48
  logger = logging.getLogger(__name__)
49
 
 
53
  try:
54
  import gradio_client.utils as gc_utils
55
  original_get_type = gc_utils.get_type
 
56
  def patched_get_type(schema):
57
  if not isinstance(schema, dict):
58
  if isinstance(schema, bool):
59
  return "boolean"
60
  return "string"
61
  return original_get_type(schema)
 
62
  gc_utils.get_type = patched_get_type
63
  logger.info("βœ… Applied Gradio schema validation monkey patch.")
64
  except (ImportError, AttributeError) as e:
 
68
  # SAM2 LOADER (Hydra search path; pass STRING config name to build_sam2)
69
  # ============================================================================ #
70
  def load_sam2_predictor(device: str = "cuda", progress: Optional[gr.Progress] = None):
71
+ """Loads SAM2 and returns SAM2ImagePredictor. Uses STRING config name for build_sam2."""
 
 
 
72
  import hydra
73
 
74
  sam_logger = logging.getLogger("SAM2Loader")
 
76
  sam_logger.info(f"Looking for SAM2 configs in absolute path: {configs_dir}")
77
 
78
  if not os.path.isdir(configs_dir):
79
+ raise gr.Error(f"FATAL: SAM2 Configs directory not found at '{configs_dir}'")
 
 
 
 
 
 
80
 
81
  def _maybe_progress(pct: float, desc: str):
82
  if progress is not None:
83
+ try: progress(pct, desc=desc)
84
+ except Exception: pass
 
 
85
 
86
  def try_load(config_name_with_yaml: str, checkpoint_name: str):
87
  try:
 
92
  sam_logger.info(f"Downloading {checkpoint_name} from Hugging Face Hub...")
93
  _maybe_progress(0.1, f"Downloading {checkpoint_name}...")
94
  from huggingface_hub import hf_hub_download
95
+ repo = f"facebook/{config_name_with_yaml.replace('.yaml','')}"
96
  checkpoint_path = hf_hub_download(
97
  repo_id=repo,
98
  filename=checkpoint_name,
 
110
  job_name=f"sam2_load_{int(time.time())}"
111
  )
112
 
113
+ # Pass STRING config name to build_sam2
114
  config_name = config_name_with_yaml.replace(".yaml", "")
115
 
116
  from sam2.build_sam import build_sam2
 
124
  predictor = SAM2ImagePredictor(sam2_model)
125
  sam_logger.info(f"βœ… Loaded {config_name_with_yaml} successfully on {device}")
126
  return predictor
127
+
128
  except Exception as e:
129
+ err = f"Failed to load {config_name_with_yaml}: {e}\nTraceback: {traceback.format_exc()}"
130
+ sam_logger.warning(err)
 
131
  return None
132
 
133
  predictor = try_load("sam2_hiera_large.yaml", "sam2_hiera_large.pt")
 
134
  if predictor is None:
135
+ raise gr.Error("SAM2 loading failed for large model. Check configs/checkpoint.")
 
 
 
136
  return predictor
137
 
138
  # ============================================================================ #
139
+ # MatAnyOne LOADER (hard requirement; auto-clone if needed; load weights)
140
  # ============================================================================ #
141
+ def _git_clone(url: str, dest: str):
142
+ os.makedirs(os.path.dirname(dest), exist_ok=True)
143
+ if not os.path.exists(dest):
144
+ logger.info(f"πŸ“₯ Cloning {url} β†’ {dest}")
145
+ subprocess.check_call(["git", "clone", "--depth", "1", url, dest])
146
+
147
+ def _ensure_repo_on_path(repo_dir: str):
148
+ if repo_dir not in sys.path:
149
+ sys.path.insert(0, repo_dir)
150
+
151
+ def _download_weight_candidates(repo_id: str, filenames: list, cache_dir="./checkpoints") -> Optional[str]:
152
+ from huggingface_hub import hf_hub_download
153
+ last_err = None
154
+ for fn in filenames:
155
+ try:
156
+ path = hf_hub_download(repo_id=repo_id, filename=fn, cache_dir=cache_dir, local_dir_use_symlinks=False)
157
+ return path
158
+ except Exception as e:
159
+ last_err = e
160
+ if last_err:
161
+ raise RuntimeError(f"Could not download MatAnyOne weights from {repo_id} (tried: {filenames}). Last error: {last_err}")
162
+ return None
163
+
164
  def load_matanyone(device: str):
165
  """
166
+ Load MatAnyOne (hard requirement):
167
+ - Try installed package
168
+ - Else auto-clone official repo to ./third_party/MatAnyOne and import
169
+ - Load cfg (or minimal), download weights, build Net + InferenceCore
170
  """
171
  from omegaconf import OmegaConf
 
172
  ma_logger = logging.getLogger("MatAnyOneLoader")
173
 
174
+ # 1) Try to import from installed package first (two common layouts)
175
+ import_attempts = [
176
+ ("matanyone.model.matanyone", "MatAnyOne", "matanyone.inference.inference_core", "InferenceCore"),
177
+ ("matanyone", "MatAnyOne", "matanyone", "InferenceCore"),
178
  ]
179
+
180
+ # 2) Prepare local clone as backup
181
+ repo_dir = os.path.abspath("./third_party/MatAnyOne")
182
+ official_git = "https://github.com/PeiqingYang/MatAnyOne"
183
+
184
  cfg = None
185
+ for p in ("Configs/matanyone.yaml", "configs/matanyone.yaml"):
186
  if os.path.exists(p):
187
  ma_logger.info(f"Loading MatAnyOne cfg: {p}")
188
  cfg = OmegaConf.load(p)
 
195
  "device": device,
196
  })
197
 
198
+ # helper to finalize core from given modules
199
+ def _build_from_modules(net_mod, core_mod):
200
+ Net = getattr(net_mod, "MatAnyOne")
201
+ Core = getattr(core_mod, "InferenceCore")
202
+ net = Net(cfg)
 
 
203
  net.to(device)
204
+
205
+ # Try to get weights if the net exposes a load function or needs explicit weights
206
+ # Try common filenames
207
+ try:
208
+ weight_path = _download_weight_candidates(
209
+ repo_id="PeiqingYang/MatAnyOne-v1.0",
210
+ filenames=[
211
+ "MatAnyOne_swinB.pth",
212
+ "MatAnyOne_SwinB.pth",
213
+ "matanyone_swinB.pth",
214
+ "weights_swinB.pth",
215
+ ],
216
+ cache_dir="./checkpoints"
217
+ )
218
+ if weight_path and hasattr(net, "load_state_dict"):
219
+ state = torch.load(weight_path, map_location=device)
220
+ # Some repos wrap state under 'state_dict' or similar
221
+ if isinstance(state, dict) and "state_dict" in state:
222
+ state = state["state_dict"]
223
+ net.load_state_dict(state, strict=False)
224
+ ma_logger.info(f"βœ… Loaded MatAnyOne weights: {os.path.basename(weight_path)}")
225
+ except Exception as e:
226
+ ma_logger.warning(f"Could not load MatAnyOne weights automatically (continuing): {e}")
227
+
228
+ core = Core(net, cfg)
229
  return core
 
 
 
230
 
231
+ # A) Installed package attempts
232
+ last_err = None
233
+ for mod_net, _, mod_core, _ in import_attempts:
234
+ try:
235
+ net_mod = importlib.import_module(mod_net)
236
+ core_mod = importlib.import_module(mod_core)
237
+ core = _build_from_modules(net_mod, core_mod)
238
+ ma_logger.info(f"βœ… MatAnyOne loaded from installed package ({mod_net} / {mod_core})")
239
+ return core
240
+ except Exception as e:
241
+ last_err = e
242
+ ma_logger.warning(f"MatAnyOne import attempt failed ({mod_net} / {mod_core}): {e}")
243
+
244
+ # B) Local clone fallback
245
  try:
246
+ _git_clone(official_git, repo_dir)
247
+ _ensure_repo_on_path(repo_dir)
248
+ # Try common in-repo layouts
249
+ clone_attempts = [
250
+ ("matanyone.model.matanyone", "matanyone.inference.inference_core"),
251
+ ("model.matanyone", "inference.inference_core"),
252
+ ("src.matanyone.model.matanyone", "src.matanyone.inference.inference_core"),
253
+ ]
254
+ for mod_net, mod_core in clone_attempts:
255
+ try:
256
+ net_mod = importlib.import_module(mod_net)
257
+ core_mod = importlib.import_module(mod_core)
258
+ core = _build_from_modules(net_mod, core_mod)
259
+ ma_logger.info(f"βœ… MatAnyOne loaded from cloned repo ({mod_net} / {mod_core})")
260
+ return core
261
+ except Exception as e:
262
+ last_err = e
263
+ ma_logger.warning(f"MatAnyOne clone import failed ({mod_net} / {mod_core}): {e}")
264
  except Exception as e:
265
  last_err = e
266
+ ma_logger.error(f"Cloning MatAnyOne failed: {e}")
267
 
268
+ # If we reach here, MatAnyOne is not usable β€” HARD FAIL by request
269
+ raise RuntimeError(f"MatAnyOne required but failed to initialize. Last error: {last_err}")
270
 
271
  # ============================================================================ #
272
  # GLOBALS & MODEL SETUP
 
277
  loading_lock = threading.Lock()
278
 
279
  def download_and_setup_models(progress: Optional[gr.Progress] = None):
280
+ """Download and setup models. BOTH SAM2 and MatAnyOne are REQUIRED."""
 
 
281
  global sam2_predictor, matanyone_model, models_loaded
282
 
283
  with loading_lock:
284
  if models_loaded:
285
+ return "βœ… SAM2 + MatAnyOne already loaded"
 
 
286
 
287
+ try:
288
+ logger.info("πŸ”„ Starting ENHANCED model loading...")
289
  device = "cuda" if torch.cuda.is_available() else "cpu"
290
 
291
+ # --- Load SAM2 (required) ---
292
  local_sam2 = load_sam2_predictor(device=device, progress=progress)
293
  sam2_predictor = local_sam2
294
 
295
+ # --- Load MatAnyOne (required) ---
296
+ local_matanyone = load_matanyone(device)
297
+ matanyone_model = local_matanyone
 
 
 
 
 
298
 
299
  models_loaded = True
300
+ logger.info("--- βœ… All models loaded successfully (SAM2 + MatAnyOne) ---")
301
  return "βœ… SAM2 + MatAnyOne loaded successfully!"
302
  except Exception as e:
303
  logger.error(f"❌ Enhanced loading failed: {str(e)}")
 
305
  return f"❌ Enhanced loading failed: {str(e)}"
306
 
307
  # ============================================================================ #
308
+ # TWO-STAGE PROCESSING PIPELINE (uses your utilities’ segmentation/compositing)
309
  # ============================================================================ #
310
+ def process_video_hq(video_path, background_choice, custom_background_path, progress: Optional[gr.Progress] = None):
 
 
 
 
 
311
  """TWO-STAGE High-quality video processing: Original β†’ Green Screen β†’ Final Background"""
312
  if not models_loaded:
313
  return None, "❌ Models not loaded. Click 'Load Models' first."
 
314
  if not video_path:
315
  return None, "❌ No video file provided."
316
 
317
  def _prog(pct: float, desc: str):
318
  if progress is not None:
319
+ try: progress(pct, desc=desc)
320
+ except Exception: pass
 
 
321
 
322
  try:
323
  _prog(0.0, "🎬 Initializing TWO-STAGE processing...")
324
 
 
325
  if not os.path.exists(video_path):
326
  return None, f"❌ Video file not found: {video_path}"
327
 
 
329
  if not cap.isOpened():
330
  return None, "❌ Could not open video file. Please check the format."
331
 
 
332
  fps = cap.get(cv2.CAP_PROP_FPS)
333
  total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
334
  frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
335
  frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
 
336
  logger.info(f"Video properties: {frame_width}x{frame_height}, {fps}fps, {total_frames} frames")
337
 
338
  if total_frames == 0:
339
  return None, "❌ Video appears to be empty or corrupted."
340
 
341
+ # Prepare final background (Stage 2)
342
  background = None
343
  background_name = ""
344
 
345
  if background_choice == "custom" and custom_background_path:
346
+ background = cv2.imread(custom_background_path)
347
+ if background is None:
348
+ return None, "❌ Could not read custom background image. Please check the file format."
349
+ background_name = "Custom Image"
350
+ logger.info("Using custom background image")
 
 
 
351
  else:
352
  if background_choice in PROFESSIONAL_BACKGROUNDS:
353
+ bg_config = PROFESSIONAL_BACKGROUNDS[background_choice]
354
+ background = create_professional_background(bg_config, frame_width, frame_height)
355
+ background_name = bg_config["name"]
356
+ logger.info(f"Using professional background: {background_name}")
 
 
 
 
357
  else:
358
  return None, f"❌ Invalid background selection: {background_choice}"
359
 
360
  if background is None:
361
  return None, "❌ Failed to create background."
362
 
 
363
  timestamp = int(time.time())
364
  fourcc = cv2.VideoWriter_fourcc(*'mp4v')
365
 
366
+ # STAGE 1: Original β†’ Green Screen
367
  _prog(0.1, "🟒 STAGE 1: Creating green screen version...")
368
  greenscreen_path = f"/tmp/greenscreen_{timestamp}.mp4"
369
  greenscreen_writer = cv2.VideoWriter(greenscreen_path, fourcc, fps, (frame_width, frame_height))
 
370
  if not greenscreen_writer.isOpened():
371
  return None, "❌ Could not create green screen video file."
372
 
373
  frame_count = 0
 
 
374
  while True:
375
  ret, frame = cap.read()
376
  if not ret:
377
  break
 
378
  try:
379
+ _prog(0.1 + (frame_count / max(1, total_frames)) * 0.4, f"🟒 Green screen frame {frame_count + 1}/{total_frames}")
380
+ mask = segment_person_hq(frame) # from utilities
381
+ refined_mask = refine_mask_hq(frame, mask) # from utilities
382
+ green_screen = create_green_screen_background(frame) # from utilities
383
+ green_screen_frame = replace_background_hq(frame, refined_mask, green_screen) # from utilities
 
 
 
 
384
  greenscreen_writer.write(green_screen_frame)
 
 
 
 
 
 
 
385
  except Exception as e:
386
  logger.warning(f"Error in Stage 1 frame {frame_count}: {e}")
387
  greenscreen_writer.write(frame)
388
+ frame_count += 1
389
+ if frame_count % 100 == 0:
390
+ gc.collect()
391
+ if torch.cuda.is_available():
392
+ torch.cuda.empty_cache()
393
 
394
  greenscreen_writer.release()
395
  cap.release()
396
 
397
+ # STAGE 2: Green Screen β†’ Final Background
398
  _prog(0.5, f"🎨 STAGE 2: Replacing green screen with {background_name}...")
 
399
  final_path = f"/tmp/final_output_{timestamp}.mp4"
400
  final_writer = cv2.VideoWriter(final_path, fourcc, fps, (frame_width, frame_height))
 
401
  if not final_writer.isOpened():
402
  return None, "❌ Could not create final output video file."
403
 
 
404
  greenscreen_cap = cv2.VideoCapture(greenscreen_path)
405
  if not greenscreen_cap.isOpened():
406
  return None, "❌ Could not open green screen video."
407
 
408
  frame_count = 0
 
 
409
  while True:
410
  ret, green_frame = greenscreen_cap.read()
411
  if not ret:
412
  break
 
413
  try:
414
+ _prog(0.5 + (frame_count / max(1, total_frames)) * 0.4, f"🎬 Final compositing frame {frame_count + 1}/{total_frames}")
 
 
 
415
  hsv = cv2.cvtColor(green_frame, cv2.COLOR_BGR2HSV)
416
  lower_green = np.array([25, 30, 30])
417
  upper_green = np.array([100, 255, 255])
418
  green_mask = cv2.inRange(hsv, lower_green, upper_green)
 
 
419
  kernel = np.ones((3, 3), np.uint8)
420
  green_mask = cv2.morphologyEx(green_mask, cv2.MORPH_OPEN, kernel)
421
  green_mask = cv2.morphologyEx(green_mask, cv2.MORPH_CLOSE, kernel)
422
+ green_mask = 255 - green_mask
423
+ result_frame = replace_background_hq(green_frame, green_mask, background) # from utilities
 
424
  final_writer.write(result_frame)
 
 
 
 
 
 
 
425
  except Exception as e:
426
  logger.warning(f"Error in Stage 2 frame {frame_count}: {e}")
427
  final_writer.write(green_frame)
428
+ frame_count += 1
429
+ if frame_count % 100 == 0:
430
+ gc.collect()
431
+ if torch.cuda.is_available():
432
+ torch.cuda.empty_cache()
433
 
434
  greenscreen_cap.release()
435
  final_writer.release()
436
 
437
+ try: os.remove(greenscreen_path)
438
+ except Exception: pass
 
 
 
439
 
440
  if frame_count == 0:
441
  return None, "❌ No frames were processed successfully."
442
 
443
  _prog(0.9, "🎡 Adding high-quality audio...")
 
 
444
  final_output = f"/tmp/final_output_hq_{timestamp}.mp4"
 
445
  try:
446
  audio_cmd = (
447
  f'ffmpeg -y -i "{final_path}" -i "{video_path}" '
 
450
  f'-map 0:v:0 -map 1:a:0? -shortest "{final_output}"'
451
  )
452
  result = os.system(audio_cmd)
 
453
  if result != 0 or not os.path.exists(final_output):
454
  logger.warning("Audio merging failed, using video without audio")
455
  shutil.copy2(final_path, final_output)
 
456
  except Exception as e:
457
  logger.warning(f"Audio processing error: {e}, using video without audio")
458
+ try: shutil.copy2(final_path, final_output)
 
459
  except Exception as e2:
460
  logger.error(f"Failed to copy video file: {e2}")
461
  return None, f"❌ Failed to finalize video: {str(e2)}"
 
464
  try:
465
  myavatar_path = "/tmp/MyAvatar/My_Videos/"
466
  os.makedirs(myavatar_path, exist_ok=True)
 
467
  saved_filename = f"two_stage_bg_replaced_{timestamp}.mp4"
468
  saved_path = os.path.join(myavatar_path, saved_filename)
469
  shutil.copy2(final_output, saved_path)
 
470
  logger.info(f"Video saved to: {saved_path}")
471
  except Exception as e:
472
  logger.warning(f"Could not save to MyAvatar directory: {e}")
473
  saved_filename = os.path.basename(final_output)
474
 
 
475
  try:
476
  if os.path.exists(final_path):
477
  os.remove(final_path)
 
479
  pass
480
 
481
  _prog(1.0, "βœ… TWO-STAGE processing complete!")
 
482
  success_message = (
483
  f"βœ… TWO-STAGE Success!\n"
484
  f"🟒 Stage 1: Original β†’ Green Screen\n"
 
488
  f"🎯 Quality: Cinema-grade with SAM2 + MatAnyOne\n"
489
  f"πŸš€ Method: Professional two-stage compositing"
490
  )
 
491
  return final_output, success_message
492
 
493
  except Exception as e:
 
494
  logger.error(f"Video processing error: {traceback.format_exc()}")
495
+ return None, f"❌ TWO-STAGE Processing Error: {str(e)}"
496
 
497
  # ============================================================================ #
498
  # GRADIO UI
499
  # ============================================================================ #
500
  def create_interface():
 
 
501
  def extract_video_path(v):
 
502
  if isinstance(v, (tuple, list)) and len(v) > 0:
503
  return v[0]
504
  return v
 
511
  .progress-bar { background: linear-gradient(90deg, #3498db, #2ecc71) !important; }
512
  """
513
  ) as demo:
 
 
514
  gr.Markdown("# 🎬 Cinema-Quality Video Background Replacement")
515
  gr.Markdown("**Upload a video β†’ Choose a background β†’ Get professional results with AI**")
516
  gr.Markdown("*Powered by SAM2 + MatAnyOne with multi-fallback loading for maximum reliability*")
517
  gr.Markdown("---")
518
 
519
  with gr.Row():
 
520
  with gr.Column(scale=1):
521
  gr.Markdown("### πŸ“₯ Step 1: Upload Your Video")
522
  gr.Markdown("*Supports MP4, MOV, AVI, and other common formats*")
523
+ video_input = gr.Video(label="πŸŽ₯ Drop your video here", height=300)
524
 
525
+ video_preview = gr.Video(label="πŸ“Ί Preview of Uploaded Video", height=200, interactive=False)
526
+ video_input.change(fn=extract_video_path, inputs=video_input, outputs=video_preview)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
527
 
528
  gr.Markdown("### 🎨 Step 2: Choose Background Method")
529
  gr.Markdown("*Select your preferred background creation method*")
 
 
530
  background_method = gr.Radio(
531
  choices=["upload", "professional", "colors", "ai"],
532
  value="professional",
533
  label="Background Method"
534
  )
 
535
  gr.Markdown(
536
  "- **upload** = πŸ“· Upload Image \n"
537
  "- **professional** = 🎨 Professional Presets \n"
 
539
  "- **ai** = πŸ€– AI Generated"
540
  )
541
 
 
542
  with gr.Group(visible=False) as upload_group:
543
  gr.Markdown("**πŸ“· Upload Your Background Image**")
544
+ custom_background = gr.Image(label="Drop your background image here", type="filepath")
 
 
 
545
 
 
546
  with gr.Group(visible=True) as professional_group:
547
  gr.Markdown("**🎨 Professional Background Presets**")
548
  professional_choice = gr.Dropdown(
 
551
  label="Select Professional Background"
552
  )
553
 
 
554
  with gr.Group(visible=False) as colors_group:
555
  gr.Markdown("**🌈 Custom Colors & Gradients**")
 
556
  gradient_type = gr.Dropdown(
557
  choices=["solid", "vertical", "horizontal", "diagonal", "radial", "soft_radial"],
558
  value="vertical",
559
  label="Gradient Type"
560
  )
 
561
  with gr.Row():
562
  color1 = gr.ColorPicker(label="🎨 Color 1", value="#3498db")
563
  color2 = gr.ColorPicker(label="🎨 Color 2", value="#2ecc71")
 
564
  with gr.Row():
565
  color3 = gr.ColorPicker(label="🎨 Color 3", value="#e74c3c")
566
  use_third_color = gr.Checkbox(label="Use 3rd color", value=False)
567
 
 
568
  with gr.Group(visible=False) as ai_group:
569
  gr.Markdown("**πŸ€– AI Generated Background**")
 
570
  ai_prompt = gr.Textbox(
571
  label="Describe your background",
572
  placeholder="e.g., 'modern office with plants', 'sunset over mountains', 'abstract tech pattern'",
573
  lines=2
574
  )
 
575
  ai_style = gr.Dropdown(
576
  choices=["photorealistic", "artistic", "abstract", "minimalist", "corporate", "nature"],
577
  value="photorealistic",
578
  label="Style"
579
  )
 
580
  with gr.Row():
581
  generate_ai_btn = gr.Button("🎨 Generate Background", variant="secondary")
582
  ai_generated_image = gr.Image(label="Generated Background", type="filepath", visible=False)
583
 
 
584
  def switch_background_method(method):
585
  return (
586
+ gr.update(visible=(method == "upload")),
587
+ gr.update(visible=(method == "professional")),
588
+ gr.update(visible=(method == "colors")),
589
+ gr.update(visible=(method == "ai"))
590
  )
 
591
  background_method.change(
592
  fn=switch_background_method,
593
  inputs=background_method,
 
596
 
597
  gr.Markdown("### 🎬 Processing Controls")
598
  gr.Markdown("*First load the AI models, then process your video*")
 
599
  with gr.Row():
600
+ load_models_btn = gr.Button("πŸš€ Step 1: Load AI Models", variant="secondary")
601
+ process_btn = gr.Button("✨ Step 2: Process Video", variant="primary")
 
 
 
 
 
 
602
 
603
+ status_text = gr.Textbox(label="πŸ”§ System Status", value=get_model_status(), interactive=False, lines=3)
 
 
 
 
 
 
604
 
 
605
  with gr.Column(scale=1):
606
  gr.Markdown("### πŸ“€ Your Results")
607
  gr.Markdown("*Processed video will appear here after Step 2*")
608
+ video_output = gr.Video(label="🎬 Your Processed Video", height=400)
 
 
 
 
 
609
  result_text = gr.Textbox(
610
  label="πŸ“Š Processing Results",
611
  interactive=False,
 
614
  )
615
 
616
  gr.Markdown("### 🎨 Professional Backgrounds Available")
 
 
617
  bg_preview_html = """
618
  <div style='display: grid; grid-template-columns: repeat(3, 1fr); gap: 8px; padding: 10px; max-height: 400px; overflow-y: auto; border: 1px solid #ddd; border-radius: 8px;'>
619
  """
620
  for key, config in PROFESSIONAL_BACKGROUNDS.items():
621
  colors = config["colors"]
622
+ gradient = f"linear-gradient(45deg, {colors[0]}, {colors[-1]})" if len(colors) >= 2 else colors[0]
 
 
 
623
  bg_preview_html += f"""
624
+ <div style='padding: 12px 8px; border: 1px solid #ddd; border-radius: 6px; text-align: center; background: {gradient};
625
+ min-height: 60px; display: flex; align-items: center; justify-content: center;'>
 
 
 
 
 
 
 
 
 
626
  <div>
627
  <strong style='color: white; text-shadow: 1px 1px 2px rgba(0,0,0,0.8); font-size: 12px; display: block;'>{config["name"]}</strong>
628
  <small style='color: rgba(255,255,255,0.9); text-shadow: 1px 1px 1px rgba(0,0,0,0.6); font-size: 10px;'>{config.get("description", "")[:30]}...</small>
 
632
  bg_preview_html += "</div>"
633
  gr.HTML(bg_preview_html)
634
 
 
635
  def generate_ai_background(prompt, style):
 
636
  if not prompt or not prompt.strip():
637
  return None, "❌ Please enter a prompt"
 
638
  try:
 
639
  bg_image = create_procedural_background(prompt, style, 1920, 1080)
 
640
  if bg_image is not None:
641
  with tempfile.NamedTemporaryFile(suffix='.png', delete=False) as tmp:
642
  cv2.imwrite(tmp.name, bg_image)
643
  return tmp.name, f"βœ… Background generated: {prompt[:50]}..."
644
+ return None, "❌ Generation failed, try different prompt"
 
645
  except Exception as e:
646
  logger.error(f"AI generation error: {e}")
647
  return None, f"❌ Generation error: {str(e)}"
648
 
 
649
  def process_video_enhanced(
650
+ video_path, bg_method, custom_img, prof_choice, grad_type,
651
+ color1, color2, color3, use_third, ai_prompt, ai_style, ai_img,
 
 
 
 
 
652
  progress: Optional[gr.Progress] = None
653
  ):
 
 
654
  if not models_loaded:
655
  return None, "❌ Models not loaded. Click 'Load Models' first."
 
656
  if not video_path:
657
  return None, "❌ No video file provided."
 
658
  try:
659
  if bg_method == "upload":
660
  if custom_img and os.path.exists(custom_img):
661
  return process_video_hq(video_path, "custom", custom_img, progress)
662
+ return None, "❌ No image uploaded. Please upload a background image."
 
 
663
  elif bg_method == "professional":
664
  if prof_choice and prof_choice in PROFESSIONAL_BACKGROUNDS:
665
  return process_video_hq(video_path, prof_choice, None, progress)
666
+ return None, f"❌ Invalid professional background: {prof_choice}"
 
 
667
  elif bg_method == "colors":
668
  try:
669
  colors = [color1 or "#3498db", color2 or "#2ecc71"]
670
  if use_third and color3:
671
  colors.append(color3)
 
672
  bg_config = {
673
  "type": "gradient" if grad_type != "solid" else "color",
674
  "colors": colors if grad_type != "solid" else [colors[0]],
675
  "direction": grad_type if grad_type != "solid" else "vertical"
676
  }
 
677
  gradient_bg = create_professional_background(bg_config, 1920, 1080)
678
  temp_path = f"/tmp/gradient_{int(time.time())}.png"
679
  cv2.imwrite(temp_path, gradient_bg)
 
680
  return process_video_hq(video_path, "custom", temp_path, progress)
681
  except Exception as e:
682
  return None, f"❌ Error creating gradient: {str(e)}"
 
683
  elif bg_method == "ai":
684
  if ai_img and os.path.exists(ai_img):
685
  return process_video_hq(video_path, "custom", ai_img, progress)
686
+ return None, "❌ No AI background generated. Click 'Generate Background' first."
 
 
687
  else:
688
  return None, f"❌ Unknown background method: {bg_method}"
 
689
  except Exception as e:
690
  logger.error(f"Enhanced processing error: {e}")
691
  return None, f"❌ Processing error: {str(e)}"
692
 
693
+ load_models_btn.click(fn=download_and_setup_models, outputs=status_text)
694
+ generate_ai_btn.click(fn=generate_ai_background, inputs=[ai_prompt, ai_style], outputs=[ai_generated_image, status_text])
 
 
 
 
 
 
 
 
 
 
695
  process_btn.click(
696
  fn=process_video_enhanced,
697
+ inputs=[video_input, background_method, custom_background, professional_choice,
698
+ gradient_type, color1, color2, color3, use_third_color,
699
+ ai_prompt, ai_style, ai_generated_image],
 
 
 
 
 
 
700
  outputs=[video_output, result_text]
701
  )
702
 
 
703
  with gr.Accordion("ℹ️ ENHANCED Quality & Features", open=False):
704
  gr.Markdown("""
705
  ### πŸ† TWO-STAGE Cinema-Quality Features:
706
  **Stage 1**: Original β†’ Green Screen (SAM2 + MatAnyOne)
707
  **Stage 2**: Green Screen β†’ Final Background (professional chroma key)
 
 
 
708
  **Quality**: Edge feathering, gamma correction, mask cleanup, H.264 CRF 18, AAC 192kbps.
709
  """)
710
 
 
717
  # MAIN
718
  # ============================================================================ #
719
  def main():
 
720
  try:
721
+ print(f"===== Application Startup at {time.strftime('%Y-%m-%d %H:%M:%S')} =====\n")
722
  print("🎬 Cinema-Quality Video Background Replacement")
723
  print("=" * 50)
 
 
724
  os.makedirs("/tmp/MyAvatar/My_Videos/", exist_ok=True)
725
  os.makedirs(os.path.expanduser("~/.cache/sam2"), exist_ok=True)
726
 
 
733
  print(" β€’ Enhanced stability & error handling")
734
  print("=" * 50)
735
 
 
736
  logger.info("🌐 Creating Gradio interface...")
737
  demo = create_interface()
738
 
739
  logger.info("πŸš€ Launching application...")
740
+ demo.launch(server_name="0.0.0.0", server_port=7860, share=True, show_error=True)
 
 
 
 
 
741
 
742
  except KeyboardInterrupt:
743
  logger.info("πŸ›‘ Application stopped by user")