Repository: nerfstudio-project/nerfstudio

Issue:
ns-process-data insta360 does not support the single-file format used by the insta360 X3
**Describe the bug**
ns-process-data insta360 throws an error looking for a second file when given single-file 360 footage (both sensor's data in a single file). does not support 360 video where footage from both sensors is in the same file.

**To Reproduce**
Steps to reproduce the behavior:
1. Record 360 footage with a camera that records to a single file instead of two files. 
    _Note that: Insta360 X3 records to a single file (only 00) in normal video mode and only uses two files (00 and 10) for certain modes, like HDR and TimeShift._
2. Attempt to process data: `ns-process-data insta360 --data .\VID_20220925_113615_00_190.insv --output outputs`
3. See error `FileNotFoundError: Could not find VID_20220925_113615_10_190.insv`

**Expected behavior**
ns-process-data should support single-file 360 footage and an error should not be thrown by `ns-process-data insta360 --data .\VID_20220925_113615_00_190.insv --output outputs` when a second file does not exist (if the single file contains data from two sensors.

**Screenshots**
Here is an example of what the expected two-file format looks like:
Note how there are two separate 1:1 aspect ratio files with "00" and "10" in the files names. This is what ns-process-data is looking for.
![image](https://user-images.githubusercontent.com/3620968/203854306-a595b2e6-779f-40a5-b5bf-ad4ad9a8dd64.png)

Whereas here is what a single-file (seemingly the new standard for insta360 cameras) 360 video looks like:
Note how it's a single file, which is 2:1 aspect ratio, and this one file has "00" in the name. This should be supported.
![image](https://user-images.githubusercontent.com/3620968/203854497-dbf3c354-df31-4241-8329-a6bbd7226e54.png)


**Additional context**
This single file format appears to be the new standard way that insta360 records videos, at least in the new X3 camera.


Discussion:

Resolving patch:
diff --git a/nerfstudio/process_data/insta360_utils.py b/nerfstudio/process_data/insta360_utils.py
--- a/nerfstudio/process_data/insta360_utils.py
+++ b/nerfstudio/process_data/insta360_utils.py
@@ -48,11 +48,6 @@ def get_insta360_filenames(data: Path) -> Tuple[Path, Path]:
     filename_back = data.parent / stem_back
     filename_front = data.parent / stem_front
 
-    if not filename_back.exists():
-        raise FileNotFoundError(f"Could not find {filename_back}")
-    if not filename_front.exists():
-        raise FileNotFoundError(f"Could not find {filename_front}")
-
     return filename_back, filename_front
 
 
@@ -71,6 +66,8 @@ def convert_insta360_to_images(
         video_back: Path to the back video.
         output_dir: Path to the output directory.
         num_frames_target: Number of frames to extract.
+        crop_percentage: Percentage used to calculate the cropped dimentions of extracted frames. Currently used to crop
+         out the curved portions of the fish-eye lens.
         verbose: If True, logs the output of the command.
     Returns:
         A tuple containing summary of the conversion and the number of extracted frames.
@@ -123,3 +120,71 @@ def convert_insta360_to_images(
     CONSOLE.log("[bold green]:tada: Done converting insta360 to images.")
 
     return summary_log, num_final_frames
+
+
+def convert_insta360_single_file_to_images(
+    video: Path,
+    image_dir: Path,
+    num_frames_target: int,
+    crop_percentage: float = 0.7,
+    verbose: bool = False,
+) -> Tuple[List[str], int]:
+    """Converts a video into a sequence of images.
+
+    Args:
+        video: Path to the video.
+        output_dir: Path to the output directory.
+        num_frames_target: Number of frames to extract.
+        crop_percentage: Percentage used to calculate the cropped dimentions of extracted frames. Currently used to crop
+         out the curved portions of the fish-eye lens.
+        verbose: If True, logs the output of the command.
+    Returns:
+        A tuple containing summary of the conversion and the number of extracted frames.
+    """
+
+    with status(msg="Converting video to images...", spinner="bouncingBall", verbose=verbose):
+        # delete existing images in folder
+        for img in image_dir.glob("*.png"):
+            if verbose:
+                CONSOLE.log(f"Deleting {img}")
+            img.unlink()
+
+        num_frames = get_num_frames_in_video(video)
+        if num_frames == 0:
+            CONSOLE.print(f"[bold red]Error: Video has no frames: {video}")
+            sys.exit(1)
+
+        spacing = num_frames // (num_frames_target // 2)
+        vf_cmds = []
+        if spacing > 1:
+            vf_cmds = [f"thumbnail={spacing}", "setpts=N/TB"]
+        else:
+            CONSOLE.print("[bold red]Can't satify requested number of frames. Extracting all frames.")
+
+        vf_cmds_back = vf_cmds.copy()
+        vf_cmds_front = vf_cmds.copy()
+
+        vf_cmds_back.append(
+            f"crop=ih*{crop_percentage}:ih*{crop_percentage}:ih*({crop_percentage}/4):ih*({crop_percentage}/4)"
+        )
+        vf_cmds_front.append(
+            f"crop=ih*{crop_percentage}:ih*{crop_percentage}:iw/2+ih*{crop_percentage/4}:ih*{crop_percentage/4}"
+        )
+
+        front_ffmpeg_cmd = f"ffmpeg -i {video} -vf {','.join(vf_cmds_front)} -r 1 {image_dir / 'frame_%05d.png'}"
+        back_ffmpeg_cmd = f"ffmpeg -i {video} -vf {','.join(vf_cmds_back)} -r 1 {image_dir / 'back_frame_%05d.png'}"
+
+        run_command(back_ffmpeg_cmd, verbose=verbose)
+        run_command(front_ffmpeg_cmd, verbose=verbose)
+
+        num_extracted_frames = len(list(image_dir.glob("frame*.png")))
+        for i, img in enumerate(image_dir.glob("back_frame_*.png")):
+            img.rename(image_dir / f"frame_{i+1+num_extracted_frames:05d}.png")
+
+    num_final_frames = len(list(image_dir.glob("*.png")))
+    summary_log = []
+    summary_log.append(f"Starting with {num_frames} video frames")
+    summary_log.append(f"We extracted {num_final_frames} images")
+    CONSOLE.log("[bold green]:tada: Done converting insta360 to images.")
+
+    return summary_log, num_final_frames
diff --git a/scripts/process_data.py b/scripts/process_data.py
--- a/scripts/process_data.py
+++ b/scripts/process_data.py
@@ -2,6 +2,7 @@
 """Processes a video or image sequence to a nerfstudio compatible dataset."""
 
 
+import json
 import sys
 import zipfile
 from dataclasses import dataclass
@@ -246,14 +247,38 @@ def main(self) -> None:
 
         filename_back, filename_front = insta360_utils.get_insta360_filenames(self.data)
 
-        # Convert video to images
-        summary_log, num_extracted_frames = insta360_utils.convert_insta360_to_images(
-            video_front=filename_front,
-            video_back=filename_back,
-            image_dir=image_dir,
-            num_frames_target=self.num_frames_target,
-            verbose=self.verbose,
-        )
+        if not filename_back.exists():
+            raise FileNotFoundError(f"Could not find {filename_back}")
+
+        ffprobe_cmd = f"ffprobe -v quiet -print_format json -show_streams -select_streams v:0 {filename_back}"
+
+        ffprobe_output = process_data_utils.run_command(ffprobe_cmd)
+
+        assert ffprobe_output is not None
+        ffprobe_decoded = json.loads(ffprobe_output)
+
+        width, height = ffprobe_decoded["streams"][0]["width"], ffprobe_decoded["streams"][0]["height"]
+
+        summary_log, num_extracted_frames = [], 0
+
+        if width / height == 1:
+            if not filename_front.exists():
+                raise FileNotFoundError(f"Could not find {filename_front}")
+            # Convert video to images
+            summary_log, num_extracted_frames = insta360_utils.convert_insta360_to_images(
+                video_front=filename_front,
+                video_back=filename_back,
+                image_dir=image_dir,
+                num_frames_target=self.num_frames_target,
+                verbose=self.verbose,
+            )
+        else:
+            summary_log, num_extracted_frames = insta360_utils.convert_insta360_single_file_to_images(
+                video=filename_back,
+                image_dir=image_dir,
+                num_frames_target=self.num_frames_target,
+                verbose=self.verbose,
+            )
 
         # Downscale images
         summary_log.append(process_data_utils.downscale_images(image_dir, self.num_downscales, verbose=self.verbose))


Test patch: