added gitignores when downloading videos/video-ids, and added download_videos.py to run_pipeline.sh

pipeline/download_videos.py

@@ -17,6 +17,7 @@ def get_id(url: str) -> str:
 
 def download_videos(video_ids: List[str]) -> None:
     VIDEO_DIR.mkdir(exist_ok=True, parents=True)
+    (VIDEO_DIR / "gitignore").write_text("*")
     for video_id in tqdm(video_ids):
         video_url = f"https://www.youtube.com/watch?v={video_id}"
         video_path = VIDEO_DIR / f"{video_id}.mp4"

pipeline/get_video_ids.py

@@ -6,6 +6,8 @@ from typing import Final, Optional
 
 import youtube_dl
 
+from pipeline.download_videos import DATA_DIR
+
 logging.basicConfig(
     level=logging.INFO,
     format="%(asctime)s - %(levelname)s - %(message)s",
@@ -26,7 +28,7 @@ PLAYLIST_URLS = [
     "https://www.youtube.com/playlist?list=PLCQCtoOJpI_A5oktQImEdDBJ50BqHXujj",  # 495, MTV Classic 2000's music videos (US Version)
 ]
 URL_FILE: Final[Optional[str]] = os.environ.get("URL_FILE")
-OUTPUT_DIR: Final[str] = os.environ.get("OUTPUT_DIR", "data/ids")
+OUTPUT_DIR: Final[str] = DATA_DIR / "ids"
 
 
 def get_all_video_ids(channel_url: str) -> list[str]:
@@ -58,12 +60,10 @@ def get_all_video_ids(channel_url: str) -> list[str]:
 def process_youtube_url(url: str):
     logging.info(f"Processing {url}")
     ids = get_all_video_ids(url)
-
-    output_dir = Path(OUTPUT_DIR)
-    output_dir.mkdir(parents=True, exist_ok=True)
-
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    (OUTPUT_DIR / ".gitignore").write_text("*")
     output = "\n".join(ids)
-    output_path = output_dir / f"{hashlib.md5(output.encode()).hexdigest()}.txt"
+    output_path = OUTPUT_DIR / f"{hashlib.md5(output.encode()).hexdigest()}.txt"
     logging.info(f"Writing {len(ids)} video IDs to {output_path}")
     with output_path.open(mode="w") as f:
         f.write(output)

run_pipeline.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
+poetry run python pipeline/get_video_ids.py
 poetry run python pipeline/download_videos.py
 poetry run python pipeline/process_videos.py