Add Space-only YouTube fallback strategies

Dockerfile

@@ -5,6 +5,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y \
     build-essential \
     curl \
+    ffmpeg \
     git \
     && rm -rf /var/lib/apt/lists/*
 

README.md

@@ -32,3 +32,28 @@ YouTube transcript loading may work locally but fail on Hugging Face Spaces beca
 - `YOUTUBE_HTTPS_PROXY`
 
 You can also use the standard `HTTP_PROXY` and `HTTPS_PROXY` environment variables if that matches your setup.
+
+## Space-Only YouTube Fallbacks
+
+The Hugging Face Space version now supports multiple YouTube retrieval strategies:
+
+- Direct transcript fetch
+- External transcript API
+- Audio transcription via `yt-dlp` + Groq Whisper
+- Manual transcript paste/upload
+
+### Optional secrets for external transcript API
+
+- `YOUTUBE_TRANSCRIPT_API_URL`
+- `YOUTUBE_TRANSCRIPT_API_KEY`
+- `YOUTUBE_TRANSCRIPT_API_METHOD` (`GET` or `POST`, default `GET`)
+- `YOUTUBE_TRANSCRIPT_API_KEY_HEADER` (default `Authorization`)
+- `YOUTUBE_TRANSCRIPT_API_TIMEOUT` (default `45`)
+
+`YOUTUBE_TRANSCRIPT_API_URL` may contain placeholders such as `{video_id}`, `{url}`, and `{language_code}`.
+
+### Optional secrets for Groq audio transcription fallback
+
+- `GROQ_AUDIO_TRANSCRIPTION_MODEL`
+
+Default model: `whisper-large-v3-turbo`

app.py

@@ -1,6 +1,7 @@
 import os
+import tempfile
 from io import BytesIO
-from urllib.parse import urlparse
+from urllib.parse import quote_plus, urlparse
 from xml.etree import ElementTree as ET
 from zipfile import ZipFile
 
@@ -25,7 +26,7 @@ from youtube_transcript_api import YouTubeTranscriptApi
 
 load_dotenv()
 
-APP_VERSION = "2026-04-23-hf-youtube-fix-2"
+APP_VERSION = "2026-04-23-hf-youtube-fallbacks-1"
 SAMPLE_YOUTUBE_URL = "https://youtu.be/ocBh08fjIfU"
 LANGUAGE_OPTIONS = ["Original", "English", "Arabic", "French", "Bahasa Malay"]
 LANGUAGE_CODE_MAP = {
@@ -46,6 +47,12 @@ YOUTUBE_PROXY_ENV_VARS = (
     "HTTP_PROXY",
     "HTTPS_PROXY",
 )
+YOUTUBE_AUDIO_EXTENSIONS = (".m4a", ".mp3", ".mp4", ".mpeg", ".mpga", ".ogg", ".wav", ".webm")
+
+
+def _is_youtube_url(url: str) -> bool:
+    host = urlparse(url).netloc.lower()
+    return "youtube.com" in host or "youtu.be" in host
 
 st.set_page_config(page_title="Summarize Text From PDF, YouTube, Website", page_icon="📝")
 st.title("📝 Summarize Text From PDF, YouTube, Website")
@@ -79,6 +86,8 @@ if "youtube_transcript_source_url" not in st.session_state:
     st.session_state.youtube_transcript_source_url = ""
 if "youtube_transcript_language_label" not in st.session_state:
     st.session_state.youtube_transcript_language_label = "Original"
+if "youtube_transcript_source_mode" not in st.session_state:
+    st.session_state.youtube_transcript_source_mode = ""
 
 summary_language = "Original"
 transcript_language = "Original"
@@ -136,6 +145,9 @@ with st.sidebar:
 
 generic_url = ""
 uploaded_files = []
+youtube_source_mode = "Auto"
+manual_transcript_text = ""
+manual_transcript_file = None
 
 if input_source_mode in {"URL", "Both"}:
     st.markdown('<div class="source-section-label">Summarize URL</div>', unsafe_allow_html=True)
@@ -161,6 +173,43 @@ if input_source_mode in {"Upload documents", "Both"}:
             "Uploaded files: " + ", ".join(uploaded_file.name for uploaded_file in uploaded_files)
         )
 
+if input_source_mode in {"URL", "Both"} and generic_url.strip() and _is_youtube_url(generic_url):
+    st.markdown('<div class="source-section-label">YouTube Fallback Options</div>', unsafe_allow_html=True)
+    youtube_source_mode = st.radio(
+        "YouTube transcript source",
+        options=[
+            "Auto",
+            "Direct transcript",
+            "External transcript API",
+            "Audio transcription (yt-dlp + Groq)",
+            "Manual transcript",
+        ],
+        index=0,
+        help=(
+            "`Auto` tries direct transcript first, then external API, then yt-dlp + Groq audio transcription. "
+            "`Manual transcript` lets you paste or upload transcript text."
+        ),
+    )
+    if youtube_source_mode == "Manual transcript":
+        manual_transcript_text = st.text_area(
+            "Paste transcript",
+            height=220,
+            placeholder="Paste the YouTube transcript here if direct fetching is blocked.",
+        )
+        manual_transcript_file = st.file_uploader(
+            "Upload transcript file",
+            type=["txt", "md", "csv", "srt", "vtt"],
+            help="Upload a transcript file to summarize when direct YouTube access is blocked.",
+        )
+    else:
+        configured_modes = []
+        if any(os.getenv(var_name) for var_name in YOUTUBE_PROXY_ENV_VARS):
+            configured_modes.append("direct transcript via proxy")
+        if os.getenv("YOUTUBE_TRANSCRIPT_API_URL"):
+            configured_modes.append("external transcript API")
+        configured_modes.append("audio transcription via yt-dlp + Groq")
+        st.caption("Available fallbacks: " + ", ".join(configured_modes) + ".")
+
 llm = ChatGroq(model="llama-3.1-8b-instant", groq_api_key=groq_api_key)
 
 REQUEST_HEADERS = {
@@ -170,12 +219,6 @@ REQUEST_HEADERS = {
     "Referer": "https://www.google.com/",
 }
 
-
-def _is_youtube_url(url: str) -> bool:
-    host = urlparse(url).netloc.lower()
-    return "youtube.com" in host or "youtu.be" in host
-
-
 def _summary_language_instruction(selected_language: str) -> str:
     if selected_language == "Original":
         return "Write the summary in the original language of the source content. If the source is mixed-language, use the dominant language."
@@ -447,6 +490,14 @@ def _make_transcript_filename(url: str) -> str:
     return f"youtube_transcript_{video_id}.txt"
 
 
+def _reset_youtube_transcript_state() -> None:
+    st.session_state.youtube_transcript_text = ""
+    st.session_state.youtube_transcript_name = "youtube_transcript.txt"
+    st.session_state.youtube_transcript_source_url = ""
+    st.session_state.youtube_transcript_language_label = "Original"
+    st.session_state.youtube_transcript_source_mode = ""
+
+
 def _store_youtube_transcript(url: str, docs: list[Document]) -> None:
     st.session_state.youtube_transcript_text = "\n\n".join(
         doc.page_content for doc in docs if doc.page_content.strip()
@@ -457,6 +508,285 @@ def _store_youtube_transcript(url: str, docs: list[Document]) -> None:
         "transcript_language_label",
         docs[0].metadata.get("language", "Original"),
     )
+    st.session_state.youtube_transcript_source_mode = docs[0].metadata.get(
+        "transcript_source_mode",
+        "Direct transcript",
+    )
+
+
+def _normalize_transcript_text(raw_text: str) -> str:
+    lines = [line.strip() for line in raw_text.splitlines()]
+    return "\n".join(line for line in lines if line)
+
+
+def _read_uploaded_text_file(uploaded_file) -> str:
+    return uploaded_file.getvalue().decode("utf-8", errors="ignore").strip()
+
+
+def _build_transcript_documents(
+    url: str,
+    transcript_text: str,
+    language_label: str,
+    source_mode: str,
+) -> list[Document]:
+    normalized_text = _normalize_transcript_text(transcript_text)
+    if not normalized_text:
+        raise ValueError("Transcript text is empty.")
+
+    return [
+        Document(
+            page_content=normalized_text,
+            metadata={
+                "source": url,
+                "video_id": YoutubeLoader.extract_video_id(url),
+                "transcript_language_label": language_label,
+                "transcript_source_mode": source_mode,
+            },
+        )
+    ]
+
+
+def _load_manual_transcript_documents(
+    url: str,
+    selected_language: str,
+    transcript_text: str,
+    transcript_file,
+) -> list[Document]:
+    combined_parts = []
+    if transcript_text.strip():
+        combined_parts.append(transcript_text.strip())
+    if transcript_file is not None:
+        combined_parts.append(_read_uploaded_text_file(transcript_file))
+
+    combined_text = "\n\n".join(part for part in combined_parts if part.strip())
+    if not combined_text.strip():
+        raise ValueError("Please paste a transcript or upload a transcript file.")
+
+    docs = _build_transcript_documents(
+        url,
+        combined_text,
+        "Original",
+        "Manual transcript",
+    )
+    if selected_language != "Original":
+        docs = _translate_documents_with_llm(docs, selected_language)
+        for doc in docs:
+            doc.metadata["transcript_language_label"] = f"{selected_language} (LLM translated)"
+    return docs
+
+
+def _extract_transcript_text_from_payload(payload) -> str:
+    if isinstance(payload, str):
+        return payload.strip()
+
+    if isinstance(payload, list):
+        text_parts = []
+        for item in payload:
+            extracted = _extract_transcript_text_from_payload(item)
+            if extracted:
+                text_parts.append(extracted)
+        return "\n".join(part for part in text_parts if part)
+
+    if isinstance(payload, dict):
+        for key in ("text", "transcript", "content", "full_text", "body"):
+            value = payload.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+
+        for key in ("data", "result", "results", "transcription", "response"):
+            if key in payload:
+                extracted = _extract_transcript_text_from_payload(payload[key])
+                if extracted:
+                    return extracted
+
+        for key in ("segments", "items", "captions", "chunks", "utterances"):
+            value = payload.get(key)
+            if isinstance(value, list):
+                extracted = _extract_transcript_text_from_payload(value)
+                if extracted:
+                    return extracted
+
+    return ""
+
+
+def _load_youtube_documents_via_external_api(url: str, selected_language: str) -> list[Document]:
+    api_url = os.getenv("YOUTUBE_TRANSCRIPT_API_URL", "").strip()
+    if not api_url:
+        raise ValueError(
+            "External transcript API is not configured. Set `YOUTUBE_TRANSCRIPT_API_URL` in Space secrets."
+        )
+
+    video_id = YoutubeLoader.extract_video_id(url)
+    language_code = LANGUAGE_CODE_MAP.get(selected_language, "")
+    formatted_url = api_url.format(
+        video_id=video_id,
+        url=quote_plus(url),
+        language_code=language_code,
+    )
+
+    method = os.getenv("YOUTUBE_TRANSCRIPT_API_METHOD", "GET").strip().upper()
+    timeout_seconds = int(os.getenv("YOUTUBE_TRANSCRIPT_API_TIMEOUT", "45"))
+    api_key = os.getenv("YOUTUBE_TRANSCRIPT_API_KEY", "").strip()
+    api_key_header = os.getenv("YOUTUBE_TRANSCRIPT_API_KEY_HEADER", "Authorization").strip()
+
+    headers = {"Accept": "application/json"}
+    if api_key:
+        if api_key_header.lower() == "authorization":
+            headers[api_key_header] = f"Bearer {api_key}"
+        else:
+            headers[api_key_header] = api_key
+
+    payload = {
+        "video_id": video_id,
+        "url": url,
+        "language": language_code or None,
+    }
+
+    if method == "POST":
+        response = requests.post(formatted_url, json=payload, headers=headers, timeout=timeout_seconds)
+    else:
+        response = requests.get(formatted_url, params=payload, headers=headers, timeout=timeout_seconds)
+    response.raise_for_status()
+
+    try:
+        parsed_payload = response.json()
+    except ValueError:
+        parsed_payload = response.text
+
+    transcript_text = _extract_transcript_text_from_payload(parsed_payload)
+    if not transcript_text:
+        raise ValueError("External transcript API response did not contain usable transcript text.")
+
+    docs = _build_transcript_documents(
+        url,
+        transcript_text,
+        selected_language if selected_language != "Original" else "Original",
+        "External transcript API",
+    )
+    if selected_language != "Original":
+        for doc in docs:
+            doc.metadata["transcript_language_label"] = selected_language
+    return docs
+
+
+def _download_youtube_audio(url: str, video_id: str) -> str:
+    try:
+        import yt_dlp
+    except ImportError as exc:
+        raise RuntimeError("`yt-dlp` is not installed in this Space build.") from exc
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        output_template = os.path.join(temp_dir, f"{video_id}.%(ext)s")
+        ydl_opts = {
+            "format": "bestaudio[ext=m4a]/bestaudio[ext=webm]/bestaudio/best",
+            "outtmpl": output_template,
+            "quiet": True,
+            "no_warnings": True,
+            "noprogress": True,
+            "skip_download": False,
+        }
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            ydl.extract_info(url, download=True)
+
+        audio_files = [
+            os.path.join(temp_dir, file_name)
+            for file_name in os.listdir(temp_dir)
+            if os.path.splitext(file_name)[1].lower() in YOUTUBE_AUDIO_EXTENSIONS
+        ]
+        if not audio_files:
+            raise RuntimeError("yt-dlp did not produce a supported audio file for transcription.")
+
+        source_path = max(audio_files, key=os.path.getsize)
+        persisted_path = os.path.join(tempfile.gettempdir(), os.path.basename(source_path))
+        with open(source_path, "rb") as source_file, open(persisted_path, "wb") as target_file:
+            target_file.write(source_file.read())
+        return persisted_path
+
+
+def _transcribe_audio_with_groq(audio_path: str, selected_language: str) -> str:
+    if not groq_api_key.strip():
+        raise ValueError("`GROQ_API_KEY` is required for audio transcription fallback.")
+
+    model_name = os.getenv("GROQ_AUDIO_TRANSCRIPTION_MODEL", "whisper-large-v3-turbo")
+    payload = {
+        "model": model_name,
+        "response_format": "json",
+        "temperature": "0",
+    }
+    if selected_language != "Original":
+        payload["language"] = LANGUAGE_CODE_MAP[selected_language]
+
+    with open(audio_path, "rb") as audio_file:
+        response = requests.post(
+            "https://api.groq.com/openai/v1/audio/transcriptions",
+            headers={"Authorization": f"Bearer {groq_api_key}"},
+            data=payload,
+            files={"file": (os.path.basename(audio_path), audio_file)},
+            timeout=300,
+        )
+    response.raise_for_status()
+    transcript_text = response.json().get("text", "").strip()
+    if not transcript_text:
+        raise ValueError("Groq audio transcription returned empty text.")
+    return transcript_text
+
+
+def _load_youtube_documents_via_audio_transcription(url: str, selected_language: str) -> list[Document]:
+    video_id = YoutubeLoader.extract_video_id(url)
+    audio_path = _download_youtube_audio(url, video_id)
+    try:
+        transcript_text = _transcribe_audio_with_groq(audio_path, selected_language)
+    finally:
+        if os.path.exists(audio_path):
+            os.remove(audio_path)
+
+    return _build_transcript_documents(
+        url,
+        transcript_text,
+        selected_language if selected_language != "Original" else "Original",
+        "Audio transcription (yt-dlp + Groq)",
+    )
+
+
+def _load_youtube_documents_with_fallbacks(
+    url: str,
+    selected_language: str,
+    source_mode: str,
+    transcript_text: str,
+    transcript_file,
+) -> list[Document]:
+    if source_mode == "Manual transcript":
+        return _load_manual_transcript_documents(url, selected_language, transcript_text, transcript_file)
+
+    strategies = []
+    if source_mode in {"Auto", "Direct transcript"}:
+        strategies.append(("Direct transcript", lambda: _load_youtube_documents(url, selected_language)))
+    if source_mode in {"Auto", "External transcript API"}:
+        strategies.append(
+            ("External transcript API", lambda: _load_youtube_documents_via_external_api(url, selected_language))
+        )
+    if source_mode in {"Auto", "Audio transcription (yt-dlp + Groq)"}:
+        strategies.append(
+            (
+                "Audio transcription (yt-dlp + Groq)",
+                lambda: _load_youtube_documents_via_audio_transcription(url, selected_language),
+            )
+        )
+
+    failures = []
+    for strategy_name, loader in strategies:
+        try:
+            return loader()
+        except Exception as exc:
+            failures.append(f"{strategy_name}: {exc}")
+
+    if source_mode == "Auto" and (transcript_text.strip() or transcript_file is not None):
+        return _load_manual_transcript_documents(url, selected_language, transcript_text, transcript_file)
+
+    if not failures:
+        raise ValueError("No YouTube transcript strategy is available for the selected mode.")
+
+    raise RuntimeError("All YouTube transcript strategies failed.\n" + "\n".join(failures))
 
 
 def _has_meaningful_content(docs: list[Document], min_chars: int = 300) -> bool:
@@ -690,13 +1020,21 @@ if input_source_mode in {"URL", "Both"} and _is_youtube_url(generic_url):
             else:
                 try:
                     with st.spinner("Loading transcript..."):
-                        docs = _load_youtube_documents(generic_url, transcript_language)
+                        docs = _load_youtube_documents_with_fallbacks(
+                            generic_url,
+                            transcript_language,
+                            youtube_source_mode,
+                            manual_transcript_text,
+                            manual_transcript_file,
+                        )
                         if not docs:
                             st.error("No transcript could be extracted from the provided YouTube video.")
                         else:
                             _store_youtube_transcript(generic_url, docs)
                             st.success(
-                                f"Transcript ready for export in {st.session_state.youtube_transcript_language_label}."
+                                "Transcript ready for export in "
+                                f"{st.session_state.youtube_transcript_language_label} "
+                                f"via {st.session_state.youtube_transcript_source_mode}."
                             )
                 except Exception as transcript_err:
                     st.error(f"Failed to load YouTube transcript: {transcript_err}")
@@ -705,7 +1043,11 @@ if input_source_mode in {"URL", "Both"} and _is_youtube_url(generic_url):
             st.session_state.youtube_transcript_text
             and st.session_state.youtube_transcript_source_url == generic_url
         ):
-            st.caption(f"Prepared transcript: `{st.session_state.youtube_transcript_language_label}`")
+            st.caption(
+                "Prepared transcript: "
+                f"`{st.session_state.youtube_transcript_language_label}` via "
+                f"`{st.session_state.youtube_transcript_source_mode}`"
+            )
             st.download_button(
                 "Export transcript",
                 data=st.session_state.youtube_transcript_text,
@@ -733,15 +1075,19 @@ if st.button("Summarize content"):
                 if input_source_mode in {"URL", "Both"} and generic_url.strip():
                     if _is_youtube_url(generic_url):
                         try:
-                            url_docs = _load_youtube_documents(generic_url, transcript_language)
+                            url_docs = _load_youtube_documents_with_fallbacks(
+                                generic_url,
+                                transcript_language,
+                                youtube_source_mode,
+                                manual_transcript_text,
+                                manual_transcript_file,
+                            )
                             _store_youtube_transcript(generic_url, url_docs)
                         except Exception as load_err:
                             st.error(f"Failed to load YouTube transcript: {load_err}")
                             st.stop()
                     else:
-                        st.session_state.youtube_transcript_text = ""
-                        st.session_state.youtube_transcript_name = "youtube_transcript.txt"
-                        st.session_state.youtube_transcript_source_url = ""
+                        _reset_youtube_transcript_state()
                         try:
                             url_docs = _load_web_documents(generic_url)
                         except Exception as load_err:
@@ -750,9 +1096,7 @@ if st.button("Summarize content"):
 
                     docs.extend(url_docs)
                 else:
-                    st.session_state.youtube_transcript_text = ""
-                    st.session_state.youtube_transcript_name = "youtube_transcript.txt"
-                    st.session_state.youtube_transcript_source_url = ""
+                    _reset_youtube_transcript_state()
 
                 if input_source_mode in {"Upload documents", "Both"} and uploaded_files:
                     try:

requirements.txt

@@ -14,3 +14,4 @@ langchain-text-splitters>=1.1.2
 youtube-transcript-api>=1.2.4
 unstructured>=0.22.22
 pytube>=15.0.0
+yt-dlp>=2025.1.15