Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/facebook.py (Python)

Code before:

    def _download_fb_webpage_and_verify(self, url, video_id):
        """
        Facebook may return a 403 with an HTML page containing a JS snippet that:
        1. Fetches a /__rd_verify_... endpoint with POST
        2. That endpoint sets an rd_challenge cookie
        3. The original request with that cookie succeeds

        We need to mimic this for us to see the true webpage

        See https://github.com/yt-dlp/yt-dlp/issues/15577 for details
        """

        try:
            return self._download_webpage(url, video_id)
        except ExtractorError as e:
            if (
                not isinstance(e.cause, HTTPError)
                or e.cause.status != 403
                or e.cause.reason != 'Client challenge'
            ):
                raise
            error_page = self._webpage_read_content(e.cause.response, url, video_id)

        # we've hit a 403 with a client challenge, we need to solve it
        self.write_debug('Found facebook ads JS challenge')

        challenge_path = self._search_regex(
            r'''fetch\s*\(\s*['"](\/__rd_verify[^'"]+)['"]''',
            error_page,
            'challenge path',
            default=None,
        )
        if not challenge_path:
            raise ExtractorError(
                'Unable to extract rd_challenge.',
                expected=True,
            )

        # we have to POST to the challenge URL to get a cookie back (rd_challenge) (the system adds it to the cookie jar)
        full_challenge_url = urljoin(url, challenge_path)
        self._request_webpage(
            full_challenge_url,
            video_id,
            note='Solving rd_challenge',
            errnote='Failed to solve rd_challenge',
            expected_status=[200],
            fatal=True,
            data=b'',
        )


Review comment (bug):
Let's not call it a "JS challenge" since we're not executing or interpreting any actual JavaScript, and "JS challenge" could make the user think they need a JS runtime like they do for YouTube's JS challenges
```suggestion
            error_page = self._webpage_read_content(e.cause.response, url, video_id)

        self.write_debug('Received a client challenge response')
```

Code after:
    }, {
        'url': 'https://es-la.facebook.com/ads/library/?id=901230958115569',
        'only_matching': True,
    }, {
        'url': 'https://m.facebook.com/ads/library/?id=901230958115569',
        'only_matching': True,
    }]

    _FORMATS_MAP = {
        'watermarked_video_sd_url': ('sd-wmk', 'SD, watermarked'),
        'video_sd_url': ('sd', None),
        'watermarked_video_hd_url': ('hd-wmk', 'HD, watermarked'),
        'video_hd_url': ('hd', None),
    }

    def _extract_formats(self, video_dict):
        formats = []
        for format_key, format_url in traverse_obj(video_dict, (
            {dict.items}, lambda _, v: v[0] in self._FORMATS_MAP and url_or_none(v[1]),
        )):
            formats.append({
                'format_id': self._FORMATS_MAP[format_key][0],
                'format_note': self._FORMATS_MAP[format_key][1],
                'url': format_url,
                'ext': 'mp4',
                'quality': qualities(tuple(self._FORMATS_MAP))(format_key),
            })
        return formats

    def _download_fb_webpage_and_verify(self, url, video_id):
        # See https://github.com/yt-dlp/yt-dlp/issues/15577

        try:
            return self._download_webpage(url, video_id)
        except ExtractorError as e:
            if (
                not isinstance(e.cause, HTTPError)
                or e.cause.status != 403
                or e.cause.reason != 'Client challenge'
            ):
                raise
            error_page = self._webpage_read_content(e.cause.response, url, video_id)

        self.write_debug('Received a client challenge response')

        challenge_path = self._search_regex(
            r'fetch\s*\(\s*["\'](/__rd_verify[^"\']+)["\']',
            error_page, 'challenge path')

        # Successful response will set the necessary cookie
        self._request_webpage(