ronantakizawa/codereview-bench huggingface dataset:
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/ted.py (Python)

Code before:
        if audio_download:
            formats.append({
                'url': audio_download,
                'format_id': 'audio',
                'vcodec': 'none',
            })

        if not formats:
            external = playerData.get('external')
            if isinstance(external, dict):
                service = external.get('service')
                if isinstance(service, compat_str):
                    ext_url = None
                    if service.lower() == 'youtube':
                        ext_url = external.get('code')
                    return self.url_result(ext_url or external['uri'])

        self._sort_formats(formats)

        # trim thumbnail resize parameters
        thumbnail = self._search_regex(r'^(http[^?]*)', playerData.get('thumb'), 'thumbnail', default=None) or self._search_regex(r'^(http[^?]*)', self._og_search_property('image', webpage), 'thumbnail', default=None)

        # tags
        tags = try_get(playerData, lambda x: x['targeting']['tag'], str)
        if tags:
            tags = tags.split(',')

        return {
            'id': video_id,
            'title': title,
            'uploader': uploader,
            'thumbnail': thumbnail,
            'description': description,
            'subtitles': subtitles,
            'formats': formats,
            'duration': duration,
            'view_count': view_count,
            'upload_date': upload_date,
            'release_date': release_date,
            'tags': tags,
        }


Review comment (suggestion):
```suggestion
        tags = try_get(playerData, lambda x: x['targeting']['tag'].split(','))
```
can be inlined in return

Code after:
    _VALID_URL = r'https?://embed(?:-ssl)?\.ted\.com/'

    _TESTS = [{
        'url': 'https://embed.ted.com/talks/janet_stovall_how_to_get_serious_about_diversity_and_inclusion_in_the_workplace',
        'info_dict': {
            'id': '21802',
            'ext': 'mp4',
            'title': 'How to get serious about diversity and inclusion in the workplace',
            'description': 'md5:0978aafe396e05341f8ecc795d22189d',
            'view_count': int,
            'tags': list,
            'uploader': 'Janet Stovall',
            'duration': 664.0,
            'upload_date': '20180822',
            'release_date': '20180719',
            'thumbnail': r're:http.*\.jpg',
        },
    }]

    @classmethod
    def _extract_urls(cls, webpage):
        return [mobj.group('url') for mobj in re.finditer(
            fr'<iframe[^>]+?src=(["\'])(?P<url>{cls._VALID_URL}.+?)\1', webpage)]

    def _real_extract(self, url):
        return self.url_result(re.sub(r'://embed(-ssl)?', '://www', url), TedTalkIE.ie_key())
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/murrtube.py (Python)

Code before:
}
}'''},
            'Downloading page {0}'.format(page + 1))
        if data is None:
            raise ExtractorError(f'Failed to retrieve video list for page {page + 1}', expected=True)

        media = data['media']

        for entry in media:
            yield self.url_result(
                'murrtube:{0}'.format(entry['id']),
                MurrtubeIE.ie_key())

    def _real_extract(self, url):
        username = self._match_id(url)
        data = self._download_gql(username, {
            'operationName': 'User',
            'variables': {
                'id': username,
            },
            'query': '''query User($id: ID!) {
  user(id: $id) {
    id
    __typename
  }
}'''},
            'Downloading user info')
        if data is None:
            raise ExtractorError('Failed to fetch user info', expected=True)

        user = data['user']

        entries = OnDemandPagedList(functools.partial(
            self._fetch_page, username, user.get('id')), self._PAGE_SIZE)

        return self.playlist_result(entries, username)


Review comment (suggestion):
```suggestion
            'query': '''\
query User($id: ID!) {
  user(id: $id) {
    id
    __typename
  }
}'''},
```
etc

Code after:
            'Downloading page {0}'.format(page + 1))
        if data is None:
            raise ExtractorError(f'Failed to retrieve video list for page {page + 1}')

        media = data['media']

        for entry in media:
            yield self.url_result('murrtube:{0}'.format(entry['id']), MurrtubeIE.ie_key())

    def _real_extract(self, url):
        username = self._match_id(url)
        data = self._download_gql(username, {
            'operationName': 'User',
            'variables': {
                'id': username,
            },
            'query': '''\
query User($id: ID!) {
  user(id: $id) {
    id
    __typename
  }
}'''},
            'Downloading user info')
        if data is None:
            raise ExtractorError('Failed to fetch user info')

        user = data['user']

        entries = OnDemandPagedList(functools.partial(
            self._fetch_page, username, user.get('id')), self._PAGE_SIZE)

        return self.playlist_result(entries, username)
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/newsy.py (Python)

Code before:
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        data_json = self._parse_json(self._html_search_regex(
            r'data-video-player\s?=\s?"({[^"]+})">', webpage, 'data'), display_id, js_to_json)
        ld_json = self._search_json_ld(webpage, display_id, fatal=False)

        formats, subtitles = [], {}
        if data_json.get('stream'):
            fmts, subs = self._extract_m3u8_formats_and_subtitles(data_json['stream'], display_id)
            formats.extend(fmts)
            subtitles = self._merge_subtitles(subtitles, subs)
        self._sort_formats(formats)
        return {
            'id': data_json['id'],
            'display_id': display_id,
            'title': ld_json.get('title') or data_json.get('headline'),
            'description': ld_json.get('description'),
            'timestamp': ld_json.get('timestamp'),
            'duration': data_json.get('duration'),
            'thumbnail': data_json.get('image'),
            'formats': formats,
            'subtitles': subtitles,
        }


Review comment (suggestion):
why not just
```py
        return merge_dicts(ld_json, {
            'id': data_json['id'],
            'display_id': display_id,
            'title': data_json.get('headline'),
            'duration': data_json.get('duration'),
            'thumbnail': data_json.get('image'),
            'formats': formats,
            'subtitles': subtitles,
        })
```

Code after:
            'upload_date': '20210518'
        },
        'params': {'skip_download': True}
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        webpage = self._download_webpage(url, display_id)
        data_json = self._parse_json(self._html_search_regex(
            r'data-video-player\s?=\s?"({[^"]+})">', webpage, 'data'), display_id, js_to_json)
        ld_json = self._search_json_ld(webpage, display_id, fatal=False)

        formats, subtitles = [], {}
        if data_json.get('stream'):
            fmts, subs = self._extract_m3u8_formats_and_subtitles(data_json['stream'], display_id)
            formats.extend(fmts)
            subtitles = self._merge_subtitles(subtitles, subs)
        self._sort_formats(formats)
        return merge_dicts(ld_json, {
            'id': data_json['id'],
            'display_id': display_id,
            'title': data_json.get('headline'),
            'duration': data_json.get('duration'),
            'thumbnail': data_json.get('image'),
            'formats': formats,
            'subtitles': subtitles,
        })
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/fujitv.py (Python)

Code before:
        self._download_webpage(url, video_id)
        json_info = {}
        token = self._get_cookies(url).get('CT')
        if token:
            json_info = self._download_json('https://fod-sp.fujitv.co.jp/apps/api/episode/detail/?ep_id=%s&is_premium=false' % video_id, video_id, headers={'x-authorization': f'Bearer {token.value}'}, fatal=False)
        else:
            self.report_warning('Unable to extract token cookie, video information is unavailable')
            print(self._LOGIN_HINTS['cookies'])
        formats = []
        subtitles = []
        src_json = self._download_json(self._BASE_URL + 'abrjson_v2/tv_android/%s' % video_id, video_id)
        for src in src_json.get('video_selector'):
            format, subtitle = self._extract_m3u8_formats_and_subtitles(src.get('url'), video_id, 'mp4')
            formats += format
            subtitles += subtitle
        self._sort_formats(formats, ['tbr'])

        return {
            'id': video_id,
            'title': json_info.get('ep_title'),
            'series': json_info.get('lu_title'),
            'series_id': series_id,
            'description': json_info.get('ep_description'),
            'formats': formats,
            'subtitles': subtitles,
            'thumbnail': self._BASE_URL + f'img/program/{series_id}/episode/{video_id}_a.jpg',
        }


Review comment (suggestion):
```suggestion
            'thumbnail': f'{self._BASE_URL}img/program/{series_id}/episode/{video_id}_a.jpg',
```

Code after:
        self._request_webpage(HEADRequest(url), video_id)
        json_info = {}
        token = self._get_cookies(url).get('CT')
        if token:
            json_info = self._download_json('https://fod-sp.fujitv.co.jp/apps/api/episode/detail/?ep_id=%s&is_premium=false' % video_id, video_id, headers={'x-authorization': f'Bearer {token.value}'}, fatal=False)
        else:
            self.report_warning(f'The token cookie is needed to extract video metadata. {self._LOGIN_HINTS["cookies"]}')
        formats, subtitles = [], {}
        src_json = self._download_json(f'{self._BASE_URL}abrjson_v2/tv_android/{video_id}', video_id)
        for src in src_json['video_selector']:
            if not src.get('url'):
                continue
            fmt, subs = self._extract_m3u8_formats_and_subtitles(src['url'], video_id, 'mp4')
            formats.extend(fmt)
            subtitles = self._merge_subtitles(subtitles, subs)
        self._sort_formats(formats, ['tbr'])

        return {
            'id': video_id,
            'title': json_info.get('ep_title'),
            'series': json_info.get('lu_title'),
            'series_id': series_id,
            'description': json_info.get('ep_description'),
            'formats': formats,
            'subtitles': subtitles,
            'thumbnail': f'{self._BASE_URL}img/program/{series_id}/episode/{video_id}_a.jpg',
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/crunchyroll.py (Python)

Code before:
)
from ..utils import (
    ExtractorError,
    bytes_to_intlist,
    extract_attributes,
    float_or_none,
    format_field,
    intlist_to_bytes,
    int_or_none,
    join_nonempty,
    lowercase_escape,
    merge_dicts,
    qualities,
    remove_end,
    sanitized_Request,
    traverse_obj,
    try_get,
    xpath_text,
)
from ..aes import (
    aes_cbc_decrypt,
)


class CrunchyrollBaseIE(InfoExtractor):
    _LOGIN_URL = 'https://www.crunchyroll.com/welcome/login'
    _NETRC_MACHINE = 'crunchyroll'

    def _call_rpc_api(self, method, video_id, note=None, data=None):
        data = data or {}
        data['req'] = 'RpcApi' + method
        data = compat_urllib_parse_urlencode(data).encode('utf-8')
        return self._download_xml(
            'https://www.crunchyroll.com/xml/',
            video_id, note, fatal=False, data=data, headers={
                'Content-Type': 'application/x-www-form-urlencoded',
            })

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return
        if self._get_cookies(self._LOGIN_URL).get('etp_rt'):
            return

        upsell_response = self._download_json(
            'https://api.crunchyroll.com/get_upsell_data.0.json', None, 'Getting session id',
            query={
                'sess_id': 1,
                'device_id': 'whatvalueshouldbeforweb',
                'device_type': 'com.crunchyroll.static',

Review comment (suggestion):
```suggestion
    _LOGIN_URL = 'https://www.crunchyroll.com/welcome/login'
    _API_BASE = 'https://api.crunchyroll.com'
```

Code after:
)
from ..utils import (
    ExtractorError,
    bytes_to_intlist,
    extract_attributes,
    float_or_none,
    format_field,
    intlist_to_bytes,
    int_or_none,
    join_nonempty,
    lowercase_escape,
    merge_dicts,
    qualities,
    remove_end,
    sanitized_Request,
    traverse_obj,
    try_get,
    xpath_text,
)
from ..aes import (
    aes_cbc_decrypt,
)


class CrunchyrollBaseIE(InfoExtractor):
    _LOGIN_URL = 'https://www.crunchyroll.com/welcome/login'
    _API_BASE = 'https://api.crunchyroll.com'
    _NETRC_MACHINE = 'crunchyroll'

    def _call_rpc_api(self, method, video_id, note=None, data=None):
        data = data or {}
        data['req'] = 'RpcApi' + method
        data = compat_urllib_parse_urlencode(data).encode('utf-8')
        return self._download_xml(
            'https://www.crunchyroll.com/xml/',
            video_id, note, fatal=False, data=data, headers={
                'Content-Type': 'application/x-www-form-urlencoded',
            })

    def _login(self):
        username, password = self._get_login_info()
        if username is None:
            return
        if self._get_cookies(self._LOGIN_URL).get('etp_rt'):
            return

        upsell_response = self._download_json(
            f'{self._API_BASE}/get_upsell_data.0.json', None, 'Getting session id',
            query={
                'sess_id': 1,
                'device_id': 'whatvalueshouldbeforweb',
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/mildom.py (Python)

Code before:
# coding: utf-8
from __future__ import unicode_literals

import base64
from datetime import datetime
import itertools
import json

from .common import InfoExtractor
from ..utils import (
    std_headers,
    update_url_query,
    random_uuidv4,
    try_get,
    float_or_none,
    dict_get
)
from ..compat import (
    compat_str
)


class MildomBaseIE(InfoExtractor):
    _GUEST_ID = None
    _DISPATCHER_CONFIG = None

    def _call_api(self, url, video_id, query={}, note='Downloading JSON metadata', init=False):
        query = query or {}
        if query:
            query['__platform'] = 'web'
        url = update_url_query(url, self._common_queries(query, init=init))
        content = self._download_json(url, video_id, note=note)
        if content['code'] == 0:
            return content['body']
        else:
            self.raise_no_formats('Video not found or premium content. Mildom error code %i, "%s"'
                                  % (content['code'], content['message']), expected=True)

    def _common_queries(self, query={}, init=False):
        dc = self._fetch_dispatcher_config()
        r = {
            'timestamp': self.iso_timestamp(),
            '__guest_id': '' if init else self.guest_id(),
            '__location': dc['location'],

Review comment (suggestion):
```suggestion
    compat_str,
```
unrelated diff

Code after:
# coding: utf-8
from __future__ import unicode_literals

import base64
from datetime import datetime
import itertools
import json

from .common import InfoExtractor
from ..utils import (
    std_headers,
    update_url_query,
    random_uuidv4,
    try_get,
    float_or_none,
    dict_get
)
from ..compat import (
    compat_str,
)


class MildomBaseIE(InfoExtractor):
    _GUEST_ID = None
    _DISPATCHER_CONFIG = None

    def _call_api(self, url, video_id, query=None, note='Downloading JSON metadata', init=False):
        query = query or {}
        if query:
            query['__platform'] = 'web'
        url = update_url_query(url, self._common_queries(query, init=init))
        content = self._download_json(url, video_id, note=note)
        if content['code'] == 0:
            return content['body']
        else:
            self.raise_no_formats(
                f'Video not found or premium content. {content["code"]} - {content["message"]}',
                expected=True)

    def _common_queries(self, query={}, init=False):
        dc = self._fetch_dispatcher_config()
        r = {
            'timestamp': self.iso_timestamp(),
            '__guest_id': '' if init else self.guest_id(),
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/nfb.py (Python)

Code before:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor


class NFBIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nfb\.ca/(?:film)/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.nfb.ca/film/trafficopter/',
        "info_dict": {
            "id": "trafficopter",
            'ext': 'mp4',
            "title": "Trafficopter",
            "description": "md5:060228455eb85cf88785c41656776bc0",
            'thumbnail': r're:^https?://.*\.jpg$',
            "uploader": "Barrie Howells",
            "release_date": "1972",
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage('https://www.nfb.ca/film/%s/' % video_id, video_id)

        title = self._html_search_regex(
            r'<[^>]+\bid=["\']titleHeader["\'][^>]*>\s*<h1[^>]*>\s*([^<]+?)\s*</h1>',
            webpage, 'title', default=None)
        director = self._html_search_regex(
            r'<[^>]+\bitemprop=["\']name["\'][^>]*>([^<]+)',
            webpage, 'director', default=None)
        year = self._html_search_regex(

Review comment (suggestion):
```suggestion
    _VALID_URL = r'https?://(?:www\.)?nfb\.ca/film/(?P<id>[^/?#&]+)'
```

Code after:
# coding: utf-8
from __future__ import unicode_literals

from .common import InfoExtractor
from ..utils import int_or_none


class NFBIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nfb\.ca/film/(?P<id>[^/?#&]+)'
    _TESTS = [{
        'url': 'https://www.nfb.ca/film/trafficopter/',
        'info_dict': {
            'id': 'trafficopter',
            'ext': 'mp4',
            'title': 'Trafficopter',
            'description': 'md5:060228455eb85cf88785c41656776bc0',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader': 'Barrie Howells',
            'release_year': 1972,
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage('https://www.nfb.ca/film/%s/' % video_id, video_id)

        iframe = self._html_search_regex(
            r'<[^>]+\bid=["\']player-iframe["\'][^>]*src=["\']([^"\']+)',
            webpage, 'iframe', default=None, fatal=True)
        if iframe.startswith('/'):
            iframe = f'https://www.nfb.ca{iframe}'
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/gettr.py (Python)

Code before:

        live_info = video_info['broadcast']
        live_url = url_or_none(live_info.get('url'))

        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
            live_url, video_id, ext='mp4',
            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if live_url else []

        thumbnails = []
        for thumbnail in try_get(video_info, lambda x: x['postData']['imgs']) or []:
            thumbnails.append({
                'url': urljoin(self._MEDIA_BASE_URL, thumbnail),
            })

        return {
            'id': video_id,
            'title': try_get(video_info, lambda x: x['postData']['ttl']),
            'description': try_get(video_info, lambda x: x['postData']['dsc']),
            'formats': formats,
            'subtitles': subtitles,
            'thumbnails': thumbnails,
            'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname']),
            'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id']),
            'view_count': int_or_none(live_info.get('viewsCount')),
            'timestamp': int_or_none(live_info.get('startAt')),
            'duration': int_or_none(live_info.get('duration')),
            'is_live': bool_or_none(live_info.get('isLive')),
        }


Review comment (suggestion):
```suggestion
            'duration': float_or_none(live_info.get('duration'), scale=1000),
```
Duration is given in miliseconds, while info_dict should set `duration` to seconds

Code after:
            'upload_date': '20220204',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        video_info = self._call_api('live/join/%s' % video_id, video_id, data={})

        live_info = video_info['broadcast']
        live_url = url_or_none(live_info.get('url'))

        formats, subtitles = self._extract_m3u8_formats_and_subtitles(
            live_url, video_id, ext='mp4',
            entry_protocol='m3u8_native', m3u8_id='hls', fatal=False) if live_url else ([], {})

        thumbnails = [{
            'url': urljoin(self._MEDIA_BASE_URL, thumbnail),
        } for thumbnail in try_get(video_info, lambda x: x['postData']['imgs']) or []]

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': try_get(video_info, lambda x: x['postData']['ttl']),
            'description': try_get(video_info, lambda x: x['postData']['dsc']),
            'formats': formats,
            'subtitles': subtitles,
            'thumbnails': thumbnails,
            'uploader': try_get(video_info, lambda x: x['liveHostInfo']['nickname']),
            'uploader_id': try_get(video_info, lambda x: x['liveHostInfo']['_id']),
            'view_count': int_or_none(live_info.get('viewsCount')),
            'timestamp': float_or_none(live_info.get('startAt'), scale=1000),
            'duration': float_or_none(live_info.get('duration'), scale=1000),
            'is_live': bool_or_none(live_info.get('isLive')),
        }