ronantakizawa/codereview-bench huggingface dataset:
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/genericembeds.py (Python)

Code before:
            'channel': 'Helen & Douglas House',
            'comment_count': int,
            'uploader_url': 'http://www.youtube.com/user/helendouglashouse',
            'duration': 253,
            'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA',
            'playable_in_embed': True,
            'age_limit': 0,
            'channel_follower_count': int,
            'channel_id': 'UCTChGezrZVmlYlpMlkmulPA',
            'tags': 'count:6',
            'categories': ['Nonprofits & Activism'],
            'like_count': int,
            'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _extract_from_webpage(self, url, webpage):
        combined = ''
        for _, html in re.findall(r'(?s)data-html=(["\'])((?:(?!\1).)+)\1', webpage):
            if not html:
                continue
            # unescapeHTML can handle &quot; etc., unquote can handle percent encoding
            unquoted_html = unescapeHTML(unquote(html))
            if unquoted_html == html:
                continue
            combined += unquoted_html
        if not combined:
            return
        yield from self._extract_generic_embeds(url, combined)


Review comment (security):
```suggestion
            unquoted_html = unescapeHTML(urllib.parse.unquote(html))
```

Code after:
            'channel': 'Helen & Douglas House',
            'comment_count': int,
            'uploader_url': 'http://www.youtube.com/user/helendouglashouse',
            'duration': 253,
            'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA',
            'playable_in_embed': True,
            'age_limit': 0,
            'channel_follower_count': int,
            'channel_id': 'UCTChGezrZVmlYlpMlkmulPA',
            'tags': 'count:6',
            'categories': ['Nonprofits & Activism'],
            'like_count': int,
            'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _extract_from_webpage(self, url, webpage):
        combined = ''
        for _, html in re.findall(r'(?s)\bdata-html=(["\'])((?:(?!\1).)+)\1', webpage):
            # unescapeHTML can handle &quot; etc., unquote can handle percent encoding
            unquoted_html = unescapeHTML(urllib.parse.unquote(html))
            if unquoted_html != html:
                combined += unquoted_html
        if combined:
            yield from self._extract_generic_embeds(url, combined)
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/genericembeds.py (Python)

Code before:
            'availability': 'public',
            'view_count': int,
            'channel': 'Helen & Douglas House',
            'comment_count': int,
            'uploader_url': 'http://www.youtube.com/user/helendouglashouse',
            'duration': 253,
            'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA',
            'playable_in_embed': True,
            'age_limit': 0,
            'channel_follower_count': int,
            'channel_id': 'UCTChGezrZVmlYlpMlkmulPA',
            'tags': 'count:6',
            'categories': ['Nonprofits & Activism'],
            'like_count': int,
            'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _extract_from_webpage(self, url, webpage):
        combined = ''
        for _, html in re.findall(r'(?s)data-html=(["\'])((?:(?!\1).)+)\1', webpage):
            if not html:
                continue
            # unescapeHTML can handle &quot; etc., unquote can handle percent encoding
            unquoted_html = unescapeHTML(unquote(html))
            if unquoted_html == html:
                continue
            combined += unquoted_html
        if not combined:
            return
        yield from self._extract_generic_embeds(url, combined)


Review comment (suggestion):
will never happen because of `.+` in regex
```suggestion
```

Code after:
            'availability': 'public',
            'view_count': int,
            'channel': 'Helen & Douglas House',
            'comment_count': int,
            'uploader_url': 'http://www.youtube.com/user/helendouglashouse',
            'duration': 253,
            'channel_url': 'https://www.youtube.com/channel/UCTChGezrZVmlYlpMlkmulPA',
            'playable_in_embed': True,
            'age_limit': 0,
            'channel_follower_count': int,
            'channel_id': 'UCTChGezrZVmlYlpMlkmulPA',
            'tags': 'count:6',
            'categories': ['Nonprofits & Activism'],
            'like_count': int,
            'thumbnail': 'https://i.ytimg.com/vi/Tc7b_JGdZfw/hqdefault.jpg',
        },
        'params': {
            'skip_download': True,
        },
    }]

    def _extract_from_webpage(self, url, webpage):
        combined = ''
        for _, html in re.findall(r'(?s)\bdata-html=(["\'])((?:(?!\1).)+)\1', webpage):
            # unescapeHTML can handle &quot; etc., unquote can handle percent encoding
            unquoted_html = unescapeHTML(urllib.parse.unquote(html))
            if unquoted_html != html:
                combined += unquoted_html
        if combined:
            yield from self._extract_generic_embeds(url, combined)
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/genius.py (Python)

Code before:
        unescaped_json = self._parse_json(escaped_json, display_id, transform_source=js_to_json)
        song_info = self._parse_json(unescaped_json, display_id)
        song_id = str_or_none(traverse_obj(song_info, ('songPage', 'song')))
        if not song_id:
            raise ExtractorError('Song id not found in webpage')

        title = traverse_obj(
            song_info, ('songPage', 'trackingData', lambda _, x: x['key'] == 'Title', 'value'),
            get_all=False, default='untitled')
        artist = traverse_obj(
            song_info, ('songPage', 'trackingData', lambda _, x: x['key'] == 'Primary Artist', 'value'),
            get_all=False, default='unknown artist')
        media = traverse_obj(
            song_info, ('entities', 'songs', song_id, 'media'), expected_type=list, default=[])

        entries = []
        for m in media:
            if m.get('type') in ('video', 'audio') and m.get('url'):
                if m.get('provider') == 'spotify':
                    continue
                elif m.get('provider') == 'youtube':
                    entries.append(self.url_result(m['url'], 'Youtube', song_id))
                elif m.get('provider') == 'soundcloud':
                    entries.append(self.url_result(m['url'], 'Soundcloud', song_id))
                else:
                    entries.append(self.url_result(m['url']))

        return self.playlist_result(entries, song_id, title, f'{title} by {artist}')


Review comment (suggestion):
Not worth it just to add `ie_key` imo
```suggestion
                else:
                    entries.append(self.url_result(m['url']))
```

Code after:
            r'window\.__PRELOADED_STATE__\s*=\s*JSON\.parse\(', webpage, 'json string',
            display_id, transform_source=js_to_json, contains_pattern=r'\'{(?s:.+)}\'')
        song_info = self._parse_json(json_string, display_id)
        song_id = str_or_none(traverse_obj(song_info, ('songPage', 'song')))
        if not song_id:
            raise ExtractorError('Song id not found in webpage')

        title = traverse_obj(
            song_info, ('songPage', 'trackingData', lambda _, x: x['key'] == 'Title', 'value'),
            get_all=False, default='untitled')
        artist = traverse_obj(
            song_info, ('songPage', 'trackingData', lambda _, x: x['key'] == 'Primary Artist', 'value'),
            get_all=False, default='unknown artist')
        media = traverse_obj(
            song_info, ('entities', 'songs', song_id, 'media'), expected_type=list, default=[])

        entries = []
        for m in media:
            if m.get('type') in ('video', 'audio') and m.get('url'):
                if m.get('provider') == 'spotify':
                    self.to_screen(f'{song_id}: Skipping Spotify audio embed')
                else:
                    entries.append(self.url_result(m['url']))

        return self.playlist_result(entries, song_id, title, f'{title} by {artist}')
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/twitter.py (Python)

Code before:
import json
import re
import urllib.error

from .common import InfoExtractor
from .periscope import PeriscopeBaseIE, PeriscopeIE
from ..compat import (
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    dict_get,
    float_or_none,
    format_field,
    int_or_none,
    make_archive_id,
    str_or_none,
    strip_or_none,
    traverse_obj,
    try_get,
    unified_timestamp,
    update_url_query,
    url_or_none,
    xpath_text,
)


class TwitterBaseIE(InfoExtractor):
    _API_BASE = 'https://api.twitter.com/1.1/'
    _GRAPHQL_API_BASE = 'https://twitter.com/i/api/graphql/'
    _TOKENS = {

Review comment (suggestion):
```suggestion
from ..compat import functools  # isort: split
from ..compat import (
```

Code after:
import json
import re
import urllib.error

from .common import InfoExtractor
from .periscope import PeriscopeBaseIE, PeriscopeIE
from ..compat import functools  # isort: split
from ..compat import (
    compat_parse_qs,
    compat_urllib_parse_unquote,
    compat_urllib_parse_urlparse,
)
from ..utils import (
    ExtractorError,
    dict_get,
    float_or_none,
    format_field,
    int_or_none,
    make_archive_id,
    str_or_none,
    strip_or_none,
    traverse_obj,
    try_call,
    try_get,
    unified_timestamp,
    update_url_query,
    url_or_none,
    xpath_text,
)


class TwitterBaseIE(InfoExtractor):
    _API_BASE = 'https://api.twitter.com/1.1/'
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/yle_areena.py (Python)

Code before:
from .common import InfoExtractor
from ..utils import (
    traverse_obj,
)


class YleAreenaIE(InfoExtractor):
    _VALID_URL = r'https?://areena\.yle\.fi/(?P<id>[0-9-]+)'
    _TESTS = [{
        'url': 'https://areena.yle.fi/1-4371942',
        'md5': '932edda0ecf5dfd6423804182d32f8ac',
        'info_dict': {
            'id': '0_a3tjk92c',
            'ext': 'mp4',
            'title': 'K1, J2: Pouchit | Modernit miehet',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader_id': 'ovp@yle.fi',
            'duration': 1435,
            'view_count': int,
            'upload_date': '20181204',
            'timestamp': 1543916210,
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_data_url = f'https://player.api.yle.fi/v1/preview/{video_id}.json?app_id=player_static_prod&app_key=8930d72170e48303cf5f3867780d549b'
        video_data = self._download_json(video_data_url, video_id)

Review comment (suggestion):
```suggestion
from ..utils import traverse_obj
```

Code after:
from .common import InfoExtractor
from .kaltura import KalturaIE
from ..utils import int_or_none, traverse_obj, url_or_none


class YleAreenaIE(InfoExtractor):
    _VALID_URL = r'https?://areena\.yle\.fi/(?P<id>[\d-]+)'
    _TESTS = [{
        'url': 'https://areena.yle.fi/1-4371942',
        'md5': '932edda0ecf5dfd6423804182d32f8ac',
        'info_dict': {
            'id': '0_a3tjk92c',
            'ext': 'mp4',
            'title': 'Pouchit',
            'description': 'md5:d487309c3abbe5650265bbd1742d2f82',
            'series': 'Modernit miehet',
            'season': 'Season 1',
            'season_number': 1,
            'episode': 'Episode 2',
            'episode_number': 2,
            'thumbnail': 'http://cfvod.kaltura.com/p/1955031/sp/195503100/thumbnail/entry_id/0_a3tjk92c/version/100061',
            'uploader_id': 'ovp@yle.fi',
            'duration': 1435,
            'view_count': int,
            'upload_date': '20181204',
            'timestamp': 1543916210,
            'subtitles': {'fin': [{'url': r're:^https?://', 'ext': 'srt'}]},
            'age_limit': 7,
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/yle_areena.py (Python)

Code before:
from .common import InfoExtractor
from ..utils import (
    traverse_obj,
)


class YleAreenaIE(InfoExtractor):
    _VALID_URL = r'https?://areena\.yle\.fi/(?P<id>[0-9-]+)'
    _TESTS = [{
        'url': 'https://areena.yle.fi/1-4371942',
        'md5': '932edda0ecf5dfd6423804182d32f8ac',
        'info_dict': {
            'id': '0_a3tjk92c',
            'ext': 'mp4',
            'title': 'K1, J2: Pouchit | Modernit miehet',
            'thumbnail': r're:^https?://.*\.jpg$',
            'uploader_id': 'ovp@yle.fi',
            'duration': 1435,
            'view_count': int,
            'upload_date': '20181204',
            'timestamp': 1543916210,
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)

        video_data_url = f'https://player.api.yle.fi/v1/preview/{video_id}.json?app_id=player_static_prod&app_key=8930d72170e48303cf5f3867780d549b'
        video_data = self._download_json(video_data_url, video_id)
        kaltura_id = traverse_obj(video_data, ('data', 'ongoing_ondemand', 'kaltura', 'id'))

        webpage = self._download_webpage(url, video_id)
        info = self._search_json_ld(webpage, video_id, default={})

Review comment (suggestion):
```suggestion
    _VALID_URL = r'https?://areena\.yle\.fi/(?P<id>[\d-]+)'
```

Code after:
from .common import InfoExtractor
from .kaltura import KalturaIE
from ..utils import int_or_none, traverse_obj, url_or_none


class YleAreenaIE(InfoExtractor):
    _VALID_URL = r'https?://areena\.yle\.fi/(?P<id>[\d-]+)'
    _TESTS = [{
        'url': 'https://areena.yle.fi/1-4371942',
        'md5': '932edda0ecf5dfd6423804182d32f8ac',
        'info_dict': {
            'id': '0_a3tjk92c',
            'ext': 'mp4',
            'title': 'Pouchit',
            'description': 'md5:d487309c3abbe5650265bbd1742d2f82',
            'series': 'Modernit miehet',
            'season': 'Season 1',
            'season_number': 1,
            'episode': 'Episode 2',
            'episode_number': 2,
            'thumbnail': 'http://cfvod.kaltura.com/p/1955031/sp/195503100/thumbnail/entry_id/0_a3tjk92c/version/100061',
            'uploader_id': 'ovp@yle.fi',
            'duration': 1435,
            'view_count': int,
            'upload_date': '20181204',
            'timestamp': 1543916210,
            'subtitles': {'fin': [{'url': r're:^https?://', 'ext': 'srt'}]},
            'age_limit': 7,
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/cinetecamilano.py (Python)

Code before:
from json import JSONDecodeError
from urllib.error import HTTPError
from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    parse_iso8601,
    strip_or_none,
    traverse_obj,
    try_call,
    try_get,
    urljoin)


class CinetecaMilanoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cinetecamilano\.it/film/(?P<id>\d+)/?'
    _TESTS = [{
        'url': 'https://www.cinetecamilano.it/film/1942',
        'info_dict': {
            'id': '1942',
            'ext': 'mp4',
            'title': "Il draghetto Gris\u00f9 (4 episodi)",
            'release_date': '20220129',
            'thumbnail': 're:.+\\.png',
            'description': 'md5:5328cbe080b93224712b6f17fcaf2c01',
            'modified_date': "20200520",
            'duration': 3139,

Review comment (suggestion):
```suggestion
import json
import urllib.error

```
make necessary changes below

Code after:
import json
import urllib.error
from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    parse_iso8601,
    strip_or_none,
    traverse_obj,
    try_get,
    urljoin,
)


class CinetecaMilanoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cinetecamilano\.it/film/(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://www.cinetecamilano.it/film/1942',
        'info_dict': {
            'id': '1942',
            'ext': 'mp4',
            'title': 'Il draghetto Gris\u00f9 (4 episodi)',
            'release_date': '20220129',
            'thumbnail': r're:.+\.png',
            'description': 'md5:5328cbe080b93224712b6f17fcaf2c01',
            'modified_date': '20200520',
            'duration': 3139,
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/cinetecamilano.py (Python)

Code before:
from json import JSONDecodeError
from urllib.error import HTTPError
from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    parse_iso8601,
    strip_or_none,
    traverse_obj,
    try_call,
    try_get,
    urljoin)


class CinetecaMilanoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cinetecamilano\.it/film/(?P<id>\d+)/?'
    _TESTS = [{
        'url': 'https://www.cinetecamilano.it/film/1942',
        'info_dict': {
            'id': '1942',
            'ext': 'mp4',
            'title': "Il draghetto Gris\u00f9 (4 episodi)",
            'release_date': '20220129',
            'thumbnail': 're:.+\\.png',
            'description': 'md5:5328cbe080b93224712b6f17fcaf2c01',
            'modified_date': "20200520",
            'duration': 3139,
            'release_timestamp': 1643446208,
            'modified_timestamp': int
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        try:
            film_json = self._download_json(
                f'https://www.cinetecamilano.it/api/catalogo/{video_id}',
                video_id, headers={
                    'Referer': url,
                    'Authorization': try_get(self._get_cookies('https://www.cinetecamilano.it'), lambda x: f"Bearer {x['cnt-token'].value}") or ''
                })

Review comment (suggestion):
```suggestion
    _VALID_URL = r'https?://(?:www\.)?cinetecamilano\.it/film/(?P<id>\d+)'
```
it does nothing

Code after:
import json
import urllib.error
from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    float_or_none,
    parse_iso8601,
    strip_or_none,
    traverse_obj,
    try_get,
    urljoin,
)


class CinetecaMilanoIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?cinetecamilano\.it/film/(?P<id>\d+)'
    _TESTS = [{
        'url': 'https://www.cinetecamilano.it/film/1942',
        'info_dict': {
            'id': '1942',
            'ext': 'mp4',
            'title': 'Il draghetto Gris\u00f9 (4 episodi)',
            'release_date': '20220129',
            'thumbnail': r're:.+\.png',
            'description': 'md5:5328cbe080b93224712b6f17fcaf2c01',
            'modified_date': '20200520',
            'duration': 3139,
            'release_timestamp': 1643446208,
            'modified_timestamp': int
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        try:
            film_json = self._download_json(
                f'https://www.cinetecamilano.it/api/catalogo/{video_id}/?',
                video_id, headers={
                    'Referer': url,
                    'Authorization': try_get(self._get_cookies('https://www.cinetecamilano.it'), lambda x: f'Bearer {x["cnt-token"].value}') or ''
                })
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/zeenews.py (Python)

Code before:
                'display_id': 'videsh-superfast-queen-elizabeth-iis-funeral-today',
                'upload_date': '20220919',
                'thumbnail': r're:^https?://.*\.jpg*',
            }
        }
    ]

    def _real_extract(self, url):
        match = self._match_valid_url(url).groupdict()
        content_id, display_id = match['id'], match['display_id']
        webpage = self._download_webpage(url, content_id)
        json_ld_list = list(self._yield_json_ld(webpage, display_id))
        info = next(json_ld for json_ld in json_ld_list if json_ld.get('@type') == 'VideoObject')

        formats = self._extract_m3u8_formats(info.get('embedUrl'), content_id, 'mp4')
        self._sort_formats(formats)

        return {
            'id': content_id,
            'description': info.get('description'),
            'title': info.get('name'),
            'display_id': display_id,
            'formats': formats,
            'upload_date': unified_strdate(info.get('uploadDate')),
            'thumbnail': info.get('thumbnailUrl'),
        }


Review comment (refactor):
```suggestion
        json_ld_list = list(self._yield_json_ld(webpage, display_id))

        embed_url = traverse_obj(json_ld_list, (lambda _, v: v['@type'] == 'VideoObject', 'embedURL'))
        formats = self._extract_m3u8_formats(embed_url, content_id, 'mp4')
        self._sort_formats(formats)

        return {
            **self._json_ld(json_ld_list, display_id),
            'id': content_id,
            'display_id': display_id,
            'formats': formats,
        }
```
does this work?

Code after:
                'title': 'Videsh Superfast: महारानी के अंतिम संस्कार की तैयारी शुरू',
                'display_id': 'videsh-superfast-queen-elizabeth-iis-funeral-today',
                'upload_date': '20220919',
                'thumbnail': r're:^https?://.*\.jpg*',
                'timestamp': 1663556881,
                'view_count': int,
                'duration': 133,
                'description': 'सेगमेंट विदेश सुपराफास्ट में देखिए देश और दुनिया की सभी बड़ी खबरें, वो भी हर खबर फटाफट अंदाज में.',
            }
        }
    ]

    def _real_extract(self, url):
        content_id, display_id = self._match_valid_url(url).group('id', 'display_id')
        webpage = self._download_webpage(url, content_id)
        json_ld_list = list(self._yield_json_ld(webpage, display_id))

        embed_url = traverse_obj(
            json_ld_list, (lambda _, v: v['@type'] == 'VideoObject', 'embedUrl'), get_all=False)
        if not embed_url:
            raise ExtractorError('No video found', expected=True)

        formats = self._extract_m3u8_formats(embed_url, content_id, 'mp4')
        self._sort_formats(formats)

        return {
            **self._json_ld(json_ld_list, display_id),
            'id': content_id,
            'display_id': display_id,
            'formats': formats,
        }