ronantakizawa/codereview-bench huggingface dataset:
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/globalplayer.py (Python)

Code before:
from .common import InfoExtractor
from ..utils import (
    clean_html,
    join_nonempty,
    parse_duration,
    str_or_none,
    traverse_obj,
    unified_strdate,
    unified_timestamp,
    urlhandle_detect_ext,
)


class GlobalPlayerBaseIE(InfoExtractor):

    def _get_page_props(self, url, video_id):
        webpage = self._download_webpage(url, video_id)
        return self._search_nextjs_data(webpage, video_id)['props']['pageProps']

    def _request_ext(self, url, video_id):
        return urlhandle_detect_ext(self._request_webpage(
            url, video_id, note='Determining source extension'))

    def _extract_audio(self, episode, series):
        return {
            'vcodec': 'none',
            **traverse_obj(series, {
                'series': 'title',
                'series_id': 'id',
                'thumbnail': 'imageUrl',
                'uploader': 'itunesAuthor',  # podcasts only
            }),
            **traverse_obj(episode, {
                'id': 'id',
                'description': ('description', {clean_html}),
                'duration': ('duration', {parse_duration}),
                'thumbnail': 'imageUrl',
                'url': 'streamUrl',
                'timestamp': (['pubDate', 'startDate'], {unified_timestamp}),
                # pubDate for podcasts, startDate for radio catchup - that's all we need to have both in one
                'title': 'title',

Review comment (refactor):
```suggestion
class GlobalPlayerBaseIE(InfoExtractor):
    def _get_page_props(self, url, video_id):
```

Code after:
from .common import InfoExtractor
from ..utils import (
    clean_html,
    join_nonempty,
    parse_duration,
    str_or_none,
    traverse_obj,
    unified_strdate,
    unified_timestamp,
    urlhandle_detect_ext,
)


class GlobalPlayerBaseIE(InfoExtractor):
    def _get_page_props(self, url, video_id):
        webpage = self._download_webpage(url, video_id)
        return self._search_nextjs_data(webpage, video_id)['props']['pageProps']

    def _request_ext(self, url, video_id):
        return urlhandle_detect_ext(self._request_webpage(  # Server rejects HEAD requests
            url, video_id, note='Determining source extension'))

    def _extract_audio(self, episode, series):
        return {
            'vcodec': 'none',
            **traverse_obj(series, {
                'series': 'title',
                'series_id': 'id',
                'thumbnail': 'imageUrl',
                'uploader': 'itunesAuthor',  # podcasts only
            }),
            **traverse_obj(episode, {
                'id': 'id',
                'description': ('description', {clean_html}),
                'duration': ('duration', {parse_duration}),
                'thumbnail': 'imageUrl',
                'url': 'streamUrl',
                'timestamp': (('pubDate', 'startDate'), {unified_timestamp}),
                'title': 'title',
            }, get_all=False)
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/nekohacker.py (Python)

Code before:
from datetime import datetime
import re

from yt_dlp.utils import ExtractorError, determine_ext, extract_attributes, get_element_by_class, unified_strdate
from .common import InfoExtractor


class NekoHackerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>[a-zA-Z0-9-_]+)'
    _ENTRY_URL = r'https?://(?:www\.)?nekohacker\.com/wp-content/uploads/\d+/\d+/(?P<id>(?:\d+-)?[a-zA-Z0-9_-]+)\.[a-zA-Z0-9]+'
    _TESTS = [{
        'url': 'https://nekohacker.com/nekoverse/',
        'info_dict': {
            'id': 'nekoverse',
            'title': 'Nekoverse',
        },
        'playlist': [
            {
                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
                'md5': '44223701ebedba0467ebda4cc07fb3aa',
                'info_dict': {
                    'id': '01-Spaceship',
                    'ext': 'mp3',
                    'title': 'Spaceship',
                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
                    'vcodec': 'none',
                    'acodec': 'mp3',
                    'upload_date': '20221101'
                }
            },
            {
                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/02-City-Runner.mp3',
                'md5': '8f853c71719389d32bbbd3f1a87b3f08',
                'info_dict': {

Review comment (suggestion):
Use `[\w-]+`, and we can exclude the path we know definitely isn't an album page
```suggestion
    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>(?!free-dl)[\w-]+)'
```

Code after:
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    determine_ext,
    extract_attributes,
    get_element_by_class,
    get_element_text_and_html_by_tag,
    parse_duration,
    traverse_obj,
    try_call,
    url_or_none,
)


class NekoHackerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>(?!free-dl)[\w-]+)'
    _TESTS = [{
        'url': 'https://nekohacker.com/nekoverse/',
        'info_dict': {
            'id': 'nekoverse',
            'title': 'Nekoverse',
        },
        'playlist': [
            {
                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
                'md5': '44223701ebedba0467ebda4cc07fb3aa',
                'info_dict': {
                    'id': '1712',
                    'ext': 'mp3',
                    'title': 'Spaceship',
                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
                    'vcodec': 'none',
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/nekohacker.py (Python)

Code before:
from datetime import datetime
import re

from yt_dlp.utils import ExtractorError, determine_ext, extract_attributes, get_element_by_class, unified_strdate
from .common import InfoExtractor


class NekoHackerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>[a-zA-Z0-9-_]+)'
    _ENTRY_URL = r'https?://(?:www\.)?nekohacker\.com/wp-content/uploads/\d+/\d+/(?P<id>(?:\d+-)?[a-zA-Z0-9_-]+)\.[a-zA-Z0-9]+'
    _TESTS = [{
        'url': 'https://nekohacker.com/nekoverse/',
        'info_dict': {
            'id': 'nekoverse',
            'title': 'Nekoverse',
        },
        'playlist': [
            {
                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
                'md5': '44223701ebedba0467ebda4cc07fb3aa',
                'info_dict': {
                    'id': '01-Spaceship',
                    'ext': 'mp3',
                    'title': 'Spaceship',
                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
                    'vcodec': 'none',
                    'acodec': 'mp3',
                    'upload_date': '20221101'
                }
            },
            {
                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/02-City-Runner.mp3',
                'md5': '8f853c71719389d32bbbd3f1a87b3f08',
                'info_dict': {
                    'id': '02-City-Runner',

Review comment (bug):
```suggestion
```
I think this is the wrong way of going about extracting an entry `id` value for a few reasons:
 - This is not necessarily going to be unique, e.g. the first test's id is `01-Spaceship`, but what if another release also has a first track named "Spaceship"?
 - The media URLs won't always match this pattern (see https://nekohacker.com/susume/)
 - The `<li>` elements have a `data-trackid` attribute that does seem to be a unique identifier for this site, and IMO we should use that instead

Code after:
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    determine_ext,
    extract_attributes,
    get_element_by_class,
    get_element_text_and_html_by_tag,
    parse_duration,
    traverse_obj,
    try_call,
    url_or_none,
)


class NekoHackerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>(?!free-dl)[\w-]+)'
    _TESTS = [{
        'url': 'https://nekohacker.com/nekoverse/',
        'info_dict': {
            'id': 'nekoverse',
            'title': 'Nekoverse',
        },
        'playlist': [
            {
                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
                'md5': '44223701ebedba0467ebda4cc07fb3aa',
                'info_dict': {
                    'id': '1712',
                    'ext': 'mp3',
                    'title': 'Spaceship',
                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
                    'vcodec': 'none',
                    'acodec': 'mp3',
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/nekohacker.py (Python)

Code before:
import re

from yt_dlp.utils import ExtractorError, determine_ext, extract_attributes, get_element_by_class, parse_duration, traverse_obj, url_or_none
from .common import InfoExtractor


class NekoHackerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>(?!free-dl)[\w-]+)'
    _TESTS = [{
        'url': 'https://nekohacker.com/nekoverse/',
        'info_dict': {
            'id': 'nekoverse',
            'title': 'Nekoverse',
        },
        'playlist': [
            {
                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
                'md5': '44223701ebedba0467ebda4cc07fb3aa',
                'info_dict': {
                    'id': '1712',
                    'ext': 'mp3',
                    'title': 'Spaceship',
                    'thumbnail': 'https://nekohacker.com/wp-content/uploads/2022/11/Nekoverse_Artwork-1024x1024.jpg',
                    'vcodec': 'none',
                    'acodec': 'mp3',
                    'release_date': '20221101',
                    'album': 'Nekoverse',
                    'artist': 'Nekohacker',
                    'track': 'Spaceship',

Review comment (refactor):
relative imports
```suggestion
from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    determine_ext,
    extract_attributes,
    get_element_by_class,
    parse_duration,
    traverse_obj,
    url_or_none,
)
```

Code after:
import re

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    determine_ext,
    extract_attributes,
    get_element_by_class,
    get_element_text_and_html_by_tag,
    parse_duration,
    traverse_obj,
    try_call,
    url_or_none,
)


class NekoHackerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?nekohacker\.com/(?P<id>(?!free-dl)[\w-]+)'
    _TESTS = [{
        'url': 'https://nekohacker.com/nekoverse/',
        'info_dict': {
            'id': 'nekoverse',
            'title': 'Nekoverse',
        },
        'playlist': [
            {
                'url': 'https://nekohacker.com/wp-content/uploads/2022/11/01-Spaceship.mp3',
                'md5': '44223701ebedba0467ebda4cc07fb3aa',
                'info_dict': {
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/nekohacker.py (Python)

Code before:
            # handle iframes instead of playlist
            if iframe is not None:
                attr = extract_attributes(iframe[1])
                iframesrc = attr['src']
                # handle spotify explicitly
                if iframesrc is not None and re.match(r'https?://(?:www\.|open\.)?spotify\.com', iframesrc):
                    raise ExtractorError('no playlist found, but iframe with spotify', expected=True)
                raise ExtractorError('no playlist element found, but a unhandled iframe')

            raise ExtractorError('no playlist element found - likely not a album', expected=True)

        entries = []
        for track_number, track in enumerate(re.findall(r'(<li[^>]+data-audiopath[^>]+>)', playlist), 1):
            entry = traverse_obj(extract_attributes(track), {
                'url': ('data-audiopath', {url_or_none}),
                'ext': ('data-audiopath', {determine_ext}),
                'id': 'data-trackid',
                'title': 'data-tracktitle',
                'track': 'data-tracktitle',
                'album': 'data-albumtitle',
                'duration': ('data-tracktime', {parse_duration}),
                'release_date': ('data-releasedate', {lambda x: re.match(r'\d{8}', x.replace('.', ''))}, 0),
                'thumbnail': ('data-albumart', {url_or_none}),
            })
            entry['track_number'] = track_number
            entry['artist'] = 'Nekohacker'
            entry['vcodec'] = 'none'
            entry['acodec'] = 'mp3' if entry['ext'] == 'mp3' else None
            entries.append(entry)

        return self.playlist_result(entries, playlist_id, traverse_obj(entries, (0, 'album')))


Review comment (suggestion):
is the artist name actually "Neko Hacker"? just fixing my own previous suggestion
```suggestion
            entry['artist'] = 'Neko Hacker'
```

Code after:
                raise ExtractorError('Spotify embeds are not supported', expected=True)
            return self.url_result(url, 'Generic')

        entries = []
        for track_number, track in enumerate(re.findall(r'(<li[^>]+data-audiopath[^>]+>)', playlist), 1):
            entry = traverse_obj(extract_attributes(track), {
                'url': ('data-audiopath', {url_or_none}),
                'ext': ('data-audiopath', {determine_ext}),
                'id': 'data-trackid',
                'title': 'data-tracktitle',
                'track': 'data-tracktitle',
                'album': 'data-albumtitle',
                'duration': ('data-tracktime', {parse_duration}),
                'release_date': ('data-releasedate', {lambda x: re.match(r'\d{8}', x.replace('.', ''))}, 0),
                'thumbnail': ('data-albumart', {url_or_none}),
            })
            entries.append({
                **entry,
                'track_number': track_number,
                'artist': 'Neko Hacker',
                'vcodec': 'none',
                'acodec': 'mp3' if entry['ext'] == 'mp3' else None,
            })

        return self.playlist_result(entries, playlist_id, traverse_obj(entries, (0, 'album')))
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/camfm.py (Python)

Code before:
from re import finditer
from urllib.parse import urljoin

from .common import InfoExtractor
from ..utils import join_nonempty, unified_timestamp


class CamFMBaseIE(InfoExtractor):
    def _absolute_url(self, path, root='https://camfm.co.uk'):
        return urljoin(root, path)


class CamFMShowIE(CamFMBaseIE):
    _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/shows/(?P<id>[^/]+)/?'
    _TESTS = [{
        'playlist_mincount': 0,
        'url': 'https://camfm.co.uk/shows/soul-mining/',
        'info_dict': {
            'id': 'soul-mining',
            'thumbnail': 'md5:6a873091f92c936f23bdcce80f75e66a',
            'title': 'Soul Mining',
            'description': 'Telling the stories of jazz, funk and soul from all corners of the world.',
        },
    }]

    def _real_extract(self, url):
        show_id = self._match_id(url)

Review comment (suggestion):
```suggestion
import re
import urllib.parse
```

We use from imports only for internal modules for clarity

Code after:
import re

from .common import InfoExtractor
from ..utils import (
    clean_html,
    get_element_by_class,
    get_elements_by_class,
    join_nonempty,
    traverse_obj,
    unified_timestamp,
    urljoin,
)


class CamFMShowIE(InfoExtractor):
    _VALID_URL = r'https://(?:www\.)?camfm\.co\.uk/shows/(?P<id>[^/]+)'
    _TESTS = [{
        'playlist_mincount': 5,
        'url': 'https://camfm.co.uk/shows/soul-mining/',
        'info_dict': {
            'id': 'soul-mining',
            'thumbnail': 'md5:6a873091f92c936f23bdcce80f75e66a',
            'title': 'Soul Mining',
            'description': 'Telling the stories of jazz, funk and soul from all corners of the world.',
        },
    }]
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/camfm.py (Python)

Code before:
            'categories': ['Entertainment'],
        }
    }]

    def _real_extract(self, url):
        episode_id = self._match_id(url)
        page = self._download_webpage(url, episode_id)
        audios = self._parse_html5_media_entries('https://audio.camfm.co.uk', page, episode_id)
        formats = []
        for i in audios:
            formats.extend(i['formats'])

        series = self._html_search_regex(r'<div[^>]+class="caption">\n\s*(.+)', page, 'title', fatal=False)
        date = self._html_search_regex('>Aired at ([^<]+)<', page, 'air date', fatal=False)
        title = join_nonempty(series, date, delim=' - ')

        return {
            'id': episode_id,
            'title': title,
            'formats': formats,
            'timestamp': unified_timestamp(date),  # this will be off by an hour when DST is in effect
            # https://www.gov.uk/when-do-the-clocks-change - too annoying to calculate, will just treat as UTC
            # it'll be right half the time at least
            'series': series,
            'description': self._html_search_regex(r'>Aired at [^<]+<.*\n\s*(.*)', page, 'description', fatal=False),
            'thumbnail': self._absolute_url(self._search_regex(r'<div[^>]+class="cover-art"[^>]+style="[^"]+url\(\'([^\']+)\'\)[^>]+>',
                                            page, 'thumbnail', fatal=False)),
            'categories': [self._html_search_regex(r'<span[^>]+class="label"[^>]+>([^<]+)<', page, 'category', fatal=False)],
            'was_live': True,
        }


Review comment (suggestion):
```suggestion
            'thumbnail': self._absolute_url(self._search_regex(r'<div[^>]+class="cover-art"[^>]+style="[^"]+url\(\'([^\']+)',
```
trailing part does nothing

Code after:
            'categories': ['Entertainment'],
        }
    }]

    def _real_extract(self, url):
        episode_id = self._match_id(url)
        page = self._download_webpage(url, episode_id)
        audios = self._parse_html5_media_entries('https://audio.camfm.co.uk', page, episode_id)

        caption = get_element_by_class('caption', page)
        series = clean_html(re.sub(r'<span[^<]+<[^<]+>', '', caption))

        card_section = get_element_by_class('card-section', page)
        date = self._html_search_regex('>Aired at ([^<]+)<', card_section, 'air date', fatal=False)

        return {
            'id': episode_id,
            'title': join_nonempty(series, date, delim=' - '),
            'formats': traverse_obj(audios, (..., 'formats', ...)),
            'timestamp': unified_timestamp(date),  # XXX: Does not account for UK's daylight savings
            'series': series,
            'description': clean_html(re.sub(r'<b>[^<]+</b><br[^>]+/>', '', card_section)),
            'thumbnail': urljoin('https://camfm.co.uk', self._search_regex(
                r'<div[^>]+class="cover-art"[^>]+style="[^"]+url\(\'([^\']+)',
                page, 'thumbnail', fatal=False)),
            'categories': get_elements_by_class('label', caption),
            'was_live': True,
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/rule34video.py (Python)

Code before:
        webpage = self._download_webpage(url, video_id)

        formats = []

        for mobj in re.finditer(r'<a[^>]+href="(?P<video_url>[^"]+download=true[^"]+)".*>(?P<ext>[^\s]+) (?P<quality>[^<]+)p</a>', webpage):
            url, ext, quality = mobj.groups()
            formats.append({
                'url': url,
                'ext': ext.lower(),
                'quality': quality,
            })

        title = self._html_extract_title(webpage)
        thumbnail = self._html_search_regex(r'preview_url:\s+\'([^\']+)\'', webpage, 'thumbnail', default=None)
        duration = self._html_search_regex(r'"icon-clock"></i>\s+<span>((?:\d+:?)+)', webpage, 'duration', default=None)
        tags = re.findall(r'<a class="tag_item"\s+href="https://rule34video\.com/tags/\d+/"[^>]*>(?P<tag>[^>]*)</a>', webpage)
        tags = [unescapeHTML(t) for t in tags]

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'thumbnail': thumbnail,
            'duration': parse_duration(duration),
            'age_limit': 18,
            'tags': tags
        }


Review comment (security):
```suggestion
            'tags': list(map(unescapeHTML, re.findall(
                r'<a class="tag_item"[^>]+\bhref="https://rule34video\.com/tags/\d+/"[^>]*>(?P<tag>[^>]*)</a>', webpage))),
```

Code after:
        webpage = self._download_webpage(url, video_id)

        formats = []

        for mobj in re.finditer(r'<a[^>]+href="(?P<video_url>[^"]+download=true[^"]+)".*>(?P<ext>[^\s]+) (?P<quality>[^<]+)p</a>', webpage):
            url, ext, quality = mobj.groups()
            formats.append({
                'url': url,
                'ext': ext.lower(),
                'quality': quality,
            })

        title = self._html_extract_title(webpage)
        thumbnail = self._html_search_regex(r'preview_url:\s+\'([^\']+)\'', webpage, 'thumbnail', default=None)
        duration = self._html_search_regex(r'"icon-clock"></i>\s+<span>((?:\d+:?)+)', webpage, 'duration', default=None)

        return {
            'id': video_id,
            'formats': formats,
            'title': title,
            'thumbnail': thumbnail,
            'duration': parse_duration(duration),
            'age_limit': 18,
            'tags': list(map(unescapeHTML, re.findall(
                r'<a class="tag_item"[^>]+\bhref="https://rule34video\.com/tags/\d+/"[^>]*>(?P<tag>[^>]*)</a>', webpage))),
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/weyyak.py (Python)

Code before:
import re

from .common import InfoExtractor
from ..utils import traverse_obj


class WeyyakIE(InfoExtractor):
    _VALID_URL = r'https?://weyyak\.com/(?P<lang>\w+)/player/(?P<type>episode|movie)/(?P<id>\d+)(?:/.+)?'
    _TESTS = [
        {
            'url': 'https://weyyak.com/en/player/episode/1341952/Ribat-Al-Hob-Episode49',
            'md5': '0caf55c1a615531c8fe60f146ae46849',
            'info_dict': {
                'id': '1341952',
                'ext': 'm3u8',
                'title': 'Ribat Al Hob',
                'duration': 2771,
            },
        }
    ]

    def _real_extract(self, url):
        match = self._match_valid_url(url)

        # Extract the video id, language and type from the url
        _id = match.group('id')
        _lang = match.group('lang')
        _type = match.group('type')

        base_url = f'https://msapifo-prod-me.weyyak.z5.com/v1/{_lang}/{"episode/" if _type=="episode" else "contents/moviedetails?contentkey="}{_id}'

        # Get the video info
        video_info = self._download_json(base_url, _id, headers={'Content-Type': 'application/json'})

Review comment (suggestion):
```suggestion
    _VALID_URL = r'https?://weyyak\.com/(?P<lang>\w+)/player/(?P<type>episode|movie)/(?P<id>\d+)'
```
It does nothing

Code after:
from .common import InfoExtractor
from ..utils import (
    float_or_none,
    int_or_none,
    parse_age_limit,
    traverse_obj,
    unified_timestamp,
    url_or_none,
)


class WeyyakIE(InfoExtractor):
    _VALID_URL = r'https?://weyyak\.com/(?P<lang>\w+)/(?:player/)?(?P<type>episode|movie)/(?P<id>\d+)'
    _TESTS = [
        {
            'url': 'https://weyyak.com/en/player/episode/1341952/Ribat-Al-Hob-Episode49',
            'md5': '0caf55c1a615531c8fe60f146ae46849',
            'info_dict': {
                'id': '1341952',
                'ext': 'mp4',
                'title': 'Ribat Al Hob',
                'duration': 2771,
                'alt_title': 'رباط الحب',
                'season': 'Season 1',
                'season_number': 1,
                'episode': 'Episode 49',
                'episode_number': 49,
                'timestamp': 1485907200,
                'upload_date': '20170201',
                'thumbnail': r're:^https://content\.weyyak\.com/.+/poster-image',
                'categories': ['Drama', 'Thrillers', 'Romance'],
                'tags': 'count:8',
            },