ronantakizawa/codereview-bench huggingface dataset:
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/ntvde.py (Python)

Code before:
import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    js_to_json,
    traverse_obj,
)


class NTVDeIE(InfoExtractor):
    IE_NAME = 'n-tv.de'
    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'

    _TESTS = [{
        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
        'md5': '6bcf2a6638cb83f45d5561659a1cb498',
        'info_dict': {
            'id': '14438086',
            'ext': 'mp4',
            'thumbnail': r're:^https?://.*\.jpg$',
            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
            'alt_title': 'Winterchaos auf deutschen Straßen',
            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
            'duration': 67,
            'timestamp': 1422892797,
            'upload_date': '20150202',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

Review comment (suggestion):
```suggestion
    url_or_none,
)
from ..utils.traversal import traverse_obj
```

Code after:
import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    js_to_json,
    url_or_none,
)
from ..utils.traversal import traverse_obj


class NTVDeIE(InfoExtractor):
    IE_NAME = 'n-tv.de'
    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/(?:videos|magazine)/[^/?#]+/[^/?#]+-article(?P<id>[^/?#]+)\.html'

    _TESTS = [{
        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
        'md5': '6bcf2a6638cb83f45d5561659a1cb498',
        'info_dict': {
            'id': '14438086',
            'ext': 'mp4',
            'thumbnail': r're:^https?://.*\.jpg$',
            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
            'alt_title': 'Winterchaos auf deutschen Straßen',
            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
            'duration': 67,
            'timestamp': 1422892797,
            'upload_date': '20150202',
        },
    }, {
        'url': 'https://www.n-tv.de/mediathek/magazine/auslandsreport/Juedische-Siedler-wollten-Rache-die-wollten-nur-toeten-article24523089.html',
        'md5': 'c5c6014c014ccc3359470e1d34472bfd',
        'info_dict': {
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/ntvde.py (Python)

Code before:
            transform_source=lambda s: js_to_json(re.sub(r'ivw:[^},]+', '', s)))
        vdata = traverse_obj(player_data, ('setup', 'source'))

        formats = []
        if vdata.get('progressive'):
            formats.append({
                'format_id': 'mp4-0',
                'url': vdata['progressive'],
            })
        if vdata.get('hls'):
            formats.extend(self._extract_m3u8_formats(
                vdata['hls'], video_id, ext='mp4', entry_protocol='m3u8_native',
                quality=1, m3u8_id='hls', fatal=False))
        if vdata.get('dash'):
            formats.extend(self._extract_mpd_formats(vdata['dash'], video_id, fatal=False))

        return {
            'id': video_id,
            **traverse_obj(info, {
                'title': 'headline',
                'description': 'intro',
                'alt_title': 'kicker',
                'timestamp': ('publishedDateAsUnixTimeStamp', {int_or_none}),
            }),
            **traverse_obj(vdata, {
                'thumbnail': 'poster',
                'duration': ('length', {int_or_none}),
            }),
            'formats': formats,
        }


Review comment (suggestion):
```suggestion
                'thumbnail': ('poster', {url_or_none}),
```

Code after:
            'timestamp': 1699688294,
            'upload_date': '20231111',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

        info = self._search_json(
            r'article:', webpage, 'info', video_id, transform_source=js_to_json)

        vdata = self._search_json(
            r'\$\(\s*"#playerwrapper"\s*\)\s*\.data\(\s*"player",',
            webpage, 'player data', video_id,
            transform_source=lambda s: js_to_json(re.sub(r'ivw:[^},]+', '', s)))['setup']['source']

        formats = []
        if vdata.get('progressive'):
            formats.append({
                'format_id': 'http',
                'url': vdata['progressive'],
            })
        if vdata.get('hls'):
            formats.extend(self._extract_m3u8_formats(
                vdata['hls'], video_id, 'mp4', m3u8_id='hls', fatal=False))
        if vdata.get('dash'):
            formats.extend(self._extract_mpd_formats(vdata['dash'], video_id, fatal=False, mpd_id='dash'))

        return {
            'id': video_id,
            **traverse_obj(info, {
                'title': 'headline',
                'description': 'intro',
                'alt_title': 'kicker',
                'timestamp': ('publishedDateAsUnixTimeStamp', {int_or_none}),
            }),
            **traverse_obj(vdata, {
                'thumbnail': ('poster', {url_or_none}),
                'duration': ('length', {int_or_none}),
            }),
            'formats': formats,
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/ntvde.py (Python)

Code before:
import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    js_to_json,
    url_or_none,
)

from ..utils.traversal import traverse_obj


class NTVDeIE(InfoExtractor):
    IE_NAME = 'n-tv.de'
    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'

    _TESTS = [{
        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
        'md5': '6bcf2a6638cb83f45d5561659a1cb498',
        'info_dict': {
            'id': '14438086',
            'ext': 'mp4',
            'thumbnail': r're:^https?://.*\.jpg$',
            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
            'alt_title': 'Winterchaos auf deutschen Straßen',
            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
            'duration': 67,
            'timestamp': 1422892797,
            'upload_date': '20150202',
        },
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        webpage = self._download_webpage(url, video_id)

Review comment (suggestion):
```suggestion
from ..utils.traversal import traverse_obj
```

Code after:
import re

from .common import InfoExtractor
from ..utils import (
    int_or_none,
    js_to_json,
    url_or_none,
)
from ..utils.traversal import traverse_obj


class NTVDeIE(InfoExtractor):
    IE_NAME = 'n-tv.de'
    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/(?:videos|magazine)/[^/?#]+/[^/?#]+-article(?P<id>[^/?#]+)\.html'

    _TESTS = [{
        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
        'md5': '6bcf2a6638cb83f45d5561659a1cb498',
        'info_dict': {
            'id': '14438086',
            'ext': 'mp4',
            'thumbnail': r're:^https?://.*\.jpg$',
            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
            'alt_title': 'Winterchaos auf deutschen Straßen',
            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
            'duration': 67,
            'timestamp': 1422892797,
            'upload_date': '20150202',
        },
    }, {
        'url': 'https://www.n-tv.de/mediathek/magazine/auslandsreport/Juedische-Siedler-wollten-Rache-die-wollten-nur-toeten-article24523089.html',
        'md5': 'c5c6014c014ccc3359470e1d34472bfd',
        'info_dict': {
            'id': '24523089',
            'ext': 'mp4',
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/periscope.py (Python)

Code before:
from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
    traverse_obj,
    unescapeHTML,
)


class PeriscopeBaseIE(InfoExtractor):
    _M3U8_HEADERS = {
        'Referer': 'https://www.periscope.tv/'
    }

    def _call_api(self, method, query, item_id):
        return self._download_json(
            'https://api.periscope.tv/api/v2/%s' % method,
            item_id, query=query)

    def _parse_broadcast_data(self, broadcast, video_id):
        title = broadcast.get('status') or 'Periscope Broadcast'
        uploader = broadcast.get('user_display_name') or broadcast.get('username')
        title = '%s - %s' % (uploader, title) if uploader else title
        thumbnails = [{
            'url': broadcast[image],
        } for image in ('image_url', 'image_url_small') if broadcast.get(image)]

        return {
            'id': broadcast.get('id') or video_id,
            'title': title,
            'timestamp': parse_iso8601(broadcast.get('created_at')) or int_or_none(
                broadcast.get('created_at_ms'), scale=1000),

Review comment (security):
```suggestion
    unescapeHTML,
)
from ..utils.traversal import traverse_obj
```

Code after:
from .common import InfoExtractor
from ..utils import (
    int_or_none,
    parse_iso8601,
    unescapeHTML,
)
from ..utils.traversal import traverse_obj


class PeriscopeBaseIE(InfoExtractor):
    _M3U8_HEADERS = {
        'Referer': 'https://www.periscope.tv/'
    }

    def _call_api(self, method, query, item_id):
        return self._download_json(
            'https://api.periscope.tv/api/v2/%s' % method,
            item_id, query=query)

    def _parse_broadcast_data(self, broadcast, video_id):
        title = broadcast.get('status') or 'Periscope Broadcast'
        uploader = broadcast.get('user_display_name') or broadcast.get('username')
        title = '%s - %s' % (uploader, title) if uploader else title
        thumbnails = [{
            'url': broadcast[image],
        } for image in ('image_url', 'image_url_medium', 'image_url_small') if broadcast.get(image)]

        return {
            'id': broadcast.get('id') or video_id,
            'title': title,
            'timestamp': parse_iso8601(broadcast.get('created_at')) or int_or_none(
                broadcast.get('created_at_ms'), scale=1000),
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/radiocomercial.py (Python)

Code before:
import re
from collections import namedtuple

from .common import InfoExtractor
from ..utils import (
    int_or_none,
)


class RadioComercialBaseExtractor(InfoExtractor):
    def _extract_page_content(self, url):
        video_id = RadioComercialIE._match_id(url)
        webpage = self._download_webpage(url, video_id)
        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
        url = self._html_search_regex(r'<a.+?isExclusivePlay=.+?href="(.+?)">', webpage, 'url')
        date = self._html_search_regex(r'<div[^"]+"date">(\d{4}-\d{2}-\d{2})</div>', webpage, 'date')
        thumbnail = self._html_search_regex(r'<source[^"]+"image/jpeg[^/]+(.+?)\">', webpage, 'thumbnail')
        season = int_or_none(self._html_search_regex(r'<h2>\w+\s(\d+)</h2>', webpage, 'season'))
        episode_id = int_or_none(self._html_search_regex(r'episodeid=(\d+)&', url, 'episode_id'))

        return {
            'id': video_id,
            'title': title,
            'date': date,
            'thumbnail': thumbnail,
            'season': season,
            'episode_id': episode_id,
            'url': url
        }


class RadioComercialIE(RadioComercialBaseExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/]+/\w\d+/(?P<id>[-\w]+)/*$'
    _TESTS = [{
        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas',

Review comment (refactor):
I don't think it makes sense to use a base class like this here.
Currently, when extracting a playlist of 100 items, first all 100 pages need to be fetched. Instead, returning URL results makes more sense.

Code after:
import itertools

from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
    ExtractorError,
    extract_attributes,
    get_element_by_class,
    get_element_html_by_class,
    get_element_text_and_html_by_tag,
    get_elements_html_by_class,
    int_or_none,
    join_nonempty,
    try_call,
    unified_strdate,
    update_url,
    urljoin
)
from ..utils.traversal import traverse_obj


class RadioComercialIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/?#]+/t?(?P<season>\d+)/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas#page-content-wrapper',
        'md5': '5f4fe8e485b29d2e8fd495605bc2c7e4',
        'info_dict': {
            'id': 'taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas',
            'ext': 'mp3',
            'title': 'Taylor Swift entranhando-se que nem uma espada no ventre dos fãs.',
            'release_date': '20231025',
            'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg',
            'season': 6
        }
    }, {
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/radiocomercial.py (Python)

Code before:
import re
from collections import namedtuple

from .common import InfoExtractor
from ..utils import (
    int_or_none,
)


class RadioComercialBaseExtractor(InfoExtractor):
    def _extract_page_content(self, url):
        video_id = RadioComercialIE._match_id(url)
        webpage = self._download_webpage(url, video_id)
        title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
        url = self._html_search_regex(r'<a.+?isExclusivePlay=.+?href="(.+?)">', webpage, 'url')
        date = self._html_search_regex(r'<div[^"]+"date">(\d{4}-\d{2}-\d{2})</div>', webpage, 'date')
        thumbnail = self._html_search_regex(r'<source[^"]+"image/jpeg[^/]+(.+?)\">', webpage, 'thumbnail')
        season = int_or_none(self._html_search_regex(r'<h2>\w+\s(\d+)</h2>', webpage, 'season'))
        episode_id = int_or_none(self._html_search_regex(r'episodeid=(\d+)&', url, 'episode_id'))

        return {
            'id': video_id,
            'title': title,
            'date': date,
            'thumbnail': thumbnail,
            'season': season,
            'episode_id': episode_id,
            'url': url
        }


class RadioComercialIE(RadioComercialBaseExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/]+/\w\d+/(?P<id>[-\w]+)/*$'
    _TESTS = [{
        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas',
        'md5': '5f4fe8e485b29d2e8fd495605bc2c7e4',
        'info_dict': {

Review comment (suggestion):
Then this will need to be
```suggestion
        video_id, season = self._match_valid_url(url).group('id', 'season')
```

Code after:
import itertools

from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
    ExtractorError,
    extract_attributes,
    get_element_by_class,
    get_element_html_by_class,
    get_element_text_and_html_by_tag,
    get_elements_html_by_class,
    int_or_none,
    join_nonempty,
    try_call,
    unified_strdate,
    update_url,
    urljoin
)
from ..utils.traversal import traverse_obj


class RadioComercialIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/?#]+/t?(?P<season>\d+)/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas#page-content-wrapper',
        'md5': '5f4fe8e485b29d2e8fd495605bc2c7e4',
        'info_dict': {
            'id': 'taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas',
            'ext': 'mp3',
            'title': 'Taylor Swift entranhando-se que nem uma espada no ventre dos fãs.',
            'release_date': '20231025',
            'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg',
            'season': 6
        }
    }, {
        'url': 'https://radiocomercial.pt/podcasts/convenca-me-num-minuto/t3/convenca-me-num-minuto-que-os-lobisomens-existem',
        'md5': '47e96c273aef96a8eb160cd6cf46d782',
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/radiocomercial.py (Python)

Code before:
import functools

from .common import InfoExtractor
from ..utils import (
    OnDemandPagedList,
    extract_attributes,
    get_element_by_class,
    get_element_html_by_class,
    get_element_text_and_html_by_tag,
    get_elements_html_by_class,
    int_or_none,
    join_nonempty,
    try_call,
    unified_strdate,
    update_url,
    urljoin
)
from ..utils.traversal import traverse_obj


class RadioComercialIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/?#]+/t?(?P<season>\d+)/(?P<id>[\w-]+)(?:$|[?#|/])'
    _TESTS = [{
        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas#page-content-wrapper',
        'md5': '5f4fe8e485b29d2e8fd495605bc2c7e4',
        'info_dict': {

Review comment (suggestion):
will need this for other suggestion
```suggestion
import itertools
```

Code after:
import itertools

from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
    ExtractorError,
    extract_attributes,
    get_element_by_class,
    get_element_html_by_class,
    get_element_text_and_html_by_tag,
    get_elements_html_by_class,
    int_or_none,
    join_nonempty,
    try_call,
    unified_strdate,
    update_url,
    urljoin
)
from ..utils.traversal import traverse_obj


class RadioComercialIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/?#]+/t?(?P<season>\d+)/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas#page-content-wrapper',
        'md5': '5f4fe8e485b29d2e8fd495605bc2c7e4',
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/radiocomercial.py (Python)

Code before:
import functools

from .common import InfoExtractor
from ..utils import (
    OnDemandPagedList,
    extract_attributes,
    get_element_by_class,
    get_element_html_by_class,
    get_element_text_and_html_by_tag,
    get_elements_html_by_class,
    int_or_none,
    join_nonempty,
    try_call,
    unified_strdate,
    update_url,
    urljoin
)
from ..utils.traversal import traverse_obj


class RadioComercialIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/?#]+/t?(?P<season>\d+)/(?P<id>[\w-]+)(?:$|[?#|/])'
    _TESTS = [{
        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas#page-content-wrapper',
        'md5': '5f4fe8e485b29d2e8fd495605bc2c7e4',
        'info_dict': {
            'id': 'taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas',
            'ext': 'mp3',
            'title': 'Taylor Swift entranhando-se que nem uma espada no ventre dos fãs.',
            'release_date': '20231025',
            'thumbnail': r're:https://radiocomercial.pt/upload/[^.]+.jpg',

Review comment (refactor):
will need these below
```suggestion
from .common import InfoExtractor
from .networking.exceptions import HTTPError
from ..utils import (
    ExtractorError,
    extract_attributes,
```

Code after:
import itertools

from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
    ExtractorError,
    extract_attributes,
    get_element_by_class,
    get_element_html_by_class,
    get_element_text_and_html_by_tag,
    get_elements_html_by_class,
    int_or_none,
    join_nonempty,
    try_call,
    unified_strdate,
    update_url,
    urljoin
)
from ..utils.traversal import traverse_obj


class RadioComercialIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?radiocomercial\.pt/podcasts/[^/?#]+/t?(?P<season>\d+)/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://radiocomercial.pt/podcasts/o-homem-que-mordeu-o-cao/t6/taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas#page-content-wrapper',
        'md5': '5f4fe8e485b29d2e8fd495605bc2c7e4',
        'info_dict': {
            'id': 'taylor-swift-entranhando-se-que-nem-uma-espada-no-ventre-dos-fas',
            'ext': 'mp3',
            'title': 'Taylor Swift entranhando-se que nem uma espada no ventre dos fãs.',
            'release_date': '20231025',