ronantakizawa/codereview-bench huggingface dataset:
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/substack.py (Python)

Code before:
import re
import urllib.parse

from .common import InfoExtractor
from ..utils import (
    str_or_none,
    traverse_obj,
)


class SubstackIE(InfoExtractor):
    _VALID_URL = r'https?://(?P<username>[\w-]+)\.substack\.com/p/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://haleynahman.substack.com/p/i-made-a-vlog?s=r',
        'md5': 'f27e4fc6252001d48d479f45e65cdfd5',
        'info_dict': {
            'id': '47660949',
            'ext': 'mp4',
            'title': 'I MADE A VLOG',
            'description': 'md5:10c01ff93439a62e70ce963b2aa0b7f6',
            'thumbnail': 'md5:bec758a34d8ee9142d43bcebdf33af18',
            'uploader': 'Maybe Baby',
            'uploader_id': '7802200',
        }
    }, {
        'url': 'https://haleynahman.substack.com/p/-dear-danny-i-found-my-boyfriends?s=r',
        'md5': '0a63eacec877a1171a62cfa69710fcea',
        'info_dict': {
            'id': '51045592',
            'ext': 'mpga',
            'title': "🎧 Dear Danny: I found my boyfriend's secret Twitter account",
            'description': 'md5:a57f2439319e56e0af92dd0c95d75797',
            'thumbnail': 'md5:daa40b6b79249417c14ff8103db29639',

Review comment (suggestion):
```suggestion
from ..utils import str_or_none, traverse_obj
```

Code after:
import re
import urllib.parse

from .common import InfoExtractor
from ..utils import str_or_none, traverse_obj


class SubstackIE(InfoExtractor):
    _VALID_URL = r'https?://(?P<username>[\w-]+)\.substack\.com/p/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://haleynahman.substack.com/p/i-made-a-vlog?s=r',
        'md5': 'f27e4fc6252001d48d479f45e65cdfd5',
        'info_dict': {
            'id': '47660949',
            'ext': 'mp4',
            'title': 'I MADE A VLOG',
            'description': 'md5:10c01ff93439a62e70ce963b2aa0b7f6',
            'thumbnail': 'md5:bec758a34d8ee9142d43bcebdf33af18',
            'uploader': 'Maybe Baby',
            'uploader_id': '33628',
        }
    }, {
        'url': 'https://haleynahman.substack.com/p/-dear-danny-i-found-my-boyfriends?s=r',
        'md5': '0a63eacec877a1171a62cfa69710fcea',
        'info_dict': {
            'id': '51045592',
            'ext': 'mpga',
            'title': "🎧 Dear Danny: I found my boyfriend's secret Twitter account",
            'description': 'md5:a57f2439319e56e0af92dd0c95d75797',
            'thumbnail': 'md5:daa40b6b79249417c14ff8103db29639',
            'uploader': 'Maybe Baby',
            'uploader_id': '33628',
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/kicker.py (Python)

Code before:
    }, {
        'url': 'https://www.kicker.de/ex-unioner-in-der-bezirksliga-felix-kroos-vereinschallenge-in-pankow-902825/video',
        'info_dict': {
            'id': 'k2omNsJKdZ3TxwxYSFJ',
            'title': 'md5:72ec24d7f84b8436fe1e89d198152adf',
            'ext': 'mp4',
            'uploader_id': 'x2dfupo',
            'duration': 331,
            'timestamp': 1652966015,
            'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/TxU4Z1YYCmtisTbMq/x1080',
            'tags': ['FELIX KROOS', 'EINFACH MAL LUPPEN', 'KROOS', 'FSV FORTUNA PANKOW', 'published', 'category.Amateurs', 'marketingpreset.Spreekick'],
            'age_limit': 0,
            'view_count': int,
            'upload_date': '20220519',
            'uploader': 'kicker.de',
            'description': 'md5:0c2060c899a91c8bf40f578f78c5846f',
            'like_count': int,
        }
    }]

    def _real_extract(self, url):
        video_slug = self._match_id(url)

        webpage = self._download_webpage(url, video_slug)
        dailymotion_video_id = self._search_regex(
            r'data-dmprivateid=\"(?P<video_id>\w+)\"', webpage,
            'video_id', group='video_id')

        return self.url_result(
            f'https://www.dailymotion.com/video/{dailymotion_video_id}',
            ie=DailymotionIE, video_title=self._html_extract_title(webpage))


Review comment (suggestion):
```suggestion
            r'data-dmprivateid\s*=\s*[\'"](?P<video_id>\w+)', webpage,
```

Code after:
    }, {
        'url': 'https://www.kicker.de/ex-unioner-in-der-bezirksliga-felix-kroos-vereinschallenge-in-pankow-902825/video',
        'info_dict': {
            'id': 'k2omNsJKdZ3TxwxYSFJ',
            'title': 'md5:72ec24d7f84b8436fe1e89d198152adf',
            'ext': 'mp4',
            'uploader_id': 'x2dfupo',
            'duration': 331,
            'timestamp': 1652966015,
            'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/TxU4Z1YYCmtisTbMq/x1080',
            'tags': ['FELIX KROOS', 'EINFACH MAL LUPPEN', 'KROOS', 'FSV FORTUNA PANKOW', 'published', 'category.Amateurs', 'marketingpreset.Spreekick'],
            'age_limit': 0,
            'view_count': int,
            'upload_date': '20220519',
            'uploader': 'kicker.de',
            'description': 'md5:0c2060c899a91c8bf40f578f78c5846f',
            'like_count': int,
        }
    }]

    def _real_extract(self, url):
        video_slug = self._match_id(url)

        webpage = self._download_webpage(url, video_slug)
        dailymotion_video_id = self._search_regex(
            r'data-dmprivateid\s*=\s*[\'"](?P<video_id>\w+)', webpage,
            'video id', group='video_id')

        return self.url_result(
            f'https://www.dailymotion.com/video/{dailymotion_video_id}',
            ie=DailymotionIE, video_title=self._html_extract_title(webpage))
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/kicker.py (Python)

Code before:
        'url': 'https://www.kicker.de/ex-unioner-in-der-bezirksliga-felix-kroos-vereinschallenge-in-pankow-902825/video',
        'info_dict': {
            'id': 'k2omNsJKdZ3TxwxYSFJ',
            'title': 'md5:72ec24d7f84b8436fe1e89d198152adf',
            'ext': 'mp4',
            'uploader_id': 'x2dfupo',
            'duration': 331,
            'timestamp': 1652966015,
            'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/TxU4Z1YYCmtisTbMq/x1080',
            'tags': ['FELIX KROOS', 'EINFACH MAL LUPPEN', 'KROOS', 'FSV FORTUNA PANKOW', 'published', 'category.Amateurs', 'marketingpreset.Spreekick'],
            'age_limit': 0,
            'view_count': int,
            'upload_date': '20220519',
            'uploader': 'kicker.de',
            'description': 'md5:0c2060c899a91c8bf40f578f78c5846f',
            'like_count': int,
        }
    }]

    def _real_extract(self, url):
        video_slug = self._match_id(url)

        webpage = self._download_webpage(url, video_slug)
        dailymotion_video_id = self._search_regex(
            r'data-dmprivateid=\"(?P<video_id>\w+)\"', webpage,
            'video_id', group='video_id')

        return self.url_result(
            f'https://www.dailymotion.com/video/{dailymotion_video_id}',
            ie=DailymotionIE, video_title=self._html_extract_title(webpage))


Review comment (suggestion):
```suggestion
            'video id', group='video_id')
```

Code after:
        'url': 'https://www.kicker.de/ex-unioner-in-der-bezirksliga-felix-kroos-vereinschallenge-in-pankow-902825/video',
        'info_dict': {
            'id': 'k2omNsJKdZ3TxwxYSFJ',
            'title': 'md5:72ec24d7f84b8436fe1e89d198152adf',
            'ext': 'mp4',
            'uploader_id': 'x2dfupo',
            'duration': 331,
            'timestamp': 1652966015,
            'thumbnail': r're:https?://s\d+\.dmcdn\.net/v/TxU4Z1YYCmtisTbMq/x1080',
            'tags': ['FELIX KROOS', 'EINFACH MAL LUPPEN', 'KROOS', 'FSV FORTUNA PANKOW', 'published', 'category.Amateurs', 'marketingpreset.Spreekick'],
            'age_limit': 0,
            'view_count': int,
            'upload_date': '20220519',
            'uploader': 'kicker.de',
            'description': 'md5:0c2060c899a91c8bf40f578f78c5846f',
            'like_count': int,
        }
    }]

    def _real_extract(self, url):
        video_slug = self._match_id(url)

        webpage = self._download_webpage(url, video_slug)
        dailymotion_video_id = self._search_regex(
            r'data-dmprivateid\s*=\s*[\'"](?P<video_id>\w+)', webpage,
            'video id', group='video_id')

        return self.url_result(
            f'https://www.dailymotion.com/video/{dailymotion_video_id}',
            ie=DailymotionIE, video_title=self._html_extract_title(webpage))
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/instagram.py (Python)

Code before:
import itertools
import hashlib
import json
import re
import time

from .common import InfoExtractor
from ..compat import (
    compat_HTTPError,
)
from ..utils import (
    ExtractorError,
    format_field,
    float_or_none,
    get_element_by_attribute,
    int_or_none,
    lowercase_escape,
    str_or_none,
    str_to_int,
    traverse_obj,
    url_or_none,
    urlencode_postdata,
)


_ENCODING_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"


class InstagramBaseIE(InfoExtractor):
    _NETRC_MACHINE = 'instagram'
    _IS_LOGGED_IN = False

    def _perform_login(self, username, password):
        if self._IS_LOGGED_IN:
            return

        login_webpage = self._download_webpage(
            'https://www.instagram.com/accounts/login/', None,
            note='Downloading login webpage', errnote='Failed to download login webpage')

        shared_data = self._parse_json(
            self._search_regex(
                r'window\._sharedData\s*=\s*({.+?});',
                login_webpage, 'shared data', default='{}'),
            None)

        login = self._download_json('https://www.instagram.com/accounts/login/ajax/', None, note='Logging in', headers={
            'Accept': '*/*',
            'X-IG-App-ID': '936619743392459',
            'X-ASBD-ID': '198387',
            'X-IG-WWW-Claim': '0',

Review comment (suggestion):
```suggestion
_ENCODING_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
```

Code after:
import hashlib
import itertools
import json
import re
import time
import urllib.error

from .common import InfoExtractor
from ..utils import (
    ExtractorError,
    decode_base_n,
    encode_base_n,
    float_or_none,
    format_field,
    get_element_by_attribute,
    int_or_none,
    lowercase_escape,
    str_or_none,
    str_to_int,
    traverse_obj,
    url_or_none,
    urlencode_postdata,
)

_ENCODING_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'


def _pk_to_id(id):
    """Source: https://stackoverflow.com/questions/24437823/getting-instagram-post-url-from-media-id"""
    return encode_base_n(int(id.split('_')[0]), table=_ENCODING_CHARS)


def _id_to_pk(shortcode):
    """Covert a shortcode to a numeric value"""
    return decode_base_n(shortcode[:11], table=_ENCODING_CHARS)


class InstagramBaseIE(InfoExtractor):
    _NETRC_MACHINE = 'instagram'
    _IS_LOGGED_IN = False

    def _perform_login(self, username, password):
        if self._IS_LOGGED_IN:
            return

        login_webpage = self._download_webpage(
            'https://www.instagram.com/accounts/login/', None,
            note='Downloading login webpage', errnote='Failed to download login webpage')

        shared_data = self._parse_json(
            self._search_regex(
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/premiershiprugby.py (Python)

Code before:
            'id': '0_mbkb7ldt',
            'title': 'Full Match: Harlequins v Newcastle Falcons',
            'ext': 'mp4',
            'thumbnail': 'https://open.http.mp.streamamg.com/p/3000914/sp/300091400/thumbnail/entry_id/0_mbkb7ldt//width/960/height/540/type/1/quality/75',
            'duration': 6093.0,
            'tags': ['video'],
        }
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        json_data = self._download_json(
            f'https://article-cms-api.incrowdsports.com/v2/articles/slug/{display_id}?clientId=PRL',
            display_id)['data']['article']

        formats, subs = self._extract_m3u8_formats_and_subtitles(
            json_data['heroMedia']['content']['videoLink'], display_id
        )

        return {
            'id': json_data['heroMedia']['content']['sourceSystemId'],
            'title': traverse_obj(json_data, ('heroMedia', 'title')),
            'formats': formats,
            'subtitles': subs,
            'thumbnail': traverse_obj(json_data, ('heroMedia', 'content', 'videoThumbnail')),
            'duration': int_or_none(traverse_obj(json_data, ('heroMedia', 'content', 'metadata', 'msDuration'))) / 1000,
            'tags': json_data.get('tags'),
        }


Review comment (refactor):
Currently, i don't know how to make this line safe (in `duration`) if the `traverse_obj` return `None`. Is there any helper function that can make this safe ?

Code after:
            'id': '0_mbkb7ldt',
            'title': 'Full Match: Harlequins v Newcastle Falcons',
            'ext': 'mp4',
            'thumbnail': 'https://open.http.mp.streamamg.com/p/3000914/sp/300091400/thumbnail/entry_id/0_mbkb7ldt//width/960/height/540/type/1/quality/75',
            'duration': 6093.0,
            'tags': ['video'],
            'categories': ['Full Match', 'Harlequins', 'Newcastle Falcons', 'gallaher premiership'],
        }
    }]

    def _real_extract(self, url):
        display_id = self._match_id(url)
        json_data = self._download_json(
            f'https://article-cms-api.incrowdsports.com/v2/articles/slug/{display_id}',
            display_id, query={'clientId': 'PRL'})['data']['article']

        formats, subs = self._extract_m3u8_formats_and_subtitles(
            json_data['heroMedia']['content']['videoLink'], display_id)

        return {
            'id': json_data['heroMedia']['content']['sourceSystemId'],
            'display_id': display_id,
            'title': traverse_obj(json_data, ('heroMedia', 'title')),
            'formats': formats,
            'subtitles': subs,
            'thumbnail': traverse_obj(json_data, ('heroMedia', 'content', 'videoThumbnail')),
            'duration': int_or_none(traverse_obj(json_data, ('heroMedia', 'content', 'metadata', 'msDuration')), scale=1000),
            'tags': json_data.get('tags'),
            'categories': traverse_obj(json_data, ('categories', ..., 'text')),
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/videocampus_sachsen.py (Python)

Code before:
import re

from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import (
    ExtractorError,
    urlencode_postdata,
)


class VideocampusSachsenIE(InfoExtractor):
    IE_NAME = 'ViMP'
    _INSTANCES = (
        'bergauf.tv',
        'campus.demo.vimp.com',
        'corporate.demo.vimp.com',
        'dancehalldatabase.com',
        'drehzahl.tv',
        'educhannel.hs-gesundheit.de',
        'emedia.ls.haw-hamburg.de',
        'globale-evolution.net',
        'hohu.tv',
        'htvideos.hightechhigh.org',
        'k210039.vimp.mivitec.net',
        'media.cmslegal.com',
        'media.hs-furtwangen.de',
        'media.hwr-berlin.de',
        'mediathek.dkfz.de',
        'mediathek.htw-berlin.de',
        'mediathek.polizei-bw.de',
        'medien.hs-merseburg.de',
        'mportal.europa-uni.de',
        'pacific.demo.vimp.com',

Review comment (refactor):
```suggestion
from ..utils import ExtractorError, urlencode_postdata
```

Code after:
import functools
import re

from .common import InfoExtractor
from ..compat import compat_HTTPError
from ..utils import ExtractorError, OnDemandPagedList, urlencode_postdata


class VideocampusSachsenIE(InfoExtractor):
    IE_NAME = 'ViMP'
    _INSTANCES = (
        'bergauf.tv',
        'campus.demo.vimp.com',
        'corporate.demo.vimp.com',
        'dancehalldatabase.com',
        'drehzahl.tv',
        'educhannel.hs-gesundheit.de',
        'emedia.ls.haw-hamburg.de',
        'globale-evolution.net',
        'hohu.tv',
        'htvideos.hightechhigh.org',
        'k210039.vimp.mivitec.net',
        'media.cmslegal.com',
        'media.hs-furtwangen.de',
        'media.hwr-berlin.de',
        'mediathek.dkfz.de',
        'mediathek.htw-berlin.de',
        'mediathek.polizei-bw.de',
        'medien.hs-merseburg.de',
        'mportal.europa-uni.de',
        'pacific.demo.vimp.com',
        'slctv.com',
        'streaming.prairiesouth.ca',
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/videocampus_sachsen.py (Python)

Code before:
            mode = 'album'
            mode_id = '4'
            content_id = album_id
            url_part = f'aid/{album_id}'
        else:
            url_parts = ('category', 'category_id') if c_mode == 'category' else ('title', 'channel')

            mode = c_mode
            mode_id = '1' if c_mode == 'category' else '3'
            content_id = c_id
            url_part = f'{url_parts[0]}/{c_name}/{url_parts[1]}/{c_id}'

        data = {
            'vars[mode]': mode,
            f'vars[{mode}]': content_id,
            'vars[context]': mode_id,
            'vars[context_id]': content_id,
            'vars[layout]': 'thumb',
            'vars[per_page][thumb]': '10',
        }

        urls, i = [], 1
        while True:
            current_page = self._download_webpage(
                f'{host}/media/ajax/component/boxList/{url_part}?page={i}&page_only=1',
                content_id, data=urlencode_postdata(data))
            current_urls = re.findall(r'"(.+?/video/.+?)"', current_page)

            # Request returns last n video URLs if existing pages have been "used up"
            if urls[-1:] != current_urls[-1:]:
                urls += current_urls
                i += 1
            else:
                break

        return self.playlist_result(
            (self.url_result(host + url, VideocampusSachsenIE) for url in urls),
            playlist_title=title, id=f'{mode}-{content_id}')


Review comment (suggestion):
```suggestion
                f'{host}/media/ajax/component/boxList/{url_part}', content_id,
                query={'page': i, 'page_only': 1}, data=urlencode_postdata(data))
```

Code after:
    def _real_extract(self, url):
        host, album_id, mode, name, id = self._match_valid_url(url).group(
            'host', 'album_id', 'mode', 'name', 'id')

        webpage = self._download_webpage(url, album_id or id, fatal=False) or ''
        title = (self._html_search_meta('title', webpage, fatal=False)
                 or self._html_extract_title(webpage))

        url_part = (f'aid/{album_id}' if album_id
                    else f'category/{name}/category_id/{id}' if mode == 'category'
                    else f'title/{name}/channel/{id}')

        mode = mode or 'album'
        data = {
            'vars[mode]': mode,
            f'vars[{mode}]': album_id or id,
            'vars[context]': '4' if album_id else '1' if mode == 'category' else '3',
            'vars[context_id]': album_id or id,
            'vars[layout]': 'thumb',
            'vars[per_page][thumb]': str(self._PAGE_SIZE),
        }

        return self.playlist_result(
            OnDemandPagedList(functools.partial(
                self._fetch_page, host, url_part, album_id or id, data), self._PAGE_SIZE),
            playlist_title=title, id=f'{mode}-{album_id or id}')
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/fuyintv.py (Python)

Code before:
from .common import InfoExtractor
from ..utils import traverse_obj


class FuyinTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?fuyin\.tv/html/(?:\d+)/(?P<id>\d+)\.html'
    _TESTS = [{
        'url': 'https://www.fuyin.tv/html/2733/44129.html',
        'info_dict': {
            'id': '44129',
            'ext': 'mp4',
            'title': '第1集',
            'description': 'md5:21a3d238dc8d49608e1308e85044b9c3',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        json_data = self._download_json(
            'https://www.fuyin.tv/api/api/tv.movie/url',
            video_id, query={'urlid': f'{video_id}'})
        webpage = self._download_webpage(url, video_id)

        return {
            'id': video_id,
            'title': traverse_obj(json_data, ('data', 'title')),
            'url': json_data['data']['url'],
            'ext': 'mp4',
            'description': self._html_search_meta('description', webpage),
        }


Review comment (suggestion):
```suggestion
        webpage = self._download_webpage(url, video_id, fatal=False)
```

Code after:
from .common import InfoExtractor
from ..utils import traverse_obj


class FuyinTVIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?fuyin\.tv/html/(?:\d+)/(?P<id>\d+)\.html'
    _TESTS = [{
        'url': 'https://www.fuyin.tv/html/2733/44129.html',
        'info_dict': {
            'id': '44129',
            'ext': 'mp4',
            'title': '第1集',
            'description': 'md5:21a3d238dc8d49608e1308e85044b9c3',
        }
    }]

    def _real_extract(self, url):
        video_id = self._match_id(url)
        json_data = self._download_json(
            'https://www.fuyin.tv/api/api/tv.movie/url',
            video_id, query={'urlid': f'{video_id}'})
        webpage = self._download_webpage(url, video_id, fatal=False)

        return {
            'id': video_id,
            'title': traverse_obj(json_data, ('data', 'title')),
            'url': json_data['data']['url'],
            'ext': 'mp4',
            'description': self._html_search_meta('description', webpage),
        }