ronantakizawa/codereview-bench huggingface dataset:
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/fourzerostudio.py (Python)

Code before:
        webpage = self._download_webpage(url, video_id)
        nuxt_data = self._search_nuxt_data(webpage, video_id, full_data=True)

        clip_info = traverse_obj(nuxt_data, ('ssrRefs', lambda _, v: v['__typename'] == 'PublicCreatorArchivedClip'), get_all=False)

        for m in self._parse_html5_media_entries(url, webpage, video_id):
            if 'mp4' in traverse_obj(m, ('formats', ..., 'ext')):
                info = m
                break
        else:
            self.report_warning('Failed to find a desired media element. Falling back to using NUXT data.')
            info = {
                'formats': [{
                    'ext': 'mp4',
                    'url': url,
                } for url in clip_info.get('mediaFiles') or [] if url],
            }
        info.update({
            'id': video_id,
            'title': clip_info.get('clipComment'),
            'timestamp': unified_timestamp(clip_info.get('createdAt')),
            'like_count': clip_info.get('likeCount'),
            'uploader_id': uploader_id,
            **self._extract_uploader_info(nuxt_data),
        })
        return info


Review comment (performance):
```suggestion
        return {
            **info,
            'id': video_id,
            'title': clip_info.get('clipComment'),
            'timestamp': unified_timestamp(clip_info.get('createdAt')),
            'like_count': clip_info.get('likeCount'),
            'uploader_id': uploader_id,
            **self._extract_uploader_info(nuxt_data),
        }
```

Code after:
        video_id, uploader_id = self._match_valid_url(url).group('id', 'uploader_id')
        webpage = self._download_webpage(url, video_id)
        nuxt_data = self._search_nuxt_data(webpage, video_id, return_full_data=True)

        clip_info = traverse_obj(nuxt_data, ('ssrRefs', lambda _, v: v['__typename'] == 'PublicCreatorArchivedClip'), get_all=False)

        info = next((
            m for m in self._parse_html5_media_entries(url, webpage, video_id)
            if 'mp4' in traverse_obj(m, ('formats', ..., 'ext'))
        ), None)
        if not info:
            self.report_warning('Failed to find a desired media element. Falling back to using NUXT data.')
            info = {
                'formats': [{
                    'ext': 'mp4',
                    'url': url,
                } for url in clip_info.get('mediaFiles') or [] if url],
            }
        return {
            **info,
            'id': video_id,
            'title': clip_info.get('clipComment'),
            'timestamp': unified_timestamp(clip_info.get('createdAt')),
            'like_count': clip_info.get('likeCount'),
            'uploader_id': uploader_id,
            'uploader': traverse_obj(nuxt_data, (
                'ssrRefs', lambda _, v: v['__typename'] == 'PublicUser', 'username'), get_all=False),
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/atscaleconf.py (Python)

Code before:
        'playlist_mincount': 13,
        'info_dict': {
            'id': 'data-scale-spring-2022',
            'title': 'Data @Scale Spring 2022',
            'description': 'md5:7d7ca1c42ac9c6d8a785092a1aea4b55'
        },
    }, {
        'url': 'https://atscaleconference.com/events/video-scale-2021/',
        'playlist_mincount': 14,
        'info_dict': {
            'id': 'video-scale-2021',
            'title': 'Video @Scale 2021',
            'description': 'md5:7d7ca1c42ac9c6d8a785092a1aea4b55'
        },
    }]

    _VIDEO_PATTERN = re.compile(r'data-url\s?=\s?"(https?://(?:www\.)?atscaleconference\.com/videos/[^/&$?]+[^"]+)"')

    def _real_extract(self, url):
        id = self._match_id(url)
        webpage = self._download_webpage(url, id)

        return self.playlist_result([self.url_result(video) for video in self._VIDEO_PATTERN.findall(webpage)],
                                    playlist_id=id,
                                    playlist_title=self._og_search_title(webpage),
                                    playlist_description=self._og_search_description(webpage))


Review comment (refactor):
```suggestion
        return self.playlist_from_matches(
            re.findall(r'data-url\s*=\s*"(https?://(?:www\.)?atscaleconference\.com/videos/[^"]+)"', webpage),
            ie='Generic', playlist_id=id,
            title=self._og_search_title(webpage), description=self._og_search_description(webpage))
```
            
* Use `playlist_from_matches`
* I don't see the need to pre-compile the regex
* `ie=Generic` to avoid having to test it against each extractor
* `[^/&$?]+[^"]+` is same as `[^"]+`

Code after:
import re

from .common import InfoExtractor


class AtScaleConfEventIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?atscaleconference\.com/events/(?P<id>[^/&$?]+)'

    _TESTS = [{
        'url': 'https://atscaleconference.com/events/data-scale-spring-2022/',
        'playlist_mincount': 13,
        'info_dict': {
            'id': 'data-scale-spring-2022',
            'title': 'Data @Scale Spring 2022',
            'description': 'md5:7d7ca1c42ac9c6d8a785092a1aea4b55'
        },
    }, {
        'url': 'https://atscaleconference.com/events/video-scale-2021/',
        'playlist_mincount': 14,
        'info_dict': {
            'id': 'video-scale-2021',
            'title': 'Video @Scale 2021',
            'description': 'md5:7d7ca1c42ac9c6d8a785092a1aea4b55'
        },
    }]

    def _real_extract(self, url):
        id = self._match_id(url)
        webpage = self._download_webpage(url, id)

        return self.playlist_from_matches(
            re.findall(r'data-url\s*=\s*"(https?://(?:www\.)?atscaleconference\.com/videos/[^"]+)"', webpage),
            ie='Generic', playlist_id=id,
            title=self._og_search_title(webpage), description=self._og_search_description(webpage))
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/scrolller.py (Python)

Code before:
        query = {
            'query':
            '''{
                getSubredditPost(url:"/%s"){
                    id
                    title
                    subredditTitle
                    subredditUrl
                    redditPath
                    isNsfw
                    mediaSources{
                        url
                        width
                        height
                    }
                }
            }''' % (video_id)
        }

        video = self._download_json(
            "https://api.scrolller.com/api/v2/graphql", video_id, data=json.dumps(query).encode(), headers={'Content-Type': 'application/json'})['data']['getSubredditPost']

        thumbnails = []
        formats = []

        for source in video.get("mediaSources"):
            if determine_ext(source.get("url")) not in KNOWN_EXTENSIONS:
                thumbnails.append(source)
            else:
                formats.append(source)

        if not formats:
            self.raise_no_formats('There is no video.', expected=True, video_id=video_id)

        self._sort_formats(formats)

        return {
            "id": video_id,
            "title": video.get("title"),
            "thumbnails": thumbnails,
            "formats": formats,
            "age_limit": 18 if video.get("isNsfw") else 0
        }


Review comment (suggestion):
```suggestion
        for source in video.get("mediaSources") or []:
```

Code after:
        query = {
            'query': '''{
                getSubredditPost(url:"/%s"){
                    id
                    title
                    isNsfw
                    mediaSources{
                        url
                        width
                        height
                    }
                }
            }''' % video_id
        }

        video_data = self._download_json(
            'https://api.scrolller.com/api/v2/graphql', video_id, data=json.dumps(query).encode(),
            headers={'Content-Type': 'application/json'})['data']['getSubredditPost']

        formats, thumbnails = [], []
        for source in video_data['mediaSources']:
            if determine_ext(source.get('url')) in ('jpg', 'png'):
                thumbnails.append({
                    'url': source['url'],
                    'width': int_or_none(source.get('width')),
                    'height': int_or_none(source.get('height')),
                })
            elif source.get('url'):
                formats.append({
                    'url': source['url'],
                    'width': int_or_none(source.get('width')),
                    'height': int_or_none(source.get('height')),
                })

        if not formats:
            self.raise_no_formats('There is no video.', expected=True, video_id=video_id)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_data.get('title'),
            'thumbnails': thumbnails,
            'formats': formats,
            'age_limit': 18 if video_data.get('isNsfw') else 0
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/scrolller.py (Python)

Code before:
from .common import InfoExtractor
from ..utils import (determine_ext, int_or_none)
import json


class ScrolllerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?scrolller\.com/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://scrolller.com/a-helping-hand-1k9pxikxkw',
        'info_dict': {
            'id': 'a-helping-hand-1k9pxikxkw',
            'ext': 'mp4',
            'thumbnail': 'https://zepto.scrolller.com/a-helping-hand-3ty9q8x094-540x960.jpg',
            'title': 'A helping hand',
            'age_limit': 0,
        }
    }, {
        'url': 'https://scrolller.com/tigers-chasing-a-drone-c5d1f2so6j',
        'info_dict': {
            'id': 'tigers-chasing-a-drone-c5d1f2so6j',
            'ext': 'mp4',
            'thumbnail': 'https://zepto.scrolller.com/tigers-chasing-a-drone-az9pkpguwe-540x303.jpg',
            'title': 'Tigers chasing a drone',
            'age_limit': 0,
        }
    }, {
        'url': 'https://scrolller.com/baby-rhino-smells-something-9chhugsv9p',
        'info_dict': {

Review comment (refactor):
```suggestion
import json

from .common import InfoExtractor
from ..utils import determine_ext, int_or_none
```

Code after:
import json

from .common import InfoExtractor
from ..utils import determine_ext, int_or_none


class ScrolllerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?scrolller\.com/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://scrolller.com/a-helping-hand-1k9pxikxkw',
        'info_dict': {
            'id': 'a-helping-hand-1k9pxikxkw',
            'ext': 'mp4',
            'thumbnail': 'https://zepto.scrolller.com/a-helping-hand-3ty9q8x094-540x960.jpg',
            'title': 'A helping hand',
            'age_limit': 0,
        }
    }, {
        'url': 'https://scrolller.com/tigers-chasing-a-drone-c5d1f2so6j',
        'info_dict': {
            'id': 'tigers-chasing-a-drone-c5d1f2so6j',
            'ext': 'mp4',
            'thumbnail': 'https://zepto.scrolller.com/tigers-chasing-a-drone-az9pkpguwe-540x303.jpg',
            'title': 'Tigers chasing a drone',
            'age_limit': 0,
        }
    }, {
        'url': 'https://scrolller.com/baby-rhino-smells-something-9chhugsv9p',
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/scrolller.py (Python)

Code before:
                    mediaSources{
                        url
                        width
                        height
                    }
                }
            }''' % (video_id)
        }

        video_data = self._download_json(
            'https://api.scrolller.com/api/v2/graphql', video_id, data=json.dumps(query).encode(),
            headers={'Content-Type': 'application/json'})['data']['getSubredditPost']

        thumbnails = []
        formats = []

        for source in video_data['mediaSources']:
            if determine_ext(source.get('url')) not in ('.jpg', '.png'):
                thumbnails.append({
                    'url': source.get('url'),
                    'width': int_or_none(source.get('width')),
                    'height': int_or_none(source.get('height')),
                })
            else:
                formats.append({
                    'url': source.get('url'),
                    'width': int_or_none(source.get('width')),
                    'height': int_or_none(source.get('height')),
                })

        if not formats:
            self.raise_no_formats('There is no video.', expected=True, video_id=video_id)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_data.get('title'),
            'thumbnails': thumbnails,
            'formats': formats,
            'age_limit': 18 if video_data.get('isNsfw') else 0
        }


Review comment (suggestion):
```suggestion
            elif source.get('url'):
                formats.append({
                    'url': source['url'],
```

Code after:
                        url
                        width
                        height
                    }
                }
            }''' % video_id
        }

        video_data = self._download_json(
            'https://api.scrolller.com/api/v2/graphql', video_id, data=json.dumps(query).encode(),
            headers={'Content-Type': 'application/json'})['data']['getSubredditPost']

        formats, thumbnails = [], []
        for source in video_data['mediaSources']:
            if determine_ext(source.get('url')) in ('jpg', 'png'):
                thumbnails.append({
                    'url': source['url'],
                    'width': int_or_none(source.get('width')),
                    'height': int_or_none(source.get('height')),
                })
            elif source.get('url'):
                formats.append({
                    'url': source['url'],
                    'width': int_or_none(source.get('width')),
                    'height': int_or_none(source.get('height')),
                })

        if not formats:
            self.raise_no_formats('There is no video.', expected=True, video_id=video_id)

        self._sort_formats(formats)

        return {
            'id': video_id,
            'title': video_data.get('title'),
            'thumbnails': thumbnails,
            'formats': formats,
            'age_limit': 18 if video_data.get('isNsfw') else 0
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/scrolller.py (Python)

Code before:
import json

from .common import InfoExtractor
from ..utils import (determine_ext, int_or_none)


class ScrolllerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?scrolller\.com/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://scrolller.com/a-helping-hand-1k9pxikxkw',
        'info_dict': {
            'id': 'a-helping-hand-1k9pxikxkw',
            'ext': 'mp4',
            'thumbnail': 'https://zepto.scrolller.com/a-helping-hand-3ty9q8x094-540x960.jpg',
            'title': 'A helping hand',
            'age_limit': 0,
        }
    }, {
        'url': 'https://scrolller.com/tigers-chasing-a-drone-c5d1f2so6j',
        'info_dict': {
            'id': 'tigers-chasing-a-drone-c5d1f2so6j',
            'ext': 'mp4',
            'thumbnail': 'https://zepto.scrolller.com/tigers-chasing-a-drone-az9pkpguwe-540x303.jpg',
            'title': 'Tigers chasing a drone',
            'age_limit': 0,
        }
    }, {
        'url': 'https://scrolller.com/baby-rhino-smells-something-9chhugsv9p',
        'info_dict': {

Review comment (suggestion):
```suggestion
from ..utils import determine_ext, int_or_none
```

Code after:
import json

from .common import InfoExtractor
from ..utils import determine_ext, int_or_none


class ScrolllerIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?scrolller\.com/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://scrolller.com/a-helping-hand-1k9pxikxkw',
        'info_dict': {
            'id': 'a-helping-hand-1k9pxikxkw',
            'ext': 'mp4',
            'thumbnail': 'https://zepto.scrolller.com/a-helping-hand-3ty9q8x094-540x960.jpg',
            'title': 'A helping hand',
            'age_limit': 0,
        }
    }, {
        'url': 'https://scrolller.com/tigers-chasing-a-drone-c5d1f2so6j',
        'info_dict': {
            'id': 'tigers-chasing-a-drone-c5d1f2so6j',
            'ext': 'mp4',
            'thumbnail': 'https://zepto.scrolller.com/tigers-chasing-a-drone-az9pkpguwe-540x303.jpg',
            'title': 'Tigers chasing a drone',
            'age_limit': 0,
        }
    }, {
        'url': 'https://scrolller.com/baby-rhino-smells-something-9chhugsv9p',
        'info_dict': {
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/substack.py (Python)

Code before:

    def _real_extract(self, url):
        display_id, username = self._match_valid_url(url).group('id', 'username')
        webpage = self._download_webpage(url, display_id)

        post_info = self._search_json(
            r'<script[^>]*>\s*window\._preloads\s*=\s*', webpage, 'preloads', display_id)['post']

        post_type = post_info.get('type')
        formats, subtitles = [], {}
        if post_type == 'podcast':
            formats, subtitles = [{'url': post_info['podcast_url']}], {}
        elif post_type == 'video':
            formats, subtitles = self._extract_video_formats(post_info['videoUpload']['id'], username)
        else:
            self.raise_no_formats(f'Page type "{post_type}" is not supported')

        self._sort_formats(formats)
        return {
            'id': str(post_info['id']),
            'formats': formats,
            'subtitles': subtitles,
            'title': post_info.get('title'),
            'description': post_info.get('description'),
            'thumbnail': post_info.get('cover_image'),
            'uploader': username,
        }


Review comment (security):
`uploader` should probably somehow point to the Substack profile page (under `https://substack.com/profile/`). What you call `username` here is actually the name of the newsletter only.

Compare mollymovieclub.com and computerenhance.substack.com being both written by the same author.

Code after:

        return formats, subtitles

    def _real_extract(self, url):
        display_id, username = self._match_valid_url(url).group('id', 'username')
        webpage = self._download_webpage(url, display_id)

        webpage_info = self._search_json(r'<script[^>]*>\s*window\._preloads\s*=', webpage, 'preloads', display_id)

        post_type = webpage_info['post']['type']
        formats, subtitles = [], {}
        if post_type == 'podcast':
            formats, subtitles = [{'url': webpage_info['post']['podcast_url']}], {}
        elif post_type == 'video':
            formats, subtitles = self._extract_video_formats(webpage_info['post']['videoUpload']['id'], username)
        else:
            self.raise_no_formats(f'Page type "{post_type}" is not supported')

        self._sort_formats(formats)
        return {
            'id': str(webpage_info['post']['id']),
            'formats': formats,
            'subtitles': subtitles,
            'title': traverse_obj(webpage_info, ('post', 'title')),
            'description': traverse_obj(webpage_info, ('post', 'description')),
            'thumbnail': traverse_obj(webpage_info, ('post', 'cover_image')),
            'uploader': traverse_obj(webpage_info, ('pub', 'name')),
            'uploader_id': str_or_none(traverse_obj(webpage_info, ('post', 'publication_id'))),
        }
Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/substack.py (Python)

Code before:
import re
from urllib.parse import urlparse

from .common import InfoExtractor
from ..utils import (
    traverse_obj,
    str_or_none,
)


class SubstackIE(InfoExtractor):
    _VALID_URL = r'https?://(?P<username>[\w-]+)\.substack\.com/p/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://haleynahman.substack.com/p/i-made-a-vlog?s=r',
        'md5': 'f27e4fc6252001d48d479f45e65cdfd5',
        'info_dict': {
            'id': '47660949',
            'ext': 'mp4',
            'title': 'I MADE A VLOG',
            'description': 'md5:10c01ff93439a62e70ce963b2aa0b7f6',
            'thumbnail': 'md5:bec758a34d8ee9142d43bcebdf33af18',
            'uploader': 'Maybe Baby',
            'uploader_id': '7802200',
        }
    }, {
        'url': 'https://haleynahman.substack.com/p/-dear-danny-i-found-my-boyfriends?s=r',
        'md5': '0a63eacec877a1171a62cfa69710fcea',
        'info_dict': {
            'id': '51045592',
            'ext': 'mpga',
            'title': "🎧 Dear Danny: I found my boyfriend's secret Twitter account",
            'description': 'md5:a57f2439319e56e0af92dd0c95d75797',
            'thumbnail': 'md5:daa40b6b79249417c14ff8103db29639',

Review comment (refactor):
```suggestion
import urllib.parse

from .common import InfoExtractor
from ..utils import (
    str_or_none,
    traverse_obj,
)
```

Code after:
import re
import urllib.parse

from .common import InfoExtractor
from ..utils import str_or_none, traverse_obj


class SubstackIE(InfoExtractor):
    _VALID_URL = r'https?://(?P<username>[\w-]+)\.substack\.com/p/(?P<id>[\w-]+)'
    _TESTS = [{
        'url': 'https://haleynahman.substack.com/p/i-made-a-vlog?s=r',
        'md5': 'f27e4fc6252001d48d479f45e65cdfd5',
        'info_dict': {
            'id': '47660949',
            'ext': 'mp4',
            'title': 'I MADE A VLOG',
            'description': 'md5:10c01ff93439a62e70ce963b2aa0b7f6',
            'thumbnail': 'md5:bec758a34d8ee9142d43bcebdf33af18',
            'uploader': 'Maybe Baby',
            'uploader_id': '33628',
        }
    }, {
        'url': 'https://haleynahman.substack.com/p/-dear-danny-i-found-my-boyfriends?s=r',
        'md5': '0a63eacec877a1171a62cfa69710fcea',
        'info_dict': {
            'id': '51045592',
            'ext': 'mpga',
            'title': "🎧 Dear Danny: I found my boyfriend's secret Twitter account",
            'description': 'md5:a57f2439319e56e0af92dd0c95d75797',
            'thumbnail': 'md5:daa40b6b79249417c14ff8103db29639',
            'uploader': 'Maybe Baby',
            'uploader_id': '33628',
        }