Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/bilibili.py (Python)

Code before:

    def _get_n_results(self, category, subcategory, n, match=None):
        query = "%s: %s" % (category, subcategory)
        # map of categories : subcategories : RIDs
        rid_map = {
            "kichiku": {
                "mad": 26,
                "manual_vocaloid": 126,
                "guide": 22,
                "theatre": 216,
                "course": 127
            },
        }

        if category not in rid_map:
            raise ExtractorError("The supplied category, %s, is not supported" % category)

        if subcategory not in rid_map[category]:
            raise ExtractorError("The supplied subcategory, %s, is not supported" % subcategory)

        rid_value = rid_map[category][subcategory]

        api_url = "https://api.bilibili.com/x/web-interface/newlist?rid=%d&type=1&ps=20&jsonp=jsonp" % rid_value
        json_str = self._download_webpage(api_url + "&pn=1", "None", query={"Search_key": query})
        parsed_json = json.loads(json_str)
        num_pages = math.ceil(parsed_json['data']['page']['count'] / parsed_json['data']['page']['size'])

        entries = []

        # desc order
        for page_number in range(1, num_pages + 1, 1):
            time.sleep(2)
            json_str = self._download_webpage(api_url + "&pn=%s" % page_number, "None", query={"Search_key": query},
                                              note='Extracting results from page %s of %s' % (page_number, num_pages))

            parsed_json = json.loads(json_str)
            # Ascending by publish date
            video_list = sorted(parsed_json['data']['archives'], key=lambda video: video['pubdate'])
            video_list_processed = list(map(lambda video: self.url_result("https://www.bilibili.com/video/%s" % video['bvid'],
                                                        'BiliBili', video['bvid']), video_list))
            should_brk = False
            if match:
                for idx, ele in enumerate(video_list_processed):
                    matches = match(ele)
                    if matches is None:
                        video_list_processed = video_list_processed[:idx]
                        should_brk = True
                        break

            entries += video_list_processed
            if(len(entries) >= n or len(entries) >= BilibiliCategoryIE._MAX_RESULTS) or should_brk:

Review comment (bug):
probably want to safely extract the count and size and raise an ExtractorError if we fail to do so.

Something like this maybe (assuming count and size should not be 0 as well):
```suggestion
        data_page = try_get(parsed_json, lambda x: x['data']['page'], dict) or {}
        count, size = int_or_none(data_page.get('count')), int_or_none(data_page.get('size'))
        if not count or not size:
            raise ExtractorError('something here')
        num_pages = count / size
```



Code after:

    def _fetch_page(self, api_url, num_pages, query, page_num):
        parsed_json = self._download_json(
            api_url, query, query={'Search_key': query, 'pn': page_num},
            note='Extracting results from page %s of %s' % (page_num, num_pages))

        video_list = try_get(parsed_json, lambda x: x['data']['archives'], list)
        if not video_list:
            raise ExtractorError('Failed to retrieve video list for page %d' % page_num)

        for video in video_list:
            yield self.url_result(
                'https://www.bilibili.com/video/%s' % video['bvid'], 'BiliBili', video['bvid'])

    def _entries(self, category, subcategory, query):
        # map of categories : subcategories : RIDs
        rid_map = {
            'kichiku': {
                'mad': 26,
                'manual_vocaloid': 126,
                'guide': 22,
                'theatre': 216,
                'course': 127
            },
        }

        if category not in rid_map:
            raise ExtractorError('The supplied category, %s, is not supported. List of supported categories: %s' % (category, list(rid_map.keys())))

        if subcategory not in rid_map[category]:
            raise ExtractorError('The subcategory, %s, isn\'t supported for this category. Supported subcategories: %s' % (subcategory, list(rid_map[category].keys())))

        rid_value = rid_map[category][subcategory]

        api_url = 'https://api.bilibili.com/x/web-interface/newlist?rid=%d&type=1&ps=20&jsonp=jsonp' % rid_value
        page_json = self._download_json(api_url, query, query={'Search_key': query, 'pn': '1'})
        page_data = try_get(page_json, lambda x: x['data']['page'], dict)
        count, size = int_or_none(page_data.get('count')), int_or_none(page_data.get('size'))
        if count is None or not size:
            raise ExtractorError('Failed to calculate either page count or size')

        num_pages = math.ceil(count / size)

        return OnDemandPagedList(functools.partial(
            self._fetch_page, api_url, num_pages, query), size)

    def _real_extract(self, url):
        u = compat_urllib_parse_urlparse(url)
        category, subcategory = u.path.split('/')[2:4]
        query = '%s: %s' % (category, subcategory)