Repository: yt-dlp/yt-dlp
File: yt_dlp/extractor/detik.py (Python)

Code before:
        player_type, video_data = self._search_regex(
            r'<script\s*[^>]+src="https?://(aws)?cdn\.detik\.net\.id/(?P<type>flowplayer|detikVideo)[^>]+>\s*(?P<video_data>{[^}]+})',
            webpage, 'playerjs', group=('type', 'video_data'), default=(None, ''))

        json_ld_data = self._search_json_ld(webpage, display_id, default={})
        extra_info_dict = {}

        if not player_type:
            return

        elif player_type == 'flowplayer':
            video_json_data = self._parse_json(video_data.replace('\'', '"'), None)
            video_url = video_json_data['videoUrl']

            extra_info_dict = {
                'id': self._search_regex(r'identifier\s*:\s*\'([^\']+)', webpage, 'identifier'),
                'thumbnail': video_json_data.get('imageUrl'),
            }

        elif player_type == 'detikVideo':
            video_url = self._search_regex(
                r'videoUrl\s*:\s*[\'"]?([^"\']+)', video_data, 'videoUrl')
            extra_info_dict = {
                'id': self._html_search_meta(['video_id', 'dtk:video_id'], webpage),
                'thumbnail': self._search_regex(r'imageUrl\s*:\s*[\'"]?([^"\']+)', video_data, 'videoUrl'),
                # 'duration': int_or_none(self._html_search_meta('duration', webpage, fatal=False), default=0),
                'release_timestamp': int_or_none(self._html_search_meta('dtk:publishdateunix', webpage, fatal=False), 1000),
            }

        formats, subtitles = self._extract_m3u8_formats_and_subtitles(video_url, display_id)
        self._sort_formats(formats)

        yield merge_dicts(json_ld_data, extra_info_dict, {
            'display_id': display_id,
            'title': self._html_search_meta(['og:title', 'originalTitle'], webpage),
            'description': self._html_search_meta(['og:description', 'twitter:description', 'description'], webpage),
            'formats': formats,
            'subtitle': subtitles,
            'tags': try_call(lambda: self._html_search_meta(
                ['keywords', 'keyword', 'dtk:keywords'], webpage).split(',')),
        })


Review comment (bug):
Does anyone know why in the test, it always complaining `duration` (when `duration` metadata uncommented). Is it bug from test runner or is it my mistake ?

<details>
<summary> log from test </summary>

```shell
(python_proj2) [REDACTED]>python test/test_download.py TestDownload.test_DetikEmbed_all
[debug] Loaded 1672 extractors 
[debug] [generic] Extracting URL: https://www.cnnindonesia.com/embed/video/846189 
[generic] 846189: Downloading webpage 
[generic] 846189: Extracting information 
[debug] Looking for Brightcove embeds 
[debug] Looking for embeds 
ERROR: [generic] [DetikEmbed] unable to extract duration; please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U; please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U
  File "[REDACTED]\yt_dlp\extractor\common.py", line 670, in extract 
    ie_result = self._real_extract(url)
  File "[REDACTED]\yt_dlp\extractor\generic.py", line 2813, in _real_extract
    current_embeds.append(next(gen))
  File "[REDACTED]\yt_dlp\extractor\common.py", line 3868, in extract_from_webpage
    for info in ie._extract_from_webpage(url, webpage) or []:
  File "[REDACTED]\yt_dlp\extractor\detik.py", line 113, in _extract_from_webpage
    'duration': int_or_none(self._html_search_meta('duration', webpage, fatal=False), default=0),
  File "[REDACTED]\yt_dlp\extractor\common.py", line 1375, in _html_search_meta
    return self._html_search_regex(
  File "[REDACTED]\yt_dlp\extractor\common.py", line 1258, in _html_search_regex
    res = self._search_regex(pattern, string, name, default, fatal, flags, group)
  File "[REDACTED]\yt_dlp\extractor\common.py", line 1224, in _search_regex
    self.report_warning('unable to extract %s' % _name + bug_reports_message())
  File "[REDACTED]\yt_dlp\extractor\common.py", line 1093, in report_warning
    self._downloader.report_warning(msg, *args, **kwargs)
  File "[REDACTED]\test\helper.py", line 313, in _report_warning
    real_warning(w, *args, **kwargs)
  File "[REDACTED]\test\test_download.py", line 51, in report_warning
    raise ExtractorError(message)

E
======================================================================
ERROR: test_DetikEmbed_all (__main__.TestDownload):
----------------------------------------------------------------------
Traceback (most recent call last):
  File "[REDACTED]\yt_dlp\extractor\common.py", line 670, in extract
    ie_result = self._real_extract(url)
  File "[REDACTED]\yt_dlp\extractor\generic.py", line 2813, in _real_extract
    current_embeds.append(next(gen))
  File "[REDACTED]\yt_dlp\extractor\common.py", line 3868, in extract_from_webpage
    for info in ie._extract_from_webpage(url, webpage) or []:
  File "[REDACTED]\yt_dlp\extractor\detik.py", line 113, in _extract_from_webpage
    'duration': int_or_none(self._html_search_meta('duration', webpage, fatal=False), default=0),
  File "[REDACTED]\yt_dlp\extractor\common.py", line 1375, in _html_search_meta
    return self._html_search_regex(
  File "[REDACTED]\yt_dlp\extractor\common.py", line 1258, in _html_search_regex
    res = self._search_regex(pattern, string, name, default, fatal, flags, group)
  File "[REDACTED]\yt_dlp\extractor\common.py", line 1224, in _search_regex
    self.report_warning('unable to extract %s' % _name + bug_reports_message())
  File "[REDACTED]\yt_dlp\extractor\common.py", line 1093, in report_warning
    self._downloader.report_warning(msg, *args, **kwargs)
  File "[REDACTED]\test\helper.py", line 313, in _report_warning
    real_warning(w, *args, **kwargs)
  File "[REDACTED]\test\test_download.py", line 51, in report_warning
    raise ExtractorError(message)
yt_dlp.utils.ExtractorError: [DetikEmbed] unable to extract duration; please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U; please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "[REDACTED]\yt_dlp\YoutubeDL.py", line 1477, in wrapper
    return func(self, *args, **kwargs)
  File "[REDACTED]\yt_dlp\YoutubeDL.py", line 1553, in __extract_info
    ie_result = ie.extract(url)
  File "[REDACTED]\yt_dlp\extractor\common.py", line 696, in extract
    raise type(e)(e.orig_msg, **kwargs)
yt_dlp.utils.ExtractorError: [generic] [DetikEmbed] unable to extract duration; please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U; please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "[REDACTED]\test\test_download.py", line 278, in test_template
    getattr(self, test_name)()
  File "[REDACTED]\test\test_download.py", line 159, in test_template
    res_dict = ydl.extract_info(
  File "[REDACTED]\yt_dlp\YoutubeDL.py", line 1466, in extract_info
    return self.__extract_info(url, self.get_info_extractor(key), download, extra_info, process)
  File "[REDACTED]\yt_dlp\YoutubeDL.py", line 1495, in wrapper
    self.report_error(str(e), e.format_traceback())
  File "[REDACTED]\yt_dlp\YoutubeDL.py", line 996, in report_error
    self.trouble(f'{self._format_err("ERROR:", self.Styles.ERROR)} {message}', *args, **kwargs)
  File "[REDACTED]\yt_dlp\YoutubeDL.py", line 936, in trouble
    raise DownloadError(message, exc_info)
yt_dlp.utils.DownloadError: ERROR: [generic] [DetikEmbed] unable to extract duration; please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U; please report this issue on  https://github.com/yt-dlp/yt-dlp/issues?q= , filling out the appropriate issue template. Confirm you are on the latest version using  yt-dlp -U

----------------------------------------------------------------------
Ran 1 test in 1.137s

FAILED (errors=1)
```
</details>

Code after:
            'ext': 'mp4',
            'title': 'Puluhan Pejabat Rusia Tuntut Putin Mundur',
            'tags': ['putin'],
            'age_limit': 0,
            'thumbnail': 'https://awsimages.detik.net.id/visual/2022/09/13/cnbc-indonesia-tv-3_169.png?w=600&q=80',
            'description': 'md5:8b9111e37555fcd95fe549a9b4ae6fdc',
        }
    }, {
        # detik shortlink (we can get it from https://dtk.id/?<url>)
        'url': 'https://dtk.id/NkISKr',
        'info_dict': {
            'id': '220914049',
            'ext': 'mp4',
            'release_timestamp': 1663114488,
            'uploader': 'Tim 20Detik',
            'title': 'Pakar Bicara soal Tim Khusus Jokowi dan Mereka yang Pro ke Bjorka',
            'age_limit': 0,
            'thumbnail': 'https://cdnv.detik.com/videoservice/AdminTV/2022/09/14/f15cae71d7b640c58e75b254ecbb1ce1-20220914071613-0s.jpg?w=400&q=80',
            'display_id': 'pakar-bicara-soal-tim-khusus-jokowi-dan-mereka-yang-pro-ke-bjorka',
            'upload_date': '20220914',
            'release_date': '20220914',
            'description': 'md5:5eb03225f7ee40207dd3a1e18a73f1ff',
            'timestamp': 1663139688,
            'duration': 213.0,
            'tags': ['hacker bjorka', 'bjorka', 'hacker bjorka bocorkan data rahasia presiden jokowi', 'jokowi'],
        }
    }]

    def _extract_from_webpage(self, url, webpage):
        display_id = url_basename(url)
        player_type, video_data = self._search_regex(
            r'<script\s*[^>]+src="https?://(aws)?cdn\.detik\.net\.id/(?P<type>flowplayer|detikVideo)[^>]+>\s*(?P<video_data>{[^}]+})',
            webpage, 'playerjs', group=('type', 'video_data'), default=(None, ''))

        json_ld_data = self._search_json_ld(webpage, display_id, default={})
        extra_info_dict = {}

        if not player_type:
            return

        elif player_type == 'flowplayer':
            video_json_data = self._parse_json(video_data.replace('\'', '"'), display_id)
            video_url = video_json_data['videoUrl']

            extra_info_dict = {
                'id': self._search_regex(r'identifier\s*:\s*\'([^\']+)', webpage, 'identifier'),
                'thumbnail': video_json_data.get('imageUrl'),
            }

        elif player_type == 'detikVideo':
            video_url = self._search_regex(