Repository: yt-dlp/yt-dlp
Pull request #4817 (2022-09-02 09:01:09)

[extractor/mediaworksnz] Add MediaWorksNZVOD extractor
### Description of your *pull request* and other information

</details>

<!--

Explanation of your *pull request* in arbitrary form goes here. Please **make sure the description explains the purpose and effect** of your *pull request* and is worded well enough to be understood. Provide as much **context and examples** as possible

-->

Adds generic support for videos on most MediaWorks NZ news sites (e.g. todayfm)

<details open><summary>Template</summary> <!-- OPEN is intentional -->

<!--

# PLEASE FOLLOW THE GUIDE BELOW

- You will be asked some questions, please read them **carefully** and answer honestly
- Put an `x` into all the boxes `[ ]` relevant to your *pull request* (like [x])
- Use *Preview* tab to see how your *pull request* will actually look like

-->

### Before submitting a *pull request* make sure you have:
- [X] At least skimmed through [contributing guidelines](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions) including [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions)
- [X] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
- [X] Checked the code with [flake8](https://pypi.python.org/pypi/flake8) and [ran relevant tests](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions)

### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options:
- [X] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)
- [ ] I am not the original author of this code but it is in public domain or released under [Unlicense](http://unlicense.org/) (provide reliable evidence)

### What is the purpose of your *pull request*?
- [ ] Fix or improvement to an extractor (Make sure to add/update tests)
- [X] New extractor ([Piracy websites will not be accepted](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#is-the-website-primarily-used-for-piracy))
- [ ] Core bug fix/improvement
- [ ] New feature (It is strongly [recommended to open an issue first](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#adding-new-feature-or-making-overarching-changes))



Patch:
diff --git a/supportedsites.md b/supportedsites.md
index d98863315..7b1e72016 100644
--- a/supportedsites.md
+++ b/supportedsites.md
@@ -66,7 +66,6 @@ # Supported sites
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **Angel**
  - **AnimalPlanet**
- - **AnimeOnDemand**: [<abbr title="netrc machine"><em>animeondemand</em></abbr>]
  - **ant1newsgr:article**: ant1news.gr articles
  - **ant1newsgr:embed**: ant1news.gr embedded videos
  - **ant1newsgr:watch**: ant1news.gr videos
diff --git a/yt_dlp/extractor/_extractors.py b/yt_dlp/extractor/_extractors.py
index 82b701a5d..aedf063f6 100644
--- a/yt_dlp/extractor/_extractors.py
+++ b/yt_dlp/extractor/_extractors.py
@@ -84,7 +84,6 @@
     AmericasTestKitchenSeasonIE,
 )
 from .angel import AngelIE
-from .animeondemand import AnimeOnDemandIE
 from .anvato import AnvatoIE
 from .aol import AolIE
 from .allocine import AllocineIE
@@ -938,6 +937,7 @@
     MediasiteCatalogIE,
     MediasiteNamedCatalogIE,
 )
+from .mediaworksnz import MediaWorksNZVODIE
 from .medici import MediciIE
 from .megaphone import MegaphoneIE
 from .meipai import MeipaiIE
diff --git a/yt_dlp/extractor/animeondemand.py b/yt_dlp/extractor/animeondemand.py
deleted file mode 100644
index de49db4ea..000000000
--- a/yt_dlp/extractor/animeondemand.py
+++ /dev/null
@@ -1,282 +0,0 @@
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_str
-from ..utils import (
-    determine_ext,
-    extract_attributes,
-    ExtractorError,
-    join_nonempty,
-    url_or_none,
-    urlencode_postdata,
-    urljoin,
-)
-
-
-class AnimeOnDemandIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
-    _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
-    _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
-    _NETRC_MACHINE = 'animeondemand'
-    # German-speaking countries of Europe
-    _GEO_COUNTRIES = ['AT', 'CH', 'DE', 'LI', 'LU']
-    _TESTS = [{
-        # jap, OmU
-        'url': 'https://www.anime-on-demand.de/anime/161',
-        'info_dict': {
-            'id': '161',
-            'title': 'Grimgar, Ashes and Illusions (OmU)',
-            'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
-        },
-        'playlist_mincount': 4,
-    }, {
-        # Film wording is used instead of Episode, ger/jap, Dub/OmU
-        'url': 'https://www.anime-on-demand.de/anime/39',
-        'only_matching': True,
-    }, {
-        # Episodes without titles, jap, OmU
-        'url': 'https://www.anime-on-demand.de/anime/162',
-        'only_matching': True,
-    }, {
-        # ger/jap, Dub/OmU, account required
-        'url': 'https://www.anime-on-demand.de/anime/169',
-        'only_matching': True,
-    }, {
-        # Full length film, non-series, ger/jap, Dub/OmU, account required
-        'url': 'https://www.anime-on-demand.de/anime/185',
-        'only_matching': True,
-    }, {
-        # Flash videos
-        'url': 'https://www.anime-on-demand.de/anime/12',
-        'only_matching': True,
-    }]
-
-    def _perform_login(self, username, password):
-        login_page = self._download_webpage(
-            self._LOGIN_URL, None, 'Downloading login page')
-
-        if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
-            self.raise_geo_restricted(
-                '%s is only available in German-speaking countries of Europe' % self.IE_NAME)
-
-        login_form = self._form_hidden_inputs('new_user', login_page)
-
-        login_form.update({
-            'user[login]': username,
-            'user[password]': password,
-        })
-
-        post_url = self._search_regex(
-            r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
-            'post url', default=self._LOGIN_URL, group='url')
-
-        if not post_url.startswith('http'):
-            post_url = urljoin(self._LOGIN_URL, post_url)
-
-        response = self._download_webpage(
-            post_url, None, 'Logging in',
-            data=urlencode_postdata(login_form), headers={
-                'Referer': self._LOGIN_URL,
-            })
-
-        if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
-            error = self._search_regex(
-                r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
-                response, 'error', default=None, group='error')
-            if error:
-                raise ExtractorError('Unable to login: %s' % error, expected=True)
-            raise ExtractorError('Unable to log in')
-
-    def _real_extract(self, url):
-        anime_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, anime_id)
-
-        if 'data-playlist=' not in webpage:
-            self._download_webpage(
-                self._APPLY_HTML5_URL, anime_id,
-                'Activating HTML5 beta', 'Unable to apply HTML5 beta')
-            webpage = self._download_webpage(url, anime_id)
-
-        csrf_token = self._html_search_meta(
-            'csrf-token', webpage, 'csrf token', fatal=True)
-
-        anime_title = self._html_search_regex(
-            r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
-            webpage, 'anime name')
-        anime_description = self._html_search_regex(
-            r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
-            webpage, 'anime description', default=None)
-
-        def extract_info(html, video_id, num=None):
-            title, description = [None] * 2
-            formats = []
-
-            for input_ in re.findall(
-                    r'<input[^>]+class=["\'].*?streamstarter[^>]+>', html):
-                attributes = extract_attributes(input_)
-                title = attributes.get('data-dialog-header')
-                playlist_urls = []
-                for playlist_key in ('data-playlist', 'data-otherplaylist', 'data-stream'):
-                    playlist_url = attributes.get(playlist_key)
-                    if isinstance(playlist_url, compat_str) and re.match(
-                            r'/?[\da-zA-Z]+', playlist_url):
-                        playlist_urls.append(attributes[playlist_key])
-                if not playlist_urls:
-                    continue
-
-                lang = attributes.get('data-lang')
-                lang_note = attributes.get('value')
-
-                for playlist_url in playlist_urls:
-                    kind = self._search_regex(
-                        r'videomaterialurl/\d+/([^/]+)/',
-                        playlist_url, 'media kind', default=None)
-                    format_id = join_nonempty(lang, kind) if lang or kind else str(num)
-                    format_note = join_nonempty(kind, lang_note, delim=', ')
-                    item_id_list = []
-                    if format_id:
-                        item_id_list.append(format_id)
-                    item_id_list.append('videomaterial')
-                    playlist = self._download_json(
-                        urljoin(url, playlist_url), video_id,
-                        'Downloading %s JSON' % ' '.join(item_id_list),
-                        headers={
-                            'X-Requested-With': 'XMLHttpRequest',
-                            'X-CSRF-Token': csrf_token,
-                            'Referer': url,
-                            'Accept': 'application/json, text/javascript, */*; q=0.01',
-                        }, fatal=False)
-                    if not playlist:
-                        continue
-                    stream_url = url_or_none(playlist.get('streamurl'))
-                    if stream_url:
-                        rtmp = re.search(
-                            r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+/))(?P<playpath>mp[34]:.+)',
-                            stream_url)
-                        if rtmp:
-                            formats.append({
-                                'url': rtmp.group('url'),
-                                'app': rtmp.group('app'),
-                                'play_path': rtmp.group('playpath'),
-                                'page_url': url,
-                                'player_url': 'https://www.anime-on-demand.de/assets/jwplayer.flash-55abfb34080700304d49125ce9ffb4a6.swf',
-                                'rtmp_real_time': True,
-                                'format_id': 'rtmp',
-                                'ext': 'flv',
-                            })
-                            continue
-                    start_video = playlist.get('startvideo', 0)
-                    playlist = playlist.get('playlist')
-                    if not playlist or not isinstance(playlist, list):
-                        continue
-                    playlist = playlist[start_video]
-                    title = playlist.get('title')
-                    if not title:
-                        continue
-                    description = playlist.get('description')
-                    for source in playlist.get('sources', []):
-                        file_ = source.get('file')
-                        if not file_:
-                            continue
-                        ext = determine_ext(file_)
-                        format_id = join_nonempty(
-                            lang, kind,
-                            'hls' if ext == 'm3u8' else None,
-                            'dash' if source.get('type') == 'video/dash' or ext == 'mpd' else None)
-                        if ext == 'm3u8':
-                            file_formats = self._extract_m3u8_formats(
-                                file_, video_id, 'mp4',
-                                entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
-                        elif source.get('type') == 'video/dash' or ext == 'mpd':
-                            continue
-                            file_formats = self._extract_mpd_formats(
-                                file_, video_id, mpd_id=format_id, fatal=False)
-                        else:
-                            continue
-                        for f in file_formats:
-                            f.update({
-                                'language': lang,
-                                'format_note': format_note,
-                            })
-                        formats.extend(file_formats)
-
-            return {
-                'title': title,
-                'description': description,
-                'formats': formats,
-            }
-
-        def extract_entries(html, video_id, common_info, num=None):
-            info = extract_info(html, video_id, num)
-
-            if info['formats']:
-                self._sort_formats(info['formats'])
-                f = common_info.copy()
-                f.update(info)
-                yield f
-
-            # Extract teaser/trailer only when full episode is not available
-            if not info['formats']:
-                m = re.search(
-                    r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>(?P<kind>Teaser|Trailer)<',
-                    html)
-                if m:
-                    f = common_info.copy()
-                    f.update({
-                        'id': '%s-%s' % (f['id'], m.group('kind').lower()),
-                        'title': m.group('title'),
-                        'url': urljoin(url, m.group('href')),
-                    })
-                    yield f
-
-        def extract_episodes(html):
-            for num, episode_html in enumerate(re.findall(
-                    r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', html), 1):
-                episodebox_title = self._search_regex(
-                    (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
-                     r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
-                    episode_html, 'episodebox title', default=None, group='title')
-                if not episodebox_title:
-                    continue
-
-                episode_number = int(self._search_regex(
-                    r'(?:Episode|Film)\s*(\d+)',
-                    episodebox_title, 'episode number', default=num))
-                episode_title = self._search_regex(
-                    r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
-                    episodebox_title, 'episode title', default=None)
-
-                video_id = 'episode-%d' % episode_number
-
-                common_info = {
-                    'id': video_id,
-                    'series': anime_title,
-                    'episode': episode_title,
-                    'episode_number': episode_number,
-                }
-
-                for e in extract_entries(episode_html, video_id, common_info):
-                    yield e
-
-        def extract_film(html, video_id):
-            common_info = {
-                'id': anime_id,
-                'title': anime_title,
-                'description': anime_description,
-            }
-            for e in extract_entries(html, video_id, common_info):
-                yield e
-
-        def entries():
-            has_episodes = False
-            for e in extract_episodes(webpage):
-                has_episodes = True
-                yield e
-
-            if not has_episodes:
-                for e in extract_film(webpage, anime_id):
-                    yield e
-
-        return self.playlist_result(
-            entries(), anime_id, anime_title, anime_description)
diff --git a/yt_dlp/extractor/mediaworksnz.py b/yt_dlp/extractor/mediaworksnz.py
new file mode 100644
index 000000000..651239bd4
--- /dev/null
+++ b/yt_dlp/extractor/mediaworksnz.py
@@ -0,0 +1,105 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    bug_reports_message,
+    float_or_none,
+    traverse_obj,
+    unified_timestamp,
+)
+
+
+class MediaWorksNZVODIE(InfoExtractor):
+    _VALID_URL_BASE_RE = r'https?://vodupload-api\.mediaworks\.nz/library/asset/published/'
+    _VALID_URL_ID_RE = r'(?P<id>[A-Za-z0-9-]+)'
+    _VALID_URL = rf'{_VALID_URL_BASE_RE}{_VALID_URL_ID_RE}'
+    _TESTS = [{
+        'url': 'https://vodupload-api.mediaworks.nz/library/asset/published/VID00359',
+        'info_dict': {
+            'id': 'VID00359',
+            'ext': 'mp4',
+            'title': 'GRG Jacinda Ardern safe drug testing 1920x1080',
+            'description': 'md5:d4d7dc366742e86d8130b257dcb520ba',
+            'duration': 142.76,
+            'timestamp': 1604268608,
+            'upload_date': '20201101',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'channel': 'George FM'
+        }
+    }, {
+        # has audio-only format
+        'url': 'https://vodupload-api.mediaworks.nz/library/asset/published/VID02627',
+        'info_dict': {
+            'id': 'VID02627',
+            'ext': 'mp3',
+            'title': 'Tova O\'Brien meets Ukraine President Volodymyr Zelensky',
+            'channel': 'Today FM',
+            'description': 'Watch in full the much anticipated interview of Volodymyr Zelensky',
+            'duration': 2061.16,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'upload_date': '20220822',
+            'timestamp': 1661152289,
+        },
+        'params': {'format': 'ba[ext=mp3]'}
+    }]
+
+    _WEBPAGE_TESTS = [{
+        'url': 'https://www.rova.nz/home/podcasts/socrates-walks-into-a-bar/the-trolley-problem---episode-1.html',
+        'info_dict': {
+            'id': 'VID02494',
+            'ext': 'mp4',
+            'title': 'The Trolley Problem',
+            'duration': 2843.56,
+            'channel': 'Other',
+            'timestamp': 1658356489,
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'Socrates Walks Into A Bar Podcast Episode 1',
+            'upload_date': '20220720',
+        }
+    }]
+
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        for mobj in re.finditer(
+            rf'''(?x)<div\s+\bid=["']Player-Attributes-JWID[^>]+\b
+            data-request-url=["']{cls._VALID_URL_BASE_RE}["'][^>]+\b
+            data-asset-id=["']{cls._VALID_URL_ID_RE}["']''', webpage
+        ):
+            yield f'https://vodupload-api.mediaworks.nz/library/asset/published/{mobj.group("id")}'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        asset = self._download_json(url, video_id)['asset']
+
+        if asset.get('drm') not in ('NonDRM', None):
+            self.report_drm(video_id)
+
+        content_type = asset.get('type')
+        if content_type and content_type != 'video':
+            self.report_warning(f'Unknown content type: {content_type}' + bug_reports_message(), video_id)
+
+        formats, subtitles = self._extract_m3u8_formats_and_subtitles(asset['streamingUrl'], video_id)
+
+        audio_streaming_url = traverse_obj(
+            asset, 'palyoutPathAudio', 'playoutpathaudio', expected_type=str)
+        if audio_streaming_url:
+            audio_formats = self._extract_m3u8_formats(audio_streaming_url, video_id, fatal=False, ext='mp3')
+            for audio_format in audio_formats:
+                # all the audio streams appear to be aac
+                audio_format.setdefault('vcodec', 'none')
+                audio_format.setdefault('acodec', 'aac')
+                formats.append(audio_format)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': asset.get('title'),
+            'description': asset.get('description'),
+            'duration': float_or_none(asset.get('duration')),
+            'timestamp': unified_timestamp(asset.get('dateadded')),
+            'channel': asset.get('brand'),
+            'thumbnails': [{'url': thumbnail_url} for thumbnail_url in asset.get('thumbnails') or []],
+            'formats': formats,
+            'subtitles': subtitles,
+        }
diff --git a/yt_dlp/jsinterp.py b/yt_dlp/jsinterp.py
index 2bb4acf3e..4caad6f74 100644
--- a/yt_dlp/jsinterp.py
+++ b/yt_dlp/jsinterp.py
@@ -117,8 +117,8 @@ def _js_ternary(cndn, if_true=True, if_false=False):
     '-': _js_arith_op(operator.sub),
 
     '*': _js_arith_op(operator.mul),
-    '/': _js_div,
     '%': _js_mod,
+    '/': _js_div,
     '**': _js_exp,
 }
 


Test patch:
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index b1c5cb2b3..c3dcb4d68 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -126,6 +126,10 @@
         'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
         'M92UUMHa8PdvPd3wyM', '3hPqLJsiNZx7yA',
     ),
+    (
+        'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js',
+        'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ',
+    ),
 ]