Repository: streamlink/streamlink

Issue:
plugins.dogan: fix broken streams
<!--
Thanks for opening a pull request!

Before you continue, please make sure that you have read and understood the contribution guidelines, otherwise your changes may be rejected:
https://github.com/streamlink/streamlink/blob/master/CONTRIBUTING.md#contributing-to-streamlink

If possible, run the tests, perform code linting and build the documentation locally on your system first to avoid unnecessary build failures:
https://streamlink.github.io/latest/developing.html#validating-changes

Also don't forget to add a meaningful description of your changes, so that the reviewing process is as simple as possible for the maintainers.

Thank you very much!
-->
Live streams other than kanald were broken. I tired to fix them. I checked all the example links from https://github.com/streamlink/streamlink/issues/3416#issuecomment-748613571 and from test file. All links are working again.


Discussion:
Please see #5053 instead.

Resolving patch:
diff --git a/src/streamlink/plugins/dogan.py b/src/streamlink/plugins/dogan.py
--- a/src/streamlink/plugins/dogan.py
+++ b/src/streamlink/plugins/dogan.py
@@ -12,124 +12,144 @@
 import re
 from urllib.parse import urljoin
 
-from streamlink.plugin import Plugin, pluginmatcher
+from streamlink.plugin import Plugin, PluginError, pluginmatcher
 from streamlink.plugin.api import validate
 from streamlink.stream.hls import HLSStream
 
 log = logging.getLogger(__name__)
 
 
-@pluginmatcher(re.compile(r"""
-    https?://(?:www\.)?
-    (?:
-        cnnturk\.com/(?:action/embedvideo/|canli-yayin|tv-cnn-turk|video/)|
-        dreamturk\.com\.tr/(?:canli|canli-yayin-izle|dream-turk-ozel/|programlar/)|
-        dreamtv\.com\.tr/dream-ozel/|
-        kanald\.com\.tr/|
-        teve2\.com\.tr/(?:canli-yayin|diziler/|embed/|filmler/|programlar/)
-    )
-""", re.VERBOSE))
+@pluginmatcher(re.compile(r"https?://(?:www\.)?cnnturk\.com/"))
+@pluginmatcher(re.compile(r"https?://(?:www\.)?(dreamturk|dreamtv)\.com\.tr/"))
+@pluginmatcher(re.compile(r"https?://(?:www\.)?teve2\.com\.tr/"))
+@pluginmatcher(re.compile(r"https?://(?:www\.)?kanald\.com\.tr/"))
 class Dogan(Plugin):
-    playerctrl_re = re.compile(r'''<div\s+id="video-element".*?>''', re.DOTALL)
-    data_id_re = re.compile(r'''data-id=(?P<quote>["'])/?(?P<id>\w+)(?P=quote)''')
-    content_id_re = re.compile(r'"content[Ii]d",\s*"(\w+)"')
-    item_id_re = re.compile(r"_itemId\s+=\s+'(\w+)';")
-    content_api = "/actions/media?id={id}"
-    dream_api = "/actions/content/media/{id}"
-    new_content_api = "/action/media/{id}"
-    content_api_schema = validate.Schema(
-        {
-            "data": {
-                "id": str,
-                "media": {
-                    "link": {
-                        validate.optional("defaultServiceUrl"): validate.any(validate.url(), ""),
-                        validate.optional("serviceUrl"): validate.any(validate.url(), ""),
-                        "securePath": str,
+    # based on the order of matchers
+    API_URLS = [
+        "/api/media?id={id}",
+        "/actions/content/media/{id}",
+        "/action/media/{id}",
+    ]
+    API_URL_OLD = "/actions/media?id={id}"
+
+    def _get_content_id(self):
+        return self.session.http.get(
+            self.url,
+            schema=validate.Schema(
+                validate.parse_html(),
+                validate.any(
+                    validate.all(
+                        validate.xml_xpath_string("""
+                            .//div[@data-id][
+                                @data-live
+                                or @id='video-element'
+                                or @id='player-container'
+                                or contains(@class, 'player-container')
+                            ][1]/@data-id
+                        """),
+                        str,
+                    ),
+                    # xpath query needs to have a lower priority
+                    validate.all(
+                        validate.xml_xpath_string(
+                            ".//body[@data-content-id][1]/@data-content-id",
+                        ),
+                        str,
+                    ),
+                ),
+            ),
+        )
+
+    def _api_query_new(self, content_id, api_url):
+        url = urljoin(self.url, api_url.format(id=content_id))
+        data = self.session.http.get(
+            url,
+            schema=validate.Schema(
+                validate.parse_json(),
+                validate.any(
+                    validate.all(
+                        str,
+                        validate.parse_json(),
+                        {"Error": str},
+                        validate.get("Error"),
+                    ),
+                    validate.all(
+                        {
+                            "Media": {
+                                "Link": {
+                                    "ContentId": str,
+                                    validate.optional("DefaultServiceUrl"): validate.any(validate.url(), ""),
+                                    validate.optional("ServiceUrl"): validate.any(validate.url(), ""),
+                                    "SecurePath": str,
+                                },
+                            },
+                        },
+                        validate.get(("Media", "Link")),
+                        validate.union_get("ServiceUrl", "DefaultServiceUrl", "SecurePath", "ContentId"),
+                    ),
+                ),
+            ),
+        )
+        if type(data) is str:
+            log.error(data)
+            return
+
+        service_url, default_service_url, secure_path, content_id = data
+
+        if default_service_url == "https://www.kanald.com.tr":
+            self.url = default_service_url
+            return self._api_query_old(content_id)
+
+        if re.match(r"^https?://", secure_path):
+            return secure_path
+
+        return urljoin(service_url or default_service_url, secure_path)
+
+    def _api_query_old(self, content_id):
+        url = urljoin(self.url, self.API_URL_OLD.format(id=content_id))
+        service_url, default_service_url, secure_path = self.session.http.get(
+            url,
+            schema=validate.Schema(
+                validate.parse_json(),
+                {
+                    "data": {
+                        "id": str,
+                        "media": {
+                            "link": {
+                                validate.optional("defaultServiceUrl"): validate.any(validate.url(), ""),
+                                validate.optional("serviceUrl"): validate.any(validate.url(), ""),
+                                "securePath": str,
+                            },
+                        },
                     },
                 },
-            },
-        },
-        validate.get("data"),
-        validate.get("media"),
-        validate.get("link"),
-    )
-    new_content_api_schema = validate.Schema(
-        {
-            "Media": {
-                "Link": {
-                    "ContentId": str,
-                    validate.optional("DefaultServiceUrl"): validate.any(validate.url(), ""),
-                    validate.optional("ServiceUrl"): validate.any(validate.url(), ""),
-                    "SecurePath": str,
-                },
-            },
-        },
-        validate.get("Media"),
-        validate.get("Link"),
-    )
+                validate.get(("data", "media", "link")),
+                validate.union_get("serviceUrl", "defaultServiceUrl", "securePath"),
+            ),
+        )
 
-    def _get_content_id(self):
-        res = self.session.http.get(self.url)
-        # find the contentId
-        content_id_m = self.content_id_re.search(res.text)
-        if content_id_m:
-            log.debug("Found contentId by contentId regex")
-            return content_id_m.group(1)
-
-        # find the PlayerCtrl div
-        player_ctrl_m = self.playerctrl_re.search(res.text)
-        if player_ctrl_m:
-            # extract the content id from the player control data
-            player_ctrl_div = player_ctrl_m.group(0)
-            content_id_m = self.data_id_re.search(player_ctrl_div)
-            if content_id_m:
-                log.debug("Found contentId by player data-id regex")
-                return content_id_m.group("id")
-
-        # find the itemId var
-        item_id_m = self.item_id_re.search(res.text)
-        if item_id_m:
-            log.debug("Found contentId by itemId regex")
-            return item_id_m.group(1)
-
-    def _get_new_content_hls_url(self, content_id, api_url):
-        log.debug("Using new content API url")
-        d = self.session.http.get(urljoin(self.url, api_url.format(id=content_id)))
-        d = self.session.http.json(d, schema=self.new_content_api_schema)
-
-        if d["DefaultServiceUrl"] == "https://www.kanald.com.tr":
-            self.url = d["DefaultServiceUrl"]
-            return self._get_content_hls_url(content_id)
-        else:
-            if d["SecurePath"].startswith("http"):
-                return d["SecurePath"]
-            else:
-                return urljoin((d["ServiceUrl"] or d["DefaultServiceUrl"]), d["SecurePath"])
-
-    def _get_content_hls_url(self, content_id):
-        d = self.session.http.get(urljoin(self.url, self.content_api.format(id=content_id)))
-        d = self.session.http.json(d, schema=self.content_api_schema)
-
-        return urljoin((d["serviceUrl"] or d["defaultServiceUrl"]), d["securePath"])
+        return urljoin(service_url or default_service_url, secure_path)
 
     def _get_hls_url(self, content_id):
-        # make the api url relative to the current domain
-        if "cnnturk.com" in self.url or "teve2.com.tr" in self.url:
-            return self._get_new_content_hls_url(content_id, self.new_content_api)
-        elif "dreamturk.com.tr" in self.url or "dreamtv.com.tr" in self.url:
-            return self._get_new_content_hls_url(content_id, self.dream_api)
-        else:
-            return self._get_content_hls_url(content_id)
+        for idx, match in enumerate(self.matches[:len(self.API_URLS)]):
+            if match:
+                return self._api_query_new(content_id, self.API_URLS[idx])
+
+        return self._api_query_old(content_id)
 
     def _get_streams(self):
-        content_id = self._get_content_id()
-        if content_id:
-            log.debug(f"Loading content: {content_id}")
-            hls_url = self._get_hls_url(content_id)
-            return HLSStream.parse_variant_playlist(self.session, hls_url)
-        else:
-            log.error("Could not find the contentId for this stream")
+        try:
+            content_id = self._get_content_id()
+        except PluginError:
+            log.error("Could not find the content ID for this stream")
+            return
+
+        log.debug(f"Loading content: {content_id}")
+        hls_url = self._get_hls_url(content_id)
+        if not hls_url:
+            return
+
+        return HLSStream.parse_variant_playlist(self.session, hls_url)
 
 
 __plugin__ = Dogan


Test patch:
diff --git a/tests/plugins/test_dogan.py b/tests/plugins/test_dogan.py
--- a/tests/plugins/test_dogan.py
+++ b/tests/plugins/test_dogan.py
@@ -6,34 +6,23 @@ class TestPluginCanHandleUrlDogan(PluginCanHandleUrl):
     __plugin__ = Dogan
 
     should_match = [
-        'https://www.cnnturk.com/action/embedvideo/',
-        'https://www.cnnturk.com/action/embedvideo/5fa56d065cf3b018a8dd0bbc',
-        'https://www.cnnturk.com/canli-yayin',
-        'https://www.cnnturk.com/tv-cnn-turk/',
-        'https://www.cnnturk.com/tv-cnn-turk/belgeseller/bir-zamanlar/bir-zamanlar-90lar-belgeseli',
-        'https://www.cnnturk.com/video/',
-        'https://www.cnnturk.com/video/turkiye/polis-otomobiliyle-tur-atan-sahisla-ilgili-islem-baslatildi-video',
-        'https://www.dreamturk.com.tr/canli',
-        'https://www.dreamturk.com.tr/canli-yayin-izle',
-        'https://www.dreamturk.com.tr/dream-turk-ozel/',
-        'https://www.dreamturk.com.tr/dream-turk-ozel/radyo-d/ilyas-yalcintas-radyo-dnin-konugu-oldu',
-        'https://www.dreamturk.com.tr/programlar/',
-        'https://www.dreamturk.com.tr/programlar/t-rap/bolumler/t-rap-keisan-ozel',
-        'https://www.dreamtv.com.tr/dream-ozel/',
-        'https://www.dreamtv.com.tr/dream-ozel/konserler/acik-sahne-dream-ozel',
-        'https://www.kanald.com.tr/canli-yayin',
-        'https://www.kanald.com.tr/sadakatsiz/fragmanlar/sadakatsiz-10-bolum-fragmani',
-        'https://www.teve2.com.tr/canli-yayin',
-        'https://www.teve2.com.tr/diziler/',
-        'https://www.teve2.com.tr/diziler/guncel/oyle-bir-gecer-zaman-ki/bolumler/oyle-bir-gecer-zaman-ki-1-bolum',
-        'https://www.teve2.com.tr/embed/',
-        'https://www.teve2.com.tr/embed/55f6d5b8402011f264ec7f64',
-        'https://www.teve2.com.tr/filmler/',
-        'https://www.teve2.com.tr/filmler/guncel/yasamak-guzel-sey',
-        'https://www.teve2.com.tr/programlar/',
-        'https://www.teve2.com.tr/programlar/guncel/kelime-oyunu/bolumler/kelime-oyunu-800-bolum-19-12-2020',
-    ]
+        "https://www.cnnturk.com/canli-yayin",
+        "https://www.cnnturk.com/action/embedvideo/5fa56d065cf3b018a8dd0bbc",
+        "https://www.cnnturk.com/tv-cnn-turk/belgeseller/bir-zamanlar/bir-zamanlar-90lar-belgeseli",
+        "https://www.cnnturk.com/video/turkiye/polis-otomobiliyle-tur-atan-sahisla-ilgili-islem-baslatildi-video",
+
+        "https://www.dreamturk.com.tr/canli-yayin-izle",
+        "https://www.dreamturk.com.tr/dream-turk-ozel/radyo-d/ilyas-yalcintas-radyo-dnin-konugu-oldu",
+        "https://www.dreamturk.com.tr/programlar/dream-10",
+        "https://www.dreamtv.com.tr/dream-ozel/konserler/acik-sahne-dream-ozel",
+
+        "https://www.teve2.com.tr/canli-yayin",
+        "https://www.teve2.com.tr/diziler/guncel/oyle-bir-gecer-zaman-ki/bolumler/oyle-bir-gecer-zaman-ki-1-bolum",
+        "https://www.teve2.com.tr/embed/55f6d5b8402011f264ec7f64",
+        "https://www.teve2.com.tr/filmler/guncel/yasamak-guzel-sey",
+        "https://www.teve2.com.tr/programlar/guncel/kelime-oyunu/bolumler/kelime-oyunu-800-bolum-19-12-2020",
 
-    should_not_match = [
-        'https://www.dreamtv.com.tr/canli-yayin',
+        "https://www.kanald.com.tr/canli-yayin",
+        "https://www.kanald.com.tr/embed/5fda41a8ebc8302048167df6",
+        "https://www.kanald.com.tr/sadakatsiz/fragmanlar/sadakatsiz-10-bolum-fragmani",
     ]