Repository: streamlink/streamlink
Pull request #1144 (2017-08-04 17:56:25)

plugins: add priority ordering to plugins
Plugins can return a priority level for the URLs that they support.
If two plugins match the same URLs then the plugin with the higher priority will be used. This means that the built-in HLS and HDS plugins can support `.m3u8` and `.f4m` URLs with a lower priority without causing issues with overlapping URL patterns.

By default plugins return NORMAL priority for any URL. It is expected that the `priority` method will only be called with URLs that are actually supported by the plugin.

Thoughts? 

codecov[bot]: # [Codecov](https://codecov.io/gh/streamlink/streamlink/pull/1144?src=pr&el=h1) Report
> Merging [#1144](https://codecov.io/gh/streamlink/streamlink/pull/1144?src=pr&el=desc) into [master](https://codecov.io/gh/streamlink/streamlink/commit/f69e7cdf015d93e458ebb0b0cd3e4b7c6d46680e?src=pr&el=desc) will **increase** coverage by `0.03%`.
> The diff coverage is `84.61%`.


```diff
@@            Coverage Diff             @@
##           master    #1144      +/-   ##
==========================================
+ Coverage   47.44%   47.47%   +0.03%     
==========================================
  Files          96       96              
  Lines        7078     7079       +1     
==========================================
+ Hits         3358     3361       +3     
+ Misses       3720     3718       -2
```

bastimeyer: This sounds like a reasonable addition, but you should also add some tests for such a change.
beardypig: Yeah, you're right - I'll add some. 
beardypig: Added a test that tests the correct plugin is returned when multiple plugins support the same URL. The other tests cover a lot of cases already. 
gravyboat: Okay, one last fix here, can you resolve that merge conflict? Sorry @beardypig.

Patch:
diff --git a/src/streamlink/plugin/plugin.py b/src/streamlink/plugin/plugin.py
index 971300f6..7c17743f 100644
--- a/src/streamlink/plugin/plugin.py
+++ b/src/streamlink/plugin/plugin.py
@@ -42,6 +42,11 @@ FILTER_OPERATORS = {
 
 PARAMS_REGEX = r"(\w+)=({.+?}|\[.+?\]|\(.+?\)|'(?:[^'\\]|\\')*'|\"(?:[^\"\\]|\\\")*\"|\S+)"
 
+HIGH_PRIORITY = 30
+NORMAL_PRIORITY = 20
+LOW_PRIORITY = 10
+NO_PRIORITY = 0
+
 
 def stream_weight(stream):
     for group, weights in QUALITY_WEIGTHS_EXTRA.items():
@@ -218,6 +223,15 @@ class Plugin(object):
 
         return decorator
 
+    @classmethod
+    def priority(cls, url):
+        """
+        Return the plugin priority for a given URL, by default it returns
+        NORMAL priority.
+        :return: priority level
+        """
+        return NORMAL_PRIORITY
+
     def streams(self, stream_types=None, sorting_excludes=None):
         """Attempts to extract available streams.
 
diff --git a/src/streamlink/plugins/hds.py b/src/streamlink/plugins/hds.py
index 00b85344..f30028e3 100644
--- a/src/streamlink/plugins/hds.py
+++ b/src/streamlink/plugins/hds.py
@@ -1,22 +1,44 @@
 import re
 
 from streamlink.plugin import Plugin
-from streamlink.plugin.plugin import parse_url_params
+from streamlink.plugin.plugin import parse_url_params, LOW_PRIORITY, NORMAL_PRIORITY, NO_PRIORITY
 from streamlink.stream import HDSStream
 from streamlink.utils import update_scheme
+from streamlink.compat import urlparse
 
 
 class HDSPlugin(Plugin):
-    _url_re = re.compile(r"hds://(.+)")
+    _url_re = re.compile(r"(hds://)?(.+(?:\.f4m)?.*)")
+
+    @classmethod
+    def priority(cls, url):
+        """
+        Returns LOW priority if the URL is not prefixed with hds:// but ends with
+        .f4m and return NORMAL priority if the URL is prefixed.
+        :param url: the URL to find the plugin priority for
+        :return: plugin priority for the given URL
+        """
+        m = cls._url_re.match(url)
+        if m:
+            prefix, url = cls._url_re.match(url).groups()
+            url_path = urlparse(url).path
+            if prefix is None and url_path.endswith(".f4m"):
+                return LOW_PRIORITY
+            elif prefix is not None:
+                return NORMAL_PRIORITY
+        return NO_PRIORITY
 
     @classmethod
     def can_handle_url(cls, url):
-        return cls._url_re.match(url) is not None
+        m = cls._url_re.match(url)
+        if m:
+            url_path = urlparse(m.group(2)).path
+            return m.group(1) is not None or url_path.endswith(".f4m")
 
     def _get_streams(self):
         url, params = parse_url_params(self.url)
 
-        urlnoproto = self._url_re.match(url).group(1)
+        urlnoproto = self._url_re.match(url).group(2)
         urlnoproto = update_scheme("http://", urlnoproto)
 
         return HDSStream.parse_manifest(self.session, urlnoproto, **params)
diff --git a/src/streamlink/plugins/hls.py b/src/streamlink/plugins/hls.py
index 503d88f5..1697be1f 100644
--- a/src/streamlink/plugins/hls.py
+++ b/src/streamlink/plugins/hls.py
@@ -1,21 +1,43 @@
 import re
 
 from streamlink.plugin import Plugin
-from streamlink.plugin.plugin import parse_url_params
+from streamlink.plugin.plugin import parse_url_params, LOW_PRIORITY, NORMAL_PRIORITY, NO_PRIORITY
 from streamlink.stream import HLSStream
 from streamlink.utils import update_scheme
+from streamlink.compat import urlparse
 
 
 class HLSPlugin(Plugin):
-    _url_re = re.compile(r"hls(?:variant)?://(.+)")
+    _url_re = re.compile(r"(hls(?:variant)?://)?(.+(?:\.m3u8)?.*)")
+
+    @classmethod
+    def priority(cls, url):
+        """
+        Returns LOW priority if the URL is not prefixed with hls:// but ends with
+        .m3u8 and return NORMAL priority if the URL is prefixed.
+        :param url: the URL to find the plugin priority for
+        :return: plugin priority for the given URL
+        """
+        m = cls._url_re.match(url)
+        if m:
+            prefix, url = cls._url_re.match(url).groups()
+            url_path = urlparse(url).path
+            if prefix is None and url_path.endswith(".m3u8"):
+                return LOW_PRIORITY
+            elif prefix is not None:
+                return NORMAL_PRIORITY
+        return NO_PRIORITY
 
     @classmethod
     def can_handle_url(cls, url):
-        return cls._url_re.match(url) is not None
+        m = cls._url_re.match(url)
+        if m:
+            url_path = urlparse(m.group(2)).path
+            return m.group(1) is not None or url_path.endswith(".m3u8")
 
     def _get_streams(self):
         url, params = parse_url_params(self.url)
-        urlnoproto = self._url_re.match(url).group(1)
+        urlnoproto = self._url_re.match(url).group(2)
         urlnoproto = update_scheme("http://", urlnoproto)
 
         self.logger.debug("URL={0}; params={1}", urlnoproto, params)
diff --git a/src/streamlink/session.py b/src/streamlink/session.py
index aba75a86..7299a2e3 100644
--- a/src/streamlink/session.py
+++ b/src/streamlink/session.py
@@ -1,16 +1,13 @@
 import imp
 import pkgutil
-import re
 import sys
 import traceback
-
 import requests
 
 from streamlink.utils import update_scheme
 from streamlink.utils.l10n import Localization
-
 from . import plugins, __version__
-from .compat import urlparse, is_win32
+from .compat import is_win32
 from .exceptions import NoPluginError, PluginError
 from .logger import Logger
 from .options import Options
@@ -350,7 +347,7 @@ class Streamlink(object):
         """
         self.logger.set_output(output)
 
-    def resolve_url(self, url):
+    def resolve_url(self, url, follow_redirect=True):
         """Attempts to find a plugin that can use this URL.
 
         The default protocol (http) will be prefixed to the URL if
@@ -359,30 +356,33 @@ class Streamlink(object):
         Raises :exc:`NoPluginError` on failure.
 
         :param url: a URL to match against loaded plugins
+        :param follow_redirect: follow redirects
 
         """
-        parsed = urlparse(url)
-
-        if len(parsed.scheme) == 0:
-            url = "http://" + url
+        url = update_scheme("http://", url)
 
+        available_plugins = []
         for name, plugin in self.plugins.items():
             if plugin.can_handle_url(url):
-                obj = plugin(url)
-                return obj
+                available_plugins.append(plugin)
 
-        # Attempt to handle a redirect URL
-        try:
-            res = self.http.head(url, allow_redirects=True, acceptable_status=[501])
+        available_plugins.sort(key=lambda x: x.priority(url), reverse=True)
+        if available_plugins:
+            return available_plugins[0](url)
 
-            # Fall back to GET request if server doesn't handle HEAD.
-            if res.status_code == 501:
-                res = self.http.get(url, stream=True)
+        if follow_redirect:
+            # Attempt to handle a redirect URL
+            try:
+                res = self.http.head(url, allow_redirects=True, acceptable_status=[501])
+
+                # Fall back to GET request if server doesn't handle HEAD.
+                if res.status_code == 501:
+                    res = self.http.get(url, stream=True)
 
-            if res.url != url:
-                return self.resolve_url(res.url)
-        except PluginError:
-            pass
+                if res.url != url:
+                    return self.resolve_url(res.url, follow_redirect=follow_redirect)
+            except PluginError:
+                pass
 
         raise NoPluginError
 
@@ -397,17 +397,7 @@ class Streamlink(object):
         :param url: a URL to match against loaded plugins
 
         """
-        parsed = urlparse(url)
-
-        if len(parsed.scheme) == 0:
-            url = "http://" + url
-
-        for name, plugin in self.plugins.items():
-            if plugin.can_handle_url(url):
-                obj = plugin(url)
-                return obj
-
-        raise NoPluginError
+        return self.resolve_url(url, follow_redirect=False)
 
     def streams(self, url, **params):
         """Attempts to find a plugin and extract streams from the *url*.


Test patch:
diff --git a/tests/test_session.py b/tests/test_session.py
index 026e2796..5fd5b106 100644
--- a/tests/test_session.py
+++ b/tests/test_session.py
@@ -1,7 +1,14 @@
 import os
 import unittest
 
-from streamlink import Streamlink, PluginError, NoPluginError
+from streamlink.plugin.plugin import HIGH_PRIORITY, LOW_PRIORITY
+
+try:
+    from unittest.mock import MagicMock
+except ImportError:
+    from mock import MagicMock
+
+from streamlink import Streamlink, NoPluginError
 from streamlink.plugins import Plugin
 from streamlink.stream import *
 
@@ -34,6 +41,30 @@ class TestSession(unittest.TestCase):
         self.assertTrue(isinstance(channel, Plugin))
         self.assertTrue(isinstance(channel, plugins["testplugin"]))
 
+    def test_resolve_url_priority(self):
+        from tests.plugins.testplugin import TestPlugin
+
+        class HighPriority(TestPlugin):
+            @classmethod
+            def priority(cls, url):
+                return HIGH_PRIORITY
+
+        class LowPriority(TestPlugin):
+            @classmethod
+            def priority(cls, url):
+                return LOW_PRIORITY
+
+        self.session.plugins = {
+            "test_plugin": TestPlugin,
+            "test_plugin_low": LowPriority,
+            "test_plugin_high": HighPriority,
+        }
+        channel = self.session.resolve_url_no_redirect("http://test.se/channel")
+        plugins = self.session.get_plugins()
+
+        self.assertTrue(isinstance(channel, plugins["test_plugin_high"]))
+        self.assertEqual(HIGH_PRIORITY, channel.priority(channel.url))
+
     def test_resolve_url_no_redirect(self):
         plugins = self.session.get_plugins()
         channel = self.session.resolve_url_no_redirect("http://test.se/channel")