Repository: streamlink/streamlink (python)

Issue:
Python 3.6: DeprecationWarning: inspect.getargspec() is deprecated
### Checklist

- [x] This is a bug report.
- [ ] This is a feature request.
- [ ] This is a plugin (improvement) request.
- [ ] I have read the contribution guidelines.

### Description

Running tests with python 3.6 triggers a deprecation warning on inspect.getargspec().
I'm not sure there are better way to handle this and still support python 2.7 though.

Maybe waiting this function to be removed or removal of python 2.7 support in streamlink is enough.

```
.pybuild/cpython3_3.6/build/tests/test_log.py::TestDeprecatedLogger::test_deprecated_session_logger
  /<<BUILDDIR>>/streamlink-0.13.0+dfsg/.pybuild/cpython3_3.6/build/streamlink/packages/flashmedia/types.py:1051: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()
    amfcls.__members__ = getargspec(amfcls.__init__).args[1:]
  /<<BUILDDIR>>/streamlink-0.13.0+dfsg/.pybuild/cpython3_3.6/build/streamlink/packages/flashmedia/types.py:1051: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() or inspect.getfullargspec()
    amfcls.__members__ = getargspec(amfcls.__init__).args[1:]
```

### Expected / Actual behavior

No python warning about this while running streamlink tests.

### Reproduction steps / Explicit stream URLs to test

Run `PYTHONPATH=src/ pytest-3 tests` on root directory with python 3.6 installed.

### Logs

See here: https://travis-ci.org/amurzeau/debian-travis/builds/393389349#L2492

[Love Streamlink? Please consider supporting our collective. Thanks!](https://opencollective.com/streamlink/donate)


Pull request:
Suppress/fix deprecated warnings for Python 3
`inspect.getargspec` was deprecated since 3.0 in favour of `inspect.getfullargspec`. For most uses the methods are interchangeable, so depending on if it exists or not can use one or the other. 

I added a decorator method to `tests` so that warnings can be suppressed (or captured) on a per-tests basis.

Fixes #1831, the only problem may be that `flashmedia` is untested. 

Patch:
diff --git a/src/streamlink/compat.py b/src/streamlink/compat.py
index 43b1d8f8..0eedef5d 100644
--- a/src/streamlink/compat.py
+++ b/src/streamlink/compat.py
@@ -1,5 +1,6 @@
 import os
 import sys
+import inspect
 
 is_py2 = (sys.version_info[0] == 2)
 is_py3 = (sys.version_info[0] == 3)
@@ -46,8 +47,18 @@ try:
 except ImportError:
     from backports.shutil_which import which
 
+try:
+    from html import unescape as html_unescape
+except ImportError:
+    from HTMLParser import HTMLParser
+    html_unescape = unescape = HTMLParser().unescape
+
+
+
+getargspec = getattr(inspect, "getfullargspec", inspect.getargspec)
+
 
 __all__ = ["is_py2", "is_py3", "is_py33", "is_win32", "str", "bytes",
            "urlparse", "urlunparse", "urljoin", "parse_qsl", "quote",
            "unquote", "queue", "range", "urlencode", "devnull", "which",
-           "izip", "urlsplit", "urlunsplit"]
+           "izip", "urlsplit", "urlunsplit", "getargspec", "html_unescape"]
diff --git a/src/streamlink/packages/flashmedia/types.py b/src/streamlink/packages/flashmedia/types.py
index cc4fccae..f6cd14bf 100644
--- a/src/streamlink/packages/flashmedia/types.py
+++ b/src/streamlink/packages/flashmedia/types.py
@@ -3,7 +3,10 @@ from .util import pack_bytes_into
 
 from collections import namedtuple
 from struct import Struct, error as struct_error
-from inspect import getargspec
+import inspect
+
+
+getargspec = getattr(inspect, "getfullargspec", inspect.getargspec)
 
 (SCRIPT_DATA_TYPE_NUMBER, SCRIPT_DATA_TYPE_BOOLEAN,
  SCRIPT_DATA_TYPE_STRING, SCRIPT_DATA_TYPE_OBJECT,
diff --git a/src/streamlink/plugin/api/support_plugin.py b/src/streamlink/plugin/api/support_plugin.py
index 96afb178..4b1d96b9 100644
--- a/src/streamlink/plugin/api/support_plugin.py
+++ b/src/streamlink/plugin/api/support_plugin.py
@@ -1,6 +1,7 @@
 import os
 import inspect
 import sys
+from streamlink.utils import load_module
 
 __all__ = ["load_support_plugin"]
 
@@ -26,26 +27,4 @@ def load_support_plugin(name):
         prefix = os.path.normpath(__file__ + "../../../../../")
         path = os.path.join(prefix, path)
 
-    # importlib is the preferred way of importing a module, but it's
-    # only available on Python 3.1+.
-    if sys.version_info[0] == 3 and sys.version_info[1] >= 3:
-        import importlib
-
-        loader = importlib.find_loader(name, [path])
-
-        if loader:
-            module = loader.load_module()
-        else:
-            raise ImportError("No module named '{0}'".format(name))
-    else:
-        import imp
-
-        fd, filename, desc = imp.find_module(name, [path])
-
-        try:
-            module = imp.load_module(name, fd, filename, desc)
-        finally:
-            if fd:
-                fd.close()
-
-    return module
+    return load_module(name, path)
diff --git a/src/streamlink/plugins/rtbf.py b/src/streamlink/plugins/rtbf.py
index cafcf5ef..513ce60a 100644
--- a/src/streamlink/plugins/rtbf.py
+++ b/src/streamlink/plugins/rtbf.py
@@ -1,14 +1,11 @@
 import datetime
-try:
-    from HTMLParser import HTMLParser
-except ImportError:
-    from html.parser import HTMLParser
 import re
 
 from streamlink.plugin import Plugin
 from streamlink.plugin.api import http, validate
 from streamlink.stream import DASHStream, HLSStream, HTTPStream
 from streamlink.utils import parse_json
+from streamlink.compat import html_unescape
 
 
 class RTBF(Plugin):
@@ -36,7 +33,7 @@ class RTBF(Plugin):
             None,
             validate.all(
                 validate.get(1),
-                validate.transform(HTMLParser().unescape),
+                validate.transform(html_unescape),
                 validate.transform(parse_json),
                 {
                     'geoLocRestriction': validate.text,
diff --git a/src/streamlink/stream/http.py b/src/streamlink/stream/http.py
index 09752111..07ba74f1 100644
--- a/src/streamlink/stream/http.py
+++ b/src/streamlink/stream/http.py
@@ -1,10 +1,9 @@
-import inspect
-
 import requests
 
-from .stream import Stream
-from .wrappers import StreamIOThreadWrapper, StreamIOIterWrapper
-from ..exceptions import StreamError
+from streamlink.compat import getargspec
+from streamlink.exceptions import StreamError
+from streamlink.stream import Stream
+from streamlink.stream.wrappers import StreamIOThreadWrapper, StreamIOIterWrapper
 
 
 def normalize_key(keyval):
@@ -15,7 +14,7 @@ def normalize_key(keyval):
 
 
 def valid_args(args):
-    argspec = inspect.getargspec(requests.Request.__init__)
+    argspec = getargspec(requests.Request.__init__)
 
     return dict(filter(lambda kv: kv[0] in argspec.args, args.items()))
 
diff --git a/src/streamlink/utils/__init__.py b/src/streamlink/utils/__init__.py
index 658ed6a8..eb4eebf2 100644
--- a/src/streamlink/utils/__init__.py
+++ b/src/streamlink/utils/__init__.py
@@ -1,8 +1,7 @@
+import functools
 import json
 import re
 import zlib
-import collections
-import functools
 
 try:
     import xml.etree.cElementTree as ET
@@ -219,6 +218,34 @@ def search_dict(data, key):
                 yield result
 
 
+def load_module(name, path=None):
+    if is_py3:
+        import importlib.machinery
+        import importlib.util
+        import sys
+
+        loader_details = [(importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES)]
+        finder = importlib.machinery.FileFinder(path, *loader_details)
+        spec = finder.find_spec(name)
+        if not spec or not spec.loader:
+            raise ImportError("no module named {0}".format(name))
+        if sys.version_info[1] > 4:
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            return mod
+        else:
+            return spec.loader.load_module(name)
+
+    else:
+        import imp
+        fd, filename, desc = imp.find_module(name, path and [path])
+        try:
+            return imp.load_module(name, fd, filename, desc)
+        finally:
+            if fd:
+                fd.close()
+
+
 #####################################
 # Deprecated functions, do not use. #
 #####################################


Test patch:
diff --git a/tests/__init__.py b/tests/__init__.py
index fc89b671..a6e20084 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,8 +1,17 @@
-import sys
+import warnings
 
-if sys.version_info[0:2] == (2, 6):
-    import unittest2 as unittest
-else:
-    import unittest
 
-__all__ = ['unittest']
+def catch_warnings(record=False, module=None):
+    def _catch_warnings_wrapper(f):
+        def _catch_warnings(*args, **kwargs):
+            with warnings.catch_warnings(record=True, module=module) as w:
+                if record:
+                    return f(*(args + (w, )), **kwargs)
+                else:
+                    return f(*args, **kwargs)
+        return _catch_warnings
+
+    return _catch_warnings_wrapper
+
+
+__all__ = ['ignore_warnings']
diff --git a/tests/streams/test_dash.py b/tests/streams/test_dash.py
index 26f3a2d0..ff489f7e 100644
--- a/tests/streams/test_dash.py
+++ b/tests/streams/test_dash.py
@@ -1,11 +1,9 @@
-from streamlink.stream.dash import DASHStreamWorker
-from streamlink.stream.dash_manifest import MPD
-
-from tests import unittest
-from tests.mock import MagicMock, patch, ANY, Mock, call
+import unittest
 
 from streamlink import PluginError
 from streamlink.stream import *
+from streamlink.stream.dash import DASHStreamWorker
+from tests.mock import MagicMock, patch, ANY, Mock, call
 
 
 class TestDASHStream(unittest.TestCase):
diff --git a/tests/streams/test_dash_parser.py b/tests/streams/test_dash_parser.py
index b7f58b88..09600859 100644
--- a/tests/streams/test_dash_parser.py
+++ b/tests/streams/test_dash_parser.py
@@ -1,15 +1,14 @@
 from __future__ import unicode_literals
 
 import datetime
-from freezegun import freeze_time
-
 import itertools
+import unittest
 from operator import attrgetter
 
+from freezegun import freeze_time
 from freezegun.api import FakeDatetime
 
-from streamlink.stream.dash_manifest import MPD, MPDParsers, MPDParsingError, utc, datetime_to_seconds
-from tests import unittest
+from streamlink.stream.dash_manifest import MPD, MPDParsers, MPDParsingError, utc
 from tests.resources import xml
 
 
@@ -20,26 +19,26 @@ class TestMPDParsers(unittest.TestCase):
         self.assertEqual(utc.utcoffset(None), datetime.timedelta(0))
 
     def test_bool_str(self):
-        self.assertEquals(MPDParsers.bool_str("true"), True)
-        self.assertEquals(MPDParsers.bool_str("TRUE"), True)
-        self.assertEquals(MPDParsers.bool_str("True"), True)
+        self.assertEqual(MPDParsers.bool_str("true"), True)
+        self.assertEqual(MPDParsers.bool_str("TRUE"), True)
+        self.assertEqual(MPDParsers.bool_str("True"), True)
 
-        self.assertEquals(MPDParsers.bool_str("0"), False)
-        self.assertEquals(MPDParsers.bool_str("False"), False)
-        self.assertEquals(MPDParsers.bool_str("false"), False)
-        self.assertEquals(MPDParsers.bool_str("FALSE"), False)
+        self.assertEqual(MPDParsers.bool_str("0"), False)
+        self.assertEqual(MPDParsers.bool_str("False"), False)
+        self.assertEqual(MPDParsers.bool_str("false"), False)
+        self.assertEqual(MPDParsers.bool_str("FALSE"), False)
 
     def test_type(self):
-        self.assertEquals(MPDParsers.type("dynamic"), "dynamic")
-        self.assertEquals(MPDParsers.type("static"), "static")
+        self.assertEqual(MPDParsers.type("dynamic"), "dynamic")
+        self.assertEqual(MPDParsers.type("static"), "static")
         with self.assertRaises(MPDParsingError):
             MPDParsers.type("other")
 
     def test_duration(self):
-        self.assertEquals(MPDParsers.duration("PT1S"), datetime.timedelta(0, 1))
+        self.assertEqual(MPDParsers.duration("PT1S"), datetime.timedelta(0, 1))
 
     def test_datetime(self):
-        self.assertEquals(MPDParsers.datetime("2018-01-01T00:00:00Z"),
+        self.assertEqual(MPDParsers.datetime("2018-01-01T00:00:00Z"),
                           datetime.datetime(2018, 1, 1, 0, 0, 0, tzinfo=utc))
 
     def test_segment_template(self):
diff --git a/tests/test_hls.py b/tests/test_hls.py
index ba049c4f..39cecc65 100644
--- a/tests/test_hls.py
+++ b/tests/test_hls.py
@@ -87,7 +87,6 @@ audio_only.m3u8
         # Set to default value to avoid a test fail if the default change
         streamlink.set_option("hls-live-edge", 3)
 
-        streamlink.logger.set_level("debug")
         masterStream = hls.HLSStream.parse_variant_playlist(streamlink, masterPlaylist, **kwargs)
         stream = masterStream["1080p (source)"].open()
         data = b"".join(iter(partial(stream.read, 8192), b""))
@@ -163,7 +162,6 @@ playlist.m3u8
 
         if audio_select:
             streamlink.set_option("hls-audio-select", audio_select)
-        streamlink.logger.set_level("debug")
 
         master_stream = hls.HLSStream.parse_variant_playlist(streamlink, playlist)
 
diff --git a/tests/test_log.py b/tests/test_log.py
index d2cc293b..08ffc45e 100644
--- a/tests/test_log.py
+++ b/tests/test_log.py
@@ -11,6 +11,9 @@ if is_py2:
 else:
     from io import StringIO
 
+from tests import catch_warnings
+
+
 
 class TestLogging(unittest.TestCase):
     @classmethod
@@ -70,17 +73,20 @@ class TestDeprecatedLogger(unittest.TestCase):
         manager.set_output(output)
         return manager, output
 
-    def test_level(self):
+    @catch_warnings()
+    def test_deprecated_level(self):
         manager, output = self._new_logger()
 
-        log = manager.new_module("test_level")
-        log.debug("test")
-        self.assertEqual(output.tell(), 0)
-        manager.set_level("debug")
-        log.debug("test")
-        self.assertNotEqual(output.tell(), 0)
+        with warnings.catch_warnings(record=True):
+            log = manager.new_module("test_level")
+            log.debug("test")
+            self.assertEqual(output.tell(), 0)
+            manager.set_level("debug")
+            log.debug("test")
+            self.assertNotEqual(output.tell(), 0)
 
-    def test_output(self):
+    @catch_warnings()
+    def test_deprecated_output(self):
         manager, output = self._new_logger()
 
         log = manager.new_module("test_output")
@@ -88,6 +94,7 @@ class TestDeprecatedLogger(unittest.TestCase):
         log.debug("test")
         self.assertEqual(output.getvalue(), "[test_output][debug] test\n")
 
+    @catch_warnings()
     def test_deprecated_session_logger(self):
         session = Streamlink()
         output = StringIO()
diff --git a/tests/test_meta.py b/tests/test_meta.py
new file mode 100644
index 00000000..f50066f5
--- /dev/null
+++ b/tests/test_meta.py
@@ -0,0 +1,30 @@
+import unittest
+import warnings
+
+from tests import catch_warnings
+
+
+class TestMeta(unittest.TestCase):
+    """
+    Meta tests, to test the tests or test utils
+    """
+    def test_catch_warnings(self):
+
+        @catch_warnings()
+        def _assert_false():
+            assert False
+
+        self.assertRaises(AssertionError, _assert_false)
+
+    def test_catch_warnings_record(self):
+
+        @catch_warnings(record=True)
+        def _includes_warnings(w):
+            def _inner():
+                warnings.warn("a warning")
+
+            _inner()
+            self.assertEqual(1, len(w))
+            return True
+
+        self.assertEqual(True, _includes_warnings())
diff --git a/tests/test_stream_to_url.py b/tests/test_stream_to_url.py
index 5e9712ed..2894be21 100644
--- a/tests/test_stream_to_url.py
+++ b/tests/test_stream_to_url.py
@@ -27,7 +27,7 @@ class TestStreamToURL(unittest.TestCase):
 
     def test_http_stream(self):
         expected = "http://test.se/stream"
-        stream = HTTPStream(self.session, expected)
+        stream = HTTPStream(self.session, expected, invalid_arg="invalid")
         self.assertEqual(expected, stream_to_url(stream))
         self.assertEqual(expected, stream.to_url())
 
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 68a6a0fd..8d2714f6 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,7 +1,8 @@
 import sys
+import os.path
 
 from streamlink.plugin.api.validate import xml_element, text
-from streamlink.utils import update_scheme, url_equal, search_dict
+from streamlink.utils import update_scheme, url_equal, search_dict, load_module
 
 try:
     import xml.etree.cElementTree as ET
@@ -11,10 +12,10 @@ from streamlink import PluginError
 from streamlink.plugin.api import validate
 from streamlink.utils import *
 
-if sys.version_info[0:2] == (2, 6):
-    import unittest2 as unittest
-else:
-    import unittest
+import unittest
+
+# used in the import test to verify that this module was imported
+__test_marker__ = "test_marker"
 
 
 class TestUtil(unittest.TestCase):
@@ -151,3 +152,12 @@ class TestUtil(unittest.TestCase):
             list(sorted(search_dict({"one": [{"inner": "test1"}], "two": {"inner": "test2"}}, "inner"))),
             list(sorted(["test1", "test2"]))
         )
+
+    def test_load_module_non_existent(self):
+        self.assertRaises(ImportError, load_module, "non_existent_module", os.path.dirname(__file__))
+
+    def test_load_module(self):
+        self.assertEqual(
+            sys.modules[__name__].__test_marker__,
+            load_module(__name__.split(".")[-1], os.path.dirname(__file__)).__test_marker__
+        )