Repository: streamlink/streamlink

Issue:
[Plugin Request] pixiv Sketch
- [ ] This is a bug report.
- [x] This is a feature request.
- [ ] This is a plugin (improvement) request.
- [x] I have read the contribution guidelines.

I'm requesting a plugin for pixiv sketch 

its streamed via HLS it serves a generated .m3u8(for 30ish seconds) along with its 3ish second chunk every second or so

| xhr | https://hls4.pixivsketch.net/2018022718/15197228728388594847a96103176027c0811ead7c5788b1297/4000000_1920x1080/506579247.ts
| xhr | https://hls4.pixivsketch.net/2018022718/15197228728388594847a96103176027c0811ead7c5788b1297/4000000_1920x1080/index.m3u8

hls4.pixivsketch.net: fixed gen'd server*
2018022718: stream start time (YYYYMMDDHH)
1519...1297: fixed UID*
4000000_1920x1080: video quality (low = 365000_480x270)

*Until the api regens (The streamers internet drops)

a look at the .m3u8
```
#EXTM3U
#EXT-X-VERSION:6
#EXT-X-MEDIA-SEQUENCE:506579247
#EXT-X-TARGETDURATION:6
#EXT-X-START:TIME-OFFSET=-3.00,PRECISE=NO
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:21.000741924+09:00
#EXTINF:3.000000,
506579247.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:24.000744924+09:00
#EXTINF:3.000000,
506579248.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:27.000747924+09:00
#EXTINF:3.000000,
506579249.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:30.000750924+09:00
#EXTINF:3.000000,
506579250.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:33.000753924+09:00
#EXTINF:3.000000,
506579251.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:36.000756924+09:00
#EXTINF:3.000000,
506579252.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:39.000759924+09:00
#EXTINF:3.000000,
506579253.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:42.000762924+09:00
#EXTINF:3.000000,
506579254.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:45.000765924+09:00
#EXTINF:3.000000,
506579255.ts
#EXT-X-PROGRAM-DATE-TIME:2018-02-27T22:22:48.000768924+09:00
#EXTINF:3.000000,
506579256.ts
```
**Update** found hlsvariant://... and it works but a proper plugin is appreciated

Comments:

Patch:
diff --git a/docs/plugin_matrix.rst b/docs/plugin_matrix.rst
index b3f1584e..8ba5f492 100644
--- a/docs/plugin_matrix.rst
+++ b/docs/plugin_matrix.rst
@@ -141,6 +141,7 @@ ovvatv              ovva.tv              Yes   No
 pandatv             panda.tv             Yes   ?
 periscope           periscope.tv         Yes   Yes   Replay/VOD is supported.
 picarto             picarto.tv           Yes   Yes
+pixiv               sketch.pixiv.net     Yes   --
 playtv              playtv.fr            Yes   --    Streams may be geo-restricted to France.
 pluzz               - france.tv          Yes   Yes   Streams may be geo-restricted to France, Andorra and Monaco.
                     - ludo.fr
diff --git a/src/streamlink/plugins/pixiv.py b/src/streamlink/plugins/pixiv.py
new file mode 100644
index 00000000..5010f563
--- /dev/null
+++ b/src/streamlink/plugins/pixiv.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+import re
+
+from streamlink import PluginError
+from streamlink.compat import urljoin
+from streamlink.exceptions import NoStreamsError
+from streamlink.plugin import Plugin
+from streamlink.plugin import PluginOptions
+from streamlink.plugin.api import http
+from streamlink.plugin.api import useragents
+from streamlink.plugin.api import validate
+from streamlink.stream import HLSStream
+from streamlink.utils import parse_json
+
+
+class Pixiv(Plugin):
+    """Plugin for https://sketch.pixiv.net/lives"""
+
+    _url_re = re.compile(r"https?://sketch\.pixiv\.net/[^/]+(?P<videopage>/lives/\d+)?")
+
+    _videopage_re = re.compile(r"""["']live-button["']><a\shref=["'](?P<path>[^"']+)["']""")
+    _data_re = re.compile(r"""<script\sid=["']state["']>[^><{]+(?P<data>{[^><]+})</script>""")
+    _post_key_re = re.compile(r"""name=["']post_key["']\svalue=["'](?P<data>[^"']+)["']""")
+
+    _data_schema = validate.Schema(
+        validate.all(
+            validate.transform(_data_re.search),
+            validate.any(
+                None,
+                validate.all(
+                    validate.get("data"),
+                    validate.transform(parse_json),
+                    validate.get("context"),
+                    validate.get("dispatcher"),
+                    validate.get("stores"),
+                )
+            )
+        )
+    )
+
+    login_url_get = "https://accounts.pixiv.net/login"
+    login_url_post = "https://accounts.pixiv.net/api/login"
+
+    options = PluginOptions({
+        "username": None,
+        "password": None
+    })
+
+    @classmethod
+    def can_handle_url(cls, url):
+        return cls._url_re.match(url) is not None
+
+    def find_videopage(self):
+        self.logger.debug("Not a videopage")
+        res = http.get(self.url)
+
+        m = self._videopage_re.search(res.text)
+        if not m:
+            self.logger.debug("No stream path, stream might be offline or invalid url.")
+            raise NoStreamsError(self.url)
+
+        path = m.group("path")
+        self.logger.debug("Found new path: {0}".format(path))
+        return urljoin(self.url, path)
+
+    def _login(self, username, password):
+        res = http.get(self.login_url_get)
+        m = self._post_key_re.search(res.text)
+        if not m:
+            raise PluginError("Missing post_key, no login posible.")
+
+        post_key = m.group("data")
+        data = {
+            "lang": "en",
+            "source": "sketch",
+            "post_key": post_key,
+            "pixiv_id": username,
+            "password": password,
+        }
+
+        res = http.post(self.login_url_post, data=data)
+        res = http.json(res)
+
+        if res["body"].get("success"):
+            return True
+        else:
+            return False
+
+    def _get_streams(self):
+        http.headers = {"User-Agent": useragents.FIREFOX}
+
+        login_username = self.get_option("username")
+        login_password = self.get_option("password")
+        if login_username and login_password:
+            self.logger.debug("Attempting login as {0}".format(login_username))
+            if self._login(login_username, login_password):
+                self.logger.info("Successfully logged in as {0}".format(login_username))
+            else:
+                self.logger.info("Failed to login as {0}".format(login_username))
+
+        videopage = self._url_re.match(self.url).group("videopage")
+        if not videopage:
+            self.url = self.find_videopage()
+
+        data = http.get(self.url, schema=self._data_schema)
+
+        if not data.get("LiveStore"):
+            self.logger.debug("No video url found, stream might be offline.")
+            return
+
+        data = data["LiveStore"]["lives"]
+
+        # get the unknown user-id
+        for _key in data.keys():
+            video_data = data.get(_key)
+
+        owner = video_data["owner"]
+        self.logger.info("Owner ID: {0}".format(owner["user_id"]))
+        self.logger.debug("HLS URL: {0}".format(owner["hls_movie"]))
+        for n, s in HLSStream.parse_variant_playlist(self.session, owner["hls_movie"]).items():
+            yield n, s
+
+        performers = video_data.get("performers")
+        if performers:
+            for p in performers:
+                self.logger.info("CO-HOST ID: {0}".format(p["user_id"]))
+                hls_url = p["hls_movie"]
+                self.logger.debug("HLS URL: {0}".format(hls_url))
+                for n, s in HLSStream.parse_variant_playlist(self.session, hls_url).items():
+                    _n = "{0}_{1}".format(n, p["user_id"])
+                    yield _n, s
+
+
+__plugin__ = Pixiv
diff --git a/src/streamlink/plugins/youtube.py b/src/streamlink/plugins/youtube.py
index 2046634b..b828036c 100644
--- a/src/streamlink/plugins/youtube.py
+++ b/src/streamlink/plugins/youtube.py
@@ -78,6 +78,8 @@ _config_schema = validate.Schema(
         validate.optional("hlsvp"): validate.text,
         validate.optional("live_playback"): validate.transform(bool),
         validate.optional("reason"): validate.text,
+        validate.optional("livestream"): validate.text,
+        validate.optional("live_playback"): validate.text,
         "status": validate.text
     }
 )
@@ -137,7 +139,7 @@ class YouTube(Plugin):
     }
 
     @classmethod
-    def can_handle_url(self, url):
+    def can_handle_url(cls, url):
         return _url_re.match(url)
 
     @classmethod
@@ -157,13 +159,58 @@ class YouTube(Plugin):
 
         return weight, group
 
+    def _create_adaptive_streams(self, info, streams, protected):
+        adaptive_streams = {}
+        best_audio_itag = None
+
+        # Extract audio streams from the DASH format list
+        for stream_info in info.get("adaptive_fmts", []):
+            if stream_info.get("s"):
+                protected = True
+                continue
+
+            stream_params = dict(parse_qsl(stream_info["url"]))
+            if "itag" not in stream_params:
+                continue
+            itag = int(stream_params["itag"])
+            # extract any high quality streams only available in adaptive formats
+            adaptive_streams[itag] = stream_info["url"]
+
+            stream_type, stream_format = stream_info["type"]
+            if stream_type == "audio":
+                stream = HTTPStream(self.session, stream_info["url"])
+                name = "audio_{0}".format(stream_format)
+                streams[name] = stream
+
+                # find the best quality audio stream m4a, opus or vorbis
+                if best_audio_itag is None or self.adp_audio[itag] > self.adp_audio[best_audio_itag]:
+                    best_audio_itag = itag
+
+        if best_audio_itag and adaptive_streams and MuxedStream.is_usable(self.session):
+            aurl = adaptive_streams[best_audio_itag]
+            for itag, name in self.adp_video.items():
+                if itag in adaptive_streams:
+                    vurl = adaptive_streams[itag]
+                    self.logger.debug("MuxedStream: v {video} a {audio} = {name}".format(
+                        audio=best_audio_itag,
+                        name=name,
+                        video=itag,
+                    ))
+                    streams[name] = MuxedStream(self.session,
+                                                HTTPStream(self.session, vurl),
+                                                HTTPStream(self.session, aurl))
+
+        return streams, protected
+
     def _find_channel_video(self):
         res = http.get(self.url)
         match = _channelid_re.search(res.text)
         if not match:
             return
 
-        return self._get_channel_video(match.group(1))
+        channel_id = match.group(1)
+        self.logger.debug("Found channel_id: {0}".format(channel_id))
+        return self._get_channel_video(channel_id)
 
     def _get_channel_video(self, channel_id):
         query = {
@@ -178,6 +225,7 @@ class YouTube(Plugin):
 
         for video in videos:
             video_id = video["id"]["videoId"]
+            self.logger.debug("Found video_id: {0}".format(video_id))
             return video_id
 
     def _find_canonical_stream_info(self):
@@ -233,10 +281,16 @@ class YouTube(Plugin):
         return info_parsed
 
     def _get_streams(self):
+        is_live = False
+
         info = self._get_stream_info(self.url)
         if not info:
             return
 
+        if info.get("livestream") == '1' or info.get("live_playback") == '1':
+            self.logger.debug("This video is live.")
+            is_live = True
+
         formats = info.get("fmt_list")
         streams = {}
         protected = False
@@ -253,40 +307,8 @@ class YouTube(Plugin):
 
             streams[name] = stream
 
-        adaptive_streams = {}
-        best_audio_itag = None
-
-        # Extract audio streams from the DASH format list
-        for stream_info in info.get("adaptive_fmts", []):
-            if stream_info.get("s"):
-                protected = True
-                continue
-
-            stream_params = dict(parse_qsl(stream_info["url"]))
-            if "itag" not in stream_params:
-                continue
-            itag = int(stream_params["itag"])
-            # extract any high quality streams only available in adaptive formats
-            adaptive_streams[itag] = stream_info["url"]
-
-            stream_type, stream_format = stream_info["type"]
-            if stream_type == "audio":
-                stream = HTTPStream(self.session, stream_info["url"])
-                name = "audio_{0}".format(stream_format)
-                streams[name] = stream
-
-                # find the best quality audio stream m4a, opus or vorbis
-                if best_audio_itag is None or self.adp_audio[itag] > self.adp_audio[best_audio_itag]:
-                    best_audio_itag = itag
-
-        if best_audio_itag and adaptive_streams and MuxedStream.is_usable(self.session):
-            aurl = adaptive_streams[best_audio_itag]
-            for itag, name in self.adp_video.items():
-                if itag in adaptive_streams:
-                    vurl = adaptive_streams[itag]
-                    streams[name] = MuxedStream(self.session,
-                                                HTTPStream(self.session, vurl),
-                                                HTTPStream(self.session, aurl))
+        if is_live is False:
+            streams, protected = self._create_adaptive_streams(info, streams, protected)
 
         hls_playlist = info.get("hlsvp")
         if hls_playlist:
diff --git a/src/streamlink_cli/argparser.py b/src/streamlink_cli/argparser.py
index 46e4ab2a..31cdbbe1 100644
--- a/src/streamlink_cli/argparser.py
+++ b/src/streamlink_cli/argparser.py
@@ -1363,6 +1363,20 @@ plugin.add_argument(
     A afreecatv.com account password to use with --afreeca-username.
     """
 )
+plugin.add_argument(
+    "--pixiv-username",
+    metavar="USERNAME",
+    help="""
+    The email/username used to register with pixiv.net
+    """
+)
+plugin.add_argument(
+    "--pixiv-password",
+    metavar="PASSWORD",
+    help="""
+    A pixiv.net account password to use with --pixiv-username
+    """
+)
 
 # Deprecated options
 stream.add_argument(
diff --git a/src/streamlink_cli/main.py b/src/streamlink_cli/main.py
index dbb1d527..315a84a9 100644
--- a/src/streamlink_cli/main.py
+++ b/src/streamlink_cli/main.py
@@ -964,6 +964,17 @@ def setup_plugin_options():
     if afreeca_password:
         streamlink.set_plugin_option("afreeca", "password", afreeca_password)
 
+    if args.pixiv_username:
+        streamlink.set_plugin_option("pixiv", "username", args.pixiv_username)
+
+    if args.pixiv_username and not args.pixiv_password:
+        pixiv_password = console.askpass("Enter pixiv account password: ")
+    else:
+        pixiv_password = args.pixiv_password
+
+    if pixiv_password:
+        streamlink.set_plugin_option("pixiv", "password", pixiv_password)
+
 
 def check_root():
     if hasattr(os, "getuid"):


Test patch:
diff --git a/tests/test_plugin_pixiv.py b/tests/test_plugin_pixiv.py
new file mode 100644
index 00000000..8d547d3d
--- /dev/null
+++ b/tests/test_plugin_pixiv.py
@@ -0,0 +1,19 @@
+import unittest
+
+from streamlink.plugins.pixiv import Pixiv
+
+
+class TestPluginPixiv(unittest.TestCase):
+    def test_can_handle_url(self):
+        should_match = [
+            'https://sketch.pixiv.net/@exampleuser',
+            'https://sketch.pixiv.net/@exampleuser/lives/000000000000000000',
+        ]
+        for url in should_match:
+            self.assertTrue(Pixiv.can_handle_url(url))
+
+        should_not_match = [
+            'https://sketch.pixiv.net',
+        ]
+        for url in should_not_match:
+            self.assertFalse(Pixiv.can_handle_url(url))
diff --git a/tests/test_plugin_youtube.py b/tests/test_plugin_youtube.py
new file mode 100644
index 00000000..4b1934d0
--- /dev/null
+++ b/tests/test_plugin_youtube.py
@@ -0,0 +1,23 @@
+import unittest
+
+from streamlink.plugins.youtube import YouTube
+
+
+class TestPluginYouTube(unittest.TestCase):
+    def test_can_handle_url(self):
+        should_match = [
+            "https://www.youtube.com/c/EXAMPLE/live",
+            "https://www.youtube.com/channel/EXAMPLE",
+            "https://www.youtube.com/v/aqz-KE-bpKQ",
+            "https://www.youtube.com/embed/aqz-KE-bpKQ",
+            "https://www.youtube.com/user/EXAMPLE/",
+            "https://www.youtube.com/watch?v=aqz-KE-bpKQ",
+        ]
+        for url in should_match:
+            self.assertTrue(YouTube.can_handle_url(url))
+
+        should_not_match = [
+            "https://www.youtube.com",
+        ]
+        for url in should_not_match:
+            self.assertFalse(YouTube.can_handle_url(url))