Repository: streamlink/streamlink
Pull request #3220 (2020-10-03 02:38:36)

plugins.twitch: remove platform access token param
This once again removes the `platform=_` parameter from the stream access token request.

See #3210
As mentioned there, we need a better solution for this. Am happy to discuss this.

----

History of the parameter:
1. Added in #2358
2. Removed in #2692
3. Added in #3173


Patch:
diff --git a/src/streamlink/plugins/twitch.py b/src/streamlink/plugins/twitch.py
index 6bdfd452..45e3199a 100644
--- a/src/streamlink/plugins/twitch.py
+++ b/src/streamlink/plugins/twitch.py
@@ -19,7 +19,7 @@ from streamlink.stream import (
 )
 from streamlink.stream.hls import HLSStreamWorker
 from streamlink.stream.hls_filtered import FilteredHLSStreamWriter, FilteredHLSStreamReader
-from streamlink.stream.hls_playlist import M3U8Parser, load as load_hls_playlist
+from streamlink.stream.hls_playlist import M3U8, M3U8Parser, load as load_hls_playlist
 from streamlink.utils.times import hours_minutes_seconds
 
 try:
@@ -125,19 +125,29 @@ Segment = namedtuple("Segment", "uri duration title key discontinuity ad byteran
 LOW_LATENCY_MAX_LIVE_EDGE = 2
 
 
-class TwitchM3U8Parser(M3U8Parser):
-    def parse_extinf(self, value):
-        duration, title = super(TwitchM3U8Parser, self).parse_extinf(value)
-        if title and str(title).startswith("Amazon"):
-            self.state["ad"] = True
+class TwitchM3U8(M3U8):
+    def __init__(self):
+        super(TwitchM3U8, self).__init__()
+        self.dateranges_ads = []
 
-        return duration, title
 
+class TwitchM3U8Parser(M3U8Parser):
     def parse_tag_ext_x_twitch_prefetch(self, value):
         segments = self.m3u8.segments
         if segments:
             segments.append(segments[-1]._replace(uri=self.uri(value), prefetch=True))
 
+    def parse_tag_ext_x_daterange(self, value):
+        super(TwitchM3U8Parser, self).parse_tag_ext_x_daterange(value)
+        daterange = self.m3u8.dateranges[-1]
+        is_ad = (
+            daterange.classname == "twitch-stitched-ad"
+            or str(daterange.id or "").startswith("stitched-ad-")
+            or any(attr_key.startswith("X-TV-TWITCH-AD-") for attr_key in daterange.x.keys())
+        )
+        if is_ad:
+            self.m3u8.dateranges_ads.append(daterange)
+
     def get_segment(self, uri):
         byterange = self.state.pop("byterange", None)
         extinf = self.state.pop("extinf", (0, None))
@@ -145,7 +155,7 @@ class TwitchM3U8Parser(M3U8Parser):
         map_ = self.state.get("map")
         key = self.state.get("key")
         discontinuity = self.state.pop("discontinuity", False)
-        ad = self.state.pop("ad", False)
+        ad = any(self.m3u8.is_date_in_daterange(date, daterange) for daterange in self.m3u8.dateranges_ads)
 
         return Segment(
             uri,
@@ -167,7 +177,7 @@ class TwitchHLSStreamWorker(HLSStreamWorker):
         super(TwitchHLSStreamWorker, self).__init__(reader, *args, **kwargs)
 
     def _reload_playlist(self, *args):
-        return load_hls_playlist(*args, parser=TwitchM3U8Parser)
+        return load_hls_playlist(*args, parser=TwitchM3U8Parser, m3u8=TwitchM3U8)
 
     def _playlist_reload_time(self, playlist, sequences):
         if self.stream.low_latency and sequences:
@@ -317,7 +327,7 @@ class TwitchAPI(object):
     # Private API calls
 
     def access_token(self, endpoint, asset, **params):
-        return self.call("/api/{0}/{1}/access_token".format(endpoint, asset), private=True, **dict(platform="_", **params))
+        return self.call("/api/{0}/{1}/access_token".format(endpoint, asset), private=True, **params)
 
     def hosted_channel(self, **params):
         return self.call_subdomain("tmi", "/hosts", format="", **params)
diff --git a/src/streamlink/stream/hls_playlist.py b/src/streamlink/stream/hls_playlist.py
index c3ea8851..130061c3 100644
--- a/src/streamlink/stream/hls_playlist.py
+++ b/src/streamlink/stream/hls_playlist.py
@@ -3,10 +3,14 @@ import logging
 
 from binascii import unhexlify
 from collections import namedtuple
+from datetime import timedelta
 from itertools import starmap
 
+from isodate import parse_datetime
+
 from streamlink.compat import urljoin, urlparse
 
+
 log = logging.getLogger(__name__)
 
 __all__ = ["load", "M3U8Parser"]
@@ -15,6 +19,9 @@ __all__ = ["load", "M3U8Parser"]
 # EXT-X-BYTERANGE
 ByteRange = namedtuple("ByteRange", "range offset")
 
+# EXT-X-DATERANGE
+DateRange = namedtuple("DateRange", "id classname start_date end_date duration planned_duration end_on_next x")
+
 # EXT-X-KEY
 Key = namedtuple("Key", "method uri iv key_format key_format_versions")
 
@@ -54,8 +61,24 @@ class M3U8(object):
 
         self.media = []
         self.playlists = []
+        self.dateranges = []
         self.segments = []
 
+    @classmethod
+    def is_date_in_daterange(cls, date, daterange):
+        if date is None or daterange.start_date is None:
+            return None
+
+        if daterange.end_date is not None:
+            return daterange.start_date <= date < daterange.end_date
+
+        duration = daterange.duration or daterange.planned_duration
+        if duration is not None:
+            end = daterange.start_date + duration
+            return daterange.start_date <= date < end
+
+        return daterange.start_date <= date
+
 
 class M3U8Parser(object):
     _extinf_re = re.compile(r"(?P<duration>\d+(\.\d+)?)(,(?P<title>.+))?")
@@ -64,8 +87,10 @@ class M3U8Parser(object):
     _tag_re = re.compile(r"#(?P<tag>[\w-]+)(:(?P<value>.+))?")
     _res_re = re.compile(r"(\d+)x(\d+)")
 
-    def __init__(self, base_uri=None, **kwargs):
+    def __init__(self, base_uri=None, m3u8=M3U8, **kwargs):
         self.base_uri = base_uri
+        self.m3u8 = m3u8()
+        self.state = {}
 
     def create_stream_info(self, streaminf, cls=None):
         program_id = streaminf.get("PROGRAM-ID")
@@ -131,6 +156,17 @@ class M3U8Parser(object):
 
         return unhexlify(value)
 
+    def parse_iso8601(self, value):
+        if value is None:
+            return None
+        try:
+            return parse_datetime(value)
+        except ValueError:
+            return None
+
+    def parse_timedelta(self, value):
+        return timedelta(seconds=float(value)) if value is not None else None
+
     def parse_resolution(self, value):
         match = self._res_re.match(value)
 
@@ -166,7 +202,21 @@ class M3U8Parser(object):
                                 attr.get("KEYFORMATVERSIONS"))
 
     def parse_tag_ext_x_program_date_time(self, value):
-        self.state["date"] = value
+        self.state["date"] = self.parse_iso8601(value)
+
+    def parse_tag_ext_x_daterange(self, value):
+        attr = self.parse_attributes(value)
+        daterange = DateRange(
+            attr.pop("ID", None),
+            attr.pop("CLASS", None),
+            self.parse_iso8601(attr.pop("START-DATE", None)),
+            self.parse_iso8601(attr.pop("END-DATE", None)),
+            self.parse_timedelta(attr.pop("DURATION", None)),
+            self.parse_timedelta(attr.pop("PLANNED-DURATION", None)),
+            self.parse_bool(attr.pop("END-ON-NEXT", None)),
+            attr
+        )
+        self.m3u8.dateranges.append(daterange)
 
     def parse_tag_ext_x_allow_cache(self, value):
         self.m3u8.allow_cache = self.parse_bool(value)
@@ -244,9 +294,6 @@ class M3U8Parser(object):
             self.m3u8.playlists.append(playlist)
 
     def parse(self, data):
-        self.state = {}
-        self.m3u8 = M3U8()
-
         lines = iter(filter(bool, data.splitlines()))
         try:
             line = next(lines)


Test patch:
diff --git a/tests/plugins/test_twitch.py b/tests/plugins/test_twitch.py
index 3056c4cc..bfc5d99e 100644
--- a/tests/plugins/test_twitch.py
+++ b/tests/plugins/test_twitch.py
@@ -1,17 +1,42 @@
-import requests_mock
+from datetime import datetime, timedelta
 import unittest
 
-from tests.mixins.stream_hls import Playlist, Segment as Segment, TestMixinStreamHLS
+import requests_mock
+
+from tests.mixins.stream_hls import Playlist, Tag, Segment as _Segment, TestMixinStreamHLS
 from tests.mock import MagicMock, call, patch
 
 from streamlink import Streamlink
 from streamlink.plugins.twitch import Twitch, TwitchHLSStream
 
 
-class SegmentAd(Segment):
-    def __init__(self, *args, **kwargs):
-        super(SegmentAd, self).__init__(*args, **kwargs)
-        self.title = "Amazon|123456789"
+DATETIME_BASE = datetime(2000, 1, 1, 0, 0, 0, 0)
+DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
+
+class TagDateRangeAd(Tag):
+    def __init__(self, start=DATETIME_BASE, duration=1, id="stitched-ad-1234", classname="twitch-stitched-ad", custom=None):
+        attrs = {
+            "ID": self.val_quoted_string(id),
+            "CLASS": self.val_quoted_string(classname),
+            "START-DATE": self.val_quoted_string(start.strftime(DATETIME_FORMAT)),
+            "DURATION": duration
+        }
+        if custom is not None:
+            attrs.update(**{key: self.val_quoted_string(value) for (key, value) in custom.items()})
+        super(TagDateRangeAd, self).__init__("EXT-X-DATERANGE", attrs)
+
+
+class Segment(_Segment):
+    def __init__(self, num, title="live", *args, **kwargs):
+        super(Segment, self).__init__(num, title, *args, **kwargs)
+        self.date = DATETIME_BASE + timedelta(seconds=num)
+
+    def build(self, namespace):
+        return "#EXT-X-PROGRAM-DATE-TIME:{0}\n{1}".format(
+            self.date.strftime(DATETIME_FORMAT),
+            super(Segment, self).build(namespace)
+        )
 
 
 class SegmentPrefetch(Segment):
@@ -51,11 +76,48 @@ class TestTwitchHLSStream(TestMixinStreamHLS, unittest.TestCase):
 
         return session
 
+    def test_hls_disable_ads_daterange_unknown(self):
+        daterange = TagDateRangeAd(start=DATETIME_BASE, duration=1, id="foo", classname="bar", custom=None)
+        thread, segments = self.subject([
+            Playlist(0, [daterange, Segment(0), Segment(1)], end=True)
+        ], disable_ads=True, low_latency=False)
+
+        self.assertEqual(self.await_read(read_all=True), self.content(segments), "Doesn't filter out segments")
+        self.assertTrue(all(self.called(s) for s in segments.values()), "Downloads all segments")
+
+    def test_hls_disable_ads_daterange_by_class(self):
+        daterange = TagDateRangeAd(start=DATETIME_BASE, duration=1, id="foo", classname="twitch-stitched-ad", custom=None)
+        thread, segments = self.subject([
+            Playlist(0, [daterange, Segment(0), Segment(1)], end=True)
+        ], disable_ads=True, low_latency=False)
+
+        self.assertEqual(self.await_read(read_all=True), segments[1].content, "Filters out ad segments")
+        self.assertTrue(all(self.called(s) for s in segments.values()), "Downloads all segments")
+
+    def test_hls_disable_ads_daterange_by_id(self):
+        daterange = TagDateRangeAd(start=DATETIME_BASE, duration=1, id="stitched-ad-1234", classname="/", custom=None)
+        thread, segments = self.subject([
+            Playlist(0, [daterange, Segment(0), Segment(1)], end=True)
+        ], disable_ads=True, low_latency=False)
+
+        self.assertEqual(self.await_read(read_all=True), segments[1].content, "Filters out ad segments")
+        self.assertTrue(all(self.called(s) for s in segments.values()), "Downloads all segments")
+
+    def test_hls_disable_ads_daterange_by_attr(self):
+        daterange = TagDateRangeAd(start=DATETIME_BASE, duration=1, id="foo", classname="/", custom={"X-TV-TWITCH-AD-URL": "/"})
+        thread, segments = self.subject([
+            Playlist(0, [daterange, Segment(0), Segment(1)], end=True)
+        ], disable_ads=True, low_latency=False)
+
+        self.assertEqual(self.await_read(read_all=True), segments[1].content, "Filters out ad segments")
+        self.assertTrue(all(self.called(s) for s in segments.values()), "Downloads all segments")
+
     @patch("streamlink.plugins.twitch.log")
     def test_hls_disable_ads_has_preroll(self, mock_log):
+        daterange = TagDateRangeAd(duration=4)
         thread, segments = self.subject([
-            Playlist(0, [SegmentAd(0), SegmentAd(1)]),
-            Playlist(2, [SegmentAd(2), SegmentAd(3)]),
+            Playlist(0, [daterange, Segment(0), Segment(1)]),
+            Playlist(2, [daterange, Segment(2), Segment(3)]),
             Playlist(4, [Segment(4), Segment(5)], end=True)
         ], disable_ads=True, low_latency=False)
 
@@ -72,9 +134,10 @@ class TestTwitchHLSStream(TestMixinStreamHLS, unittest.TestCase):
 
     @patch("streamlink.plugins.twitch.log")
     def test_hls_disable_ads_has_midstream(self, mock_log):
+        daterange = TagDateRangeAd(start=DATETIME_BASE + timedelta(seconds=2), duration=2)
         thread, segments = self.subject([
             Playlist(0, [Segment(0), Segment(1)]),
-            Playlist(2, [SegmentAd(2), SegmentAd(3)]),
+            Playlist(2, [daterange, Segment(2), Segment(3)]),
             Playlist(4, [Segment(4), Segment(5)], end=True)
         ], disable_ads=True, low_latency=False)
 
@@ -90,8 +153,9 @@ class TestTwitchHLSStream(TestMixinStreamHLS, unittest.TestCase):
 
     @patch("streamlink.plugins.twitch.log")
     def test_hls_no_disable_ads_has_preroll(self, mock_log):
+        daterange = TagDateRangeAd(duration=2)
         thread, segments = self.subject([
-            Playlist(0, [SegmentAd(0), SegmentAd(1)]),
+            Playlist(0, [daterange, Segment(0), Segment(1)]),
             Playlist(2, [Segment(2), Segment(3)], end=True)
         ], disable_ads=False, low_latency=False)
 
@@ -161,8 +225,9 @@ class TestTwitchHLSStream(TestMixinStreamHLS, unittest.TestCase):
 
     @patch("streamlink.plugins.twitch.log")
     def test_hls_low_latency_has_prefetch_has_preroll(self, mock_log):
+        daterange = TagDateRangeAd(duration=4)
         thread, segments = self.subject([
-            Playlist(0, [SegmentAd(0), SegmentAd(1), SegmentAd(2), SegmentAd(3)]),
+            Playlist(0, [daterange, Segment(0), Segment(1), Segment(2), Segment(3)]),
             Playlist(4, [Segment(4), Segment(5), Segment(6), Segment(7), SegmentPrefetch(8), SegmentPrefetch(9)], end=True)
         ], disable_ads=False, low_latency=True)
 
@@ -179,8 +244,9 @@ class TestTwitchHLSStream(TestMixinStreamHLS, unittest.TestCase):
 
     @patch("streamlink.plugins.twitch.log")
     def test_hls_low_latency_has_prefetch_disable_ads_has_preroll(self, mock_log):
+        daterange = TagDateRangeAd(duration=4)
         self.subject([
-            Playlist(0, [SegmentAd(0), SegmentAd(1), SegmentAd(2), SegmentAd(3)]),
+            Playlist(0, [daterange, Segment(0), Segment(1), Segment(2), Segment(3)]),
             Playlist(4, [Segment(4), Segment(5), Segment(6), Segment(7), SegmentPrefetch(8), SegmentPrefetch(9)], end=True)
         ], disable_ads=True, low_latency=True)
 
@@ -193,8 +259,9 @@ class TestTwitchHLSStream(TestMixinStreamHLS, unittest.TestCase):
 
     @patch("streamlink.plugins.twitch.log")
     def test_hls_low_latency_no_prefetch_disable_ads_has_preroll(self, mock_log):
+        daterange = TagDateRangeAd(duration=4)
         self.subject([
-            Playlist(0, [SegmentAd(0), SegmentAd(1), SegmentAd(2), SegmentAd(3)]),
+            Playlist(0, [daterange, Segment(0), Segment(1), Segment(2), Segment(3)]),
             Playlist(4, [Segment(4), Segment(5), Segment(6), Segment(7)], end=True)
         ], disable_ads=True, low_latency=True)
 
diff --git a/tests/resources/hls/test_date.m3u8 b/tests/resources/hls/test_date.m3u8
new file mode 100644
index 00000000..e3657e50
--- /dev/null
+++ b/tests/resources/hls/test_date.m3u8
@@ -0,0 +1,33 @@
+#EXTM3U
+
+#EXT-X-TARGETDURATION:120
+
+#EXT-X-DATERANGE:ID="start-invalid",START-DATE="invalid"
+#EXT-X-DATERANGE:ID="start-no-frac",START-DATE="2000-01-01T00:00:00Z"
+#EXT-X-DATERANGE:ID="start-with-frac",START-DATE="2000-01-01T00:00:00.000Z"
+
+#EXT-X-DATERANGE:ID="with-class",CLASS="bar",START-DATE="2000-01-01T00:00:00.000Z"
+
+#EXT-X-DATERANGE:ID="duration",START-DATE="2000-01-01T00:00:00.000Z",DURATION=30.5
+#EXT-X-DATERANGE:ID="planned-duration",START-DATE="2000-01-01T00:00:00.000Z",PLANNED-DURATION=15
+#EXT-X-DATERANGE:ID="duration-precedence",START-DATE="2000-01-01T00:00:00.000Z",DURATION=30.5,PLANNED-DURATION=15
+#EXT-X-DATERANGE:ID="end",START-DATE="2000-01-01T00:00:00.000Z",END-DATE="2000-01-01T00:01:00.000Z"
+#EXT-X-DATERANGE:ID="end-precedence",START-DATE="2000-01-01T00:00:00.000Z",END-DATE="2000-01-01T00:01:00.000Z",DURATION=30.5
+
+#EXT-X-DATERANGE:X-CUSTOM="value"
+
+#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:00.000Z
+#EXTINF:15.000,live
+segment0-15.ts
+
+#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:15.000Z
+#EXTINF:15.500,live
+segment15-30.5.ts
+
+#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:00:30.500Z
+#EXTINF:29.500,live
+segment30.5-60.ts
+
+#EXT-X-PROGRAM-DATE-TIME:2000-01-01T00:01:00.000Z
+#EXTINF:60.000,live
+segment60-.ts
diff --git a/tests/streams/test_hls_playlist.py b/tests/streams/test_hls_playlist.py
index 1b1357d6..0f39479a 100644
--- a/tests/streams/test_hls_playlist.py
+++ b/tests/streams/test_hls_playlist.py
@@ -1,10 +1,15 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-from streamlink.stream.hls_playlist import load, StreamInfo, Resolution, Media
+from datetime import datetime, timedelta
 import unittest
+
+from isodate import tzinfo
+
 from tests.resources import text
 
+from streamlink.stream.hls_playlist import load, StreamInfo, Resolution, Media, DateRange, Segment
+
 
 class TestHLSPlaylist(unittest.TestCase):
     def test_load(self):
@@ -66,3 +71,81 @@ class TestHLSPlaylist(unittest.TestCase):
                            subtitles='subs')
             ]
         )
+
+    def test_parse_date(self):
+        with text("hls/test_date.m3u8") as m3u8_fh:
+            playlist = load(m3u8_fh.read(), "http://test.se/")
+
+        start_date = datetime(year=2000, month=1, day=1, hour=0, minute=0, second=0, microsecond=0, tzinfo=tzinfo.UTC)
+        end_date = datetime(year=2000, month=1, day=1, hour=0, minute=1, second=0, microsecond=0, tzinfo=tzinfo.UTC)
+        delta_15 = timedelta(seconds=15)
+        delta_30 = timedelta(seconds=30, milliseconds=500)
+        delta_60 = timedelta(seconds=60)
+
+        self.assertEqual(playlist.target_duration, 120)
+
+        self.assertEqual(
+            [daterange for daterange in playlist.dateranges],
+            [
+                DateRange(id="start-invalid",
+                          start_date=None,
+                          classname=None, end_date=None, duration=None, planned_duration=None, end_on_next=False, x={}),
+                DateRange(id="start-no-frac",
+                          start_date=start_date,
+                          classname=None, end_date=None, duration=None, planned_duration=None, end_on_next=False, x={}),
+                DateRange(id="start-with-frac",
+                          start_date=start_date,
+                          classname=None, end_date=None, duration=None, planned_duration=None, end_on_next=False, x={}),
+                DateRange(id="with-class",
+                          start_date=start_date, classname="bar",
+                          end_date=None, duration=None, planned_duration=None, end_on_next=False, x={}),
+                DateRange(id="duration",
+                          start_date=start_date, duration=delta_30,
+                          classname=None, end_date=None, planned_duration=None, end_on_next=False, x={}),
+                DateRange(id="planned-duration",
+                          start_date=start_date, planned_duration=delta_15,
+                          classname=None, end_date=None, duration=None, end_on_next=False, x={}),
+                DateRange(id="duration-precedence",
+                          start_date=start_date, duration=delta_30, planned_duration=delta_15,
+                          classname=None, end_date=None, end_on_next=False, x={}),
+                DateRange(id="end",
+                          start_date=start_date, end_date=end_date,
+                          classname=None, duration=None, planned_duration=None, end_on_next=False, x={}),
+                DateRange(id="end-precedence",
+                          start_date=start_date, end_date=end_date, duration=delta_30,
+                          classname=None, planned_duration=None, end_on_next=False, x={}),
+                DateRange(x={"X-CUSTOM": "value"},
+                          id=None, start_date=None, end_date=None, duration=None,
+                          classname=None, planned_duration=None, end_on_next=False)
+            ]
+        )
+        self.assertEqual(
+            [segment for segment in playlist.segments],
+            [
+                Segment(uri="http://test.se/segment0-15.ts", duration=15.0, title="live", date=start_date,
+                        key=None, discontinuity=False, byterange=None, map=None),
+                Segment(uri="http://test.se/segment15-30.5.ts", duration=15.5, title="live", date=start_date + delta_15,
+                        key=None, discontinuity=False, byterange=None, map=None),
+                Segment(uri="http://test.se/segment30.5-60.ts", duration=29.5, title="live", date=start_date + delta_30,
+                        key=None, discontinuity=False, byterange=None, map=None),
+                Segment(uri="http://test.se/segment60-.ts", duration=60.0, title="live", date=start_date + delta_60,
+                        key=None, discontinuity=False, byterange=None, map=None)
+            ]
+        )
+
+        self.assertEqual(
+            [playlist.is_date_in_daterange(playlist.segments[0].date, daterange) for daterange in playlist.dateranges],
+            [None, True, True, True, True, True, True, True, True, None]
+        )
+        self.assertEqual(
+            [playlist.is_date_in_daterange(playlist.segments[1].date, daterange) for daterange in playlist.dateranges],
+            [None, True, True, True, True, False, True, True, True, None]
+        )
+        self.assertEqual(
+            [playlist.is_date_in_daterange(playlist.segments[2].date, daterange) for daterange in playlist.dateranges],
+            [None, True, True, True, False, False, False, True, True, None]
+        )
+        self.assertEqual(
+            [playlist.is_date_in_daterange(playlist.segments[3].date, daterange) for daterange in playlist.dateranges],
+            [None, True, True, True, False, False, False, False, False, None]
+        )