Repository: globocom/m3u8

Issue:
#EXT-OATCLS-SCTE35 tag is getting removed
According to documentation, this library supports the '#EXT-OATCLS-SCTE35' tag. But, 
when I used this parser to trim manifest files, SCTE tags were getting removed.

Below are the steps I follow,
1. Downloaded master manifest file[using python request]
2. Passed downloaded master manifest string to m3u8.loads method and got parsed master manifest object[say parsed_master_manifest]
3. Iterated over playlists of parsed_master_manifest
    a. Downloaded child manifest file[using python request]
    b. Passed downloaded child manifest string to m3u8.loads method and got parsed child manifest object[say parsed_child_manifest]
    c. Initialized Segmentlist[say desired_segments]
    d. Appended desired segments in  desired_segments
    e. Replaced original segments list with desired_segments
4. Store Master and all child manifest on local machine using m3u8.dump method
 
Please expedite this. and me know If I am doing anything wrong. I would much appreciate if any of you suggest a solution for this.

Discussion:

Patch:
diff --git a/m3u8/model.py b/m3u8/model.py
index 50bd2d5..6fee06f 100644
--- a/m3u8/model.py
+++ b/m3u8/model.py
@@ -6,7 +6,13 @@ import decimal
 import os
 import errno
 
-from m3u8.protocol import ext_x_start, ext_x_key, ext_x_session_key, ext_x_map
+from m3u8.protocol import (
+    ext_x_key,
+    ext_x_map,
+    ext_oatcls_scte35,
+    ext_x_session_key,
+    ext_x_start,
+)
 from m3u8.parser import parse, format_date_time
 from m3u8.mixins import BasePathMixin, GroupedBasePathMixin
 
@@ -443,8 +449,8 @@ class Segment(BasePathMixin):
     def __init__(self, uri=None, base_uri=None, program_date_time=None, current_program_date_time=None,
                  duration=None, title=None, bitrate=None, byterange=None, cue_out=False,
                  cue_out_start=False, cue_in=False, discontinuity=False, key=None, scte35=None,
-                 scte35_duration=None, scte35_elapsedtime=None, keyobject=None, parts=None,
-                 init_section=None, dateranges=None, gap_tag=None, custom_parser_values=None):
+                 oatcls_scte35=None, scte35_duration=None, scte35_elapsedtime=None, keyobject=None,
+                 parts=None, init_section=None, dateranges=None, gap_tag=None, custom_parser_values=None):
         self.uri = uri
         self.duration = duration
         self.title = title
@@ -458,6 +464,7 @@ class Segment(BasePathMixin):
         self.cue_out = cue_out
         self.cue_in = cue_in
         self.scte35 = scte35
+        self.oatcls_scte35 = oatcls_scte35
         self.scte35_duration = scte35_duration
         self.scte35_elapsedtime = scte35_elapsedtime
         self.key = keyobject
@@ -506,7 +513,11 @@ class Segment(BasePathMixin):
             output.append(str(self.dateranges))
             output.append('\n')
 
+
         if self.cue_out_start:
+            if self.oatcls_scte35:
+                output.append(f'{ext_oatcls_scte35}:{self.oatcls_scte35}\n')
+
             output.append('#EXT-X-CUE-OUT{}\n'.format(
                 (':' + self.scte35_duration) if self.scte35_duration else ''))
         elif self.cue_out:
diff --git a/m3u8/parser.py b/m3u8/parser.py
index b30b8d3..6ebd5a7 100644
--- a/m3u8/parser.py
+++ b/m3u8/parser.py
@@ -111,10 +111,13 @@ def parse(content, strict=False, custom_tags_parser=None):
             state['cue_out'] = True
 
         elif line.startswith(protocol.ext_x_cue_out):
-            _parse_cueout(line, state, string_to_lines(content)[lineno - 2])
+            _parse_cueout(line, state)
             state['cue_out_start'] = True
             state['cue_out'] = True
 
+        elif line.startswith(f'{protocol.ext_oatcls_scte35}:'):
+            _parse_oatcls_scte35(line, state)
+
         elif line.startswith(protocol.ext_x_cue_in):
             state['cue_in'] = True
 
@@ -269,6 +272,7 @@ def _parse_ts_chunk(line, data, state):
     segment['cue_out_start'] = state.pop('cue_out_start', False)
     scte_op = state.pop if segment['cue_in'] else state.get
     segment['scte35'] = scte_op('current_cue_out_scte35', None)
+    segment['oatcls_scte35'] = scte_op('current_cue_out_oatcls_scte35', None)
     segment['scte35_duration'] = scte_op('current_cue_out_duration', None)
     segment['scte35_elapsedtime'] = scte_op('current_cue_out_elapsedtime', None)
     segment['discontinuity'] = state.pop('discontinuity', False)
@@ -397,15 +401,7 @@ def _cueout_no_duration(line):
     if line == protocol.ext_x_cue_out:
         return (None, None)
 
-def _cueout_elemental(line, state, prevline):
-    param, value = line.split(':', 1)
-    res = re.match('.*EXT-OATCLS-SCTE35:(.*)$', prevline)
-    if res:
-        return (res.group(1), value)
-    else:
-        return None
-
-def _cueout_envivio(line, state, prevline):
+def _cueout_envivio(line, state):
     param, value = line.split(':', 1)
     res = re.match('.*DURATION=(.*),.*,CUE="(.*)"', value)
     if res:
@@ -414,8 +410,6 @@ def _cueout_envivio(line, state, prevline):
         return None
 
 def _cueout_duration(line):
-    # this needs to be called after _cueout_elemental
-    # as it would capture those cues incompletely
     # This was added separately rather than modifying "simple"
     param, value = line.split(':', 1)
     res = re.match(r'DURATION=(.*)', value)
@@ -423,22 +417,21 @@ def _cueout_duration(line):
         return (None, res.group(1))
 
 def _cueout_simple(line):
-    # this needs to be called after _cueout_elemental
-    # as it would capture those cues incompletely
     param, value = line.split(':', 1)
     res = re.match(r'^(\d+(?:\.\d)?\d*)$', value)
     if res:
         return (None, res.group(1))
 
-def _parse_cueout(line, state, prevline):
+def _parse_cueout(line, state):
     _cueout_state = (_cueout_no_duration(line)
-                     or _cueout_elemental(line, state, prevline)
-                     or _cueout_envivio(line, state, prevline)
+                     or _cueout_envivio(line, state)
                      or _cueout_duration(line)
                      or _cueout_simple(line))
     if _cueout_state:
-        state['current_cue_out_scte35'] = _cueout_state[0]
-        state['current_cue_out_duration'] = _cueout_state[1]
+        cue_out_scte35, cue_out_duration = _cueout_state
+        current_cue_out_scte35 = state.get('current_cue_out_scte35')
+        state['current_cue_out_scte35'] = cue_out_scte35 or current_cue_out_scte35
+        state['current_cue_out_duration'] = cue_out_duration
 
 def _parse_server_control(line, data, state):
     attribute_parser = {
@@ -552,6 +545,13 @@ def _parse_content_steering(line, data, state):
         protocol.ext_x_content_steering, line, attribute_parser
     )
 
+
+def _parse_oatcls_scte35(line, state):
+    scte35_cue = line.split(':', 1)[1]
+    state['current_cue_out_oatcls_scte35'] = scte35_cue
+    state['current_cue_out_scte35'] = scte35_cue
+
+
 def string_to_lines(string):
     return string.strip().splitlines()
 
diff --git a/m3u8/protocol.py b/m3u8/protocol.py
index 87e6ca1..b6007b8 100644
--- a/m3u8/protocol.py
+++ b/m3u8/protocol.py
@@ -25,7 +25,7 @@ ext_x_cue_out = '#EXT-X-CUE-OUT'
 ext_x_cue_out_cont = '#EXT-X-CUE-OUT-CONT'
 ext_x_cue_in = '#EXT-X-CUE-IN'
 ext_x_cue_span = '#EXT-X-CUE-SPAN'
-ext_x_scte35 = '#EXT-OATCLS-SCTE35'
+ext_oatcls_scte35 = '#EXT-OATCLS-SCTE35'
 ext_is_independent_segments = '#EXT-X-INDEPENDENT-SEGMENTS'
 ext_x_map = '#EXT-X-MAP'
 ext_x_start = '#EXT-X-START'


Test patch:
diff --git a/tests/test_model.py b/tests/test_model.py
index 54aeb41..8f59c37 100755
--- a/tests/test_model.py
+++ b/tests/test_model.py
@@ -159,6 +159,24 @@ def test_segment_cue_out_cont_attributes_dumps():
     )
     assert expected in result
 
+def test_segment_oatcls_scte35_dumps():
+    obj = m3u8.M3U8(playlists.CUE_OUT_ELEMENTAL_PLAYLIST)
+    result = obj.dumps()
+
+    # Only insert OATCLS-SCTE35 at cue out
+    cue_out_line = (
+        '#EXT-OATCLS-SCTE35:/DAlAAAAAAAAAP/wFAUAAAABf+//wpiQkv4ARKogAAEBAQAAQ6sodg==\n'
+        '#EXT-X-CUE-OUT'
+    )
+    assert cue_out_line in result
+    
+    # Don't insert it for continued cue outs
+    cue_out_cont_line =  (
+        '#EXT-OATCLS-SCTE35:/DAlAAAAAAAAAP/wFAUAAAABf+//wpiQkv4ARKogAAEBAQAAQ6sodg==\n'
+        '#EXT-X-CUE-OUT-CONT'
+    )
+    assert cue_out_cont_line not in result
+
 def test_segment_cue_out_start_dumps():
     obj = m3u8.M3U8(playlists.CUE_OUT_WITH_DURATION_PLAYLIST)
 
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 13033c1..d693d8b 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -285,11 +285,32 @@ def test_should_parse_program_date_time_from_playlist():
 
 def test_should_parse_scte35_from_playlist():
     data = m3u8.parse(playlists.CUE_OUT_ELEMENTAL_PLAYLIST)
-    assert not data['segments'][2]['cue_out']
-    assert data['segments'][3]['scte35']
-    assert data['segments'][3]['cue_out']
-    assert '/DAlAAAAAAAAAP/wFAUAAAABf+//wpiQkv4ARKogAAEBAQAAQ6sodg==' == data['segments'][4]['scte35']
-    assert '50' == data['segments'][4]['scte35_duration']
+
+    # cue_out should be maintained from [EXT-X-CUE-OUT, EXT-X-CUE-IN)
+    actual_cue_status = [s['cue_out'] for s in data['segments']]
+    expected_cue_status = [
+        False, False, False, True, True, True, True, True, True, False, False
+    ]
+    assert actual_cue_status == expected_cue_status
+
+    # scte35 should be maintained from [EXT-X-CUE-OUT, EXT-X-CUE-IN]
+    cue = '/DAlAAAAAAAAAP/wFAUAAAABf+//wpiQkv4ARKogAAEBAQAAQ6sodg=='
+    actual_scte35 = [s['scte35'] for s in data['segments']]
+    expected_scte35 = [None, None, None, cue, cue, cue, cue, cue, cue, cue, None]
+    assert actual_scte35 == expected_scte35
+
+    # oatcls_scte35 should be maintained from [EXT-X-CUE-OUT, EXT-X-CUE-IN]
+    actual_oatcls_scte35 = [s['oatcls_scte35'] for s in data['segments']]
+    expected_oatcls_scte35 = [None, None, None, cue, cue, cue, cue, cue, cue, cue, None]
+    assert actual_oatcls_scte35 == expected_oatcls_scte35
+
+    # durations should be maintained from  from [EXT-X-CUE-OUT, EXT-X-CUE-IN]
+    actual_scte35_duration = [s['scte35_duration'] for s in data['segments']]
+    expected_scte35_duration = [
+        None, None, None, '50.000', '50', '50', '50', '50', '50', '50', None
+    ]
+    assert actual_scte35_duration == expected_scte35_duration
+
 
 def test_should_parse_envivio_cue_playlist():
     data = m3u8.parse(playlists.CUE_OUT_ENVIVIO_PLAYLIST)