Repository: just-work/fffw

Issue:
Separator is not found, and chunk exceed the limit
I am trying to transcode using `django-video-transcoding` which uses fffw library, and for a few streams I am getting `Separator is not found, and chunk exceed the limit`. I tried running ffmpeg with the exact command which`django-video-transcoding` logs and it works fine.

Comments:

Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a23163e..b69d937 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -8,7 +8,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
index 5c6a729..1266714 100644
--- a/.github/workflows/codecov.yml
+++ b/.github/workflows/codecov.yml
@@ -8,7 +8,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@master
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Generate coverage report
       run: |
         pip install coverage
diff --git a/SECURITY.md b/SECURITY.md
index 6d5c5da..1a3963c 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -2,11 +2,12 @@
 
 ## Supported Versions
 
-Currently only 2.0.x branch has full support, 1.0.x branch is shut down.
+Currently only 2.x/3x branches has full support, 1.0.x branch is shut down.
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 2.0.x   | :white_check_mark: |
+| 3.x     | :white_check_mark: |
+| 2.x     | :white_check_mark: |
 | 1.0.x   | :x:                |
 
 
diff --git a/fffw/encoding/filters.py b/fffw/encoding/filters.py
index 2564388..f42e5b2 100644
--- a/fffw/encoding/filters.py
+++ b/fffw/encoding/filters.py
@@ -19,9 +19,33 @@ __all__ = [
     'Split',
     'Trim',
     'Upload',
+    'ensure_video',
+    'ensure_audio',
 ]
 
 
+def ensure_video(meta: Meta, *_: Meta) -> VideoMeta:
+    """
+    Checks that first passed stream is a video stream
+
+    :returns: first passed stream
+    """
+    if not isinstance(meta, VideoMeta):
+        raise TypeError(meta)
+    return meta
+
+
+def ensure_audio(meta: Meta, *_: Meta) -> AudioMeta:
+    """
+    Checks that first passed stream is a audio stream
+
+    :returns: first passed stream
+    """
+    if not isinstance(meta, AudioMeta):
+        raise TypeError(meta)
+    return meta
+
+
 @dataclass
 class Filter(mixins.StreamValidationMixin, base.Node, Params):
     """
@@ -163,9 +187,7 @@ class Scale(VideoFilter):
     height: int = param(name='h')
 
     def transform(self, *metadata: Meta) -> Meta:
-        meta = metadata[0]
-        if not isinstance(meta, VideoMeta):
-            raise TypeError(meta)
+        meta = ensure_video(*metadata)
         par = meta.dar / (self.width / self.height)
         return replace(meta, width=self.width, height=self.height, par=par)
 
@@ -331,16 +353,8 @@ class Concat(Filter):
         duration = TS(0)
         scenes = []
         streams: List[str] = []
-        samples = 0
-        sampling_rate = None
         for meta in metadata:
             duration += meta.duration
-            if isinstance(meta, AudioMeta):
-                samples += meta.samples
-                if sampling_rate is None:
-                    sampling_rate = meta.sampling_rate
-                else:
-                    assert sampling_rate == meta.sampling_rate
             scenes.extend(meta.scenes)
             for stream in meta.streams:
                 if not streams or streams[-1] != stream:
@@ -348,8 +362,11 @@ class Concat(Filter):
                     # contiguous duplicates.
                     streams.append(stream)
         kwargs = dict(duration=duration, scenes=scenes, streams=streams)
-        if samples != 0:
-            kwargs['samples'] = samples
+        meta = metadata[0]
+        if isinstance(meta, AudioMeta):
+            # Recompute samples and sampling rate: sampling rate from first
+            # input, samples count corresponds duration.
+            kwargs['samples'] = round(meta.sampling_rate * duration)
         return replace(metadata[0], **kwargs)
 
 
@@ -389,7 +406,5 @@ class Upload(VideoFilter):
 
     def transform(self, *metadata: Meta) -> VideoMeta:
         """ Marks a stream as uploaded to a device."""
-        meta = super().transform(*metadata)
-        if not isinstance(meta, VideoMeta):
-            raise ValueError(meta)
+        meta = ensure_video(*metadata)
         return replace(meta, device=self.device)
diff --git a/fffw/wrapper/base.py b/fffw/wrapper/base.py
index b595a09..d8326b8 100644
--- a/fffw/wrapper/base.py
+++ b/fffw/wrapper/base.py
@@ -5,13 +5,78 @@ from asyncio.subprocess import Process
 from dataclasses import dataclass
 from logging import getLogger
 from types import TracebackType
-from typing import Tuple, List, Any, Optional, cast, Callable, Union, TextIO, \
-    Type
+from typing import Tuple, List, Any, Optional, cast, Callable, Union, TextIO
+from typing import Type, AsyncIterator
 
 from fffw.wrapper.helpers import quote, ensure_binary, ensure_text
 from fffw.wrapper.params import Params
 
 
+class UniversalLineReader:
+    """
+    Reads bytes from asyncio.StreamReader and splits it to lines with either
+    CR or LF, or even CRLF.
+
+    https://docs.python.org/3/glossary.html#term-universal-newlines
+
+    >>> # noinspection PyUnresolvedReferences
+    ... line_iter = UniversalLineReader(process.stderr)
+    >>> async for line in line_iter:
+    ...    print(line)
+
+    """
+
+    def __init__(self,
+                 reader: asyncio.StreamReader,
+                 bufsize: int = 10 * io.DEFAULT_BUFFER_SIZE,
+                 blocksize: int = io.DEFAULT_BUFFER_SIZE,
+                 encoding: str = 'utf8') -> None:
+        """
+        :param reader: asynchronous stream reader, i.e. stdout/stderr of
+            asyncio Process instance.
+        :param bufsize: max buffer size
+        :param blocksize: read block size
+        :param encoding: text encoding
+        """
+        self.blocksize = blocksize
+        self.reader = reader
+        self.bufsize = bufsize
+        self.buffer = b''
+        self.encoding = encoding
+        self.at_eof = False
+
+    def __aiter__(self) -> AsyncIterator[str]:
+        return self.readlines()
+
+    async def readlines(self) -> AsyncIterator[str]:
+        while not self.at_eof:
+            # StreamReader supports only LF line separator. This leads to buffer
+            # overrun when it contains only CR-terminated lines. Thus, we read
+            # blocks manually and then split it to lines with universal line
+            # separator.
+            block = await self.reader.read(self.blocksize)
+            # empty read means that stream is closed
+            self.at_eof = len(block) == 0
+
+            # checking max buffer size
+            buffered = len(self.buffer) + len(block)
+            if buffered > self.bufsize:
+                raise asyncio.LimitOverrunError("buffer overrun", buffered)
+
+            self.buffer += block
+            if self.buffer:
+                # Split buffer to line with any of CR, LF of CRLF separators
+                # Last line is buffered to handle the case when CRLF sequence is
+                # being split to subsequent reads.
+                [*lines, self.buffer] = self.buffer.splitlines(keepends=True)
+                for line in lines:
+                    yield line.decode(self.encoding)
+        # We always leave one non-empty line above, but stream may be empty. In
+        # this case we don't want to yield empty line.
+        if self.buffer:
+            yield self.buffer.decode(self.encoding)
+
+
 class Runner:
     """ Wrapper for Popen process for non-blocking streams handling."""
 
@@ -69,7 +134,7 @@ class Runner:
         """
         Handles read from stdout/stderr.
 
-        Reads lines from reader and feeds it to callback. Values, filtered by
+        Reads lines from stream and feeds it to callback. Values, filtered by
         callback, are written to output buffer.
 
         :param reader: Process.stdout or Process.stderr instance
@@ -78,14 +143,10 @@ class Runner:
         """
         if callback is None or reader is None:
             return
-        while True:
-            line = await reader.readline()
-            if line:
-                data = callback(line.decode())
-                if data:
-                    output.write(data)
-            else:
-                break
+        async for line in UniversalLineReader(reader):
+            data = callback(line)
+            if data:
+                output.write(data)
 
     @staticmethod
     async def write(writer: Optional[asyncio.StreamWriter],
diff --git a/setup.py b/setup.py
index 95909f1..a4ad5a9 100644
--- a/setup.py
+++ b/setup.py
@@ -85,6 +85,7 @@ setup(
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Multimedia :: Video :: Conversion',
     ]
 


Test patch:
diff --git a/tests/test_graph.py b/tests/test_graph.py
index dd68ec9..e8c25d9 100644
--- a/tests/test_graph.py
+++ b/tests/test_graph.py
@@ -49,6 +49,18 @@ class FilterGraphTestCase(TestCase):
         self.output_list = outputs.OutputList((self.output,))
         self.fc = FilterComplex(self.input_list, self.output_list)
 
+    def test_ensure_video(self):
+        """ Test video stream type assertion helper."""
+        with self.assertRaises(TypeError):
+            ensure_video(self.audio_metadata)
+        self.assertIs(ensure_video(self.video_metadata), self.video_metadata)
+
+    def test_ensure_audio(self):
+        """ Test audio stream type assertion helper."""
+        with self.assertRaises(TypeError):
+            ensure_audio(self.video_metadata)
+        self.assertIs(ensure_audio(self.audio_metadata), self.audio_metadata)
+
     def test_filter_graph(self):
         """ Filter complex smoke test and features demo.
 
@@ -246,8 +258,8 @@ class FilterGraphTestCase(TestCase):
         """
         Concat filter sums samples count for audio streams.
         """
-        audio_meta = audio_meta_data(duration=1000.0, sampling_rate=48000,
-                                     samples_count=48000 * 1000)
+        audio_meta = audio_meta_data(duration=1000.0, sampling_rate=24000,
+                                     samples_count=24000 * 1000)
         a = inputs.Stream(AUDIO, meta=audio_meta)
         self.input_list.append(inputs.input_file('second.mp4', a))
         concat = a | Concat(AUDIO)
@@ -258,7 +270,7 @@ class FilterGraphTestCase(TestCase):
         am = cast(AudioMeta, self.output.codecs[-1].get_meta_data())
         self.assertEqual(self.audio_metadata.duration + audio_meta.duration,
                          am.duration)
-        self.assertEqual(self.audio_metadata.samples + audio_meta.samples,
+        self.assertEqual(round(am.duration * audio_meta.sampling_rate),
                          am.samples)
 
     def test_trim_metadata(self):
diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py
new file mode 100644
index 0000000..ba68852
--- /dev/null
+++ b/tests/test_wrapper.py
@@ -0,0 +1,121 @@
+import asyncio
+import io
+import random
+import sys
+import time
+from dataclasses import dataclass
+from typing import List
+from unittest import TestCase, mock
+
+from fffw.wrapper import param
+from fffw.wrapper.base import UniversalLineReader, BaseWrapper
+
+
+@dataclass
+class Python(BaseWrapper):
+    command = 'python'
+    module: str = param(name='m')
+
+
+def script():
+    line = input()
+    sys.stdout.write(f'stdout: {line}\n')
+    sys.stderr.write(f'stderr: {line}\n')
+    time.sleep(int(line) // 100)
+    return int(line)
+
+
+if __name__ == '__main__':
+    sys.exit(script())
+
+
+class WrapperTestCase(TestCase):
+
+    def test_run_child_process(self):
+        p = Python(module='tests.test_wrapper')
+        ret, out, err = p.run('1')
+        self.assertEqual(ret, 1)
+        self.assertEqual(out, 'stdout: 1\n')
+        self.assertEqual(err, 'stderr: 1\n')
+
+    def test_child_timeout(self):
+        p = Python(module='tests.test_wrapper')
+        ret, out, err = p.run('100', timeout=0.01)
+        self.assertEqual(ret, -9)
+
+
+class UniversalLineReaderTestCase(TestCase):
+    def setUp(self) -> None:
+        self.data = io.BytesIO()
+        self.stream = mock.MagicMock(read=self.read)
+        self.reader = UniversalLineReader(self.stream, bufsize=100, blocksize=5)
+
+    async def read(self, n=-1):
+        return self.data.read(n)
+
+    async def iterate(self) -> List[str]:
+        result = []
+        async for line in self.reader:
+            result.append(line)
+        return result
+
+    def assert_lines(self, lines: List[str]):
+        for line in lines:
+            self.data.write(line.encode('utf-8'))
+        self.data.seek(0)
+        result = asyncio.get_event_loop().run_until_complete(self.iterate())
+        self.assertListEqual(lines, result)
+
+    def test_read_lf(self):
+        lines = []
+        for c in 'abcdefghijklmnopqrstuvwxyz':
+            length = random.randint(0, 15)
+            lines.append(c * length + '\n')
+        for line in lines:
+            self.data.write(line.encode('utf-8'))
+        self.data.seek(0)
+
+        self.assert_lines(lines)
+
+    def test_read_cr(self):
+        lines = []
+        for c in 'abcdefghijklmnopqrstuvwxyz':
+            length = random.randint(0, 15)
+            lines.append(c * length + '\r')
+
+        self.assert_lines(lines)
+
+    def test_read_crlf(self):
+        lines = []
+        for c in 'abcdefghijklmnopqrstuvwxyz':
+            length = random.randint(0, 15)
+            lines.append(c * length + '\r\n')
+
+        self.assert_lines(lines)
+
+    def test_empty_lines(self):
+        lines = [
+            'a\n',
+            '\n',
+            'b\n'
+        ]
+
+        self.assert_lines(lines)
+
+    def test_last_incomplete_line(self):
+        lines = [
+            'aaaaa\n',
+            'b'
+        ]
+
+        self.assert_lines(lines)
+
+    def test_empty_stream(self):
+        self.assert_lines([])
+
+    def test_buffer_overrun(self):
+        max_line = 'a' * self.reader.bufsize
+        lines = [max_line + '\n']
+
+        with self.assertRaises(asyncio.LimitOverrunError):
+            self.assert_lines(lines)