Repository: encode/httpx

Issue:
Feature Request: support iterating stream lines
I think it is better to implement a api for iterating stream line by line. This scene is very common (e.g., kubernetes watch api). Now we need to create a wrapper for this.

```Python
import json
import httpx


class StreamWrapper(object):
    def __init__(self, stream):
        self._stream = stream

    def __iter__(self):
        pending = ""

        for chunk in self._stream:
            chunk = pending + chunk
            lines = chunk.splitlines()

            if chunk and lines and lines[-1] and lines[-1][-1] == chunk[-1]:
                pending = lines.pop()
            else:
                pending = ""

            for line in lines:
                yield line

        if pending:
            yield pending


timeout = httpx.TimeoutConfig(
    connect_timeout=5, read_timeout=None, write_timeout=5
)


resp = httpx.get(
    "http://127.0.0.1:18081/api/v1/watch/namespaces/default/pods",
    stream=True,
    timeout=timeout,
)

for chunk in StreamWrapper(resp.stream_text()):
    print(json.loads(chunk))
```

Comments:

Patch:
diff --git a/httpx/decoders.py b/httpx/decoders.py
index 3ae822b..22c6104 100644
--- a/httpx/decoders.py
+++ b/httpx/decoders.py
@@ -211,6 +211,67 @@ class TextDecoder:
         return result
 
 
+class LineDecoder:
+    """
+    Handles incrementally reading lines from text.
+
+    Uses universal line decoding, supporting any of `\n`, `\r`, or `\r\n`
+    as line endings, normalizing to `\n`.
+    """
+
+    def __init__(self) -> None:
+        self.buffer = ""
+
+    def decode(self, text: str) -> typing.List[str]:
+        lines = []
+
+        if text.startswith("\n") and self.buffer and self.buffer[-1] == "\r":
+            # Handle the case where we have an "\r\n" split across
+            # our previous input, and our new chunk.
+            lines.append(self.buffer[:-1] + "\n")
+            self.buffer = ""
+            text = text[1:]
+
+        while text:
+            num_chars = len(text)
+            for idx in range(num_chars):
+                char = text[idx]
+                next_char = None if idx + 1 == num_chars else text[idx + 1]
+                if char == "\n":
+                    lines.append(self.buffer + text[: idx + 1])
+                    self.buffer = ""
+                    text = text[idx + 1 :]
+                    break
+                elif char == "\r" and next_char == "\n":
+                    lines.append(self.buffer + text[:idx] + "\n")
+                    self.buffer = ""
+                    text = text[idx + 2 :]
+                    break
+                elif char == "\r" and next_char is not None:
+                    lines.append(self.buffer + text[:idx] + "\n")
+                    self.buffer = ""
+                    text = text[idx + 1 :]
+                    break
+                elif next_char is None:
+                    self.buffer = text
+                    text = ""
+                    break
+
+        return lines
+
+    def flush(self) -> typing.List[str]:
+        if self.buffer.endswith("\r"):
+            # Handle the case where we had a trailing '\r', which could have
+            # been a '\r\n' pair.
+            lines = [self.buffer[:-1] + "\n"]
+        elif self.buffer:
+            lines = [self.buffer]
+        else:
+            lines = []
+        self.buffer = ""
+        return lines
+
+
 SUPPORTED_DECODERS = {
     "identity": IdentityDecoder,
     "gzip": GZipDecoder,
diff --git a/httpx/models.py b/httpx/models.py
index 5469efc..33cfb83 100644
--- a/httpx/models.py
+++ b/httpx/models.py
@@ -17,6 +17,7 @@ from .decoders import (
     SUPPORTED_DECODERS,
     Decoder,
     IdentityDecoder,
+    LineDecoder,
     MultiDecoder,
     TextDecoder,
 )
@@ -936,6 +937,14 @@ class Response:
             yield decoder.decode(chunk)
         yield decoder.flush()
 
+    async def stream_lines(self) -> typing.AsyncIterator[str]:
+        decoder = LineDecoder()
+        async for text in self.stream_text():
+            for line in decoder.decode(text):
+                yield line
+        for line in decoder.flush():
+            yield line
+
     async def raw(self) -> typing.AsyncIterator[bytes]:
         """
         A byte-iterator over the raw response content.


Test patch:
diff --git a/tests/models/test_responses.py b/tests/models/test_responses.py
index e7be487..9b60c67 100644
--- a/tests/models/test_responses.py
+++ b/tests/models/test_responses.py
@@ -164,6 +164,18 @@ async def test_stream_text():
     assert content == "Hello, world!"
 
 
+@pytest.mark.asyncio
+async def test_stream_lines():
+    response = httpx.Response(200, content=b"Hello,\nworld!")
+
+    await response.read()
+
+    content = []
+    async for line in response.stream_lines():
+        content.append(line)
+    assert content == ["Hello,\n", "world!"]
+
+
 @pytest.mark.asyncio
 async def test_stream_interface_after_read():
     response = httpx.Response(200, content=b"Hello, world!")
diff --git a/tests/test_decoders.py b/tests/test_decoders.py
index 7525239..a599ce0 100644
--- a/tests/test_decoders.py
+++ b/tests/test_decoders.py
@@ -9,6 +9,7 @@ from httpx.decoders import (
     DeflateDecoder,
     GZipDecoder,
     IdentityDecoder,
+    LineDecoder,
     TextDecoder,
 )
 
@@ -167,6 +168,48 @@ def test_text_decoder_empty_cases():
     assert decoder.flush() == ""
 
 
+def test_line_decoder_nl():
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\n\nb\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c"]
+
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\n\nb\nc\n") == ["a\n", "\n", "b\n", "c\n"]
+    assert decoder.flush() == []
+
+
+def test_line_decoder_cr():
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r\rb\rc") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c"]
+
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r\rb\rc\r") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c\n"]
+
+
+def test_line_decoder_crnl():
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r\n\r\nb\r\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c"]
+
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r\n\r\nb\r\nc\r\n") == ["a\n", "\n", "b\n", "c\n"]
+    assert decoder.flush() == []
+
+    decoder = LineDecoder()
+    assert decoder.decode("") == []
+    assert decoder.decode("a\r") == []
+    assert decoder.decode("\n\r\nb\r\nc") == ["a\n", "\n", "b\n"]
+    assert decoder.flush() == ["c"]
+
+
 def test_invalid_content_encoding_header():
     headers = [(b"Content-Encoding", b"invalid-header")]
     body = b"test 123"