Repository: python-trio/trio

Issue:
Should we somehow expose Pipe*Stream on Unix, and if so, how?
We have `PipeReceiveStream` and `PipeSendStream` classes on Unix, that are currently not exposed publicly; they're only used internally for subprocesses.

The actual code is a pretty generic building block: they can wrap around any Unix file-descriptor that supports non-blocking mode. So they could potentially be useful for talking to [FIFOs](http://man7.org/linux/man-pages/man7/fifo.7.html), TTYs, stdio streams (#174), etc.

Should we expose them more directly? If so, how? In `trio.hazmat`, with some sort of [`os.fdopen`](https://docs.python.org/3/library/os.html#os.fdopen) equivalent?

If we do we'll want to think through the ownership semantics: should they take ownership of a passed in fd, or should they dup it?

Also, they should restore the fd's blocking/non-blocking state before closing it.

Comments:

Patch:
diff --git a/docs-requirements.txt b/docs-requirements.txt
index 02ecb74a..d1466815 100644
--- a/docs-requirements.txt
+++ b/docs-requirements.txt
@@ -11,7 +11,7 @@ babel==2.7.0              # via sphinx
 certifi==2019.6.16        # via requests
 chardet==3.0.4            # via requests
 click==7.0                # via towncrier
-docutils==0.14            # via sphinx
+docutils==0.15            # via sphinx
 idna==2.8
 imagesize==1.1.0          # via sphinx
 immutables==0.9
@@ -21,7 +21,7 @@ markupsafe==1.1.1         # via jinja2
 outcome==1.0.0
 packaging==19.0           # via sphinx
 pygments==2.4.2           # via sphinx
-pyparsing==2.4.0          # via packaging
+pyparsing==2.4.1          # via packaging
 pytz==2019.1              # via babel
 requests==2.22.0          # via sphinx
 six==1.12.0               # via packaging
diff --git a/docs/source/reference-hazmat.rst b/docs/source/reference-hazmat.rst
index 6cad8476..e12d3601 100644
--- a/docs/source/reference-hazmat.rst
+++ b/docs/source/reference-hazmat.rst
@@ -174,6 +174,26 @@ All environments provide the following functions:
    yourself afterwards.
 
 
+Unix-specific API
+-----------------
+
+`FdStream` supports wrapping Unix files (such as a pipe or TTY) as
+a stream.
+
+If you have two different file descriptors for sending and receiving,
+and want to bundle them together into a single bidirectional
+`~trio.abc.Stream`, then use `trio.StapledStream`::
+
+    bidirectional_stream = trio.StapledStream(
+        trio.hazmat.FdStream(write_fd),
+        trio.hazmat.FdStream(read_fd)
+    )
+
+.. autoclass:: FdStream
+   :show-inheritance:
+   :members:
+
+
 Kqueue-specific API
 -------------------
 
diff --git a/newsfragments/829.feature.rst b/newsfragments/829.feature.rst
new file mode 100644
index 00000000..34090857
--- /dev/null
+++ b/newsfragments/829.feature.rst
@@ -0,0 +1,1 @@
+Add `trio.hazmat.FdStream` for wrapping a Unix file descriptor as a `~trio.abc.Stream`.
diff --git a/trio/_subprocess.py b/trio/_subprocess.py
index ae530e3a..f9bc0647 100644
--- a/trio/_subprocess.py
+++ b/trio/_subprocess.py
@@ -1,9 +1,8 @@
 import os
-import select
 import subprocess
-from functools import partial
+from typing import Optional
 
-from ._abc import AsyncResource
+from ._abc import AsyncResource, SendStream, ReceiveStream
 from ._highlevel_generic import StapledStream
 from ._sync import Lock
 from ._subprocess_platform import (
@@ -101,9 +100,10 @@ class Process(AsyncResource):
                     .format(key)
                 )
 
-        self.stdin = None
-        self.stdout = None
-        self.stderr = None
+        self.stdin = None  # type: Optional[SendStream]
+        self.stdout = None  # type: Optional[ReceiveStream]
+        self.stderr = None  # type: Optional[ReceiveStream]
+        self.stdio = None  # type: Optional[StapledStream]
 
         if os.name == "posix":
             if isinstance(command, str) and not options.get("shell"):
@@ -153,8 +153,6 @@ class Process(AsyncResource):
 
         if self.stdin is not None and self.stdout is not None:
             self.stdio = StapledStream(self.stdin, self.stdout)
-        else:
-            self.stdio = None
 
         self.args = self._proc.args
         self.pid = self._proc.pid
diff --git a/trio/_subprocess_platform/__init__.py b/trio/_subprocess_platform/__init__.py
index 1507ec12..b1db8499 100644
--- a/trio/_subprocess_platform/__init__.py
+++ b/trio/_subprocess_platform/__init__.py
@@ -67,15 +67,15 @@ except ImportError as ex:  # pragma: no cover
 
 try:
     if os.name == "posix":
-        from .._unix_pipes import PipeSendStream, PipeReceiveStream
+        from ..hazmat import FdStream
 
         def create_pipe_to_child_stdin():  # noqa: F811
             rfd, wfd = os.pipe()
-            return PipeSendStream(wfd), rfd
+            return FdStream(wfd), rfd
 
         def create_pipe_from_child_output():  # noqa: F811
             rfd, wfd = os.pipe()
-            return PipeReceiveStream(rfd), wfd
+            return FdStream(rfd), wfd
 
     elif os.name == "nt":
         from .._windows_pipes import PipeSendStream, PipeReceiveStream
diff --git a/trio/_unix_pipes.py b/trio/_unix_pipes.py
index 7cd205e9..b212b2a9 100644
--- a/trio/_unix_pipes.py
+++ b/trio/_unix_pipes.py
@@ -1,12 +1,16 @@
-import fcntl
 import os
 import errno
 
-from ._abc import SendStream, ReceiveStream
+from ._abc import Stream
 from ._util import ConflictDetector
 
 import trio
 
+if os.name != "posix":
+    # We raise an error here rather than gating the import in hazmat.py
+    # in order to keep jedi static analysis happy.
+    raise ImportError
+
 
 class _FdHolder:
     # This class holds onto a raw file descriptor, in non-blocking mode, and
@@ -33,9 +37,9 @@ class _FdHolder:
         if not isinstance(fd, int):
             raise TypeError("file descriptor must be an int")
         self.fd = fd
-        # Flip the fd to non-blocking mode
-        flags = fcntl.fcntl(self.fd, fcntl.F_GETFL)
-        fcntl.fcntl(self.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+        # Store original state, and ensure non-blocking mode is enabled
+        self._original_is_blocking = os.get_blocking(fd)
+        os.set_blocking(fd, False)
 
     @property
     def closed(self):
@@ -53,6 +57,7 @@ class _FdHolder:
             return
         fd = self.fd
         self.fd = -1
+        os.set_blocking(fd, self._original_is_blocking)
         os.close(fd)
 
     def __del__(self):
@@ -65,21 +70,53 @@ class _FdHolder:
         await trio.hazmat.checkpoint()
 
 
-class PipeSendStream(SendStream):
-    """Represents a send stream over an os.pipe object."""
+class FdStream(Stream):
+    """
+    Represents a stream given the file descriptor to a pipe, TTY, etc.
+
+    *fd* must refer to a file that is open for reading and/or writing and
+    supports non-blocking I/O (pipes and TTYs will work, on-disk files probably
+    not).  The returned stream takes ownership of the fd, so closing the stream
+    will close the fd too.  As with `os.fdopen`, you should not directly use
+    an fd after you have wrapped it in a stream using this function.
+
+    To be used as a Trio stream, an open file must be placed in non-blocking
+    mode.  Unfortunately, this impacts all I/O that goes through the
+    underlying open file, including I/O that uses a different
+    file descriptor than the one that was passed to Trio. If other threads
+    or processes are using file descriptors that are related through `os.dup`
+    or inheritance across `os.fork` to the one that Trio is using, they are
+    unlikely to be prepared to have non-blocking I/O semantics suddenly
+    thrust upon them.  For example, you can use ``FdStream(os.dup(0))`` to
+    obtain a stream for reading from standard input, but it is only safe to
+    do so with heavy caveats: your stdin must not be shared by any other
+    processes and you must not make any calls to synchronous methods of
+    `sys.stdin` until the stream returned by `FdStream` is closed. See
+    `issue #174 <https://github.com/python-trio/trio/issues/174>`__ for a
+    discussion of the challenges involved in relaxing this restriction.
+
+    Args:
+      fd (int): The fd to be wrapped.
+
+    Returns:
+      A new `FdStream` object.
+    """
 
     def __init__(self, fd: int):
         self._fd_holder = _FdHolder(fd)
-        self._conflict_detector = ConflictDetector(
-            "another task is using this pipe"
+        self._send_conflict_detector = ConflictDetector(
+            "another task is using this stream for send"
+        )
+        self._receive_conflict_detector = ConflictDetector(
+            "another task is using this stream for receive"
         )
 
     async def send_all(self, data: bytes):
-        with self._conflict_detector:
+        with self._send_conflict_detector:
             # have to check up front, because send_all(b"") on a closed pipe
             # should raise
             if self._fd_holder.closed:
-                raise trio.ClosedResourceError("this pipe was already closed")
+                raise trio.ClosedResourceError("file was already closed")
             await trio.hazmat.checkpoint()
             length = len(data)
             # adapted from the SocketStream code
@@ -94,15 +131,15 @@ class PipeSendStream(SendStream):
                         except OSError as e:
                             if e.errno == errno.EBADF:
                                 raise trio.ClosedResourceError(
-                                    "this pipe was closed"
+                                    "file was already closed"
                                 ) from None
                             else:
                                 raise trio.BrokenResourceError from e
 
     async def wait_send_all_might_not_block(self) -> None:
-        with self._conflict_detector:
+        with self._send_conflict_detector:
             if self._fd_holder.closed:
-                raise trio.ClosedResourceError("this pipe was already closed")
+                raise trio.ClosedResourceError("file was already closed")
             try:
                 await trio.hazmat.wait_writable(self._fd_holder.fd)
             except BrokenPipeError as e:
@@ -110,24 +147,8 @@ class PipeSendStream(SendStream):
                 # of sending, which is annoying
                 raise trio.BrokenResourceError from e
 
-    async def aclose(self):
-        await self._fd_holder.aclose()
-
-    def fileno(self):
-        return self._fd_holder.fd
-
-
-class PipeReceiveStream(ReceiveStream):
-    """Represents a receive stream over an os.pipe object."""
-
-    def __init__(self, fd: int):
-        self._fd_holder = _FdHolder(fd)
-        self._conflict_detector = ConflictDetector(
-            "another task is using this pipe"
-        )
-
     async def receive_some(self, max_bytes: int) -> bytes:
-        with self._conflict_detector:
+        with self._receive_conflict_detector:
             if not isinstance(max_bytes, int):
                 raise TypeError("max_bytes must be integer >= 1")
 
@@ -143,7 +164,7 @@ class PipeReceiveStream(ReceiveStream):
                 except OSError as e:
                     if e.errno == errno.EBADF:
                         raise trio.ClosedResourceError(
-                            "this pipe was closed"
+                            "file was already closed"
                         ) from None
                     else:
                         raise trio.BrokenResourceError from e
diff --git a/trio/hazmat.py b/trio/hazmat.py
index 69070a79..5fe32c03 100644
--- a/trio/hazmat.py
+++ b/trio/hazmat.py
@@ -3,6 +3,7 @@ This namespace represents low-level functionality not intended for daily use,
 but useful for extending Trio's functionality.
 """
 
+import os
 import sys
 
 # This is the union of a subset of trio/_core/ and some things from trio/*.py.
@@ -22,6 +23,12 @@ from ._core import (
     spawn_system_task, wait_readable, wait_writable, notify_closing
 )
 
+# Unix-specific symbols
+try:
+    from ._unix_pipes import FdStream
+except ImportError:
+    pass
+
 # Kqueue-specific symbols
 try:
     from ._core import (


Test patch:
diff --git a/test-requirements.txt b/test-requirements.txt
index 5aab7fc4..3237f14f 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -26,7 +26,7 @@ isort==4.3.21             # via pylint
 jedi==0.14.1
 lazy-object-proxy==1.4.1  # via astroid
 mccabe==0.6.1             # via flake8, pylint
-more-itertools==7.1.0     # via pytest
+more-itertools==7.2.0     # via pytest
 outcome==1.0.0
 packaging==19.0           # via pytest
 parso==0.5.1              # via jedi
@@ -42,7 +42,7 @@ pyflakes==2.1.1           # via flake8
 pygments==2.4.2           # via ipython
 pylint==2.3.1
 pyopenssl==19.0.0
-pyparsing==2.4.0          # via packaging
+pyparsing==2.4.1          # via packaging
 pytest-cov==2.7.1
 pytest==5.0.1
 six==1.12.0               # via astroid, cryptography, packaging, prompt-toolkit, pyopenssl, traitlets
diff --git a/trio/tests/test_unix_pipes.py b/trio/tests/test_unix_pipes.py
index 7728a57f..46f28d54 100644
--- a/trio/tests/test_unix_pipes.py
+++ b/trio/tests/test_unix_pipes.py
@@ -11,14 +11,17 @@ from ..testing import wait_all_tasks_blocked, check_one_way_stream
 posix = os.name == "posix"
 pytestmark = pytest.mark.skipif(not posix, reason="posix only")
 if posix:
-    from .._unix_pipes import PipeSendStream, PipeReceiveStream
+    from .._unix_pipes import FdStream
+else:
+    with pytest.raises(ImportError):
+        from .._unix_pipes import FdStream
 
 
 # Have to use quoted types so import doesn't crash on windows
-async def make_pipe() -> "Tuple[PipeSendStream, PipeReceiveStream]":
+async def make_pipe() -> "Tuple[FdStream, FdStream]":
     """Makes a new pair of pipes."""
     (r, w) = os.pipe()
-    return PipeSendStream(w), PipeReceiveStream(r)
+    return FdStream(w), FdStream(r)
 
 
 async def make_clogged_pipe():
@@ -49,7 +52,7 @@ async def make_clogged_pipe():
 
 async def test_send_pipe():
     r, w = os.pipe()
-    async with PipeSendStream(w) as send:
+    async with FdStream(w) as send:
         assert send.fileno() == w
         await send.send_all(b"123")
         assert (os.read(r, 8)) == b"123"
@@ -59,7 +62,7 @@ async def test_send_pipe():
 
 async def test_receive_pipe():
     r, w = os.pipe()
-    async with PipeReceiveStream(r) as recv:
+    async with FdStream(r) as recv:
         assert (recv.fileno()) == r
         os.write(w, b"123")
         assert (await recv.receive_some(8)) == b"123"
@@ -93,10 +96,10 @@ async def test_pipes_combined():
 
 async def test_pipe_errors():
     with pytest.raises(TypeError):
-        PipeReceiveStream(None)
+        FdStream(None)
 
     with pytest.raises(ValueError):
-        await PipeReceiveStream(0).receive_some(0)
+        await FdStream(0).receive_some(0)
 
 
 async def test_del():
@@ -146,7 +149,7 @@ async def test_misdirected_aclose_regression():
     if r2_fd != old_r_fd:  # pragma: no cover
         os.dup2(r2_fd, old_r_fd)
         os.close(r2_fd)
-    async with PipeReceiveStream(old_r_fd) as r2:
+    async with FdStream(old_r_fd) as r2:
         assert r2.fileno() == old_r_fd
 
         # And now set up a background task that's working on the new receive