././@PaxHeader0000000000000000000000000000003400000000000011452 xustar000000000000000028 mtime=1629878364.9168835 wurlitzer-3.0.2/0000755000175100001710000000000000000000000014363 5ustar00runnerdocker00000000000000././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/Demo.ipynb0000644000175100001710000001333300000000000016315 0ustar00runnerdocker00000000000000{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Capturing C-level stdout/stderr with `wurlitzer`\n", "\n", "Sometimes in Python you are calling some C code.\n", "Sometimes that C code makes calls to `printf`,\n", "or otherwise writes to the stdout/stderr of the process." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": false }, "outputs": [], "source": [ "import ctypes\n", "libc = ctypes.CDLL(None)\n", "\n", "try:\n", " c_stderr_p = ctypes.c_void_p.in_dll(libc, 'stderr')\n", "except ValueError:\n", " # libc.stdout is has a funny name on OS X\n", " c_stderr_p = ctypes.c_void_p.in_dll(libc, '__stderrp')\n", "\n", "\n", "def printf(msg):\n", " \"\"\"Call C printf\"\"\"\n", " libc.printf((msg + '\\n').encode('utf8'))\n", "\n", "def printf_err(msg):\n", " \"\"\"Cal C fprintf on stderr\"\"\"\n", " libc.fprintf(c_stderr_p, (msg + '\\n').encode('utf8'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "IPython forwards the Python-level `sys.stdout` and `sys.stderr`,\n", "but it leaves the process-level file descriptors that C code will write to untouched.\n", "That means that in a context like this notebook, these functions will print to the terminal, because they are not captured:" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": false }, "outputs": [], "source": [ "printf(\"Hello?\")\n", "printf_err(\"Stderr? Anybody?\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "With wurlitzer, we can capture these C-level functions:" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from wurlitzer import pipes, sys_pipes, STDOUT, PIPE" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "with pipes() as (stdout, stderr):\n", " printf(\"Hello, stdout!\")\n", " printf_err(\"Hello, stderr!\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "and redisplay them if we like:" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Hello, stdout!\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Hello, stderr!\n" ] } ], "source": [ "import sys\n", "sys.stdout.write(stdout.read())\n", "sys.stderr.write(stderr.read())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Some tools, such as the IPython kernel for Jupyter,\n", "capture the Python-level `sys.stdout` and `sys.stderr` and forward them somewhere.\n", "In the case of Jupyter, this is over a network socket, so that it ends up in the browser.\n", "\n", "If we know that's going on, we can easily hook up the C outputs to the Python-forwarded ones with a single call:" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Hello from C, 0!\n", "Hello from C, 1!\n", "Hello from C, 2!\n", "Hello from C, 3!\n", "Hello from C, 4!\n" ] } ], "source": [ "import time\n", "\n", "with sys_pipes():\n", " for i in range(5):\n", " time.sleep(1)\n", " printf(\"Hello from C, %i!\" % i)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We can also capture the pipes to any writeable streams, such as a `StringIO` object:" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Hello, stdout!\n", "Hello, stderr!\n", "\n" ] } ], "source": [ "import io\n", "\n", "stdout = io.StringIO()\n", "with pipes(stdout=stdout, stderr=STDOUT):\n", " printf(\"Hello, stdout!\")\n", " printf_err(\"Hello, stderr!\")\n", "\n", "print(stdout.getvalue())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## IPython extension\n", "\n", "You can also enable wurlitzer as an IPython extension,\n", "so that it always forwards C-level output during execution:" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%load_ext wurlitzer" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Hello from C, 0!\n", "Hello from C, 1!\n", "Hello from C, 2!\n", "Hello from C, 3!\n", "Hello from C, 4!\n" ] } ], "source": [ "for i in range(5):\n", " time.sleep(1)\n", " printf(\"Hello from C, %i!\" % i)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.1" } }, "nbformat": 4, "nbformat_minor": 1 } ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/LICENSE0000644000175100001710000000206100000000000015367 0ustar00runnerdocker00000000000000The MIT License (MIT) Copyright (c) 2016 Min RK Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/MANIFEST.in0000644000175100001710000000017500000000000016124 0ustar00runnerdocker00000000000000include README.md include test.py include LICENSE include Demo.ipynb include dev-requirements.txt prune htmlcov prune build ././@PaxHeader0000000000000000000000000000003400000000000011452 xustar000000000000000028 mtime=1629878364.9168835 wurlitzer-3.0.2/PKG-INFO0000644000175100001710000000352300000000000015463 0ustar00runnerdocker00000000000000Metadata-Version: 2.1 Name: wurlitzer Version: 3.0.2 Summary: Capture C-level output in context managers Home-page: https://github.com/minrk/wurlitzer Author: Min RK Author-email: benjaminrk@gmail.com License: MIT Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 3 Requires-Python: >=3.5 Description-Content-Type: text/markdown License-File: LICENSE # Wurlitzer Capture C-level stdout/stderr pipes in Python via `os.dup2`. For more details on why this is needed, please read [this blog post](https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/). ## Install pip install wurlitzer ## Usage Capture stdout/stderr in pipes: ```python from wurlitzer import pipes with pipes() as (out, err): call_some_c_function() stdout = out.read() ``` Capture stdout/stderr in StringIO: ```python from io import StringIO from wurlitzer import pipes, STDOUT out = StringIO() with pipes(stdout=out, stderr=STDOUT): call_some_c_function() stdout = out.getvalue() ``` Forward C-level stdout/stderr to Python sys.stdout/stderr, which may already be forwarded somewhere by the environment, e.g. IPython: ```python from wurlitzer import sys_pipes with sys_pipes(): call_some_c_function() ``` Or even simpler, enable it as an IPython extension: ``` %load_ext wurlitzer ``` To forward all C-level output to IPython during execution. ## Acknowledgments This package is based on stuff we learned with @takluyver and @karies while working on capturing output from the [Cling Kernel](https://github.com/root-mirror/cling/tree/master/tools/Jupyter/kernel) for Jupyter. ## Wurlitzer?! [Wurlitzer](https://en.wikipedia.org/wiki/Wurlitzer) makes pipe organs. Get it? Pipes? Naming is hard. ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/README.md0000644000175100001710000000253100000000000015643 0ustar00runnerdocker00000000000000# Wurlitzer Capture C-level stdout/stderr pipes in Python via `os.dup2`. For more details on why this is needed, please read [this blog post](https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/). ## Install pip install wurlitzer ## Usage Capture stdout/stderr in pipes: ```python from wurlitzer import pipes with pipes() as (out, err): call_some_c_function() stdout = out.read() ``` Capture stdout/stderr in StringIO: ```python from io import StringIO from wurlitzer import pipes, STDOUT out = StringIO() with pipes(stdout=out, stderr=STDOUT): call_some_c_function() stdout = out.getvalue() ``` Forward C-level stdout/stderr to Python sys.stdout/stderr, which may already be forwarded somewhere by the environment, e.g. IPython: ```python from wurlitzer import sys_pipes with sys_pipes(): call_some_c_function() ``` Or even simpler, enable it as an IPython extension: ``` %load_ext wurlitzer ``` To forward all C-level output to IPython during execution. ## Acknowledgments This package is based on stuff we learned with @takluyver and @karies while working on capturing output from the [Cling Kernel](https://github.com/root-mirror/cling/tree/master/tools/Jupyter/kernel) for Jupyter. ## Wurlitzer?! [Wurlitzer](https://en.wikipedia.org/wiki/Wurlitzer) makes pipe organs. Get it? Pipes? Naming is hard. ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/dev-requirements.txt0000644000175100001710000000007500000000000020425 0ustar00runnerdocker00000000000000codecov mock # python_version < '3.0' pytest>=2.8 pytest-cov ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/pyproject.toml0000644000175100001710000000022300000000000017274 0ustar00runnerdocker00000000000000[tool.isort] profile = "black" [tool.black] skip-string-normalization = true target_version = [ "py27", "py36", "py37", "py38", ] ././@PaxHeader0000000000000000000000000000003400000000000011452 xustar000000000000000028 mtime=1629878364.9168835 wurlitzer-3.0.2/setup.cfg0000644000175100001710000000004600000000000016204 0ustar00runnerdocker00000000000000[egg_info] tag_build = tag_date = 0 ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/setup.py0000644000175100001710000000265100000000000016101 0ustar00runnerdocker00000000000000#!/usr/bin/env python import sys from setuptools import setup from setuptools.command.bdist_egg import bdist_egg with open("README.md") as f: long_description = f.read() version_ns = {} with open("wurlitzer.py") as f: for line in f: if line.startswith("__version__"): exec(line, version_ns) class bdist_egg_disabled(bdist_egg): """Disabled version of bdist_egg Prevents setup.py install from performing setuptools' default easy_install, which it should never ever do. """ def run(self): sys.exit( "Aborting implicit building of eggs. Use `pip install .` to install from source." ) setup_args = dict( name="wurlitzer", version=version_ns["__version__"], author="Min RK", author_email="benjaminrk@gmail.com", description="Capture C-level output in context managers", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/minrk/wurlitzer", py_modules=["wurlitzer"], python_requires=">=3.5", license="MIT", cmdclass={ "bdist_egg": bdist_egg if "bdist_egg" in sys.argv else bdist_egg_disabled }, classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", ], ) if __name__ == "__main__": setup(**setup_args) ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/test.py0000644000175100001710000001127500000000000015722 0ustar00runnerdocker00000000000000# coding: utf-8 from __future__ import print_function import io import os import platform import sys import time from fcntl import fcntl from tempfile import TemporaryFile from unittest import mock import pytest import wurlitzer from wurlitzer import ( PIPE, STDOUT, Wurlitzer, c_stderr_p, c_stdout_p, libc, pipes, stop_sys_pipes, sys_pipes, sys_pipes_forever, ) def printf(msg): """Call C printf""" libc.printf((msg + '\n').encode('utf8')) def printf_err(msg): """Cal C fprintf on stderr""" libc.fprintf(c_stderr_p, (msg + '\n').encode('utf8')) def test_pipes(): with pipes(stdout=PIPE, stderr=PIPE) as (stdout, stderr): printf(u"Hellø") printf_err(u"Hi, stdérr") assert stdout.read() == u"Hellø\n" assert stderr.read() == u"Hi, stdérr\n" def test_pipe_bytes(): with pipes(encoding=None) as (stdout, stderr): printf(u"Hellø") printf_err(u"Hi, stdérr") assert stdout.read() == u"Hellø\n".encode('utf8') assert stderr.read() == u"Hi, stdérr\n".encode('utf8') def test_forward(): stdout = io.StringIO() stderr = io.StringIO() with pipes(stdout=stdout, stderr=stderr) as (_stdout, _stderr): printf(u"Hellø") printf_err(u"Hi, stdérr") assert _stdout is stdout assert _stderr is stderr assert stdout.getvalue() == u"Hellø\n" assert stderr.getvalue() == u"Hi, stdérr\n" def test_pipes_stderr(): stdout = io.StringIO() with pipes(stdout=stdout, stderr=STDOUT) as (_stdout, _stderr): printf(u"Hellø") libc.fflush(c_stdout_p) time.sleep(0.1) printf_err(u"Hi, stdérr") assert _stdout is stdout assert _stderr is None assert stdout.getvalue() == u"Hellø\nHi, stdérr\n" def test_flush(): stdout = io.StringIO() w = Wurlitzer(stdout=stdout, stderr=STDOUT) with w: printf_err(u"Hellø") time.sleep(0.5) assert stdout.getvalue().strip() == u"Hellø" def test_sys_pipes(): stdout = io.StringIO() stderr = io.StringIO() with mock.patch('sys.stdout', stdout), mock.patch( 'sys.stderr', stderr ), sys_pipes(): printf(u"Hellø") printf_err(u"Hi, stdérr") assert stdout.getvalue() == u"Hellø\n" assert stderr.getvalue() == u"Hi, stdérr\n" def test_redirect_everything(): stdout = io.StringIO() stderr = io.StringIO() with mock.patch('sys.stdout', stdout), mock.patch('sys.stderr', stderr): sys_pipes_forever() printf(u"Hellø") printf_err(u"Hi, stdérr") stop_sys_pipes() assert stdout.getvalue() == u"Hellø\n" assert stderr.getvalue() == u"Hi, stdérr\n" def count_fds(): """utility for counting file descriptors""" proc_fds = '/proc/{}/fd'.format(os.getpid()) if os.path.isdir(proc_fds): return len(proc_fds) else: # this is an approximate count, # but it should at least be stable if we aren't leaking with TemporaryFile() as tf: return tf.fileno() def test_fd_leak(): base_count = count_fds() with pipes(): print('ok') assert count_fds() == base_count for i in range(10): with pipes(): print('ok') assert count_fds() == base_count def test_buffer_full(): with pipes(stdout=None, stderr=io.StringIO()) as (stdout, stderr): long_string = "x" * 100000 # create a long string (longer than 65536) printf_err(long_string) # Test never reaches here as the process hangs. assert stderr.getvalue() == long_string + "\n" def test_buffer_full_default(): with pipes() as (stdout, stderr): long_string = "x" * 100000 # create a long string (longer than 65536) printf(long_string) # Test never reaches here as the process hangs. assert stdout.read() == long_string + "\n" def test_pipe_max_size(): max_pipe_size = wurlitzer._get_max_pipe_size() if platform.system() == 'Linux': assert 65535 <= max_pipe_size <= 1024 * 1024 else: assert max_pipe_size is None @pytest.mark.skipif( wurlitzer._get_max_pipe_size() is None, reason="requires _get_max_pipe_size" ) def test_bufsize(): default_bufsize = wurlitzer._get_max_pipe_size() with wurlitzer.pipes() as (stdout, stderr): assert fcntl(sys.__stdout__, wurlitzer.F_GETPIPE_SZ) == default_bufsize assert fcntl(sys.__stderr__, wurlitzer.F_GETPIPE_SZ) == default_bufsize bufsize = 32768 # seems to only accept powers of two? with wurlitzer.pipes(bufsize=bufsize) as (stdout, stderr): assert fcntl(sys.__stdout__, wurlitzer.F_GETPIPE_SZ) == bufsize assert fcntl(sys.__stderr__, wurlitzer.F_GETPIPE_SZ) == bufsize ././@PaxHeader0000000000000000000000000000003400000000000011452 xustar000000000000000028 mtime=1629878364.9168835 wurlitzer-3.0.2/wurlitzer.egg-info/0000755000175100001710000000000000000000000020124 5ustar00runnerdocker00000000000000././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878364.0 wurlitzer-3.0.2/wurlitzer.egg-info/PKG-INFO0000644000175100001710000000352300000000000021224 0ustar00runnerdocker00000000000000Metadata-Version: 2.1 Name: wurlitzer Version: 3.0.2 Summary: Capture C-level output in context managers Home-page: https://github.com/minrk/wurlitzer Author: Min RK Author-email: benjaminrk@gmail.com License: MIT Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 3 Requires-Python: >=3.5 Description-Content-Type: text/markdown License-File: LICENSE # Wurlitzer Capture C-level stdout/stderr pipes in Python via `os.dup2`. For more details on why this is needed, please read [this blog post](https://eli.thegreenplace.net/2015/redirecting-all-kinds-of-stdout-in-python/). ## Install pip install wurlitzer ## Usage Capture stdout/stderr in pipes: ```python from wurlitzer import pipes with pipes() as (out, err): call_some_c_function() stdout = out.read() ``` Capture stdout/stderr in StringIO: ```python from io import StringIO from wurlitzer import pipes, STDOUT out = StringIO() with pipes(stdout=out, stderr=STDOUT): call_some_c_function() stdout = out.getvalue() ``` Forward C-level stdout/stderr to Python sys.stdout/stderr, which may already be forwarded somewhere by the environment, e.g. IPython: ```python from wurlitzer import sys_pipes with sys_pipes(): call_some_c_function() ``` Or even simpler, enable it as an IPython extension: ``` %load_ext wurlitzer ``` To forward all C-level output to IPython during execution. ## Acknowledgments This package is based on stuff we learned with @takluyver and @karies while working on capturing output from the [Cling Kernel](https://github.com/root-mirror/cling/tree/master/tools/Jupyter/kernel) for Jupyter. ## Wurlitzer?! [Wurlitzer](https://en.wikipedia.org/wiki/Wurlitzer) makes pipe organs. Get it? Pipes? Naming is hard. ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878364.0 wurlitzer-3.0.2/wurlitzer.egg-info/SOURCES.txt0000644000175100001710000000035600000000000022014 0ustar00runnerdocker00000000000000Demo.ipynb LICENSE MANIFEST.in README.md dev-requirements.txt pyproject.toml setup.py test.py wurlitzer.py wurlitzer.egg-info/PKG-INFO wurlitzer.egg-info/SOURCES.txt wurlitzer.egg-info/dependency_links.txt wurlitzer.egg-info/top_level.txt././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878364.0 wurlitzer-3.0.2/wurlitzer.egg-info/dependency_links.txt0000644000175100001710000000000100000000000024172 0ustar00runnerdocker00000000000000 ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878364.0 wurlitzer-3.0.2/wurlitzer.egg-info/top_level.txt0000644000175100001710000000001200000000000022647 0ustar00runnerdocker00000000000000wurlitzer ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1629878357.0 wurlitzer-3.0.2/wurlitzer.py0000644000175100001710000003507000000000000017011 0ustar00runnerdocker00000000000000"""Capture C-level FD output on pipes Use `wurlitzer.pipes` or `wurlitzer.sys_pipes` as context managers. """ from __future__ import print_function __version__ = '3.0.2' __all__ = [ 'pipes', 'sys_pipes', 'sys_pipes_forever', 'stop_sys_pipes', 'Wurlitzer', ] import ctypes import errno import io import os import platform import selectors import sys import threading import time import warnings from contextlib import contextmanager from fcntl import F_GETFL, F_SETFL, fcntl from functools import lru_cache from queue import Queue try: from fcntl import F_GETPIPE_SZ, F_SETPIPE_SZ except ImportError: # ref: linux uapi/linux/fcntl.h F_SETPIPE_SZ = 1024 + 7 F_GETPIPE_SZ = 1024 + 8 libc = ctypes.CDLL(None) def _get_streams_cffi(): """Use CFFI to lookup stdout/stderr pointers Should work ~everywhere, but requires compilation """ try: import cffi except ImportError: raise ImportError( "Failed to lookup stdout symbols in libc. Fallback requires cffi." ) try: _ffi = cffi.FFI() _ffi.cdef("const size_t c_stdout_p();") _ffi.cdef("const size_t c_stderr_p();") _lib = _ffi.verify( '\n'.join( [ "#include ", "const size_t c_stdout_p() { return (size_t) (void*) stdout; }", "const size_t c_stderr_p() { return (size_t) (void*) stderr; }", ] ) ) c_stdout_p = ctypes.c_void_p(_lib.c_stdout_p()) c_stderr_p = ctypes.c_void_p(_lib.c_stderr_p()) except Exception as e: warnings.warn( "Failed to lookup stdout with cffi: {}.\nStreams may not be flushed.".format( e ) ) return (None, None) else: return c_stdout_p, c_stderr_p c_stdout_p = c_stderr_p = None try: c_stdout_p = ctypes.c_void_p.in_dll(libc, 'stdout') c_stderr_p = ctypes.c_void_p.in_dll(libc, 'stderr') except ValueError: # libc.stdout has a funny name on macOS try: c_stdout_p = ctypes.c_void_p.in_dll(libc, '__stdoutp') c_stderr_p = ctypes.c_void_p.in_dll(libc, '__stderrp') except ValueError: c_stdout_p, c_stderr_p = _get_streams_cffi() STDOUT = 2 PIPE = 3 _default_encoding = getattr(sys.stdin, 'encoding', None) or 'utf8' if _default_encoding.lower() == 'ascii': # don't respect ascii _default_encoding = 'utf8' # pragma: no cover def dup2(a, b, timeout=3): """Like os.dup2, but retry on EBUSY""" dup_err = None # give FDs 3 seconds to not be busy anymore for i in range(int(10 * timeout)): try: return os.dup2(a, b) except OSError as e: dup_err = e if e.errno == errno.EBUSY: time.sleep(0.1) else: raise if dup_err: raise dup_err @lru_cache() def _get_max_pipe_size(): """Get max pipe size Reads /proc/sys/fs/pipe-max-size on Linux. Always returns None elsewhere. Returns integer (up to 1MB), or None if no value can be determined. Adapted from wal-e, (c) 2018, WAL-E Contributors used under BSD-3-clause """ if platform.system() != 'Linux': return # If Linux procfs (or something that looks like it) exposes its # maximum F_SETPIPE_SZ, adjust the default buffer sizes. try: with open('/proc/sys/fs/pipe-max-size', 'r') as f: # Figure out OS max pipe size pipe_max_size = int(f.read()) except Exception: pass else: if pipe_max_size > 1024 * 1024: # avoid unusually large values, limit to 1MB return 1024 * 1024 elif pipe_max_size <= 65536: # smaller than default, don't do anything return None else: return pipe_max_size class Wurlitzer: """Class for Capturing Process-level FD output via dup2 Typically used via `wurlitzer.pipes` """ flush_interval = 0.2 def __init__( self, stdout=None, stderr=None, encoding=_default_encoding, bufsize=_get_max_pipe_size(), ): """ Parameters ---------- stdout: stream or None The stream for forwarding stdout. stderr = stream or None The stream for forwarding stderr. encoding: str or None The encoding to use, if streams should be interpreted as text. bufsize: int or None Set pipe buffer size using fcntl F_SETPIPE_SZ (linux only) default: use /proc/sys/fs/pipe-max-size up to a max of 1MB if 0, will do nothing. """ self._stdout = stdout if stderr == STDOUT: self._stderr = self._stdout else: self._stderr = stderr self.encoding = encoding if bufsize is None: bufsize = _get_max_pipe_size() self._bufsize = bufsize self._save_fds = {} self._real_fds = {} self._handlers = {} self._handlers['stderr'] = self._handle_stderr self._handlers['stdout'] = self._handle_stdout def _setup_pipe(self, name): real_fd = getattr(sys, '__%s__' % name).fileno() save_fd = os.dup(real_fd) self._save_fds[name] = save_fd pipe_out, pipe_in = os.pipe() # set max pipe buffer size (linux only) if self._bufsize: try: fcntl(pipe_in, F_SETPIPE_SZ, self._bufsize) except OSError: warnings.warn("Failed to set pipe buffer size", RuntimeWarning) dup2(pipe_in, real_fd) os.close(pipe_in) self._real_fds[name] = real_fd # make pipe_out non-blocking flags = fcntl(pipe_out, F_GETFL) fcntl(pipe_out, F_SETFL, flags | os.O_NONBLOCK) return pipe_out def _decode(self, data): """Decode data, if any Called before passing to stdout/stderr streams """ if self.encoding: data = data.decode(self.encoding, 'replace') return data def _handle_stdout(self, data): if self._stdout: self._stdout.write(self._decode(data)) def _handle_stderr(self, data): if self._stderr: self._stderr.write(self._decode(data)) def _setup_handle(self): """Setup handle for output, if any""" self.handle = (self._stdout, self._stderr) def _finish_handle(self): """Finish handle, if anything should be done when it's all wrapped up.""" pass def _flush(self): """flush sys.stdout/err and low-level FDs""" if self._stdout and sys.stdout: sys.stdout.flush() if self._stderr and sys.stderr: sys.stderr.flush() if c_stdout_p is not None: libc.fflush(c_stdout_p) if c_stderr_p is not None: libc.fflush(c_stderr_p) def __enter__(self): # flush anything out before starting self._flush() # setup handle self._setup_handle() self._control_r, self._control_w = os.pipe() # create pipe for stdout pipes = [self._control_r] names = {self._control_r: 'control'} if self._stdout: pipe = self._setup_pipe('stdout') pipes.append(pipe) names[pipe] = 'stdout' if self._stderr: pipe = self._setup_pipe('stderr') pipes.append(pipe) names[pipe] = 'stderr' # flush pipes in a background thread to avoid blocking # the reader thread when the buffer is full flush_queue = Queue() def flush_main(): while True: msg = flush_queue.get() if msg == 'stop': return self._flush() flush_thread = threading.Thread(target=flush_main) flush_thread.daemon = True flush_thread.start() def forwarder(): """Forward bytes on a pipe to stream messages""" draining = False flush_interval = 0 poller = selectors.DefaultSelector() for pipe_ in pipes: poller.register(pipe_, selectors.EVENT_READ) while pipes: events = poller.select(flush_interval) if events: # found something to read, don't block select until # we run out of things to read flush_interval = 0 else: # nothing to read if draining: # if we are draining and there's nothing to read, stop break else: # nothing to read, get ready to wait. # flush the streams in case there's something waiting # to be written. flush_queue.put('flush') flush_interval = self.flush_interval continue for selector_key, flags in events: fd = selector_key.fd if fd == self._control_r: draining = True pipes.remove(self._control_r) poller.unregister(self._control_r) os.close(self._control_r) continue name = names[fd] data = os.read(fd, 1024) if not data: # pipe closed, stop polling it pipes.remove(fd) poller.unregister(fd) os.close(fd) else: handler = getattr(self, '_handle_%s' % name) handler(data) if not pipes: # pipes closed, we are done break # stop flush thread flush_queue.put('stop') flush_thread.join() # cleanup pipes [os.close(pipe) for pipe in pipes] poller.close() self.thread = threading.Thread(target=forwarder) self.thread.daemon = True self.thread.start() return self.handle def __exit__(self, exc_type, exc_value, traceback): # flush before exiting self._flush() # signal output is complete on control pipe os.write(self._control_w, b'\1') self.thread.join() os.close(self._control_w) # restore original state for name, real_fd in self._real_fds.items(): save_fd = self._save_fds[name] dup2(save_fd, real_fd) os.close(save_fd) # finalize handle self._finish_handle() @contextmanager def pipes(stdout=PIPE, stderr=PIPE, encoding=_default_encoding, bufsize=None): """Capture C-level stdout/stderr in a context manager. The return value for the context manager is (stdout, stderr). .. versionchanged:: 3.0 when using `PIPE` (default), the type of captured output is `io.StringIO/BytesIO` instead of an OS pipe. This eliminates max buffer size issues (and hang when output exceeds 65536 bytes), but also means the buffer cannot be read with `.read()` methods until after the context exits. Examples -------- >>> with capture() as (stdout, stderr): ... printf("C-level stdout") ... output = stdout.read() """ stdout_pipe = stderr_pipe = False if encoding: PipeIO = io.StringIO else: PipeIO = io.BytesIO # setup stdout if stdout == PIPE: stdout_r = stdout_w = PipeIO() stdout_pipe = True else: stdout_r = stdout_w = stdout # setup stderr if stderr == STDOUT: stderr_r = None stderr_w = stdout_w elif stderr == PIPE: stderr_r = stderr_w = PipeIO() stderr_pipe = True else: stderr_r = stderr_w = stderr w = Wurlitzer(stdout=stdout_w, stderr=stderr_w, encoding=encoding, bufsize=bufsize) try: with w: yield stdout_r, stderr_r finally: # close pipes if stdout_pipe: # seek to 0 so that it can be read after exit stdout_r.seek(0) if stderr_pipe: # seek to 0 so that it can be read after exit stderr_r.seek(0) def sys_pipes(encoding=_default_encoding, bufsize=None): """Redirect C-level stdout/stderr to sys.stdout/stderr This is useful of sys.sdout/stderr are already being forwarded somewhere. DO NOT USE THIS if sys.stdout and sys.stderr are not already being forwarded. """ return pipes(sys.stdout, sys.stderr, encoding=encoding, bufsize=bufsize) _mighty_wurlitzer = None _mighty_lock = threading.Lock() def sys_pipes_forever(encoding=_default_encoding, bufsize=None): """Redirect all C output to sys.stdout/err This is not a context manager; it turns on C-forwarding permanently. """ global _mighty_wurlitzer with _mighty_lock: if _mighty_wurlitzer is None: _mighty_wurlitzer = sys_pipes(encoding, bufsize) _mighty_wurlitzer.__enter__() def stop_sys_pipes(): """Stop permanent redirection started by sys_pipes_forever""" global _mighty_wurlitzer with _mighty_lock: if _mighty_wurlitzer is not None: _mighty_wurlitzer.__exit__(None, None, None) _mighty_wurlitzer = None _extension_enabled = False def load_ipython_extension(ip): """Register me as an IPython extension Captures all C output during execution and forwards to sys. Does nothing on terminal IPython. Use: %load_ext wurlitzer """ global _extension_enabled if not getattr(ip, 'kernel', None): warnings.warn("wurlitzer extension doesn't do anything in terminal IPython") return for name in ("__stdout__", "__stderr__"): if getattr(sys, name) is None: warnings.warn("sys.{} is None. Wurlitzer can't capture output without it.") return ip.events.register('pre_execute', sys_pipes_forever) ip.events.register('post_execute', stop_sys_pipes) _extension_enabled = True def unload_ipython_extension(ip): """Unload me as an IPython extension Use: %unload_ext wurlitzer """ global _extension_enabled if not _extension_enabled: return ip.events.unregister('pre_execute', sys_pipes_forever) ip.events.unregister('post_execute', stop_sys_pipes) # sys_pipes_forever was called in pre_execute # after unregister we need to call it explicitly: stop_sys_pipes() _extension_enabled = False