How to get a printout of the output of ctypes C functions in Jupyter / IPython?

Question

Introduction

Suppose I have this C code:

#include <stdio.h>

// Of course, these functions are simplified for the purposes of this question.
// The actual functions are more complex and may receive additional arguments.

void printout() {
    puts("Hello");
}
void printhere(FILE* f) {
    fputs("Hello\n", f);
}

What I am compiling as a shared object (DLL): gcc -Wall -std=c99 -fPIC -shared example.c -o example.so

And then I import it into Python 3.x, running in Jupyter or IPython notepad :

import ctypes
example = ctypes.cdll.LoadLibrary('./example.so')

printout = example.printout
printout.argtypes = ()
printout.restype = None

printhere = example.printhere
printhere.argtypes = (ctypes.c_void_p)  # Should have been FILE* instead
printhere.restype = None

Question

How can I execute functions ctypes printout()and printhere()C (via ctypes) and get print output in Jupyter / IPython notepad?

If possible, I want to avoid writing more C code. I would prefer a purely Python solution.

I would also prefer to avoid writing to a temporary file. However, writing to a pipe / socket may be reasonable.

Expected Status, Current Status

If I type the following code in one cell of a laptop:

print("Hi")           # Python-style print
printout()            # C-style print
printhere(something)  # C-style print
print("Bye")          # Python-style print

I want to get this output:

Hi
Hello
Hello
Bye

Python . C , .

, Jupyter/IPython sys.stdout :

import sys

sys.stdout

# Output in command-line Python/IPython shell:
<_io.TextIOWrapper name='<stdout>' mode='w' encoding='UTF-8'>
# Output in IPython Notebook:
<IPython.kernel.zmq.iostream.OutStream at 0x7f39c6930438>
# Output in Jupyter:
<ipykernel.iostream.OutStream at 0x7f6dc8f2de80>

sys.stdout.fileno()

# Output in command-line Python/IPython shell:
1
# Output in command-line Jupyter and IPython notebook:
UnsupportedOperation: IOStream has no fileno.

:

• Python ctypes: Python <-> C FILE *
• Python 3 PyFile_AsFile
• fopen, fwrite fclose ctypes
• Python ctypes DLL
• Python: StringIO Popen - fileno() StringIO, subprocess.Popen.

, . , Python, C .

• C stdout python?
• Python

?

, C open_memstream() FILE* stdout, , stdout .

fileno() , open_memstream(), , .

freopen(), API .

Python tempfile.SpooledTemporaryFile(), . , fileno().

-o. , . ( , , .)

os.pipe(), .

Answer 1

- . ( C). , .

GitHub Gist: https://gist.github.com/denilsonsa/9c8f5c44bf2038fd000f

---

1. C Python

import ctypes

# use_errno parameter is optional, because I'm not checking errno anyway.
libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True)

class FILE(ctypes.Structure):
    pass

FILE_p = ctypes.POINTER(FILE)

# Alternatively, we can just use:
# FILE_p = ctypes.c_void_p

# These variables, defined inside the C library, are readonly.
cstdin = FILE_p.in_dll(libc, 'stdin')
cstdout = FILE_p.in_dll(libc, 'stdout')
cstderr = FILE_p.in_dll(libc, 'stderr')

# C function to disable buffering.
csetbuf = libc.setbuf
csetbuf.argtypes = (FILE_p, ctypes.c_char_p)
csetbuf.restype = None

# C function to flush the C library buffer.
cfflush = libc.fflush
cfflush.argtypes = (FILE_p,)
cfflush.restype = ctypes.c_int

2.

import io
import os
import sys
import tempfile
from contextlib import contextmanager

@contextmanager
def capture_c_stdout(encoding='utf8'):
    # Flushing, it a good practice.
    sys.stdout.flush()
    cfflush(cstdout)

    # We need to use a actual file because we need the file descriptor number.
    with tempfile.TemporaryFile(buffering=0) as temp:
        # Saving a copy of the original stdout.
        prev_sys_stdout = sys.stdout
        prev_stdout_fd = os.dup(1)
        os.close(1)

        # Duplicating the temporary file fd into the stdout fd.
        # In other words, replacing the stdout.
        os.dup2(temp.fileno(), 1)

        # Replacing sys.stdout for Python code.
        #
        # IPython Notebook version of sys.stdout is actually an
        # in-memory OutStream, so it does not have a file descriptor.
        # We need to replace sys.stdout so that interleaved Python
        # and C output gets captured in the correct order.
        #
        # We enable line_buffering to force a flush after each line.
        # And write_through to force all data to be passed through the
        # wrapper directly into the binary temporary file.
        temp_wrapper = io.TextIOWrapper(
            temp, encoding=encoding, line_buffering=True, write_through=True)
        sys.stdout = temp_wrapper

        # Disabling buffering of C stdout.
        csetbuf(cstdout, None)

        yield

        # Must flush to clear the C library buffer.
        cfflush(cstdout)

        # Restoring stdout.
        os.dup2(prev_stdout_fd, 1)
        os.close(prev_stdout_fd)
        sys.stdout = prev_sys_stdout

        # Printing the captured output.
        temp_wrapper.seek(0)
        print(temp_wrapper.read(), end='')

---

Fun: !

libfoo = ctypes.CDLL('./foo.so')

printout = libfoo.printout
printout.argtypes = ()
printout.restype = None

printhere = libfoo.printhere
printhere.argtypes = (FILE_p,)
printhere.restype = None


print('Python Before capturing')
printout()  # Not captured, goes to the terminal

with capture_c_stdout():
    print('Python First')
    printout()
    print('Python Second')
    printhere(cstdout)
    print('Python Third')

print('Python After capturing')
printout()  # Not captured, goes to the terminal

:

Python Before capturing
Python First
C printout puts
Python Second
C printhere fputs
Python Third
Python After capturing

- , , .

stdout, stdout stderr. . ;)

, ( , ).

Answer 2

, python2, , , , io.open. , Logger python stdout.

# -*- coding: utf-8 -*-

import ctypes
# from ctypes import *
from ctypes import util

# use_errno parameter is optional, because I'm not checking errno anyway.
libraryC = ctypes.util.find_library('c')
libc = ctypes.CDLL(libraryC, use_errno=True)


# libc = cdll.msvcrt


class FILE(ctypes.Structure):
    pass


FILE_p = ctypes.POINTER(FILE)

# Alternatively, we can just use:
# FILE_p = ctypes.c_void_p

# These variables, defined inside the C library, are readonly.
##cstdin = FILE_p.in_dll(libc, 'stdin')
##cstdout = FILE_p.in_dll(libc, 'stdout')
##cstderr = FILE_p.in_dll(libc, 'stderr')

# C function to disable buffering.
csetbuf = libc.setbuf
csetbuf.argtypes = (FILE_p, ctypes.c_char_p)
csetbuf.restype = None

# C function to flush the C library buffer.
cfflush = libc.fflush
cfflush.argtypes = (FILE_p,)
cfflush.restype = ctypes.c_int

import io
import os
import sys
import tempfile
from contextlib import contextmanager
#import cStringIO


def read_as_encoding(fileno, encoding="utf-8"):
    fp = io.open(fileno, mode="r+", encoding=encoding, closefd=False)
    return fp


class Logger(object):
    def __init__(self, file, encoding='utf-8'):
        self.file = file
        self.encoding = encoding

    def write(self, message):
        self.file.flush()  # Meed to flush
        # python2 temp file is always binary
        # msg_unicode = message.('utf-8')
        self.file.write(message)


@contextmanager
def capture_c_stdout(on_output, on_error=None, encoding='utf8'):
    # Flushing, it a good practice.
    sys.stdout.flush()
    sys.stderr.flush()
    ##cfflush(cstdout)
    # cfflush(cstdcerr)

    # We need to use a actual file because we need the file descriptor number.
    with tempfile.NamedTemporaryFile() as temp:
        with tempfile.NamedTemporaryFile() as temp_err:
            # print "TempName:", temp.name
            # print "TempErrName:", temp_err.name

            # Saving a copy of the original stdout.
            prev_sys_stdout = sys.stdout
            prev_stdout_fd = os.dup(1)
            os.close(1)
            # Duplicating the temporary file fd into the stdout fd.
            # In other words, replacing the stdout.
            os.dup2(temp.fileno(), 1)

            if on_error:
                prev_sys_stderr = sys.stderr
                prev_stderr_fd = os.dup(2)
                os.close(2)
                os.dup2(temp_err.fileno(), 2)

            # Replacing sys.stdout for Python code.
            #
            # IPython Notebook version of sys.stdout is actually an
            # in-memory OutStream, so it does not have a file descriptor.
            # We need to replace sys.stdout so that interleaved Python
            # and C output gets captured in the correct order.
            #
            # We enable line_buffering to force a flush after each line.
            # And write_through to force all data to be passed through the
            # wrapper directly into the binary temporary file.
            # No need to use TextIOWrapper in python2, in python2, tempFile is always binary according to official document
            ##temp_wrapper = io.TextIOWrapper(
            ##   read_as_encoding(temp.fileno(), encoding=encoding), encoding=encoding, line_buffering=True) ##, write_through=True)

            # temp_wrapper_python = io.TextIOWrapper(
            #    read_as_encoding(temp.fileno(), encoding=encoding), encoding='ascii', line_buffering=True)
            temp_wrapper_python = Logger(temp, encoding=encoding)
            sys.stdout = temp_wrapper_python

            if on_error:
                # temp_wrapper_err = io.TextIOWrapper(
                #   read_as_encoding(temp_err.fileno(), encoding=encoding), encoding=encoding, line_buffering=True) ##, write_through=True)
                temp_wrapper_python_err = Logger(temp_err, encoding=encoding)
                # string_str_err = cStringIO.StringIO()
                sys.stderr = temp_wrapper_python_err

            # Disabling buffering of C stdout.
            ##csetbuf(cstdout, None)

            yield

            # Must flush to clear the C library buffer.
            ##cfflush(cstdout)

            # Restoring stdout.
            os.dup2(prev_stdout_fd, 1)
            os.close(prev_stdout_fd)
            sys.stdout = prev_sys_stdout

            if on_error:
                os.dup2(prev_stderr_fd, 2)
                os.close(prev_stderr_fd)
                sys.stderr = prev_sys_stderr

            # Printing the captured output.
            # temp_wrapper.seek(0)
            # print "Reading: "
            # print temp_wrapper.read()
            if on_output:
                temp.flush()
                temp.seek(0)
                on_output(temp.read())
            temp.close()

            if on_error:
                temp_err.flush()
                temp_err.seek(0)
                on_error(temp_err.read())
                temp_err.close()


import repo_checker_cpp


def on_capture_output(input_stream):
    if input_stream:
        print "Here is captured stdout: \n", input_stream


def on_capture_err(input_stream):
    if input_stream:
        print "Here is captured stderr: \n", input_stream


if __name__ == '__main__':
    with capture_c_stdout(on_capture_output, on_capture_err) as custom_output:  # redirection here
        # repo_checker_cpp is a ctypes.CDll module
        print >> sys.stderr, "Hello World in python err\n"
        repo_checker_cpp.test_exception()  # throw an exception an capture inside cpp module then output to std::cerr
        print "Hello World in python\n"
        repo_checker_cpp.hello_world()  # simple std::cout << "Hello World" << std::endl; std::cerr << "Hello World in cerr" << std::endl;

cstdin = FILE_p.in_dll(libc, 'stdin'). ##, , . .

Window10 + Python 2.7, :

Here is captured stdout: 
Hello World in python
Hello World(C++)


Here is captured stderr: 
Hello World in python err
RepoCheckCpp_TestException, Reason: ensure failed : false
xxxxx\repocheckercpp.cpp(38)
context variables:
    error : This is a test exception


Hello World(C++) in cerr