In Python, how can you determine if a module comes from a C extension?

Question

What is the right or most reliable way to tell from Python whether the imported module is from a C extension and not from a pure Python module? This is useful, for example, if the Python package has a module with both a pure Python implementation and a C implementation, and you want to be able to determine at runtime which one is used.

One idea is to study the file extension module.__file__, but I'm not sure if you need to check all file extensions, and whether this approach is necessarily the most reliable.

Answer 1

-, , . , Python C , , Python C, , Python, .

: numpy - Python, C; bintrees - Python, C, Python , ; .

stdlib 3.2 . , import pickle, C ( cpickle 2.7) CPython, pure-Python PyPy, pickle Python.

---

, :

• , sys.
• C, 2.x cpickle.
• Pure Python, 2.x pickle.

, CPython; , , Jython IronPython, JVM .NET, .

__file__ :

• __file__. ( , , inspect.) , - py2app cx_freeze, , "", .
• pure-Python .pyc/.pyo .py .
• , ( easy_install, pip) , __file__.
• , , zip , , .

---

3.1+ , Python Python.

, importlib, , , BuiltinImporter ( (2 → ).2 >

, , , importlib.machinery. , , any(pathname.endswith(suffix) for suffix in importlib.machinery.EXTENSION_SUFFIXES)), , , /-, .

---

, , - , inspect, .

getsource, getsourcefile getfile; , .

TypeError .

getsourcefile. , 2.5-3.4, , 2.4. getsource, , .so , IOError. ( 3.x UnicodeError SyntaxError, , , ...)

Pure Python getsourcefile, egg/zip/etc. getsource, , egg/zip/etc., - (.pyc/etc.), IOError.

, (-), .

Answer 2

;

. " " .

, C, Qaru Productions ™ ... .

C -C , Python . :

• , Python - . PyInstaller . C . , , C, , C, . ^{: PyInstaller.}
• , (, Cython), (, Numba). Python C Python.
• , . (Numpy), , (, BLAS), , . ? , - , . ^{- , !}
• , . , ?

, , . Ergo, C .

abarnert , :

, - , - , inspect, .

. , - , . stdlib (, inspect) . :

• inspect.getsource() inspect.getsourcefile() None C ( Python), , Python (, -)). .
• importlib , PEP 302- , , importlib . , . PEP 302 , . , , __import__() ? ^{Python - , .}

abarnert :

... .

. Triforce of Hyrulean, .

.

Python True C: ^{Python 3.x.}

import inspect, os
from importlib.machinery import ExtensionFileLoader, EXTENSION_SUFFIXES
from types import ModuleType

def is_c_extension(module: ModuleType) -> bool:
    '''
    'True' only if the passed module is a C extension implemented as a
    dynamically linked shared library specific to the current platform.

    Parameters
    ----------
    module : ModuleType
        Previously imported module object to be tested.

    Returns
    ----------
    bool
        'True' only if this module is a C extension.
    '''
    assert isinstance(module, ModuleType), '"{}" not a module.'.format(module)

    # If this module was loaded by a PEP 302-compliant CPython-specific loader
    # loading only C extensions, this module is a C extension.
    if isinstance(getattr(module, '__loader__', None), ExtensionFileLoader):
        return True

    # Else, fallback to filetype matching heuristics.
    #
    # Absolute path of the file defining this module.
    module_filename = inspect.getfile(module)

    # "."-prefixed filetype of this path if any or the empty string otherwise.
    module_filetype = os.path.splitext(module_filename)[1]

    # This module is only a C extension if this path filetype is that of a
    # C extension specific to the current platform.
    return module_filetype in EXTENSION_SUFFIXES

, , , . . , .

:

• stdlib pure-Python os.__init__. ^{, C.}
• stdlib pure-Python importlib.machinery. ^{, C.}
• stdlib _elementtree C.
• numpy.core.multiarray numpy.core.multiarray C.

:

>>> import os
>>> import importlib.machinery as im
>>> import _elementtree as et
>>> import numpy.core.multiarray as ma
>>> for module in (os, im, et, ma):
...     print('Is "{}" a C extension? {}'.format(
...         module.__name__, is_c_extension(module)))
Is "os" a C extension? False
Is "importlib.machinery" a C extension? False
Is "_elementtree" a C extension? True
Is "numpy.core.multiarray" a C extension? True

, .

?

. , ?

1. , PEP 302 ( ), PEP 302 , , , __loader__, , . :
   1. importlib.machinery.ExtensionFileLoader importlib.machinery.ExtensionFileLoader, C.
2. , (A) Python CPython (, PyPy), (B) Python CPython, , PEP 302, - __import__() (, , Python ). , C, .

. .

Answer 3

( , abarnert, ) False " " , , C (, numpy vs. numpy.core.multiarray).

, , , , , :

def is_c(module):

    # if module is part of the main python library (e.g. os), it won't have a path
    try:
        for path, subdirs, files in os.walk(module.__path__[0]):

            for f in files:
                ftype = f.split('.')[-1]
                if ftype == 'so':
                    is_c = True
                    break
        return is_c

    except AttributeError:

        path = inspect.getfile(module)
        suffix = path.split('.')[-1]

        if suffix != 'so':

            return False

        elif suffix == 'so':

            return True

is_c(os), is_c(im), is_c(et), is_c_extension(ma), is_c(numpy)
# (False, False, True, True, True)

Answer 4

@Cecil Curry . : - _elementtree _elementtree TypeError _elementtree Python 3.5.6. -, @crld, , C, . :

import importlib
from importlib.machinery import ExtensionFileLoader, EXTENSION_SUFFIXES
import inspect
import os
import os.path
import pkgutil
from types import ModuleType
from typing import List


def is_builtin_module(module: ModuleType) -> bool:
    """
    Is this module a built-in module, like ''os''?
    Method is as per :func:'inspect.getfile'.
    """
    return not hasattr(module, "__file__")


def contains_c_extension(module: ModuleType,
                         import_all_submodules: bool = True,
                         include_external_imports: bool = False,
                         seen: List[ModuleType] = None) -> bool:
    """
    Extends :func:'is_c_extension' by asking: is this module, or any of its
    submodules, a C extension?

    Args:
        module: Previously imported module object to be tested.
        import_all_submodules: explicitly import all submodules of this module?
        include_external_imports: check modules in other packages that this
            module imports?
        seen: used internally for recursion (to deal with recursive modules);
            should be ''None'' when called by users

    Returns:
        bool: ''True'' only if this module or one of its submodules is a C
        extension.

    Examples:

    .. code-block:: python

        import logging

        import _elementtree as et
        import os

        import arrow
        import alembic
        import django
        import numpy
        import numpy.core.multiarray as numpy_multiarray

        log = logging.getLogger(__name__)
        logging.basicConfig(level=logging.DEBUG)  # be verbose

        contains_c_extension(os)  # False
        contains_c_extension(et)  # False

        contains_c_extension(numpy)  # True -- different from is_c_extension()
        contains_c_extension(numpy_multiarray)  # True

        contains_c_extension(arrow)  # False

        contains_c_extension(alembic)  # False
        contains_c_extension(alembic, include_external_imports=True)  # True
        # ... this example shows that Alembic imports hashlib, which can import
        #     _hashlib, which is a C extension; however, that doesn't stop us (for
        #     example) installing Alembic on a machine with no C compiler

        contains_c_extension(django)

    """  # noqa
    assert inspect.ismodule(module), '"{}" not a module.'.format(module)

    if seen is None:  # only true for the top-level call
        seen = []  # type: List[ModuleType]
    if module in seen:  # modules can "contain" themselves
        # already inspected; avoid infinite loops
        return False
    seen.append(module)

    # Check the thing we were asked about
    is_c_ext = is_c_extension(module)
    log.info("Is module {!r} a C extension? {}".format(module, is_c_ext))
    if is_c_ext:
        return True
    if is_builtin_module(module):
        # built-in, therefore we stop searching it
        return False

    # Now check any children, in a couple of ways

    top_level_module = seen[0]
    top_path = os.path.dirname(top_level_module.__file__)

    # Recurse using dir(). This picks up modules that are automatically
    # imported by our top-level model. But it won't pick up all submodules;
    # try e.g. for django.
    for candidate_name in dir(module):
        candidate = getattr(module, candidate_name)
        try:
            if not inspect.ismodule(candidate):
                # not a module
                continue
        except Exception:
            # e.g. a Django module that won't import until we configure its
            # settings
            log.error("Failed to test ismodule() status of {!r}".format(
                candidate))
            continue
        if is_builtin_module(candidate):
            # built-in, therefore we stop searching it
            continue

        candidate_fname = getattr(candidate, "__file__")
        if not include_external_imports:
            if os.path.commonpath([top_path, candidate_fname]) != top_path:
                log.debug("Skipping, not within the top-level module "
                          "directory: {!r}".format(candidate))
                continue
        # Recurse:
        if contains_c_extension(
                module=candidate,
                import_all_submodules=False,  # only done at the top level, below  # noqa
                include_external_imports=include_external_imports,
                seen=seen):
            return True

    if import_all_submodules:
        if not is_module_a_package(module):
            log.debug("Top-level module is not a package: {!r}".format(module))
            return False

        # Otherwise, for things like Django, we need to recurse in a different
        # way to scan everything.
        # See https://stackoverflow.com/questions/3365740/how-to-import-all-submodules.  # noqa
        log.debug("Walking path: {!r}".format(top_path))
        try:
            for loader, module_name, is_pkg in pkgutil.walk_packages([top_path]):  # noqa
                if not is_pkg:
                    log.debug("Skipping, not a package: {!r}".format(
                        module_name))
                    continue
                log.debug("Manually importing: {!r}".format(module_name))
                try:
                    candidate = loader.find_module(module_name)\
                        .load_module(module_name)  # noqa
                except Exception:
                    # e.g. Alembic "autogenerate" gives: "ValueError: attempted
                    # relative import beyond top-level package"; or Django
                    # "django.core.exceptions.ImproperlyConfigured"
                    log.error("Package failed to import: {!r}".format(
                        module_name))
                    continue
                if contains_c_extension(
                        module=candidate,
                        import_all_submodules=False,  # only done at the top level  # noqa
                        include_external_imports=include_external_imports,
                        seen=seen):
                    return True
        except Exception:
            log.error("Unable to walk packages further; no C extensions "
                      "detected so far!")
            raise

    return False