cpython/Tools/scripts/generate_module_names.py

# This script lists the names of standard library modules
# to update Python/module_names.h
import os.path
import re
import subprocess
import sys
import sysconfig


SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
MODULES_SETUP = os.path.join(SRC_DIR, 'Modules', 'Setup')
SETUP_PY = os.path.join(SRC_DIR, 'setup.py')

IGNORE = {
    '__init__',
    '__pycache__',
    'site-packages',

    # Helper modules of public modules.
    # For example, sysconfig uses _osx_support.
    '_aix_support',
    '_collections_abc',
    '_compat_pickle',
    '_compression',
    '_markupbase',
    '_osx_support',
    '_sitebuiltins',
    '_strptime',
    '_threading_local',
    '_weakrefset',

    # Used to bootstrap setup.py
    '_bootsubprocess',

    # pure Python implementation
    '_py_abc',
    '_pydecimal',
    '_pyio',

    # test modules
    '__phello__.foo',
    '_ctypes_test',
    '_testbuffer',
    '_testcapi',
    '_testconsole',
    '_testimportmultiple',
    '_testinternalcapi',
    '_testmultiphase',
    '_xxtestfuzz',
    'distutils.tests',
    'idlelib.idle_test',
    'lib2to3.tests',
    'test',
    'xxlimited',
    'xxlimited_35',
    'xxsubtype',
}

# Windows extension modules
WINDOWS_MODULES = (
    '_msi',
    '_testconsole',
    '_winapi',
    'msvcrt',
    'nt',
    'winreg',
    'winsound'
)


def write_comment(fp, comment):
    print(f"// {comment}", file=fp)


def write_modules(fp, names):
    for name in sorted(names):
        if name in IGNORE:
            continue
        print(f'"{name}",', file=fp)
    print(file=fp)


def list_builtin_modules(fp):
    write_comment(fp, "Built-in modules")
    write_modules(fp, sys.builtin_module_names)


# Pure Python modules (Lib/*.py)
def list_python_modules(fp):
    write_comment(fp, "Pure Python modules (Lib/*.py)")
    names = []
    for filename in os.listdir(STDLIB_PATH):
        if not filename.endswith(".py"):
            continue
        name = filename.removesuffix(".py")
        names.append(name)
    write_modules(fp, names)


def _list_sub_packages(path, names, parent=None):
    for name in os.listdir(path):
        package_path = os.path.join(path, name)
        if name in IGNORE:
            continue
        if not os.path.isdir(package_path):
            continue
        if not any(package_file.endswith(".py")
                   for package_file in os.listdir(package_path)):
            continue
        if parent:
            qualname = f"{parent}.{name}"
        else:
            qualname = name
        if qualname in IGNORE:
            continue
        names.append(qualname)
        _list_sub_packages(package_path, names, qualname)


# Packages and sub-packages
def list_packages(fp):
    write_comment(fp, "Packages and sub-packages")
    names = []
    _list_sub_packages(STDLIB_PATH, names)
    write_modules(fp, names)


# Windows extensions
def list_windows_extensions(fp):
    write_comment(fp, "Windows extension modules")
    write_modules(fp, WINDOWS_MODULES)


# Extension modules built by setup.py
def list_setup(fp):
    cmd = [sys.executable, SETUP_PY, "-q", "build", "--list-module-names"]
    output = subprocess.check_output(cmd)
    output = output.decode("utf8")
    names = output.splitlines()

    write_comment(fp, "Extension modules built by setup.py")
    write_modules(fp, names)


# Built-in and extension modules built by Modules/Setup
def list_modules_setup(fp):
    assign_var = re.compile("^[A-Z]+=")

    names = []
    with open(MODULES_SETUP, encoding="utf-8") as modules_fp:
        for line in modules_fp:
            # Strip comment
            line = line.partition("#")[0]
            line = line.rstrip()
            if not line:
                continue
            if assign_var.match(line):
                # Ignore "VAR=VALUE"
                continue
            if line in ("*disabled*", "*shared*"):
                continue
            parts = line.split()
            if len(parts) < 2:
                continue
            # "errno errnomodule.c" => write "errno"
            name = parts[0]
            names.append(name)

    write_comment(fp, "Built-in and extension modules built by Modules/Setup")
    write_modules(fp, names)


def list_modules(fp):
    print("// Auto-generated by Tools/scripts/generate_module_names.py.", file=fp)
    print(file=fp)
    print("static const char* _Py_module_names[] = {", file=fp)
    print(file=fp)

    list_builtin_modules(fp)
    list_python_modules(fp)
    list_packages(fp)
    list_setup(fp)
    list_modules_setup(fp)
    list_windows_extensions(fp)

    print("};", file=fp)


def main():
    if not sysconfig.is_python_build():
        print(f"ERROR: {sys.executable} is not a Python build",
              file=sys.stderr)
        sys.exit(1)

    list_modules(sys.stdout)


if __name__ == "__main__":
    main()