bpo-22257: Private C-API for main interpreter initialization (PEP 432). (#1729)

(patch by Nick Coghlan)

Include/pylifecycle.h

@@ -23,7 +23,8 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
 /* PEP 432 Multi-phase initialization API (Private while provisional!) */
 PyAPI_FUNC(void) _Py_InitializeCore(const _PyCoreConfig *);
 PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
-PyAPI_FUNC(int) _Py_InitializeMainInterpreter(int install_sigs);
+PyAPI_FUNC(int) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *);
+PyAPI_FUNC(int) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *);
 #endif
 
 /* Initialization and finalization */

Include/pystate.h

@@ -33,6 +33,18 @@ typedef struct {
 
 #define _PyCoreConfig_INIT {0, -1, 0, 0}
 
+/* Placeholders while working on the new configuration API
+ *
+ * See PEP 432 for final anticipated contents
+ *
+ * For the moment, just handle the args to _Py_InitializeEx
+ */
+typedef struct {
+    int install_signal_handlers;
+} _PyMainInterpreterConfig;
+
+#define _PyMainInterpreterConfig_INIT {-1}
+
 typedef struct _is {
 
     struct _is *next;
@@ -53,6 +65,7 @@ typedef struct _is {
     int fscodec_initialized;
 
     _PyCoreConfig core_config;
+    _PyMainInterpreterConfig config;
 #ifdef HAVE_DLOPEN
     int dlopenflags;
 #endif

Modules/main.c

@@ -768,8 +768,21 @@ Py_Main(int argc, wchar_t **argv)
 #else
     Py_SetProgramName(argv[0]);
 #endif
-    if (_Py_InitializeMainInterpreter(1))
+    /* Replaces previous call to Py_Initialize()
-        Py_FatalError("Py_Main: Py_InitializeMainInterpreter failed");
+     *
+     * TODO: Move environment queries (etc) into Py_ReadConfig
+     */
+    {
+        _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT;
+
+        /* TODO: Moar config options! */
+        config.install_signal_handlers = 1;
+        /* TODO: Print any exceptions raised by these operations */
+        if (_Py_ReadMainInterpreterConfig(&config))
+            Py_FatalError("Py_Main: Py_ReadMainInterpreterConfig failed");
+        if (_Py_InitializeMainInterpreter(&config))
+            Py_FatalError("Py_Main: Py_InitializeMainInterpreter failed");
+    }
 
     /* TODO: Move this to _PyRun_PrepareMain */
     if (!Py_QuietFlag && (Py_VerboseFlag ||

Python/pylifecycle.c

@@ -69,6 +69,7 @@ extern void _PyFaulthandler_Fini(void);
 extern void _PyHash_Fini(void);
 extern int _PyTraceMalloc_Init(void);
 extern int _PyTraceMalloc_Fini(void);
+extern void _Py_ReadyTypes(void);
 
 #ifdef WITH_THREAD
 extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *);
@@ -112,6 +113,7 @@ PyModule_GetWarningsModule(void)
     return PyImport_ImportModule("warnings");
 }
 
+
 /* APIs to access the initialization flags
  *
  * Can be called prior to Py_Initialize.
@@ -366,8 +368,8 @@ void _Py_InitializeCore(const _PyCoreConfig *config)
     PyThreadState *tstate;
     PyObject *bimod, *sysmod, *pstderr;
     char *p;
-    extern void _Py_ReadyTypes(void);
     _PyCoreConfig core_config = _PyCoreConfig_INIT;
+    _PyMainInterpreterConfig preinit_config = _PyMainInterpreterConfig_INIT;
 
     if (config != NULL) {
         core_config = *config;
@@ -428,6 +430,7 @@ void _Py_InitializeCore(const _PyCoreConfig *config)
     if (interp == NULL)
         Py_FatalError("Py_InitializeCore: can't make main interpreter");
     interp->core_config = core_config;
+    interp->config = preinit_config;
 
     tstate = PyThreadState_New(interp);
     if (tstate == NULL)
@@ -518,21 +521,62 @@ void _Py_InitializeCore(const _PyCoreConfig *config)
     _Py_CoreInitialized = 1;
 }
 
-int
+/* Read configuration settings from standard locations
-_Py_InitializeMainInterpreter(int install_sigs)
+ *
+ * This function doesn't make any changes to the interpreter state - it
+ * merely populates any missing configuration settings. This allows an
+ * embedding application to completely override a config option by
+ * setting it before calling this function, or else modify the default
+ * setting before passing the fully populated config to Py_EndInitialization.
+ *
+ * More advanced selective initialization tricks are possible by calling
+ * this function multiple times with various preconfigured settings.
+ */
+
+int _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config)
+{
+    /* Signal handlers are installed by default */
+    if (config->install_signal_handlers < 0) {
+        config->install_signal_handlers = 1;
+    }
+
+    return 0;
+}
+
+/* Update interpreter state based on supplied configuration settings
+ *
+ * After calling this function, most of the restrictions on the interpreter
+ * are lifted. The only remaining incomplete settings are those related
+ * to the main module (sys.argv[0], __main__ metadata)
+ *
+ * Calling this when the interpreter is not initializing, is already
+ * initialized or without a valid current thread state is a fatal error.
+ * Other errors should be reported as normal Python exceptions with a
+ * non-zero return code.
+ */
+int _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
 {
     PyInterpreterState *interp;
     PyThreadState *tstate;
 
+    if (!_Py_CoreInitialized) {
+        Py_FatalError("Py_InitializeMainInterpreter: runtime core not initialized");
+    }
+    if (_Py_Initialized) {
+        Py_FatalError("Py_InitializeMainInterpreter: main interpreter already initialized");
+    }
+
     /* Get current thread state and interpreter pointer */
     tstate = PyThreadState_GET();
     if (!tstate)
-        Py_FatalError("Py_Initialize: failed to read thread state");
+        Py_FatalError("Py_InitializeMainInterpreter: failed to read thread state");
     interp = tstate->interp;
     if (!interp)
-        Py_FatalError("Py_Initialize: failed to get interpreter");
+        Py_FatalError("Py_InitializeMainInterpreter: failed to get interpreter");
 
     /* Now finish configuring the main interpreter */
+    interp->config = *config;
+
     if (interp->core_config._disable_importlib) {
         /* Special mode for freeze_importlib: run with no import system
          *
@@ -545,7 +589,7 @@ _Py_InitializeMainInterpreter(int install_sigs)
     /* TODO: Report exceptions rather than fatal errors below here */
 
     if (_PyTime_Init() < 0)
-        Py_FatalError("Py_Initialize: can't initialize time");
+        Py_FatalError("Py_InitializeMainInterpreter: can't initialize time");
 
     /* Finish setting up the sys module and import system */
     /* GetPath may initialize state that _PySys_EndInit locks
@@ -559,21 +603,21 @@ _Py_InitializeMainInterpreter(int install_sigs)
 
     /* initialize the faulthandler module */
     if (_PyFaulthandler_Init())
-        Py_FatalError("Py_Initialize: can't initialize faulthandler");
+        Py_FatalError("Py_InitializeMainInterpreter: can't initialize faulthandler");
 
     if (initfsencoding(interp) < 0)
-        Py_FatalError("Py_Initialize: unable to load the file system codec");
+        Py_FatalError("Py_InitializeMainInterpreter: unable to load the file system codec");
 
-    if (install_sigs)
+    if (config->install_signal_handlers)
         initsigs(); /* Signal handling stuff, including initintr() */
 
     if (_PyTraceMalloc_Init() < 0)
-        Py_FatalError("Py_Initialize: can't initialize tracemalloc");
+        Py_FatalError("Py_InitializeMainInterpreter: can't initialize tracemalloc");
 
     initmain(interp); /* Module __main__ */
     if (initstdio() < 0)
         Py_FatalError(
-            "Py_Initialize: can't initialize sys standard streams");
+            "Py_InitializeMainInterpreter: can't initialize sys standard streams");
 
     /* Initialize warnings. */
     if (PySys_HasWarnOptions()) {
@@ -590,19 +634,27 @@ _Py_InitializeMainInterpreter(int install_sigs)
     if (!Py_NoSiteFlag)
         initsite(); /* Module site */
 
-	return 0;
+    return 0;
 }
 
+#undef _INIT_DEBUG_PRINT
+
 void
 _Py_InitializeEx_Private(int install_sigs, int install_importlib)
 {
     _PyCoreConfig core_config = _PyCoreConfig_INIT;
+    _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT;
 
     /* TODO: Moar config options! */
     core_config.ignore_environment = Py_IgnoreEnvironmentFlag;
     core_config._disable_importlib = !install_importlib;
+    config.install_signal_handlers = install_sigs;
     _Py_InitializeCore(&core_config);
-    _Py_InitializeMainInterpreter(install_sigs);
+    /* TODO: Print any exceptions raised by these operations */
+    if (_Py_ReadMainInterpreterConfig(&config))
+        Py_FatalError("Py_Initialize: Py_ReadMainInterpreterConfig failed");
+    if (_Py_InitializeMainInterpreter(&config))
+        Py_FatalError("Py_Initialize: Py_InitializeMainInterpreter failed");
 }
 
 
@@ -932,10 +984,12 @@ Py_NewInterpreter(void)
     /* Copy the current interpreter config into the new interpreter */
     if (save_tstate != NULL) {
         interp->core_config = save_tstate->interp->core_config;
+        interp->config = save_tstate->interp->config;
     } else {
         /* No current thread state, copy from the main interpreter */
         PyInterpreterState *main_interp = PyInterpreterState_Main();
         interp->core_config = main_interp->core_config;
+        interp->config = main_interp->config;
     }
 
     /* XXX The following is lax in error checking */

Python/pythonrun.c

@@ -1,5 +1,12 @@
 
-/* Python interpreter top-level routines, including init/exit */
+/* Top level execution of Python code (including in __main__) */
+
+/* To help control the interfaces between the startup, execution and
+ * shutdown code, the phases are split across separate modules (boostrap,
+ * pythonrun, shutdown)
+ */
+
+/* TODO: Cull includes following phase split */
 
 #include "Python.h"
 
@@ -59,7 +66,6 @@ static void err_input(perrdetail *);
 static void err_free(perrdetail *);
 
 /* Parse input from a file and execute it */
-
 int
 PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
                      PyCompilerFlags *flags)