From b34c0628b4e44b7a278c1552c1cc5bac1b84509a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 14 Mar 2019 20:37:33 +0100 Subject: [PATCH] bpo-36301: Add _Py_PreInitialize() function * Add _PyRuntimeState.preconfig field. _PyPreConfig_Write() now writes the applied pre-configuration into _PyRuntimeState.preconfig. * Add _Py_PreInitialize() and _Py_PreInitializeFromPreConfig() functions * _PyCoreConfig_Read() now calls _Py_PreInitialize() to ensure that Python is pre-configured. It reads the pre-configuration from _PyRuntime. --- Include/cpython/pylifecycle.h | 3 + Include/internal/pycore_coreconfig.h | 6 +- Include/internal/pycore_pystate.h | 10 ++- Modules/main.c | 74 +++++++++++----------- Python/coreconfig.c | 64 +++++-------------- Python/pathconfig.c | 2 +- Python/preconfig.c | 12 ++++ Python/pylifecycle.c | 92 ++++++++++++++++++++-------- Python/pystate.c | 3 + 9 files changed, 149 insertions(+), 117 deletions(-) diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 3bffc80c9376dcf..c17616e576fabb6 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -37,6 +37,9 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter( /* Initialization and finalization */ +PyAPI_FUNC(_PyInitError) _Py_PreInitialize(void); +PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromPreConfig( + const _PyPreConfig *config); PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig( const _PyCoreConfig *config); PyAPI_FUNC(void) _Py_NO_RETURN _Py_ExitInitError(_PyInitError err); diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h index 8c5a072e56771df..bf08d0bd008753a 100644 --- a/Include/internal/pycore_coreconfig.h +++ b/Include/internal/pycore_coreconfig.h @@ -79,11 +79,9 @@ PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( wchar_t **dest, wchar_t *wname, char *name); -PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config, - const _PyPreConfig *preconfig); +PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config); PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, - const _PyArgv *args, - const _PyPreConfig *preconfig); + const _PyArgv *args); PyAPI_FUNC(void) _PyCoreConfig_Write(const _PyCoreConfig *config); #ifdef __cplusplus diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 456dda2e8490830..eaba8108483b3b9 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_BUILTIN define" #endif +#include "cpython/coreconfig.h" #include "pystate.h" #include "pythread.h" @@ -185,8 +186,9 @@ struct _gilstate_runtime_state { /* Full Python runtime state */ typedef struct pyruntimestate { - int initialized; + int pre_initialized; int core_initialized; + int initialized; PyThreadState *finalizing; struct pyinterpreters { @@ -220,10 +222,14 @@ typedef struct pyruntimestate { struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; + _PyPreConfig preconfig; // XXX Consolidate globals found via the check-c-globals script. } _PyRuntimeState; -#define _PyRuntimeState_INIT {.initialized = 0, .core_initialized = 0} +#define _PyRuntimeState_INIT \ + {.pre_initialized = 0, \ + .core_initialized = 0, \ + .initialized = 0} /* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */ PyAPI_DATA(_PyRuntimeState) _PyRuntime; diff --git a/Modules/main.c b/Modules/main.c index 5c7f7e45673a122..dd007ba3fb2961a 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -283,32 +283,50 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config, /* --- pymain_init() ---------------------------------------------- */ static _PyInitError -preconfig_read_write(_PyPreConfig *config, const _PyArgv *args) +pymain_init_preconfig(const _PyArgv *args) { - _PyPreConfig_GetGlobalConfig(config); + _PyInitError err; + _PyPreConfig config = _PyPreConfig_INIT; - _PyInitError err = _PyPreConfig_ReadFromArgv(config, args); + err = _PyPreConfig_ReadFromArgv(&config, args); if (_Py_INIT_FAILED(err)) { - return err; + goto done; + } + + err = _Py_PreInitializeFromPreConfig(&config); + if (_Py_INIT_FAILED(err)) { + goto done; } - return _PyPreConfig_Write(config); +done: + _PyPreConfig_Clear(&config); + return err; } static _PyInitError -config_read_write(_PyCoreConfig *config, const _PyArgv *args, - const _PyPreConfig *preconfig) +pymain_init_coreconfig(const _PyArgv *args, PyInterpreterState **interp_p) { - _PyCoreConfig_GetGlobalConfig(config); + _PyInitError err; + _PyCoreConfig config = _PyCoreConfig_INIT; - _PyInitError err = _PyCoreConfig_ReadFromArgv(config, args, preconfig); + err = _PyCoreConfig_ReadFromArgv(&config, args); if (_Py_INIT_FAILED(err)) { - return err; + goto done; } - _PyCoreConfig_Write(config); - return _Py_INIT_OK(); + _PyCoreConfig_Write(&config); + + err = _Py_InitializeCore(interp_p, &config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = _Py_INIT_OK(); + +done: + _PyCoreConfig_Clear(&config); + return err; } @@ -350,40 +368,22 @@ pymain_init(const _PyArgv *args, PyInterpreterState **interp_p) fedisableexcept(FE_OVERFLOW); #endif - _PyPreConfig local_preconfig = _PyPreConfig_INIT; - _PyPreConfig *preconfig = &local_preconfig; - - _PyCoreConfig local_config = _PyCoreConfig_INIT; - _PyCoreConfig *config = &local_config; - - err = preconfig_read_write(preconfig, args); - if (_Py_INIT_FAILED(err)) { - goto done; - } - - err = config_read_write(config, args, preconfig); + err = pymain_init_preconfig(args); if (_Py_INIT_FAILED(err)) { - goto done; + return err; } - PyInterpreterState *interp; - err = _Py_InitializeCore(&interp, config); + err = pymain_init_coreconfig(args, interp_p); if (_Py_INIT_FAILED(err)) { - goto done; + return err; } - *interp_p = interp; - err = pymain_init_python_main(interp); + err = pymain_init_python_main(*interp_p); if (_Py_INIT_FAILED(err)) { - goto done; + return err; } - err = _Py_INIT_OK(); - -done: - _PyPreConfig_Clear(preconfig); - _PyCoreConfig_Clear(config); - return err; + return _Py_INIT_OK(); } diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 08273765098e07f..98b67b09de05243 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -3,9 +3,10 @@ #include "pycore_coreconfig.h" #include "pycore_fileutils.h" #include "pycore_getopt.h" +#include "pycore_pathconfig.h" #include "pycore_pylifecycle.h" #include "pycore_pymem.h" -#include "pycore_pathconfig.h" +#include "pycore_pystate.h" /* _PyRuntime */ #include /* setlocale() */ #ifdef HAVE_LANGINFO_H # include /* nl_langinfo(CODESET) */ @@ -1327,35 +1328,6 @@ config_init_fs_encoding(_PyCoreConfig *config) } -static _PyInitError -_PyCoreConfig_ReadPreConfig(_PyCoreConfig *config) -{ - _PyInitError err; - _PyPreConfig local_preconfig = _PyPreConfig_INIT; - _PyPreConfig_GetGlobalConfig(&local_preconfig); - - if (_PyPreConfig_Copy(&local_preconfig, &config->preconfig) < 0) { - err = _Py_INIT_NO_MEMORY(); - goto done; - } - - err = _PyPreConfig_Read(&local_preconfig); - if (_Py_INIT_FAILED(err)) { - goto done; - } - - if (_PyPreConfig_Copy(&config->preconfig, &local_preconfig) < 0) { - err = _Py_INIT_NO_MEMORY(); - goto done; - } - err = _Py_INIT_OK(); - -done: - _PyPreConfig_Clear(&local_preconfig); - return err; -} - - /* Read the configuration into _PyCoreConfig from: * Environment variables @@ -1363,22 +1335,20 @@ _PyCoreConfig_ReadPreConfig(_PyCoreConfig *config) See _PyCoreConfig_ReadFromArgv() to parse also command line arguments. */ _PyInitError -_PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig) +_PyCoreConfig_Read(_PyCoreConfig *config) { _PyInitError err; + err = _Py_PreInitialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } + _PyCoreConfig_GetGlobalConfig(config); - if (preconfig != NULL) { - if (_PyPreConfig_Copy(&config->preconfig, preconfig) < 0) { - return _Py_INIT_NO_MEMORY(); - } - } - else { - err = _PyCoreConfig_ReadPreConfig(config); - if (_Py_INIT_FAILED(err)) { - return err; - } + /* Read pre-configuration written by _PyPreConfig_Write() */ + if (_PyPreConfig_Copy(&config->preconfig, &_PyRuntime.preconfig) < 0) { + return _Py_INIT_NO_MEMORY(); } assert(config->preconfig.use_environment >= 0); @@ -2019,12 +1989,13 @@ config_usage(int error, const wchar_t* program) /* Parse command line options and environment variables. */ static _PyInitError -config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline, - const _PyPreConfig *preconfig) +config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline) { int need_usage = 0; _PyInitError err; + _PyCoreConfig_GetGlobalConfig(config); + err = config_init_program(config, cmdline); if (_Py_INIT_FAILED(err)) { return err; @@ -2056,7 +2027,7 @@ config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline, return err; } - err = _PyCoreConfig_Read(config, preconfig); + err = _PyCoreConfig_Read(config); if (_Py_INIT_FAILED(err)) { return err; } @@ -2086,8 +2057,7 @@ config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline, * Environment variables * Py_xxx global configuration variables */ _PyInitError -_PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args, - const _PyPreConfig *preconfig) +_PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args) { _PyInitError err; @@ -2099,7 +2069,7 @@ _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args, goto done; } - err = config_from_cmdline(config, &cmdline, preconfig); + err = config_from_cmdline(config, &cmdline); if (_Py_INIT_FAILED(err)) { goto done; } diff --git a/Python/pathconfig.c b/Python/pathconfig.c index fb2d19e2797a7d2..3b4cac9873c3893 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -389,7 +389,7 @@ pathconfig_global_init(void) _PyInitError err; _PyCoreConfig config = _PyCoreConfig_INIT; - err = _PyCoreConfig_Read(&config, NULL); + err = _PyCoreConfig_Read(&config); if (_Py_INIT_FAILED(err)) { goto error; } diff --git a/Python/preconfig.c b/Python/preconfig.c index a86ece57cfced93..9263978be10aca3 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -666,6 +666,8 @@ _PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args) int locale_coerced = 0; int loops = 0; + _PyPreConfig_GetGlobalConfig(config); + err = get_ctype_locale(&init_ctype_locale); if (_Py_INIT_FAILED(err)) { goto done; @@ -839,5 +841,15 @@ _PyPreConfig_Write(_PyPreConfig *config) /* Set LC_CTYPE to the user preferred locale */ _Py_SetLocaleFromEnv(LC_CTYPE); + /* Write the new pre-configuration into _PyRuntime */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + int res = _PyPreConfig_Copy(&_PyRuntime.preconfig, config); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (res < 0) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0902508429a3e73..546d43ae0139583 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -713,41 +713,58 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p, } -static _PyInitError -pyinit_preconfig(_PyPreConfig *preconfig, const _PyPreConfig *src_preconfig) +_PyInitError +_Py_PreInitializeFromPreConfig(const _PyPreConfig *src_preconfig) { - if (_PyPreConfig_Copy(preconfig, src_preconfig) < 0) { - return _Py_INIT_ERR("failed to copy pre config"); - } + _PyInitError err; - _PyInitError err = _PyPreConfig_Read(preconfig); + err = _PyRuntime_Initialize(); if (_Py_INIT_FAILED(err)) { return err; } - return _PyPreConfig_Write(preconfig); + _PyPreConfig config = _PyPreConfig_INIT; + if (_PyPreConfig_Copy(&config, src_preconfig) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + + err = _PyPreConfig_Read(&config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = _PyPreConfig_Write(&config); + + _PyRuntime.pre_initialized = 1; + +done: + _PyPreConfig_Clear(&config); + return err; } -static _PyInitError -pyinit_coreconfig(_PyCoreConfig *config, const _PyCoreConfig *src_config, - PyInterpreterState **interp_p) +_PyInitError +_Py_PreInitialize(void) { - if (_PyCoreConfig_Copy(config, src_config) < 0) { - return _Py_INIT_ERR("failed to copy core config"); - } + _PyInitError err; - _PyInitError err = _PyCoreConfig_Read(config, NULL); + err = _PyRuntime_Initialize(); if (_Py_INIT_FAILED(err)) { return err; } - if (!_PyRuntime.core_initialized) { - return _Py_InitializeCore_impl(interp_p, config); - } - else { - return _Py_Initialize_ReconfigureCore(interp_p, config); + if (_PyRuntime.pre_initialized) { + /* Already initialized */ + return _Py_INIT_OK(); } + + _PyPreConfig config = _PyPreConfig_INIT; + + err = _Py_PreInitializeFromPreConfig(&config); + + _PyPreConfig_Clear(&config); + return err; } @@ -767,29 +784,41 @@ pyinit_coreconfig(_PyCoreConfig *config, const _PyCoreConfig *src_config, * Any code invoked from this function should *not* assume it has access * to the Python C API (unless the API is explicitly listed as being * safe to call without calling Py_Initialize first) + * + * Must be called after _Py_PreInitialize(). */ _PyInitError _Py_InitializeCore(PyInterpreterState **interp_p, const _PyCoreConfig *src_config) { - _PyInitError err; + assert(_PyRuntime.pre_initialized); - assert(src_config != NULL); + _PyInitError err; + _PyCoreConfig config = _PyCoreConfig_INIT; - _PyCoreConfig local_config = _PyCoreConfig_INIT; + if (_PyCoreConfig_Copy(&config, src_config) < 0) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } - err = pyinit_preconfig(&local_config.preconfig, &src_config->preconfig); + err = _PyCoreConfig_Read(&config); if (_Py_INIT_FAILED(err)) { goto done; } - err = pyinit_coreconfig(&local_config, src_config, interp_p); + if (!_PyRuntime.core_initialized) { + err = _Py_InitializeCore_impl(interp_p, &config); + } + else { + err = _Py_Initialize_ReconfigureCore(interp_p, &config); + } done: - _PyCoreConfig_Clear(&local_config); + _PyCoreConfig_Clear(&config); return err; } + /* Py_Initialize() has already been called: update the main interpreter configuration. Example of bpo-34008: Py_Main() called after Py_Initialize(). */ @@ -924,8 +953,19 @@ _Py_InitializeMainInterpreter(PyInterpreterState *interp, _PyInitError _Py_InitializeFromConfig(const _PyCoreConfig *config) { - PyInterpreterState *interp = NULL; _PyInitError err; + + err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = _Py_PreInitializeFromPreConfig(&config->preconfig); + if (_Py_INIT_FAILED(err)) { + return err; + } + + PyInterpreterState *interp = NULL; err = _Py_InitializeCore(&interp, config); if (_Py_INIT_FAILED(err)) { return err; diff --git a/Python/pystate.c b/Python/pystate.c index 3978baa7af89d8a..bda075c46be91a5 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -41,6 +41,7 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime) _PyGC_Initialize(&runtime->gc); _PyEval_Initialize(&runtime->ceval); + runtime->preconfig = _PyPreConfig_INIT; runtime->gilstate.check_enabled = 1; @@ -92,6 +93,8 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) runtime->interpreters.mutex = NULL; } + _PyPreConfig_Clear(&runtime->preconfig); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); }