Re: PEP 489: Multi-phase extension module initialization; version 5

On Tue, May 19, 2015 at 8:40 AM, Petr Viktorin <[email protected]> wrote:
> Here is an overview of how the modified importers will operate.
> Details such as logging or handling of errors and invalid states
> are left out, and C code is presented with a concise Python-like syntax.
>
> The framework that calls the importers is explained in PEP 451
> [#pep-0451-loading]_.

I know.  I wrote that PEP. :)

>
> importlib/_bootstrap.py:
>
>     class BuiltinImporter:
>         def create_module(self, spec):
>             module = _imp.create_builtin(spec)
>
>         def exec_module(self, module):
>             _imp.exec_dynamic(module)
>
>         def load_module(self, name):
>             # use a backwards compatibility shim
>             _load_module_shim(self, name)

Won't frozen modules be likewise affected?

>
> importlib/_bootstrap_external.py:
>
>     class ExtensionFileLoader:
>         def create_module(self, spec):
>             module = _imp.create_dynamic(spec)
>
>         def exec_module(self, module):
>             _imp.exec_dynamic(module)
>
>         def load_module(self, name):
>             # use a backwards compatibility shim
>             _load_module_shim(self, name)
>
> Python/import.c (the _imp module):
>
>     def create_dynamic(spec):
>         name = spec.name
>         path = spec.origin
>
>         # Find an already loaded module that used single-phase init.
>         # For multi-phase initialization, mod is NULL, so a new module
>         # is always created.
>         mod = _PyImport_FindExtensionObject(name, name)
>         if mod:
>             return mod
>
>         return _PyImport_LoadDynamicModuleWithSpec(spec)
>
>     def exec_dynamic(module):
>         def = PyModule_GetDef(module)

This is the point where custom module types get ignored, right?

>         state = PyModule_GetState(module)
>         if state is NULL:
>             PyModule_ExecDef(module, def)

Ah, it is idempotent.

>
>     def create_builtin(spec):
>         name = spec.name
>
>         # Find an already loaded module that used single-phase init.
>         # For multi-phase initialization, mod is NULL, so a new module
>         # is always created.
>         mod = _PyImport_FindExtensionObject(name, name)
>         if mod:
>             return mod
>
>         for initname, initfunc in PyImport_Inittab:
>             if name == initname:
>                 m = initfunc()
>                 if isinstance(m, PyModuleDef):
>                     def = m
>                     return PyModule_FromDefAndSpec(def, spec)
>                 else:
>                     # fall back to single-phase initialization
>                     module = m
>                     _PyImport_FixupExtensionObject(module, name, name)
>                     return module
>
> Python/importdl.c:
>
>     def _PyImport_LoadDynamicModuleWithSpec(spec):
>         path = spec.origin
>         package, dot, name = spec.name.rpartition('.')
>
>         # see the "Non-ASCII module names" section for export_hook_name
>         hook_name = export_hook_name(name)
>
>         # call platform-specific function for loading exported function
>         # from shared library
>         exportfunc = _find_shared_funcptr(hook_name, path)
>
>         m = exportfunc()
>         if isinstance(m, PyModuleDef):
>             def = m
>             return PyModule_FromDefAndSpec(def, spec)
>
>         module = m
>
>         # fall back to single-phase initialization
>         ....
>
> Objects/moduleobject.c:
>
>     def PyModule_FromDefAndSpec(def, spec):
>         name = spec.name
>         create = None
>         for slot, value in def.m_slots:
>             if slot == Py_mod_create:
>                 create = value
>         if create:
>             m = create(spec, def)
>         else:
>             m = PyModule_New(name)
>
>         if isinstance(m, types.ModuleType):
>             m.md_state = None
>             m.md_def = def
>
>         if def.m_methods:
>             PyModule_AddFunctions(m, def.m_methods)
>         if def.m_doc:
>             PyModule_SetDocString(m, def.m_doc)
>
>     def PyModule_ExecDef(module, def):
>         if isinstance(module, types.module_type):
>             if module.md_state is NULL:
>                 # allocate a block of zeroed-out memory
>                 module.md_state = _alloc(module.md_size)
>
>         if def.m_slots is NULL:
>             return
>
>         for slot, value in def.m_slots:
>             if slot == Py_mod_exec:
>                 value(module)
>
>

It may also be worth outlining how PyModuleDef_Init will work.

-eric