From 9c10d36094d5ecd3d7b2b7ed0d9ea39ed8590528 Mon Sep 17 00:00:00 2001 From: Jeremy Andrews Date: Fri, 20 Mar 2026 13:29:08 -0500 Subject: [PATCH] Issue #2974 - Part 51: Replace imp with importlib. imp was actually still available up until Python 3.9 or so. Ripping it out made me go through the headache of debugging our import logic. Granted, it was deprecated long ago and there was plenty of warning, but since we are doing the migration now it's all hitting at once if we want to support any semi-modern Python. --- mach | 22 +++- python/mach/mach/main.py | 21 +++- .../mozbuild/mozbuild/action/file_generate.py | 20 +++- python/virtualenv/virtualenv.py | 106 ++++++++++++++---- 4 files changed, 137 insertions(+), 32 deletions(-) diff --git a/mach b/mach index c9da14c99e..d41f08f4c0 100755 --- a/mach +++ b/mach @@ -22,13 +22,23 @@ def ancestors(path): break def load_mach(dir_path, mach_path): - import imp - with open(mach_path, 'r') as fh: - imp.load_module('mach_bootstrap', fh, mach_path, - ('.py', 'r', imp.PY_SOURCE)) - import mach_bootstrap - return mach_bootstrap.bootstrap(dir_path) + import types + import importlib.util + import importlib.machinery + module_name = "mach_bootstrap" + loader = importlib.machinery.SourceFileLoader(module_name, mach_path) + mod = types.ModuleType(module_name) + mod.__loader__ = loader + mod.__file__ = mach_path + mod.__package__ = module_name.rpartition('.')[0] + mod.__spec__ = None + sys.modules[module_name] = mod + + code = loader.get_code(module_name) + exec(code, mod.__dict__) + + return mod.bootstrap(dir_path) def check_and_get_mach(dir_path): bootstrap_paths = ( diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py index 41cc950773..5ee961a24c 100644 --- a/python/mach/mach/main.py +++ b/python/mach/mach/main.py @@ -10,11 +10,13 @@ from collections import Iterable import argparse import codecs -import imp +import importlib.util +import importlib.machinery import logging import os import sys import traceback +import types import uuid from .base import ( @@ -102,6 +104,19 @@ to implement a mach command. Please fix the error, or uninstall the module from your system. '''.lstrip() +def load_source(module_name, path): + loader = importlib.machinery.SourceFileLoader(module_name, path) + module = types.ModuleType(module_name) + module.__loader__ = loader + module.__file__ = path + module.__package__ = module_name.rpartition('.')[0] + + sys.modules[module_name] = module + + code = loader.get_code(module_name) + exec(code, module.__dict__) + return module + class ArgumentParser(argparse.ArgumentParser): """Custom implementation argument parser to make things look pretty.""" @@ -257,12 +272,12 @@ To see more help for a specific command, run: # Ensure parent module is present otherwise we'll (likely) get # an error due to unknown parent. if 'mach.commands' not in sys.modules: - mod = imp.new_module('mach.commands') + mod = types.ModuleType('mach.commands') sys.modules['mach.commands'] = mod module_name = 'mach.commands.%s' % uuid.uuid1().hex - imp.load_source(module_name, path) + load_source(module_name, path) def load_commands_from_entry_point(self, group='mach.providers'): """Scan installed packages for mach command provider entry points. An diff --git a/python/mozbuild/mozbuild/action/file_generate.py b/python/mozbuild/mozbuild/action/file_generate.py index 3bdbc264bd..0f970c7a0f 100644 --- a/python/mozbuild/mozbuild/action/file_generate.py +++ b/python/mozbuild/mozbuild/action/file_generate.py @@ -9,7 +9,9 @@ from __future__ import absolute_import, print_function import argparse -import imp +import importlib.util +import importlib.machinery +import types import os import sys import traceback @@ -48,9 +50,19 @@ def main(argv): # Since we're invoking the script in a roundabout way, we provide this # bit of convenience. sys.path.append(os.path.dirname(script)) - with open(script, 'r') as fh: - module = imp.load_module('script', fh, script, - ('.py', 'r', imp.PY_SOURCE)) + module_name = "script" + loader = importlib.machinery.SourceFileLoader(module_name, script) + module = types.ModuleType(module_name) + module.__loader__ = loader + module.__file__ = script + module.__package__ = module_name.rpartition('.')[0] + module.__spec__ = None + + sys.modules[module_name] = module + + code = loader.get_code(module_name) + exec(code, module.__dict__) + method = args.method_name if not hasattr(module, method): print('Error: script "{0}" is missing a {1} method'.format(script, method), diff --git a/python/virtualenv/virtualenv.py b/python/virtualenv/virtualenv.py index 64a3566872..69a4b0b6eb 100755 --- a/python/virtualenv/virtualenv.py +++ b/python/virtualenv/virtualenv.py @@ -1050,38 +1050,106 @@ def change_prefix(filename, dst_prefix): assert False, "Filename %s does not start with any of these prefixes: %s" % \ (filename, prefixes) +def find_module_info(modname): + for finder in sys.meta_path: + try: + loader = finder.find_module(modname) + except Exception: + continue + + if loader is None: + continue + + # get filename + try: + filename = loader.get_filename(modname) + except Exception: + continue + + # detect package + is_pkg = False + try: + is_pkg = loader.is_package(modname) + except Exception: + pass + + return filename, is_pkg + + raise ImportError("Cannot find module %s" % modname) + def copy_required_modules(dst_prefix, symlink): - import imp + import importlib + import importlib.machinery + from shutil import copytree for modname in REQUIRED_MODULES: if modname in sys.builtin_module_names: logger.info("Ignoring built-in bootstrap module: %s" % modname) continue - try: - f, filename, _ = imp.find_module(modname) - except ImportError: + + filename = None + is_pkg = False + + for finder in sys.meta_path: + try: + loader = finder.find_module(modname) + except Exception: + continue + + if loader is None: + continue + + # get filename (__init__.py for packages) + try: + filename = loader.get_filename(modname) + except Exception: + continue + + # detect package + try: + is_pkg = loader.is_package(modname) + except Exception: + is_pkg = False + + break + + if filename is None: logger.info("Cannot import bootstrap module: %s" % modname) - else: - if f is not None: - f.close() - # special-case custom readline.so on OS X, but not for pypy: - if modname == 'readline' and sys.platform == 'darwin' and not ( - is_pypy or filename.endswith(join('lib-dynload', 'readline.so'))): - dst_filename = join(dst_prefix, 'lib', 'python%s' % sys.version[:3], 'readline.so') - elif modname == 'readline' and sys.platform == 'win32': - # special-case for Windows, where readline is not a - # standard module, though it may have been installed in - # site-packages by a third-party package - pass - else: - dst_filename = change_prefix(filename, dst_prefix) + continue + + # special-case custom readline.so on OS X, but not for pypy: + if (modname == 'readline' and sys.platform == 'darwin' and + not (is_pypy or filename.endswith(join('lib-dynload', 'readline.so')))): + dst_filename = join(dst_prefix, 'lib', + 'python%s' % sys.version[:3], + 'readline.so') + + # copy single file copyfile(filename, dst_filename, symlink) + continue + + if modname == 'readline' and sys.platform == 'win32': + # special-case for Windows, where readline is not a + # standard module, though it may have been installed in + # site-packages by a third-party package + continue + + if is_pkg: + # copy entire package directory + pkg_dir = os.path.dirname(filename) + dst_dir = change_prefix(pkg_dir, dst_prefix) + copytree(pkg_dir, dst_dir, symlink) + else: + # copy single module file + dst_filename = change_prefix(filename, dst_prefix) + copyfile(filename, dst_filename, symlink) + + # also copy .py if we got a .pyc if filename.endswith('.pyc'): pyfile = filename[:-1] if os.path.exists(pyfile): copyfile(pyfile, dst_filename[:-1], symlink) - def subst_path(prefix_path, prefix, home_dir): prefix_path = os.path.normpath(prefix_path) prefix = os.path.normpath(prefix)