Issue #2974 - Part 51: Replace imp with importlib.

imp was actually still available up until Python 3.9 or so. Ripping
it out made me go through the headache of debugging our import logic.
Granted, it was deprecated long ago and there was plenty of warning, but
since we are doing the migration now it's all hitting at once if we want
to support any semi-modern Python.

mach

@@ -22,13 +22,23 @@ def ancestors(path):
             break
 
 def load_mach(dir_path, mach_path):
-    import imp
+    import types
-    with open(mach_path, 'r') as fh:
+    import importlib.util
-        imp.load_module('mach_bootstrap', fh, mach_path,
+    import importlib.machinery
-                        ('.py', 'r', imp.PY_SOURCE))
+    module_name = "mach_bootstrap"
-    import mach_bootstrap
+    loader = importlib.machinery.SourceFileLoader(module_name, mach_path)
-    return mach_bootstrap.bootstrap(dir_path)
+    mod = types.ModuleType(module_name)
+    mod.__loader__ = loader
+    mod.__file__ = mach_path
+    mod.__package__ = module_name.rpartition('.')[0]
+    mod.__spec__ = None
 
+    sys.modules[module_name] = mod
+
+    code = loader.get_code(module_name)
+    exec(code, mod.__dict__)
+
+    return mod.bootstrap(dir_path)
 
 def check_and_get_mach(dir_path):
     bootstrap_paths = (

python/mach/mach/main.py

@@ -10,11 +10,13 @@ from collections import Iterable
 
 import argparse
 import codecs
-import imp
+import importlib.util
+import importlib.machinery
 import logging
 import os
 import sys
 import traceback
+import types
 import uuid
 
 from .base import (
@@ -102,6 +104,19 @@ to implement a mach command. Please fix the error, or uninstall the module from
 your system.
 '''.lstrip()
 
+def load_source(module_name, path):
+    loader = importlib.machinery.SourceFileLoader(module_name, path)
+    module = types.ModuleType(module_name)
+    module.__loader__ = loader
+    module.__file__ = path
+    module.__package__ = module_name.rpartition('.')[0]
+
+    sys.modules[module_name] = module
+
+    code = loader.get_code(module_name)
+    exec(code, module.__dict__)
+    return module
+
 class ArgumentParser(argparse.ArgumentParser):
     """Custom implementation argument parser to make things look pretty."""
 
@@ -257,12 +272,12 @@ To see more help for a specific command, run:
             # Ensure parent module is present otherwise we'll (likely) get
             # an error due to unknown parent.
             if 'mach.commands' not in sys.modules:
-                mod = imp.new_module('mach.commands')
+                mod = types.ModuleType('mach.commands')
                 sys.modules['mach.commands'] = mod
 
             module_name = 'mach.commands.%s' % uuid.uuid1().hex
 
-        imp.load_source(module_name, path)
+        load_source(module_name, path)
 
     def load_commands_from_entry_point(self, group='mach.providers'):
         """Scan installed packages for mach command provider entry points. An

python/mozbuild/mozbuild/action/file_generate.py

@@ -9,7 +9,9 @@
 from __future__ import absolute_import, print_function
 
 import argparse
-import imp
+import importlib.util
+import importlib.machinery
+import types
 import os
 import sys
 import traceback
@@ -48,9 +50,19 @@ def main(argv):
     # Since we're invoking the script in a roundabout way, we provide this
     # bit of convenience.
     sys.path.append(os.path.dirname(script))
-    with open(script, 'r') as fh:
+    module_name = "script"
-        module = imp.load_module('script', fh, script,
+    loader = importlib.machinery.SourceFileLoader(module_name, script)
-                                 ('.py', 'r', imp.PY_SOURCE))
+    module = types.ModuleType(module_name)
+    module.__loader__ = loader
+    module.__file__ = script
+    module.__package__ = module_name.rpartition('.')[0]
+    module.__spec__ = None
+
+    sys.modules[module_name] = module
+
+    code = loader.get_code(module_name)
+    exec(code, module.__dict__)
+
     method = args.method_name
     if not hasattr(module, method):
         print('Error: script "{0}" is missing a {1} method'.format(script, method),

python/virtualenv/virtualenv.py

@@ -1050,38 +1050,106 @@ def change_prefix(filename, dst_prefix):
     assert False, "Filename %s does not start with any of these prefixes: %s" % \
         (filename, prefixes)
 
+def find_module_info(modname):
+    for finder in sys.meta_path:
+        try:
+            loader = finder.find_module(modname)
+        except Exception:
+            continue
+
+        if loader is None:
+            continue
+
+        # get filename
+        try:
+            filename = loader.get_filename(modname)
+        except Exception:
+            continue
+
+        # detect package
+        is_pkg = False
+        try:
+            is_pkg = loader.is_package(modname)
+        except Exception:
+            pass
+
+        return filename, is_pkg
+
+    raise ImportError("Cannot find module %s" % modname)
+
 def copy_required_modules(dst_prefix, symlink):
-    import imp
+    import importlib
+    import importlib.machinery
+    from shutil import copytree
 
     for modname in REQUIRED_MODULES:
         if modname in sys.builtin_module_names:
             logger.info("Ignoring built-in bootstrap module: %s" % modname)
             continue
-        try:
+
-            f, filename, _ = imp.find_module(modname)
+        filename = None
-        except ImportError:
+        is_pkg = False
+
+        for finder in sys.meta_path:
+            try:
+                loader = finder.find_module(modname)
+            except Exception:
+                continue
+
+            if loader is None:
+                continue
+
+            # get filename (__init__.py for packages)
+            try:
+                filename = loader.get_filename(modname)
+            except Exception:
+                continue
+
+            # detect package
+            try:
+                is_pkg = loader.is_package(modname)
+            except Exception:
+                is_pkg = False
+
+            break
+
+        if filename is None:
             logger.info("Cannot import bootstrap module: %s" % modname)
-        else:
+            continue
-            if f is not None:
+
-                f.close()
+        # special-case custom readline.so on OS X, but not for pypy:
-            # special-case custom readline.so on OS X, but not for pypy:
+        if (modname == 'readline' and sys.platform == 'darwin' and
-            if modname == 'readline' and sys.platform == 'darwin' and not (
+            not (is_pypy or filename.endswith(join('lib-dynload', 'readline.so')))):
-                    is_pypy or filename.endswith(join('lib-dynload', 'readline.so'))):
+            dst_filename = join(dst_prefix, 'lib',
-                dst_filename = join(dst_prefix, 'lib', 'python%s' % sys.version[:3], 'readline.so')
+                                'python%s' % sys.version[:3],
-            elif modname == 'readline' and sys.platform == 'win32':
+                                'readline.so')
-                # special-case for Windows, where readline is not a
+
-                # standard module, though it may have been installed in
+            # copy single file
-                # site-packages by a third-party package
-                pass
-            else:
-                dst_filename = change_prefix(filename, dst_prefix)
             copyfile(filename, dst_filename, symlink)
+            continue
+
+        if modname == 'readline' and sys.platform == 'win32':
+            # special-case for Windows, where readline is not a
+            # standard module, though it may have been installed in
+            # site-packages by a third-party package
+            continue
+
+        if is_pkg:
+            # copy entire package directory
+            pkg_dir = os.path.dirname(filename)
+            dst_dir = change_prefix(pkg_dir, dst_prefix)
+            copytree(pkg_dir, dst_dir, symlink)
+        else:
+            # copy single module file
+            dst_filename = change_prefix(filename, dst_prefix)
+            copyfile(filename, dst_filename, symlink)
+
+            # also copy .py if we got a .pyc
             if filename.endswith('.pyc'):
                 pyfile = filename[:-1]
                 if os.path.exists(pyfile):
                     copyfile(pyfile, dst_filename[:-1], symlink)
 
-
 def subst_path(prefix_path, prefix, home_dir):
     prefix_path = os.path.normpath(prefix_path)
     prefix = os.path.normpath(prefix)