Whitelist virtual GPUs (VirtualBox, VMware, VirtIO (QEMU and forks like UTM), Parallels)

Resolves #2506

CLOBBER

@@ -22,4 +22,4 @@
 # changes to stick? As of bug 928195, this shouldn't be necessary! Please
 # don't change CLOBBER for WebIDL changes any more.
 
-Clobber for ICU 78.1
+Clobber for Python 3 migration

Makefile.in

@@ -14,7 +14,7 @@ endif
 export TOPLEVEL_BUILD := 1
 
 default::
-
+BACKEND_ONCE_STAMP := backend.once.stamp
 ifndef TEST_MOZBUILD
 ifdef MOZ_BUILD_APP
 include $(wildcard $(topsrcdir)/$(MOZ_BUILD_APP)/build.mk)
@@ -46,7 +46,7 @@ ifndef TEST_MOZBUILD
 ifndef MOZ_PROFILE_USE
 # We need to explicitly put BUILD_BACKEND_FILES here otherwise the rule in
 # rules.mk doesn't run early enough.
-$(TIERS) binaries:: CLOBBER $(configure_dir)/configure config.status $(BUILD_BACKEND_FILES)
+$(TIERS) binaries:: CLOBBER $(configure_dir)/configure config.status $(BACKEND_ONCE_STAMP)
 
 # While our mozbuild backend has some kind of crazy install_manfests junk to
 # try and make this not have to happen, we actually want dist/bin to be
@@ -100,7 +100,11 @@ config.status js/src/config.status:
 ifndef TEST_MOZBUILD
 
 .PHONY: backend
-backend: $(BUILD_BACKEND_FILES)
+backend: $(BACKEND_ONCE_STAMP)
+# Create a per-invocation stamp so backend regenerates only once
+$(BACKEND_ONCE_STAMP):
+	@echo "Stamping backend regeneration."
+	@$(TOUCH) $@
 
 # A traditional rule would look like this:
 #    backend.%:
@@ -122,7 +126,7 @@ $(subst .,%,$(BUILD_BACKEND_FILES)):
 	@echo 'Build configuration changed. Regenerating backend.'
 	$(PYTHON) config.status
 
-Makefile: $(BUILD_BACKEND_FILES)
+Makefile: $(BACKEND_ONCE_STAMP)
 	@$(TOUCH) $@
 
 define build_backend_rule

accessible/base/AccIterator.h

@@ -55,7 +55,7 @@ private:
   {
     explicit IteratorState(Accessible* aParent, IteratorState* mParentState = nullptr);
 
-    Accessible* mParent;
+    RefPtr<Accessible> mParent;
     int32_t mIndex;
     IteratorState* mParentState;
   };

accessible/xpcom/AccEventGen.py

@@ -40,7 +40,7 @@ def loadEventIDL(parser, includePath, eventname):
 class Configuration:
     def __init__(self, filename):
         config = {}
-        execfile(filename, config)
+        exec(compile(open(filename, "rb").read(), filename, 'exec'), config)
         self.simple_events = config.get('simple_events', [])
 
 def firstCap(str):
@@ -64,7 +64,7 @@ def print_header_file(fd, conf, incdirs):
     for e in conf.simple_events:
         idl, idl_path = loadEventIDL(p, incdirs, e)
         idl_paths.add(idl_path)
-        for iface in filter(lambda p: p.kind == "interface", idl.productions):
+        for iface in [p for p in idl.productions if p.kind == "interface"]:
             classname = ("xpcAcc%s" % e)
             baseinterfaces = interfaces(iface)
 
@@ -73,7 +73,7 @@ def print_header_file(fd, conf, incdirs):
             fd.write("public:\n")
 
             attributes = allAttributes(iface)
-            args = map(writeAttributeParams, attributes)
+            args = list(map(writeAttributeParams, attributes))
             fd.write("  %s(%s) :\n" % (classname, ", ".join(args)))
 
             initializers = []
@@ -83,7 +83,7 @@ def print_header_file(fd, conf, incdirs):
             fd.write("  NS_DECL_CYCLE_COLLECTING_ISUPPORTS\n")
             fd.write("  NS_DECL_CYCLE_COLLECTION_CLASS(%s)\n" % (classname))
 
-            for iface in filter(lambda i: i.name != "nsISupports", baseinterfaces):
+            for iface in [i for i in baseinterfaces if i.name != "nsISupports"]:
                 fd.write("  NS_DECL_%s\n" % iface.name.upper())
 
             fd.write("\nprivate:\n")
@@ -97,13 +97,13 @@ def print_header_file(fd, conf, incdirs):
     return idl_paths
 
 def interfaceAttributeTypes(idl):
-    ifaces = filter(lambda p: p.kind == "interface", idl.productions)
+    ifaces = [p for p in idl.productions if p.kind == "interface"]
     attributes = []
     for i in ifaces:
         ifaceAttributes = allAttributes(i)
         attributes.extend(ifaceAttributes)
-    ifaceAttrs = filter(lambda a: a.realtype.nativeType("in").endswith("*"), attributes)
-    return map(lambda a: a.realtype.nativeType("in").strip(" *"), ifaceAttrs)
+    ifaceAttrs = [a for a in attributes if a.realtype.nativeType("in").endswith("*")]
+    return [a.realtype.nativeType("in").strip(" *") for a in ifaceAttrs]
 
 def print_cpp(idl, fd, conf, eventname):
     for p in idl.productions:
@@ -186,7 +186,7 @@ def interfaces(iface):
 def allAttributes(iface):
     attributes = []
     for i in interfaces(iface):
-        attrs = filter(lambda m: isinstance(m, xpidl.Attribute), i.members)
+        attrs = [m for m in i.members if isinstance(m, xpidl.Attribute)]
         attributes.extend(attrs)
 
     return attributes
@@ -194,7 +194,7 @@ def allAttributes(iface):
 def write_cpp(eventname, iface, fd):
     classname = "xpcAcc%s" % eventname
     attributes = allAttributes(iface)
-    ccattributes = filter(lambda m: m.realtype.nativeType('in').endswith('*'), attributes)
+    ccattributes = [m for m in attributes if m.realtype.nativeType('in').endswith('*')]
     fd.write("NS_IMPL_CYCLE_COLLECTION(%s" % classname)
     for c in ccattributes:
         fd.write(", m%s" % firstCap(c.name))

build/appini_header.py

@@ -5,11 +5,11 @@
 '''Parses a given application.ini file and outputs the corresponding
    XULAppData structure as a C++ header file'''
 
-import ConfigParser
+import configparser
 import sys
 
 def main(output, file):
-    config = ConfigParser.RawConfigParser()
+    config = configparser.RawConfigParser()
     config.read(file)
     flags = set()
     try:
@@ -27,8 +27,7 @@ def main(output, file):
                 'App:id', 'Gecko:minversion', 'Gecko:maxversion')
     missing = [var for var in expected if var not in appdata]
     if missing:
-        print >>sys.stderr, \
-            "Missing values in %s: %s" % (file, ', '.join(missing))
+        print("Missing values in %s: %s" % (file, ', '.join(missing)), file=sys.stderr)
         sys.exit(1)
 
     if not 'Crash Reporter:serverurl' in appdata:
@@ -57,4 +56,4 @@ if __name__ == '__main__':
     if len(sys.argv) != 1:
         main(sys.stdout, sys.argv[1])
     else:
-        print >>sys.stderr, "Usage: %s /path/to/application.ini" % sys.argv[0]
+        print("Usage: %s /path/to/application.ini" % sys.argv[0], file=sys.stderr)

build/build-clang/build-clang.py

@@ -34,7 +34,7 @@ def symlink(source, link_name):
 def check_run(args):
     global DEBUG
     if DEBUG:
-        print >> sys.stderr, ' '.join(args)
+        print(' '.join(args), file=sys.stderr)
     r = subprocess.call(args)
     assert r == 0
 
@@ -43,11 +43,11 @@ def run_in(path, args):
     d = os.getcwd()
     global DEBUG
     if DEBUG:
-        print >> sys.stderr, 'cd "%s"' % path
+        print('cd "%s"' % path, file=sys.stderr)
     os.chdir(path)
     check_run(args)
     if DEBUG:
-        print >> sys.stderr, 'cd "%s"' % d
+        print('cd "%s"' % d, file=sys.stderr)
     os.chdir(d)
 
 
@@ -377,7 +377,7 @@ if __name__ == "__main__":
         extra_cflags2 = ["-fPIC"]
         extra_cxxflags2 = ["-fPIC", "-static-libstdc++"]
 
-        if os.environ.has_key('LD_LIBRARY_PATH'):
+        if 'LD_LIBRARY_PATH' in os.environ:
             os.environ['LD_LIBRARY_PATH'] = '%s/lib64/:%s' % (gcc_dir, os.environ['LD_LIBRARY_PATH']);
         else:
             os.environ['LD_LIBRARY_PATH'] = '%s/lib64/' % gcc_dir

build/checksums.py

@@ -81,9 +81,9 @@ def process_files(files, output_filename, digests, strip):
                         short_file = short_file.lstrip('/')
                     else:
                         short_file = file
-                    print >>output, '%s %s %s %s' % (hash, digest,
+                    print('%s %s %s %s' % (hash, digest,
                                                      os.path.getsize(file),
-                                                     short_file)
+                                                     short_file), file=output)
 
 def setup_logging(level=logging.DEBUG):
     '''This function sets up the logging module using a speficiable logging
@@ -140,7 +140,7 @@ def main():
     try:
         for digest in options.digests:
             hashlib.new(digest)
-    except ValueError, ve:
+    except ValueError as ve:
         logger.error('Could not create a "%s" hash object (%s)' %
                      (digest, ve.args[0]))
         exit(1)

build/compare-mozconfig/compare-mozconfigs.py

@@ -11,7 +11,7 @@ import logging
 import os
 import site
 import sys
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 import difflib
 
 FAILURE_CODE = 1
@@ -40,10 +40,10 @@ def make_hg_url(hgHost, repoPath, protocol='https', revision=None,
 
 def readConfig(configfile, keys=[], required=[]):
     c = {}
-    execfile(configfile, c)
+    exec(compile(open(configfile, "rb").read(), configfile, 'exec'), c)
     for k in keys:
         c = c[k]
-    items = c.keys()
+    items = list(c.keys())
     err = False
     for key in required:
         if key not in items:
@@ -126,7 +126,7 @@ def get_mozconfig(path, options):
     else:
         url = make_hg_url(options.hghost, options.branch, 'http',
                     options.revision, path)
-        return urllib2.urlopen(url).readlines()
+        return urllib.request.urlopen(url).readlines()
 
 if __name__ == '__main__':
     from optparse import OptionParser

build/gyp.mozbuild

@@ -73,8 +73,6 @@ if CONFIG['ARM_ARCH']:
     if int(CONFIG['ARM_ARCH']) < 7:
         gyp_vars['armv7'] = 0
         gyp_vars['arm_neon_optional'] = 0
-    elif os == 'Android':
-        gyp_vars['armv7'] = 1
     else:
         # CPU detection for ARM works on Android only.  armv7 always uses CPU
         # detection, so we have to set armv7=0 for non-Android target

build/gyp_base.mozbuild

@@ -18,7 +18,7 @@ flavors = {
     'Linux': 'linux',
     'Darwin': 'mac',
     'SunOS': 'solaris',
-    'GNU/kFreeBSD': 'freebsd',
+    'kFreeBSD': 'freebsd',
     'DragonFly': 'dragonfly',
     'FreeBSD': 'freebsd',
     'NetBSD': 'netbsd',

build/mach_bootstrap.py

@@ -12,7 +12,7 @@ import random
 import subprocess
 import sys
 import uuid
-import __builtin__
+import builtins
 
 from types import ModuleType
 
@@ -170,11 +170,16 @@ def bootstrap(topsrcdir, mozilla_dir=None):
     if mozilla_dir is None:
         mozilla_dir = topsrcdir
 
-    # Ensure we are running Python 2.7+. We put this check here so we generate a
-    # user-friendly error message rather than a cryptic stack trace on module
-    # import.
-    if sys.version_info[0] != 2 or sys.version_info[1] < 7:
-        print('Python 2.7 or above (but not Python 3) is required to run mach.')
+    # Ensure we are not running Python 2.
+    # We put this check here so we generate a user-friendly error message
+    # rather than a cryptic stack trace on module import.
+    if sys.version_info[0] < 3 or sys.version_info[1] < 3:
+        print('Python 3.3 or above is required to run mach.')
+        print('You are running Python', platform.python_version())
+        sys.exit(1)
+    # This sanity check is overkill, but why not?
+    if sys.version_info[0] > 3:
+        print('Python 4.x or higher not supported. Use Python 3 instead.')
         print('You are running Python', platform.python_version())
         sys.exit(1)
 
@@ -335,7 +340,7 @@ def bootstrap(topsrcdir, mozilla_dir=None):
     # always load local repository configuration
     mach.settings_paths.append(mozilla_dir)
 
-    for category, meta in CATEGORIES.items():
+    for category, meta in list(CATEGORIES.items()):
         mach.define_category(category, meta['short'], meta['long'],
             meta['priority'])
 
@@ -360,7 +365,7 @@ class ImportHook(object):
         self._modules = set()
 
     def __call__(self, name, globals=None, locals=None, fromlist=None,
-                 level=-1):
+                 level=0):
         # name might be a relative import. Instead of figuring out what that
         # resolves to, which is complex, just rely on the real import.
         # Since we don't know the full module name, we can't check sys.modules,
@@ -409,4 +414,4 @@ class ImportHook(object):
 
 
 # Install our hook
-__builtin__.__import__ = ImportHook(__builtin__.__import__)
+builtins.__import__ = ImportHook(builtins.__import__)

build/moz.configure/checks.configure

@@ -11,7 +11,7 @@
 # _declare_exceptions template, and add it to the return statement. Then
 # destructure in the assignment below the function declaration.
 @template
-@imports(_from='__builtin__', _import='Exception')
+@imports(_from='builtins', _import='Exception')
 def _declare_exceptions():
     class FatalCheckError(Exception):
         '''An exception to throw from a function decorated with @checking.

build/moz.configure/init.configure

@@ -133,7 +133,8 @@ option(env='PYTHON', nargs=1, help='Python interpreter')
 @imports(_from='mozbuild.configure.util', _import='LineIO')
 @imports(_from='mozbuild.virtualenv', _import='VirtualenvManager')
 @imports(_from='mozbuild.virtualenv', _import='verify_python_version')
-@imports('distutils.sysconfig')
+@imports(_from='mozpack.path', _import='normsep')
+@imports('sysconfig')
 def virtualenv_python(env_python, build_env, mozconfig, help):
     if help:
         return
@@ -165,6 +166,9 @@ def virtualenv_python(env_python, build_env, mozconfig, help):
             os.path.join(topobjdir, '_virtualenv'), out,
             os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
 
+    if normsep(sys.executable) == normsep(manager.python_path):
+        return normsep(manager.python_path)
+
     if python:
         # If we're not in the virtualenv, we need the which module for
         # find_program.
@@ -193,10 +197,10 @@ def virtualenv_python(env_python, build_env, mozconfig, help):
         sys.exit(subprocess.call([python] + sys.argv))
 
     # We are now in the virtualenv
-    if not distutils.sysconfig.get_python_lib():
+    if not sysconfig.get_path("purelib"):
         die('Could not determine python site packages directory')
 
-    return python
+    return normsep(python)
 
 set_config('PYTHON', virtualenv_python)
 add_old_configure_assignment('PYTHON', virtualenv_python)
@@ -212,7 +216,7 @@ def early_options():
     def early_options():
         return set(
             option.env
-            for option in __sandbox__._options.itervalues()
+            for option in __sandbox__._options.values()
             if option.env
         )
     return early_options
@@ -240,17 +244,17 @@ def mozconfig_options(mozconfig, help):
                 log.info('  %s' % arg)
                 helper.add(arg, origin='mozconfig', args=helper._args)
 
-        for key, value in mozconfig['env']['added'].iteritems():
+        for key, value in mozconfig['env']['added'].items():
             add(key, value)
             os.environ[key] = value
-        for key, (_, value) in mozconfig['env']['modified'].iteritems():
+        for key, (_, value) in mozconfig['env']['modified'].items():
             add(key, value)
             os.environ[key] = value
-        for key, value in mozconfig['vars']['added'].iteritems():
+        for key, value in mozconfig['vars']['added'].items():
             # mozconfig_loader adds _IS_SET variables that are irrelevant
             if not key.endswith('_IS_SET'):
                 add(key, value)
-        for key, (_, value) in mozconfig['vars']['modified'].iteritems():
+        for key, (_, value) in mozconfig['vars']['modified'].items():
             add(key, value)
 
 
@@ -290,6 +294,7 @@ option('--target', nargs=1,
 @imports(_from='mozbuild.configure.constants', _import='Endianness')
 @imports(_from='mozbuild.configure.constants', _import='Kernel')
 @imports(_from='mozbuild.configure.constants', _import='OS')
+@imports(_from='mozbuild.configure.constants', _import='OS_and_Kernel')
 def split_triplet(triplet):
     # The standard triplet is defined as
     #   CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
@@ -385,8 +390,8 @@ def split_triplet(triplet):
         alias=triplet,
         cpu=CPU(canonical_cpu),
         bitness=CPU_bitness[canonical_cpu],
-        kernel=Kernel(canonical_kernel),
-        os=OS(canonical_os),
+        kernel=OS_and_Kernel(canonical_kernel),
+        os=OS_and_Kernel(canonical_os),
         endianness=Endianness(endianness),
         raw_cpu=cpu,
         raw_os=os,
@@ -400,7 +405,8 @@ def split_triplet(triplet):
 def config_sub(shell, triplet):
     config_sub = os.path.join(os.path.dirname(__file__), '..',
                               'autoconf', 'config.sub')
-    return subprocess.check_output([shell, config_sub, triplet]).strip()
+    return subprocess.check_output([shell, config_sub, triplet],
+                                   universal_newlines=True).strip()
 
 
 @depends('--host', shell)
@@ -410,7 +416,8 @@ def host(value, shell):
     if not value:
         config_guess = os.path.join(os.path.dirname(__file__), '..',
                                     'autoconf', 'config.guess')
-        host = subprocess.check_output([shell, config_guess]).strip()
+        host = subprocess.check_output([shell, config_guess],
+                                       universal_newlines=True).strip()
     else:
         host = value[0]
 
@@ -761,7 +768,7 @@ def enabled_in_nightly(milestone, _):
 def all_configure_options(_):
     result = []
     previous = None
-    for option in __sandbox__._options.itervalues():
+    for option in __sandbox__._options.values():
         # __sandbox__._options contains items for both option.name and
         # option.env. But it's also an OrderedDict, meaning both are
         # consecutive.

build/moz.configure/old.configure

@@ -3,13 +3,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-@imports('codecs')
-@imports('sys')
-def encoded_open(path, mode):
-    encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
-    return codecs.open(path, mode, encoding)
-
-
 m4 = check_prog(
     "M4",
     (
@@ -31,8 +24,10 @@ m4 = check_prog(
 @imports(_from='os.path', _import='exists')
 @imports(_from='mozbuild.shellutil', _import='quote')
 @imports(_from='os', _import='environ')
+@imports(_from='mozbuild.getencoding', _import='getencoding')
 def prepare_configure(old_configure, mozconfig, awk, m4, build_env, shell,
                       old_configure_assignments, build_project):
+    encoding = getencoding()
     # os.path.abspath in the sandbox will ensure forward slashes on Windows,
     # which is actually necessary because this path actually ends up literally
     # as $0, and backslashes there breaks autoconf's detection of the source
@@ -43,7 +38,6 @@ def prepare_configure(old_configure, mozconfig, awk, m4, build_env, shell,
         if not old_configure_dir.endswith('/js/src'):
             old_configure = os.path.join(old_configure_dir, 'js', 'src',
                                          os.path.basename(old_configure))
-
     refresh = True
     if exists(old_configure):
         mtime = getmtime(old_configure)
@@ -67,25 +61,21 @@ def prepare_configure(old_configure, mozconfig, awk, m4, build_env, shell,
         env['AWK'] = awk
         env['AC_MACRODIR'] = os.path.join(build_env.topsrcdir, 'build', 'autoconf')
         my_env = env
-        # Handle Python 2.7 not allowing unicode in the environment on Windows
-        if sys.platform == 'win32':
-            my_env = dict()
-            for e in env:
-                my_env[e.encode('ascii','ignore')] = env[e].encode('ascii','ignore')
         script = subprocess.check_output([
             shell, autoconf,
             '--localdir=%s' % os.path.dirname(old_configure),
             old_configure + '.in'], env=my_env)
+        script = script.decode(encoding, errors='replace')
 
         # Make old-configure append to config.log, where we put our own log.
         # This could be done with a m4 macro, but it's way easier this way
         script = script.replace('>./config.log', '>>./config.log')
 
-        with open(old_configure, 'wb') as fh:
+        with open(old_configure, 'w', encoding=encoding, errors='replace') as fh:
             fh.write(script)
 
     cmd = [shell, old_configure]
-    with encoded_open('old-configure.vars', 'w') as out:
+    with open('old-configure.vars', 'w', encoding=encoding, errors='replace') as out:
         log.debug('Injecting the following to old-configure:')
         def inject(command):
             print(command, file=out)
@@ -127,7 +117,6 @@ def old_configure_options(*options):
     '--enable-alsa',
     '--enable-appcompat-guid',
     '--enable-av1',
-    '--enable-jxl',
     '--enable-b2g-bt',
     '--enable-b2g-camera',
     '--enable-b2g-ril',
@@ -295,8 +284,11 @@ def old_configure_options(*options):
 @imports('subprocess')
 @imports('sys')
 @imports(_from='mozbuild.shellutil', _import='quote')
+@imports(_from='mozbuild.util', _import='exec_')
+@imports(_from='mozbuild.getencoding', _import='getencoding')
 def old_configure(prepare_configure, extra_old_configure_args, all_options,
                   *options):
+    encoding = getencoding()
     cmd = prepare_configure
 
     # old-configure only supports the options listed in @old_configure_options
@@ -333,7 +325,7 @@ def old_configure(prepare_configure, extra_old_configure_args, all_options,
     ret = subprocess.call(cmd)
     if ret:
         with log.queue_debug():
-            with encoded_open('config.log', 'r') as fh:
+            with open('config.log', 'r', encoding=encoding, errors='replace') as fh:
                 fh.seek(log_size)
                 for line in fh:
                     log.debug(line.rstrip())
@@ -341,12 +333,12 @@ def old_configure(prepare_configure, extra_old_configure_args, all_options,
         sys.exit(ret)
 
     raw_config = {}
-    with encoded_open('config.data', 'r') as fh:
+    with open('config.data', 'r', encoding=encoding, errors='replace') as fh:
         code = compile(fh.read(), 'config.data', 'exec')
         # Every variation of the exec() function I tried led to:
         # SyntaxError: unqualified exec is not allowed in function 'main' it
         # contains a nested function with free variables
-        exec code in raw_config
+        exec_(code, raw_config)
 
     # Ensure all the flags known to old-configure appear in the
     # @old_configure_options above.
@@ -380,9 +372,9 @@ def set_old_configure_define(name, value):
 def post_old_configure(raw_config):
     for k, v in raw_config['substs']:
         set_old_configure_config(
-            k[1:-1], v[1:-1] if isinstance(v, types.StringTypes) else v)
+            k[1:-1], v[1:-1] if isinstance(v, str) else v)
 
-    for k, v in dict(raw_config['defines']).iteritems():
+    for k, v in dict(raw_config['defines']).items():
         set_old_configure_define(k[1:-1], v[1:-1])
 
     set_old_configure_config('non_global_defines',

build/moz.configure/toolchain.configure

@@ -236,7 +236,7 @@ def get_compiler_info(compiler, language):
         ('CPU', CPU_preprocessor_checks),
         ('KERNEL', kernel_preprocessor_checks),
     ):
-        for n, (value, condition) in enumerate(preprocessor_checks.iteritems()):
+        for n, (value, condition) in enumerate(preprocessor_checks.items()):
             check += dedent('''\
                 #%(if)s %(condition)s
                 %%%(name)s "%(value)s"
@@ -269,7 +269,7 @@ def get_compiler_info(compiler, language):
     # have non-ASCII characters. Treat the output as bytearray.
     data = {}
     for line in result.splitlines():
-        if line.startswith(b'%'):
+        if line.startswith('%'):
             k, _, v = line.partition(' ')
             k = k.lstrip('%')
             data[k] = v.replace(' ', '').lstrip('"').rstrip('"')
@@ -397,10 +397,11 @@ def check_compiler(compiler, language, target):
 @imports('json')
 @imports('subprocess')
 @imports('sys')
+@imports(_from='locale', _import='getpreferredencoding')
 def get_vc_paths(topsrcdir):
+    encoding = getpreferredencoding(False)
     def vswhere(args):
-        encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
-        return json.loads(subprocess.check_output([os.path.join(topsrcdir, 'build/win32/vswhere.exe'), '-format', 'json'] + args).decode(encoding, 'replace'))
+        return json.loads(subprocess.check_output([os.path.join(topsrcdir, 'build/win32/vswhere.exe'), '-format', 'json'] + args).decode(encoding, errors='replace'))
 
     # Can't pass -requires with -legacy, so query each separately.
     # Legacy versions first (VS2015)
@@ -419,7 +420,7 @@ def get_vc_paths(topsrcdir):
     # Then VS2017 and newer.
     for install in vswhere(['-requires', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64']):
         path = install['installationPath']
-        tools_version = open(os.path.join(path, r'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt'), 'rb').read().strip()
+        tools_version = open(os.path.join(path, r'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt'), 'r', encoding=encoding, errors='replace').read().strip()
         tools_path = os.path.join(path, r'VC\Tools\MSVC', tools_version, r'bin\HostX64')
         yield (Version(install['installationVersion']), {
             'x64': [os.path.join(tools_path, 'x64')],
@@ -601,7 +602,7 @@ def compiler(language, host_or_target, c_compiler=None, other_compiler=None,
             if cross_compiling and other_compiler.type == 'clang':
                 return namespace(**{
                     k: [] if k == 'flags' else v
-                    for k, v in other_compiler.__dict__.iteritems()
+                    for k, v in other_compiler.__dict__.items()
                 })
 
     # Normally, we'd use `var` instead of `_var`, but the interaction with

build/moz.configure/util.configure

@@ -25,15 +25,22 @@ def configure_error(message):
 @imports('sys')
 @imports(_from='mozbuild.configure.util', _import='LineIO')
 @imports(_from='mozbuild.shellutil', _import='quote')
+@imports(_from='mozbuild.getencoding', _import='getencoding')
 def check_cmd_output(*args, **kwargs):
+    encoding = getencoding()
     onerror = kwargs.pop('onerror', None)
 
     with log.queue_debug():
         log.debug('Executing: `%s`', quote(*args))
         proc = subprocess.Popen(args, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
+
         stdout, stderr = proc.communicate()
         retcode = proc.wait()
+
+        stdout = stdout.decode(encoding, errors='replace')
+        stderr = stderr.decode(encoding, errors='replace')
+
         if retcode == 0:
             return stdout
 
@@ -70,7 +77,7 @@ def windows_binary_type(path):
     Returns WindowsBinaryType constant.
     """
     GetBinaryTypeW = ctypes.windll.kernel32.GetBinaryTypeW
-    GetBinaryTypeW.argtypes = [wintypes.LPWSTR, wintypes.POINTER(wintypes.DWORD)]
+    GetBinaryTypeW.argtypes = [wintypes.LPWSTR, ctypes.POINTER(wintypes.DWORD)]
     GetBinaryTypeW.restype = wintypes.BOOL
 
     bin_type = wintypes.DWORD()
@@ -232,7 +239,7 @@ def unique_list(l):
 #     ('19.0', 'arm', r'C:\...\amd64_arm\cl.exe')
 #     ('19.0', 'x64', r'C:\...\amd64\cl.exe')
 #     ('19.0', 'x86', r'C:\...\amd64_x86\cl.exe')
-@imports(_import='_winreg', _as='winreg')
+@imports(_import='winreg', _as='winreg')
 @imports(_from='__builtin__', _import='WindowsError')
 @imports(_from='fnmatch', _import='fnmatch')
 def get_registry_values(pattern):
@@ -295,7 +302,7 @@ def get_registry_values(pattern):
         keys = next_keys
 
 
-@imports(_from='mozbuild.configure.util', _import='Version', _as='_Version')
+@imports(_from='mozbuild.version', _import='RichVersion', _as='_Version')
 def Version(v):
     'A version number that can be compared usefully.'
     return _Version(v)

build/moz.configure/windows.configure

@@ -408,9 +408,12 @@ mt = check_prog('MT', depends_win()(lambda: ('mt.exe',)), input='MT',
 @depends_win(mt)
 @checking('whether MT is really Microsoft Manifest Tool', lambda x: bool(x))
 @imports('subprocess')
+@imports(_from='locale', _import='getpreferredencoding')
 def valid_mt(path):
+    encoding = getpreferredencoding(False)
     try:
-        out = subprocess.check_output([path]).splitlines()
+        out = subprocess.check_output([path])
+        out = out.decode(encoding, errors='replace').splitlines()
         out = '\n'.join(l for l in out
                         if 'Microsoft (R) Manifest Tool' in l)
         if out:

build/pgo/genpgocert.py

@@ -58,7 +58,7 @@ def runUtil(util, args, inputdata = None):
 
 
 def createRandomFile(randomFile):
-  for count in xrange(0, 2048):
+  for count in range(0, 2048):
     randomFile.write(chr(random.randint(0, 255)))
 
 
@@ -125,7 +125,7 @@ def createSSLServerCertificate(build, srcDir):
     iterator = iter(locations)
 
     # Skips the first entry, I don't know why: bug 879740
-    iterator.next()
+    next(iterator)
 
     locationsParam = ""
     firstLocation = ""
@@ -148,7 +148,7 @@ def createSSLServerCertificate(build, srcDir):
             firstLocation = loc.host
 
     if not firstLocation:
-      print "Nothing to generate, no automatic secure hosts specified"
+      print("Nothing to generate, no automatic secure hosts specified")
     else:
       createRandomFile(rndfile)
 
@@ -162,7 +162,7 @@ def createSSLServerCertificate(build, srcDir):
   return 0
 
 if len(sys.argv) == 1:
-  print "Specify --gen-server or --gen-ca"
+  print("Specify --gen-server or --gen-ca")
   sys.exit(1)
 
 build = MozbuildObject.from_environment()
@@ -170,23 +170,23 @@ certdir = os.path.join(build.topsrcdir, "build", "pgo", "certs")
 if sys.argv[1] == "--gen-server":
   certificateStatus = createSSLServerCertificate(build, certdir)
   if certificateStatus:
-    print "TEST-UNEXPECTED-FAIL | SSL Server Certificate generation"
+    print("TEST-UNEXPECTED-FAIL | SSL Server Certificate generation")
 
   sys.exit(certificateStatus)
 
 if sys.argv[1] == "--gen-ca":
   certificateStatus = createCertificateAuthority(build, certdir)
   if certificateStatus:
-    print "TEST-UNEXPECTED-FAIL | Certificate Authority generation"
+    print("TEST-UNEXPECTED-FAIL | Certificate Authority generation")
   else:
-    print "\n\n"
-    print "==================================================="
-    print " IMPORTANT:"
-    print " To use this new certificate authority in tests"
-    print " run 'make' at testing/mochitest"
-    print "==================================================="
+    print("\n\n")
+    print("===================================================")
+    print(" IMPORTANT:")
+    print(" To use this new certificate authority in tests")
+    print(" run 'make' at testing/mochitest")
+    print("===================================================")
 
   sys.exit(certificateStatus)
 
-print "Invalid option specified"
+print("Invalid option specified")
 sys.exit(1)

build/pgo/profileserver.py

@@ -77,7 +77,7 @@ if __name__ == '__main__':
     jarlog = os.getenv("JARLOG_FILE")
     if jarlog:
       env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
-      print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"]
+      print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
 
     cmdargs = ["http://localhost:%d/index.html" % PORT]
     runner = FirefoxRunner(profile=profile,

build/ppCheck.py

@@ -3,7 +3,7 @@
 from __future__ import print_function
 import os, sys
 
-if not len(sys.argv) is 2 or not os.path.exists(sys.argv[1]):
+if not len(sys.argv) == 2 or not os.path.exists(sys.argv[1]):
   print("\nYou did not supply a valid path to check.")
   exit(1)
 else:
@@ -44,7 +44,8 @@ PP_DIRECTIVES = [
 ]
 
 PP_FILES = []
-PP_BAD_FILES = []
+PP_BAD_DIRECTIVE_FILES = []
+PP_BAD_ENCODING_FILES = []
 
 for root, directories, filenames in os.walk(DIST_PATH):
   for filename in filenames: 
@@ -52,21 +53,31 @@ for root, directories, filenames in os.walk(DIST_PATH):
       PP_FILES += [ os.path.join(root, filename).replace(os.sep, '/') ]
 
 for file in PP_FILES:
-  with open(file) as fp:
+  with open(file, encoding='utf-8') as fp:
     marker = '%' if file.endswith(PP_SPECIAL_TYPES) else '#'
     directives = tuple(marker + directive for directive in PP_DIRECTIVES)
-    for line in fp:
-      if line.startswith(directives):
-        PP_BAD_FILES += [ file.replace(DIST_PATH + '/', '') ]
+    try:
+        for line in fp:
+          if line.startswith(directives):
+            PP_BAD_DIRECTIVE_FILES += [ file.replace(DIST_PATH + '/', '') ]
+    except UnicodeDecodeError:
+        PP_BAD_ENCODING_FILES += [ file.replace(DIST_PATH+ '/' , '') ]
+
   fp.close()
 
-PP_BAD_FILES = list(dict.fromkeys(PP_BAD_FILES))
+PP_BAD_DIRECTIVE_FILES = list(dict.fromkeys(PP_BAD_DIRECTIVE_FILES))
+PP_BAD_ENCODING_FILES = list(dict.fromkeys(PP_BAD_ENCODING_FILES))
 
 print('Done!')
 
-if len(PP_BAD_FILES) > 0:
-  print("\nWARNING: The following {0} file(s) in {1} may require preprocessing:\n".format(len(PP_BAD_FILES), DIST_PATH))
-  for file in PP_BAD_FILES:
+if len(PP_BAD_DIRECTIVE_FILES) > 0:
+  print("\nWARNING: The following {0} file(s) in {1} may require preprocessing:\n".format(len(PP_BAD_DIRECTIVE_FILES), DIST_PATH))
+  for file in PP_BAD_DIRECTIVE_FILES:
     print(file)
 
-exit(0)
+if len(PP_BAD_ENCODING_FILES) > 0:
+  print("\nWARNING: The following {0} file(s) in {1} are not valid UTF-8:\n".format(len(PP_BAD_ENCODING_FILES), DIST_PATH))
+  for file in PP_BAD_ENCODING_FILES:
+    print(file)
+
+sys.exit(0)

build/pymake/mkformat.py

@@ -10,4 +10,4 @@ with open(filename, 'rU') as fh:
     source = fh.read()
 
 statements = pymake.parser.parsestring(source, filename)
-print statements.to_source()
+print(statements.to_source())

build/pymake/mkparse.py

@@ -4,9 +4,9 @@ import sys
 import pymake.parser
 
 for f in sys.argv[1:]:
-    print "Parsing %s" % f
+    print("Parsing %s" % f)
     fd = open(f, 'rU')
     s = fd.read()
     fd.close()
     stmts = pymake.parser.parsestring(s, f)
-    print stmts
+    print(stmts)

build/pymake/pymake/builtins.py

@@ -2,7 +2,7 @@
 import errno, sys, os, shutil, time
 from getopt import getopt, GetoptError
 
-from process import PythonException
+from .process import PythonException
 
 __all__ = ["mkdir", "rm", "sleep", "touch"]
 
@@ -13,8 +13,8 @@ def mkdir(args):
   """
   try:
     opts, args = getopt(args, "p", ["parents"])
-  except GetoptError, e:
-    raise PythonException, ("mkdir: %s" % e, 1)
+  except GetoptError as e:
+    raise PythonException("mkdir: %s" % e, 1)
   parents = False
   for o, a in opts:
     if o in ('-p', '--parents'):
@@ -25,11 +25,11 @@ def mkdir(args):
         os.makedirs(f)
       else:
         os.mkdir(f)
-    except OSError, e:
+    except OSError as e:
       if e.errno == errno.EEXIST and parents:
         pass
       else:
-        raise PythonException, ("mkdir: %s" % e, 1)
+        raise PythonException("mkdir: %s" % e, 1)
 
 def rm(args):
   """
@@ -38,8 +38,8 @@ def rm(args):
   """
   try:
     opts, args = getopt(args, "rRf", ["force", "recursive"])
-  except GetoptError, e:
-    raise PythonException, ("rm: %s" % e, 1)
+  except GetoptError as e:
+    raise PythonException("rm: %s" % e, 1)
   force = False
   recursive = False
   for o, a in opts:
@@ -50,7 +50,7 @@ def rm(args):
   for f in args:
     if os.path.isdir(f):
       if not recursive:
-        raise PythonException, ("rm: cannot remove '%s': Is a directory" % f, 1)
+        raise PythonException("rm: cannot remove '%s': Is a directory" % f, 1)
       else:
         shutil.rmtree(f, force)
     elif os.path.exists(f):
@@ -58,9 +58,9 @@ def rm(args):
         os.unlink(f)
       except:
         if not force:
-          raise PythonException, ("rm: failed to remove '%s': %s" % (f, sys.exc_info()[0]), 1)
+          raise PythonException("rm: failed to remove '%s': %s" % (f, sys.exc_info()[0]), 1)
     elif not force:
-      raise PythonException, ("rm: cannot remove '%s': No such file or directory" % f, 1)
+      raise PythonException("rm: cannot remove '%s': No such file or directory" % f, 1)
 
 def sleep(args):
     """
@@ -70,7 +70,7 @@ def sleep(args):
     values = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400}
     for a in args:
         multiplier = 1
-        for k, v in values.iteritems():
+        for k, v in values.items():
             if a.endswith(k):
                 a = a[:-1]
                 multiplier = v
@@ -79,7 +79,7 @@ def sleep(args):
             f = float(a)
             total += f * multiplier
         except ValueError:
-            raise PythonException, ("sleep: invalid time interval '%s'" % a, 1)
+            raise PythonException("sleep: invalid time interval '%s'" % a, 1)
     time.sleep(total)
 
 def touch(args):
@@ -88,8 +88,8 @@ def touch(args):
     """
     try:
         opts, args = getopt(args, "t:")
-    except GetoptError, e:
-        raise PythonException, ("touch: %s" % e, 1)
+    except GetoptError as e:
+        raise PythonException("touch: %s" % e, 1)
     opts = dict(opts)
     times = None
     if '-t' in opts:
@@ -97,7 +97,7 @@ def touch(args):
         from time import mktime, localtime
         m = re.match('^(?P<Y>(?:\d\d)?\d\d)?(?P<M>\d\d)(?P<D>\d\d)(?P<h>\d\d)(?P<m>\d\d)(?:\.(?P<s>\d\d))?$', opts['-t'])
         if not m:
-            raise PythonException, ("touch: invalid date format '%s'" % opts['-t'], 1)
+            raise PythonException("touch: invalid date format '%s'" % opts['-t'], 1)
         def normalized_field(m, f):
             if f == 'Y':
                 if m.group(f) is None:

build/pymake/pymake/command.py

@@ -11,7 +11,7 @@ except when a submake specifies -j1 when the parent make is building in parallel
 
 import os, subprocess, sys, logging, time, traceback, re
 from optparse import OptionParser
-import data, parserdata, process, util
+from . import data, parserdata, process, util
 
 # TODO: If this ever goes from relocatable package to system-installed, this may need to be
 # a configured-in path.
@@ -62,7 +62,7 @@ def parsemakeflags(env):
     return opts
 
 def _version(*args):
-    print """pymake: GNU-compatible make program
+    print("""pymake: GNU-compatible make program
 Copyright (C) 2009 The Mozilla Foundation <http://www.mozilla.org/>
 This is free software; see the source for copying conditions.
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
@@ -71,7 +71,7 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE."""
+DEALINGS IN THE SOFTWARE.""")
 
 _log = logging.getLogger('pymake.execution')
 
@@ -95,7 +95,7 @@ class _MakeContext(object):
 
     def remakecb(self, remade, error=None):
         if error is not None:
-            print error
+            print(error)
             self.context.defer(self.cb, 2)
             return
 
@@ -124,15 +124,15 @@ class _MakeContext(object):
                     self.makefile.include(f)
                 self.makefile.finishparsing()
                 self.makefile.remakemakefiles(self.remakecb)
-            except util.MakeError, e:
-                print e
+            except util.MakeError as e:
+                print(e)
                 self.context.defer(self.cb, 2)
 
             return
 
         if len(self.targets) == 0:
             if self.makefile.defaulttarget is None:
-                print "No target specified and no default target found."
+                print("No target specified and no default target found.")
                 self.context.defer(self.cb, 2)
                 return
 
@@ -154,7 +154,7 @@ class _MakeContext(object):
 
         if not len(self.realtargets):
             if self.options.printdir:
-                print "make.py[%i]: Leaving directory '%s'" % (self.makelevel, self.workdir)
+                print("make.py[%i]: Leaving directory '%s'" % (self.makelevel, self.workdir))
             sys.stdout.flush()
 
             self.context.defer(self.cb, 0)
@@ -255,24 +255,24 @@ def main(args, env, cwd, cb):
         context = process.getcontext(options.jobcount)
 
         if options.printdir:
-            print "make.py[%i]: Entering directory '%s'" % (makelevel, workdir)
+            print("make.py[%i]: Entering directory '%s'" % (makelevel, workdir))
             sys.stdout.flush()
 
         if len(options.makefiles) == 0:
             if os.path.exists(util.normaljoin(workdir, 'Makefile')):
                 options.makefiles.append('Makefile')
             else:
-                print "No makefile found"
+                print("No makefile found")
                 cb(2)
                 return
 
         ostmts, targets, overrides = parserdata.parsecommandlineargs(arguments)
 
         _MakeContext(makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb)
-    except (util.MakeError), e:
-        print e
+    except (util.MakeError) as e:
+        print(e)
         if options.printdir:
-            print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir)
+            print("make.py[%i]: Leaving directory '%s'" % (makelevel, workdir))
         sys.stdout.flush()
         cb(2)
         return

build/pymake/pymake/data.py

@@ -3,8 +3,9 @@ A representation of makefile data structures.
 """
 
 import logging, re, os, sys
-import parserdata, parser, functions, process, util, implicit
-from cStringIO import StringIO
+from . import parserdata, parser, functions, process, util, implicit
+from io import StringIO
+from functools import reduce
 
 if sys.version_info[0] < 3:
     str_type = basestring
@@ -405,7 +406,7 @@ class Expansion(BaseExpansion, list):
         if len(a) != len(b):
             return False
 
-        for i in xrange(len(self)):
+        for i in range(len(self)):
             e1, is_func1 = a[i]
             e2, is_func2 = b[i]
 
@@ -447,7 +448,7 @@ class Variables(object):
         self.parent = parent
 
     def readfromenvironment(self, env):
-        for k, v in env.iteritems():
+        for k, v in env.items():
             self.set(k, self.FLAVOR_RECURSIVE, self.SOURCE_ENVIRONMENT, v)
 
     def get(self, name, expand=True):
@@ -547,7 +548,7 @@ class Variables(object):
             self.set(k, flavor, source, value)
 
     def __iter__(self):
-        for k, (flavor, source, value, valueexp) in self._map.iteritems():
+        for k, (flavor, source, value, valueexp) in self._map.items():
             yield k, flavor, source, value
 
     def __contains__(self, item):
@@ -864,7 +865,7 @@ class RemakeRuleContext(object):
         assert error in (True, False)
 
         if error:
-            print "<%s>: Found error" % self.target.target
+            print("<%s>: Found error" % self.target.target)
             self.error = True
         if didanything:
             self.didanything = True
@@ -949,8 +950,8 @@ class RemakeRuleContext(object):
             self.target.didanything = True
             try:
                 self.commands = [c for c in self.rule.getcommands(self.target, self.makefile)]
-            except util.MakeError, e:
-                print e
+            except util.MakeError as e:
+                print(e)
                 sys.stdout.flush()
                 cb(error=True)
                 return
@@ -1283,9 +1284,9 @@ class Target(object):
 
         try:
             self.resolvedeps(makefile, targetstack, [], False)
-        except util.MakeError, e:
+        except util.MakeError as e:
             if printerror:
-                print e
+                print(e)
             self.error = True
             self.notifydone(makefile)
             return
@@ -1397,7 +1398,7 @@ class _CommandWrapper(object):
 
     def _cb(self, res):
         if res != 0 and not self.ignoreErrors:
-            print "%s: command '%s' failed, return code %i" % (self.loc, self.cline, res)
+            print("%s: command '%s' failed, return code %i" % (self.loc, self.cline, res))
             self.usercb(error=True)
         else:
             self.usercb(error=False)
@@ -1591,7 +1592,7 @@ class _RemakeContext(object):
         assert error in (True, False)
 
         if error and self.required:
-            print "Error remaking makefiles (ignored)"
+            print("Error remaking makefiles (ignored)")
 
         if len(self.toremake):
             target, self.required = self.toremake.pop(0)
@@ -1678,7 +1679,7 @@ class Makefile(object):
         self.variables.set('MAKECMDGOALS', Variables.FLAVOR_SIMPLE,
                            Variables.SOURCE_AUTOMATIC, ' '.join(targets))
 
-        for vname, val in implicit.variables.iteritems():
+        for vname, val in implicit.variables.items():
             self.variables.set(vname,
                                Variables.FLAVOR_SIMPLE,
                                Variables.SOURCE_IMPLICIT, val)
@@ -1750,11 +1751,10 @@ class Makefile(object):
         if value is None:
             self._vpath = []
         else:
-            self._vpath = filter(lambda e: e != '',
-                                 re.split('[%s\s]+' % os.pathsep,
-                                          value.resolvestr(self, self.variables, ['VPATH'])))
+            self._vpath = [e for e in re.split('[%s\s]+' % os.pathsep,
+                                          value.resolvestr(self, self.variables, ['VPATH'])) if e != '']
 
-        targets = list(self._targets.itervalues())
+        targets = list(self._targets.values())
         for t in targets:
             t.explicit = True
             for r in t.rules:
@@ -1825,7 +1825,7 @@ class Makefile(object):
 
     def getsubenvironment(self, variables):
         env = dict(self.env)
-        for vname, v in self.exportedvars.iteritems():
+        for vname, v in self.exportedvars.items():
             if v:
                 flavor, source, val = variables.get(vname)
                 if val is None:

build/pymake/pymake/functions.py

@@ -2,10 +2,10 @@
 Makefile functions.
 """
 
-import parser, util
+from . import parser, util
 import subprocess, os, logging, sys
-from globrelative import glob
-from cStringIO import StringIO
+from .globrelative import glob
+from io import StringIO
 
 log = logging.getLogger('pymake.data')
 
@@ -135,7 +135,7 @@ class Function(object):
         if len(self._arguments) != len(other._arguments):
             return False
 
-        for i in xrange(len(self._arguments)):
+        for i in range(len(self._arguments)):
             # According to the GNU make manual Section 8.1, whitespace around
             # arguments is *not* part of the argument's value. So, we do a
             # whitespace-agnostic comparison.
@@ -525,7 +525,7 @@ class JoinFunction(Function):
 
     @staticmethod
     def iterjoin(l1, l2):
-        for i in xrange(0, max(len(l1), len(l2))):
+        for i in range(0, max(len(l1), len(l2))):
             i1 = i < len(l1) and l1[i] or ''
             i2 = i < len(l2) and l2[i] or ''
             yield i1 + i2
@@ -670,7 +670,7 @@ class CallFunction(Function):
 
         v = data.Variables(parent=variables)
         v.set('0', data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, vname)
-        for i in xrange(1, len(self._arguments)):
+        for i in range(1, len(self._arguments)):
             param = self._arguments[i].resolvestr(makefile, variables, setting)
             v.set(str(i), data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, param)
 
@@ -770,7 +770,7 @@ class ShellFunction(Function):
     __slots__ = Function.__slots__
 
     def resolve(self, makefile, variables, fd, setting):
-        from process import prepare_command
+        from .process import prepare_command
         cline = self._arguments[0].resolvestr(makefile, variables, setting)
         executable, cline = prepare_command(cline, makefile.workdir, self.loc)
 
@@ -785,8 +785,8 @@ class ShellFunction(Function):
         try:
             p = subprocess.Popen(cline, executable=executable, env=makefile.env, shell=False,
                                  stdout=subprocess.PIPE, cwd=makefile.workdir)
-        except OSError, e:
-            print >>sys.stderr, "Error executing command %s" % cline[0], e
+        except OSError as e:
+            print("Error executing command %s" % cline[0], e, file=sys.stderr)
             return
         finally:
             os.environ['PATH'] = oldpath
@@ -830,7 +830,7 @@ class InfoFunction(Function):
 
     def resolve(self, makefile, variables, fd, setting):
         v = self._arguments[0].resolvestr(makefile, variables, setting)
-        print v
+        print(v)
 
 functionmap = {
     'subst': SubstFunction,
@@ -870,4 +870,4 @@ functionmap = {
     'info': InfoFunction,
 }
 
-import data
+from . import data

build/pymake/pymake/globrelative.py

@@ -7,7 +7,7 @@ Filename globbing like the python glob module with minor differences:
 """
 
 import os, re, fnmatch
-import util
+from . import util
 
 _globcheck = re.compile('[[*?]')
 
@@ -62,7 +62,7 @@ def globpattern(dir, pattern):
                   if not leaf.startswith('.')]
 
     leaves = fnmatch.filter(leaves, pattern)
-    leaves = filter(lambda l: os.path.exists(util.normaljoin(dir, l)), leaves)
+    leaves = [l for l in leaves if os.path.exists(util.normaljoin(dir, l))]
 
     leaves.sort()
     return leaves

build/pymake/pymake/parser.py

@@ -34,7 +34,7 @@ coming.
 """
 
 import logging, re, os, sys
-import data, functions, util, parserdata
+from . import data, functions, util, parserdata
 
 _log = logging.getLogger('pymake.parser')
 
@@ -108,7 +108,7 @@ _alltokens = re.compile(r'''\\*\# | # hash mark preceeded by any number of backs
                             :: |
                             (?:\$(?:$|[\(\{](?:%s)\s+|.)) | # dollar sign followed by EOF, a function keyword with whitespace, or any character
                             :(?![\\/]) | # colon followed by anything except a slash (Windows path detection)
-                            [=#{}();,|'"]''' % '|'.join(functions.functionmap.iterkeys()), re.VERBOSE)
+                            [=#{}();,|'"]''' % '|'.join(iter(functions.functionmap.keys())), re.VERBOSE)
 
 def iterdata(d, offset, tokenlist, it):
     """
@@ -336,7 +336,7 @@ _conditionkeywords = {
     'ifndef': ifndef
     }
 
-_conditiontokens = tuple(_conditionkeywords.iterkeys())
+_conditiontokens = tuple(_conditionkeywords.keys())
 _conditionre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_conditiontokens))
 
 _directivestokenlist = _conditiontokens + \
@@ -691,7 +691,7 @@ def parsemakesyntax(d, offset, stopon, iterfunc):
     while True: # this is not a for loop because `di` changes during the function
         assert stacktop is not None
         try:
-            s, token, tokenoffset, offset = di.next()
+            s, token, tokenoffset, offset = next(di)
         except StopIteration:
             break
 

build/pymake/pymake/parserdata.py

@@ -1,6 +1,6 @@
 import logging, re, os
-import data, parser, functions, util
-from cStringIO import StringIO
+from . import data, parser, functions, util
+from io import StringIO
 from pymake.globrelative import hasglob, glob
 
 _log = logging.getLogger('pymake.data')
@@ -76,7 +76,7 @@ def parsecommandlineargs(args):
     overrides = []
     stmts = StatementList()
     r = []
-    for i in xrange(0, len(args)):
+    for i in range(0, len(args)):
         a = args[i]
 
         vname, t, val = util.strpartition(a, ':=')
@@ -195,7 +195,7 @@ class Rule(Statement):
 
         deps = list(_expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables))))
         if ispattern:
-            rule = data.PatternRule(targets, map(data.Pattern, deps), self.doublecolon, loc=self.targetexp.loc)
+            rule = data.PatternRule(targets, list(map(data.Pattern, deps)), self.doublecolon, loc=self.targetexp.loc)
             makefile.appendimplicitrule(rule)
         else:
             rule = data.Rule(deps, self.doublecolon, loc=self.targetexp.loc, weakdeps=False)
@@ -207,7 +207,7 @@ class Rule(Statement):
         context.currule = rule
 
     def dump(self, fd, indent):
-        print >>fd, "%sRule %s: %s" % (indent, self.targetexp, self.depexp)
+        print("%sRule %s: %s" % (indent, self.targetexp, self.depexp), file=fd)
 
     def to_source(self):
         sep = ':'
@@ -285,7 +285,7 @@ class StaticPatternRule(Statement):
         context.currule = rule
 
     def dump(self, fd, indent):
-        print >>fd, "%sStaticPatternRule %s: %s: %s" % (indent, self.targetexp, self.patternexp, self.depexp)
+        print("%sStaticPatternRule %s: %s: %s" % (indent, self.targetexp, self.patternexp, self.depexp), file=fd)
 
     def to_source(self):
         sep = ':'
@@ -339,7 +339,7 @@ class Command(Statement):
         context.currule.addcommand(self.exp)
 
     def dump(self, fd, indent):
-        print >>fd, "%sCommand %s" % (indent, self.exp,)
+        print("%sCommand %s" % (indent, self.exp,), file=fd)
 
     def to_source(self):
         # Commands have some interesting quirks when it comes to source
@@ -438,7 +438,7 @@ class SetVariable(Statement):
             v.set(vname, flavor, self.source, value)
 
     def dump(self, fd, indent):
-        print >>fd, "%sSetVariable<%s> %s %s\n%s %r" % (indent, self.valueloc, self.vnameexp, self.token, indent, self.value)
+        print("%sSetVariable<%s> %s %s\n%s %r" % (indent, self.valueloc, self.vnameexp, self.token, indent, self.value), file=fd)
 
     def __eq__(self, other):
         if not isinstance(other, SetVariable):
@@ -452,7 +452,7 @@ class SetVariable(Statement):
 
     def to_source(self):
         chars = []
-        for i in xrange(0, len(self.value)):
+        for i in range(0, len(self.value)):
             c = self.value[i]
 
             # Literal # is escaped in variable assignment otherwise it would be
@@ -651,14 +651,14 @@ class ConditionBlock(Statement):
             i += 1
 
     def dump(self, fd, indent):
-        print >>fd, "%sConditionBlock" % (indent,)
+        print("%sConditionBlock" % (indent,), file=fd)
 
         indent2 = indent + '  '
         for c, statements in self._groups:
-            print >>fd, "%s Condition %s" % (indent, c)
+            print("%s Condition %s" % (indent, c), file=fd)
             statements.dump(fd, indent2)
-            print >>fd, "%s ~Condition" % (indent,)
-        print >>fd, "%s~ConditionBlock" % (indent,)
+            print("%s ~Condition" % (indent,), file=fd)
+        print("%s~ConditionBlock" % (indent,), file=fd)
 
     def to_source(self):
         lines = []
@@ -681,7 +681,7 @@ class ConditionBlock(Statement):
         if len(self) != len(other):
             return False
 
-        for i in xrange(0, len(self)):
+        for i in range(0, len(self)):
             our_condition, our_statements = self[i]
             other_condition, other_statements = other[i]
 
@@ -793,7 +793,7 @@ class Include(Statement):
             makefile.include(f, self.required, loc=self.exp.loc, weak=self.weak)
 
     def dump(self, fd, indent):
-        print >>fd, "%sInclude %s" % (indent, self.exp)
+        print("%sInclude %s" % (indent, self.exp), file=fd)
 
     def to_source(self):
         prefix = ''
@@ -840,7 +840,7 @@ class VPathDirective(Statement):
                     makefile.addvpath(pattern, dirs)
 
     def dump(self, fd, indent):
-        print >>fd, "%sVPath %s" % (indent, self.exp)
+        print("%sVPath %s" % (indent, self.exp), file=fd)
 
     def to_source(self):
         return 'vpath %s' % self.exp.to_source()
@@ -885,7 +885,7 @@ class ExportDirective(Statement):
             makefile.exportedvars[v] = True
 
     def dump(self, fd, indent):
-        print >>fd, "%sExport (single=%s) %s" % (indent, self.single, self.exp)
+        print("%sExport (single=%s) %s" % (indent, self.single, self.exp), file=fd)
 
     def to_source(self):
         return ('export %s' % self.exp.to_source()).rstrip()
@@ -915,7 +915,7 @@ class UnexportDirective(Statement):
             makefile.exportedvars[v] = False
 
     def dump(self, fd, indent):
-        print >>fd, "%sUnexport %s" % (indent, self.exp)
+        print("%sUnexport %s" % (indent, self.exp), file=fd)
 
     def to_source(self):
         return 'unexport %s' % self.exp.to_source()
@@ -947,7 +947,7 @@ class EmptyDirective(Statement):
             raise data.DataError("Line expands to non-empty value", self.exp.loc)
 
     def dump(self, fd, indent):
-        print >>fd, "%sEmptyDirective: %s" % (indent, self.exp)
+        print("%sEmptyDirective: %s" % (indent, self.exp), file=fd)
 
     def to_source(self):
         return self.exp.to_source()

build/pymake/pymake/process.py

@@ -10,9 +10,9 @@ import site
 from collections import deque
 # XXXkhuey Work around http://bugs.python.org/issue1731717
 subprocess._cleanup = lambda: None
-import command, util
+from . import command, util
 if sys.platform=='win32':
-    import win32process
+    from . import win32process
 
 _log = logging.getLogger('pymake.process')
 
@@ -28,7 +28,7 @@ def tokens2re(tokens):
     # which matches the pattern and captures it in a named match group.
     # The group names and patterns come are given as a dict in the function
     # argument.
-    nonescaped = r'(?<!\\)(?:%s)' % '|'.join('(?P<%s>%s)' % (name, value) for name, value in tokens.iteritems())
+    nonescaped = r'(?<!\\)(?:%s)' % '|'.join('(?P<%s>%s)' % (name, value) for name, value in tokens.items())
     # The final pattern matches either the above pattern, or an escaped
     # backslash, captured in the "escape" match group.
     return re.compile('(?:%s|%s)' % (nonescaped, r'(?P<escape>\\\\)'))
@@ -115,7 +115,7 @@ class ClineSplitter(list):
                 self._push(self.cline[:m.start()])
             self.cline = self.cline[m.end():]
 
-            match = dict([(name, value) for name, value in m.groupdict().items() if value])
+            match = dict([(name, value) for name, value in list(m.groupdict().items()) if value])
             if 'quote' in match:
                 # " or ' start a quoted string
                 if match['quote'] == '"':
@@ -129,7 +129,7 @@ class ClineSplitter(list):
             elif 'special' in match:
                 # Unquoted, non-escaped special characters need to be sent to a
                 # shell.
-                raise MetaCharacterException, match['special']
+                raise MetaCharacterException(match['special'])
             elif 'whitespace' in match:
                 # Whitespaces terminate current argument.
                 self._next()
@@ -145,7 +145,7 @@ class ClineSplitter(list):
                 self.glob = True
                 self._push(m.group(0))
             else:
-                raise Exception, "Shouldn't reach here"
+                raise Exception("Shouldn't reach here")
         if self.arg:
             self._next()
 
@@ -153,20 +153,20 @@ class ClineSplitter(list):
         # Single quoted strings are preserved, except for the final quote
         index = self.cline.find("'")
         if index == -1:
-            raise Exception, 'Unterminated quoted string in command'
+            raise Exception('Unterminated quoted string in command')
         self._push(self.cline[:index])
         self.cline = self.cline[index+1:]
 
     def _parse_doubly_quoted(self):
         if not self.cline:
-            raise Exception, 'Unterminated quoted string in command'
+            raise Exception('Unterminated quoted string in command')
         while self.cline:
             m = _doubly_quoted_tokens.search(self.cline)
             if not m:
-                raise Exception, 'Unterminated quoted string in command'
+                raise Exception('Unterminated quoted string in command')
             self._push(self.cline[:m.start()])
             self.cline = self.cline[m.end():]
-            match = dict([(name, value) for name, value in m.groupdict().items() if value])
+            match = dict([(name, value) for name, value in list(m.groupdict().items()) if value])
             if 'quote' in match:
                 # a double quote ends the quoted string, so go back to
                 # unquoted parsing
@@ -175,7 +175,7 @@ class ClineSplitter(list):
                 # Unquoted, non-escaped special characters in a doubly quoted
                 # string still have a special meaning and need to be sent to a
                 # shell.
-                raise MetaCharacterException, match['special']
+                raise MetaCharacterException(match['special'])
             elif 'escape' in match:
                 # Escaped backslashes turn into a single backslash
                 self._push('\\')
@@ -194,7 +194,7 @@ def clinetoargv(cline, cwd):
     str = _escapednewlines.sub('', cline)
     try:
         args = ClineSplitter(str, cwd)
-    except MetaCharacterException, e:
+    except MetaCharacterException as e:
         return None, e.char
 
     if len(args) and args[0].find('=') != -1:
@@ -327,12 +327,12 @@ class PopenJob(Job):
         # general overview of "subprocess PATH semantics and portability".
         oldpath = os.environ['PATH']
         try:
-            if self.env is not None and self.env.has_key('PATH'):
+            if self.env is not None and 'PATH' in self.env:
                 os.environ['PATH'] = self.env['PATH']
             p = subprocess.Popen(self.argv, executable=self.executable, shell=self.shell, env=self.env, cwd=self.cwd)
             return p.wait()
-        except OSError, e:
-            print >>sys.stderr, e
+        except OSError as e:
+            print(e, file=sys.stderr)
             return -127
         finally:
             os.environ['PATH'] = oldpath
@@ -383,30 +383,30 @@ class PythonJob(Job):
                 try:
                     __import__(self.module)
                 except Exception as e:
-                    print >>sys.stderr, 'Error importing %s: %s' % (
-                        self.module, e)
+                    print('Error importing %s: %s' % (
+                        self.module, e), file=sys.stderr)
                     return -127
 
             m = sys.modules[self.module]
             if self.method not in m.__dict__:
-                print >>sys.stderr, "No method named '%s' in module %s" % (self.method, self.module)
+                print("No method named '%s' in module %s" % (self.method, self.module), file=sys.stderr)
                 return -127
             rv = m.__dict__[self.method](self.argv)
             if rv != 0 and rv is not None:
-                print >>sys.stderr, (
+                print((
                     "Native command '%s %s' returned value '%s'" %
-                    (self.module, self.method, rv))
+                    (self.module, self.method, rv)), file=sys.stderr)
                 return (rv if isinstance(rv, int) else 1)
 
-        except PythonException, e:
-            print >>sys.stderr, e
+        except PythonException as e:
+            print(e, file=sys.stderr)
             return e.exitcode
         except:
             e = sys.exc_info()[1]
             if isinstance(e, SystemExit) and (e.code == 0 or e.code is None):
                 pass # sys.exit(0) is not a failure
             else:
-                print >>sys.stderr, e
+                print(e, file=sys.stderr)
                 traceback.print_exc()
                 return -127
         finally:
@@ -461,7 +461,7 @@ class ParallelContext(object):
 
     def _docall_generic(self, pool, job, cb, echo, justprint):
         if echo is not None:
-            print echo
+            print(echo)
         processcb = job.get_callback(ParallelContext._condition)
         if justprint:
             processcb(0)
@@ -491,7 +491,7 @@ class ParallelContext(object):
         def _checkdone():
             jobs = []
             for c in ParallelContext._allcontexts:
-                for i in xrange(0, len(c.running)):
+                for i in range(0, len(c.running)):
                     if c.running[i][0].done:
                         jobs.append(c.running[i])
                 for j in jobs:

build/pymake/pymake/util.py

@@ -82,7 +82,7 @@ else:
         return str[:offset], token, str[offset + len(token):]
 
 try:
-    from __builtin__ import any
+    from builtins import any
 except ImportError:
     def any(it):
         for i in it:
@@ -138,7 +138,7 @@ class MostUsedCache(object):
         return item.o
 
     def verify(self):
-        for k, v in self.d.iteritems():
+        for k, v in self.d.items():
             if v.o:
                 assert v in self.active
             else:

build/pymake/tests/datatests.py

@@ -1,10 +1,10 @@
 import pymake.data, pymake.functions, pymake.util
 import unittest
 import re
-from cStringIO import StringIO
+from io import StringIO
 
 def multitest(cls):
-    for name in cls.testdata.iterkeys():
+    for name in cls.testdata.keys():
         def m(self, name=name):
             return self.runSingle(*self.testdata[name])
 
@@ -63,7 +63,7 @@ class LRUTest(unittest.TestCase):
         c = pymake.util.LRUCache(3, self.spaceFunc, lambda k, v: k % 2)
         self.assertEqual(tuple(c.debugitems()), ())
 
-        for i in xrange(0, len(self.expected)):
+        for i in range(0, len(self.expected)):
             k, e, fc, di = self.expected[i]
 
             v = c.get(k)

build/pymake/tests/formattingtests.py

@@ -277,7 +277,7 @@ class MakefileCorupusTest(TestBase):
 
             self.assertEqual(len(statements), len(new_statements))
 
-            for i in xrange(0, len(statements)):
+            for i in range(0, len(statements)):
                 original = statements[i]
                 formatted = new_statements[i]
 

build/pymake/tests/parsertests.py

@@ -2,10 +2,10 @@ import pymake.data, pymake.parser, pymake.parserdata, pymake.functions
 import unittest
 import logging
 
-from cStringIO import StringIO
+from io import StringIO
 
 def multitest(cls):
-    for name in cls.testdata.iterkeys():
+    for name in cls.testdata.keys():
         def m(self, name=name):
             return self.runSingle(*self.testdata[name])
 
@@ -134,7 +134,7 @@ class IterTest(TestBase):
         self.assertEqual(actual, expected)
 
         if ifunc == pymake.parser.itermakefilechars:
-            print "testing %r" % expected
+            print("testing %r" % expected)
             self.assertEqual(pymake.parser.flattenmakesyntax(d, 0), expected)
 
 multitest(IterTest)
@@ -207,7 +207,7 @@ class MakeSyntaxTest(TestBase):
     def compareRecursive(self, actual, expected, path):
         self.assertEqual(len(actual), len(expected),
                          "compareRecursive: %s %r" % (path, actual))
-        for i in xrange(0, len(actual)):
+        for i in range(0, len(actual)):
             ipath = path + [i]
 
             a, isfunc = actual[i]
@@ -217,7 +217,7 @@ class MakeSyntaxTest(TestBase):
             else:
                 self.assertEqual(type(a), getattr(pymake.functions, e['type']),
                                  "compareRecursive: %s" % (ipath,))
-                for k, v in e.iteritems():
+                for k, v in e.items():
                     if k == 'type':
                         pass
                     elif k[0] == '[':
@@ -264,7 +264,7 @@ class VariableTest(TestBase):
 
         m = pymake.data.Makefile()
         stmts.execute(m)
-        for k, v in self.expected.iteritems():
+        for k, v in self.expected.items():
             flavor, source, val = m.variables.get(k)
             if val is None:
                 self.assertEqual(val, v, 'variable named %s' % k)

build/pymake/tests/runtests.py

@@ -63,7 +63,7 @@ def runTest(makefile, make, logfile, options):
     """
 
     if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir)
-    os.mkdir(opts.tempdir, 0755)
+    os.mkdir(opts.tempdir, 0o755)
 
     logfd = open(logfile, 'w')
     p = Popen(make + options['commandline'], stdout=logfd, stderr=STDOUT, env=options['env'])
@@ -78,15 +78,15 @@ def runTest(makefile, make, logfile, options):
     logfd.close()
 
     if stdout.find('TEST-FAIL') != -1:
-        print stdout
+        print(stdout)
         return False, "FAIL (TEST-FAIL printed)"
 
     if options['grepfor'] and stdout.find(options['grepfor']) == -1:
-        print stdout
+        print(stdout)
         return False, "FAIL (%s not in output)" % options['grepfor']
 
     if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1:
-        print stdout
+        print(stdout)
         return False, 'FAIL (No TEST-PASS printed)'
 
     if options['returncode'] != 0:
@@ -94,7 +94,7 @@ def runTest(makefile, make, logfile, options):
 
     return True, 'PASS'
 
-print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:")
+print("%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:"))
 
 gmakefails = 0
 pymakefails = 0
@@ -144,7 +144,7 @@ for makefile in makefiles:
             if sys.platform in data:
                 d['returncode'] = data[sys.platform]
         elif key == 'environment':
-            for k, v in data.iteritems():
+            for k, v in data.items():
                 d['env'][k] = v
         elif key == 'grep-for':
             d['grepfor'] = data
@@ -153,7 +153,7 @@ for makefile in makefiles:
         elif key == 'skip':
             d['skip'] = True
         else:
-            print >>sys.stderr, "%s: Unexpected #T key: %s" % (makefile, key)
+            print("%s: Unexpected #T key: %s" % (makefile, key), file=sys.stderr)
             sys.exit(1)
 
     mdata.close()
@@ -190,12 +190,12 @@ for makefile in makefiles:
         else:
             pymakemsg = "OK (known fail)"
 
-    print "%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile),
-                                        gmakemsg, pymakemsg)
+    print("%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile),
+                                        gmakemsg, pymakemsg))
 
-print
-print "Summary:"
-print "%-30s%-28s%-28s" % ("", "gmake:", "pymake:")
+print()
+print("Summary:")
+print("%-30s%-28s%-28s" % ("", "gmake:", "pymake:"))
 
 if gmakefails == 0:
     gmakemsg = 'PASS'
@@ -207,7 +207,7 @@ if pymakefails == 0:
 else:
     pymakemsg = 'FAIL (%i failures)' % pymakefails
 
-print "%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg)
+print("%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg))
 
 shutil.rmtree(opts.tempdir)
 

build/release/info.py

@@ -4,7 +4,7 @@ from os import path
 import re
 import shutil
 import sys
-from urllib2 import urlopen
+from urllib.request import urlopen
 
 from release.paths import makeCandidatesDir
 
@@ -50,7 +50,7 @@ def findOldBuildIDs(product, version, buildNumber, platforms,
                 id = getBuildID(platform, product, version, n, nightlyDir,
                                 server)
                 ids[platform].append(id)
-            except Exception, e:
+            except Exception as e:
                 log.error("Hit exception: %s" % e)
     return ids
 
@@ -87,10 +87,10 @@ def readBranchConfig(dir, localconfig, branch, required=[]):
 
 def readConfig(configfile, keys=[], required=[]):
     c = {}
-    execfile(configfile, c)
+    exec(compile(open(configfile, "rb").read(), configfile, 'exec'), c)
     for k in keys:
         c = c[k]
-    items = c.keys()
+    items = list(c.keys())
     err = False
     for key in required:
         if key not in items:
@@ -137,7 +137,7 @@ def getReleaseName(product, version, buildNumber):
 
 
 def getRepoMatchingBranch(branch, sourceRepositories):
-    for sr in sourceRepositories.values():
+    for sr in list(sourceRepositories.values()):
         if branch in sr['path']:
             return sr
     return None

build/release/sanity.py

@@ -1,7 +1,7 @@
 import difflib
 import logging
 import re
-import urllib2
+import urllib.request, urllib.error, urllib.parse
 from util.commands import run_cmd, get_output
 from util.hg import get_repo_name, make_hg_url
 from subprocess import CalledProcessError

build/subconfigure.py

@@ -13,9 +13,11 @@ import re
 import subprocess
 import sys
 import pickle
+from locale import getpreferredencoding
 
 import mozpack.path as mozpath
 
+encoding = getpreferredencoding(False)
 
 class Pool(object):
     def __new__(cls, size):
@@ -27,7 +29,7 @@ class Pool(object):
             return super(Pool, cls).__new__(cls)
 
     def imap_unordered(self, fn, iterable):
-        return itertools.imap(fn, iterable)
+        return map(fn, iterable)
 
     def close(self):
         pass
@@ -60,7 +62,7 @@ class File(object):
 
         modified = True
         if os.path.exists(self._path):
-            if open(self._path, 'rb').read() == self._content:
+            if open(self._path, 'r', encoding=encoding, errors='replace').read() == self._content:
                 modified = False
         self._modified = modified
         return modified
@@ -131,11 +133,11 @@ def maybe_clear_cache(data):
         is_set = cache.get('ac_cv_env_%s_set' % precious) == 'set'
         value = cache.get('ac_cv_env_%s_value' % precious) if is_set else None
         if value != env.get(precious):
-            print 'Removing %s because of %s value change from:' \
-                % (data['cache-file'], precious)
-            print '  %s' % (value if value is not None else 'undefined')
-            print 'to:'
-            print '  %s' % env.get(precious, 'undefined')
+            print('Removing %s because of %s value change from:' \
+                % (data['cache-file'], precious))
+            print('  %s' % (value if value is not None else 'undefined'))
+            print('to:')
+            print('  %s' % env.get(precious, 'undefined'))
             os.remove(data['cache-file'])
             return True
     return False
@@ -160,9 +162,11 @@ def get_config_files(data):
 
     # Scan the config.status output for information about configuration files
     # it generates.
-    config_status_output = subprocess.check_output(
+    raw_output = subprocess.check_output(
         [data['shell'], '-c', '%s --help' % config_status],
-        stderr=subprocess.STDOUT).splitlines()
+        stderr=subprocess.STDOUT)
+    config_status_output = raw_output.decode(encoding,
+                                             errors='replace').splitlines()
     state = None
     for line in config_status_output:
         if line.startswith('Configuration') and line.endswith(':'):
@@ -183,7 +187,6 @@ def get_config_files(data):
 
     return config_files, command_files
 
-
 def prepare(srcdir, objdir, shell, args):
     parser = argparse.ArgumentParser()
     parser.add_argument('--target', type=str)
@@ -232,7 +235,7 @@ def prepare(srcdir, objdir, shell, args):
         'args': others,
         'shell': shell,
         'srcdir': srcdir,
-        'env': environ,
+        'env': {str(k): str(v) for k, v in environ.items()},
     }
 
     if args.cache_file:
@@ -249,9 +252,9 @@ def prepare(srcdir, objdir, shell, args):
     except OSError as e:
         if e.errno != errno.EEXIST:
             raise
-
-    with open(data_file, 'wb') as f:
-        pickle.dump(data, f)
+    for k, v in data.items():
+        with open(data_file, 'wb') as f:
+            pickle.dump(data, f)
 
 
 def prefix_lines(text, prefix):
@@ -261,7 +264,6 @@ def prefix_lines(text, prefix):
 def run(objdir):
     ret = 0
     output = ''
-
     with open(os.path.join(objdir, CONFIGURE_DATA), 'rb') as f:
         data = pickle.load(f)
 
@@ -333,14 +335,16 @@ def run(objdir):
         # We're going to run it ourselves.
         command += ['--no-create']
 
-        print prefix_lines('configuring', relobjdir)
-        print prefix_lines('running %s' % ' '.join(command[:-1]), relobjdir)
+        print(prefix_lines('configuring', relobjdir))
+        print(prefix_lines('running %s' % ' '.join(command[:-1]), relobjdir))
         sys.stdout.flush()
         try:
-            output += subprocess.check_output(command,
+            raw_output = subprocess.check_output(command,
                 stderr=subprocess.STDOUT, cwd=objdir, env=data['env'])
+            output += raw_output.decode(encoding, errors='replace')
         except subprocess.CalledProcessError as e:
-            return relobjdir, e.returncode, e.output
+            err = e.output.decode(encoding, errors='replace')
+            return relobjdir, e.returncode, err
 
         # Leave config.status with a new timestamp if configure is newer than
         # its original mtime.
@@ -368,12 +372,13 @@ def run(objdir):
 
     if not skip_config_status:
         if skip_configure:
-            print prefix_lines('running config.status', relobjdir)
+            print(prefix_lines('running config.status', relobjdir))
             sys.stdout.flush()
         try:
-            output += subprocess.check_output([data['shell'], '-c',
+            raw_output = subprocess.check_output([data['shell'], '-c',
                 './config.status'], stderr=subprocess.STDOUT, cwd=objdir,
                 env=data['env'])
+            output += raw_output.decode(encoding, errors='replace')
         except subprocess.CalledProcessError as e:
             ret = e.returncode
             output += e.output
@@ -395,9 +400,9 @@ def subconfigure(args):
     args, others = parser.parse_known_args(args)
     subconfigures = args.subconfigures
     if args.list:
-        subconfigures.extend(open(args.list, 'rb').read().splitlines())
+        subconfigures.extend(open(args.list, 'r', encoding=encoding, errors='replace').read().splitlines())
     if args.skip:
-        skips = set(open(args.skip, 'rb').read().splitlines())
+        skips = set(open(args.skip, 'r', encoding=encoding, errors='replace').read().splitlines())
         subconfigures = [s for s in subconfigures if s not in skips]
 
     if not subconfigures:
@@ -410,7 +415,7 @@ def subconfigure(args):
     pool = Pool(len(subconfigures))
     for relobjdir, returncode, output in \
             pool.imap_unordered(run, subconfigures):
-        print prefix_lines(output, relobjdir)
+        print(prefix_lines(output, relobjdir))
         sys.stdout.flush()
         ret = max(returncode, ret)
         if ret:

build/upload.py

@@ -104,10 +104,10 @@ def DoSSHCommand(command, user, host, port=None, ssh_key=None):
         try:
             output = f(cmdline, stderr=STDOUT).strip()
         except CalledProcessError as e:
-            print "failed ssh command output:"
-            print '=' * 20
-            print e.output
-            print '=' * 20
+            print("failed ssh command output:")
+            print('=' * 20)
+            print(e.output)
+            print('=' * 20)
             raise
         return output
 
@@ -118,7 +118,7 @@ def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None,
     """Upload file to user@host:remote_path using scp. Optionally use
     port and ssh_key, if provided."""
     if log:
-        print 'Uploading %s' % file
+        print('Uploading %s' % file)
     cmdline = ["scp"]
     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
     cmdline.extend([WindowsPathToMsysPath(file),
@@ -221,8 +221,8 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None,
     if not host or not user:
         return {}
     if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
-        print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
-                "defined."
+        print("One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
+                "defined.")
         sys.exit(1)
 
     if upload_to_temp_dir:
@@ -269,18 +269,18 @@ def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None,
 
         if post_upload_command is not None:
             if verbose:
-                print "Running post-upload command: " + post_upload_command
+                print("Running post-upload command: " + post_upload_command)
             file_list = '"' + '" "'.join(remote_files) + '"'
             output = DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
             # We print since mozharness may parse URLs from the output stream.
-            print output
+            print(output)
             properties = GetUrlProperties(output, package)
     finally:
         if upload_to_temp_dir:
             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
                          ssh_key=ssh_key)
     if verbose:
-        print "Upload complete"
+        print("Upload complete")
     return properties
 
 def CopyFilesLocally(path, files, verbose=False, base_path=None, package=None):
@@ -299,7 +299,7 @@ def CopyFilesLocally(path, files, verbose=False, base_path=None, package=None):
         if not os.path.exists(target_path):
             os.makedirs(target_path)
         if verbose:
-            print "Copying " + file + " to " + target_path
+            print("Copying " + file + " to " + target_path)
         shutil.copy(file, target_path)
 
 def WriteProperties(files, properties_file, url_properties, package):
@@ -341,20 +341,20 @@ if __name__ == '__main__':
                       help="Name of the main package.")
     (options, args) = parser.parse_args()
     if len(args) < 1:
-        print "You must specify at least one file to upload"
+        print("You must specify at least one file to upload")
         sys.exit(1)
     if not options.properties_file:
-        print "You must specify a --properties-file"
+        print("You must specify a --properties-file")
         sys.exit(1)
 
     if host == "localhost":
         if upload_to_temp_dir:
-            print "Cannot use UPLOAD_TO_TEMP with UPLOAD_HOST=localhost"
+            print("Cannot use UPLOAD_TO_TEMP with UPLOAD_HOST=localhost")
             sys.exit(1)
         if post_upload_command:
             # POST_UPLOAD_COMMAND is difficult to extract from the mozharness
             # scripts, so just ignore it until it's no longer used anywhere
-            print "Ignoring POST_UPLOAD_COMMAND with UPLOAD_HOST=localhost"
+            print("Ignoring POST_UPLOAD_COMMAND with UPLOAD_HOST=localhost")
 
     try:
         if host == "localhost":
@@ -370,6 +370,6 @@ if __name__ == '__main__':
                                          package=options.package, verbose=True)
 
             WriteProperties(args, options.properties_file, url_properties, options.package)
-    except IOError, (strerror):
-        print strerror
+    except IOError as strerror:
+        print(strerror)
         sys.exit(1)

build/util/count_ctors.py

@@ -36,7 +36,7 @@ def count_ctors(filename):
         # Even if we have .ctors, we shouldn't have any constructors in .ctors.
         # Complain if .ctors does not look how we expect it to.
         if have_ctors and n_ctors_ctors != 0:
-            print >>sys.stderr, "Unexpected .ctors contents for", filename
+            print("Unexpected .ctors contents for", filename, file=sys.stderr)
             sys.exit(1)
         return n_init_array_ctors
     if have_ctors:
@@ -44,7 +44,7 @@ def count_ctors(filename):
 
     # We didn't find anything; somebody switched initialization mechanisms on
     # us, or the binary is completely busted.  Complain either way.
-    print >>sys.stderr, "Couldn't find .init_array or .ctors in", filename
+    print("Couldn't find .init_array or .ctors in", filename, file=sys.stderr)
     sys.exit(1)
 
 if __name__ == '__main__':
@@ -60,5 +60,5 @@ if __name__ == '__main__':
                 }]}
             ]
         }
-        print "PERFHERDER_DATA: %s" % json.dumps(perfherder_data)
+        print("PERFHERDER_DATA: %s" % json.dumps(perfherder_data))
 

build/util/hg.py

@@ -2,8 +2,8 @@
 import os
 import re
 import subprocess
-from urlparse import urlsplit
-from ConfigParser import RawConfigParser
+from urllib.parse import urlsplit
+from configparser import RawConfigParser
 
 from util.commands import run_cmd, get_output, remove_path
 from util.retry import retry
@@ -105,7 +105,7 @@ def purge(dest):
     try:
         run_cmd(['hg', '--config', 'extensions.purge=', 'purge',
                  '-a', '--all', dest], cwd=dest)
-    except subprocess.CalledProcessError, e:
+    except subprocess.CalledProcessError as e:
         log.debug('purge failed: %s' % e)
         raise
 
@@ -306,7 +306,7 @@ def out(src, remote, **kwargs):
                     branch = "default"
                 revs.append((rev, branch))
             return revs
-        except subprocess.CalledProcessError, inst:
+        except subprocess.CalledProcessError as inst:
             # In some situations, some versions of Mercurial return "1"
             # if no changes are found, so we need to ignore this return code
             if inst.returncode == 1:
@@ -500,7 +500,7 @@ def apply_and_push(localrepo, remote, changer, max_attempts=10,
             push(src=localrepo, remote=remote, ssh_username=ssh_username,
                  ssh_key=ssh_key, force=force)
             return
-        except subprocess.CalledProcessError, e:
+        except subprocess.CalledProcessError as e:
             log.debug("Hit error when trying to push: %s" % str(e))
             if n == max_attempts:
                 log.debug("Tried %d times, giving up" % max_attempts)
@@ -515,7 +515,7 @@ def apply_and_push(localrepo, remote, changer, max_attempts=10,
             try:
                 run_cmd(['hg', '--config', 'ui.merge=internal:merge',
                          'rebase'], cwd=localrepo)
-            except subprocess.CalledProcessError, e:
+            except subprocess.CalledProcessError as e:
                 log.debug("Failed to rebase: %s" % str(e))
                 update(localrepo, branch=branch)
                 for r in reversed(new_revs):
@@ -578,7 +578,7 @@ def adjust_paths(dest, **paths):
         config.add_section('paths')
 
     changed = False
-    for path_name, path_value in paths.items():
+    for path_name, path_value in list(paths.items()):
         if (not config.has_option('paths', path_name) or
                 config.get('paths', path_name) != path_value):
             changed = True

build/virtualenv_packages.txt

@@ -11,7 +11,6 @@ mozbuild.pth:python/mozbuild
 mozversioncontrol.pth:python/mozversioncontrol
 mozlint.pth:python/mozlint
 pymake.pth:build/pymake
-optional:setup.py:python/psutil:build_ext:--inplace
 optional:psutil.pth:python/psutil
 which.pth:python/which
 ply.pth:other-licenses/ply/
@@ -29,6 +28,7 @@ moztreedocs.pth:tools/docs
 packages.txt:testing/mozbase/packages.txt
 objdir:build
 gyp.pth:media/webrtc/trunk/tools/gyp/pylib
+six.pth:python/requests/requests/packages/urllib3/packages
 pyasn1.pth:python/pyasn1
 pyasn1_modules.pth:python/pyasn1-modules
 bitstring.pth:python/bitstring

build/win32/autobinscope.py

@@ -23,8 +23,8 @@ BINSCOPE_OUTPUT_LOGFILE = r".\binscope_xml_output.log"
 
 # usage
 if len(sys.argv) < 3:
-  print """usage : autobinscope.by path_to_binary path_to_symbols [log_file_path]"
-		log_file_path is optional, log will be written to .\binscope_xml_output.log by default"""
+  print("""usage : autobinscope.by path_to_binary path_to_symbols [log_file_path]"
+		log_file_path is optional, log will be written to .\binscope_xml_output.log by default""")
   sys.exit(0)
 
 binary_path = sys.argv[1]
@@ -40,7 +40,7 @@ else:
 try:
   binscope_path = os.environ['BINSCOPE']
 except KeyError:
-  print "BINSCOPE environment variable is not set, can't check DEP/ASLR etc. status."
+  print("BINSCOPE environment variable is not set, can't check DEP/ASLR etc. status.")
   sys.exit(0)
   
 try:    
@@ -51,13 +51,15 @@ try:
     "/c", "CompilerVersionCheck", "/c", "SafeSEHCheck", "/c", "SNCheck",
     "/c", "DBCheck"], stdout=subprocess.PIPE)
 
-except WindowsError, (errno, strerror): 
+except OSError as e: 
+  errno = e.errno
+  strerror = e.strerror or str(e)
   if errno != 2 and errno != 3:
-    print "Unexpected error ! \nError " + str(errno) + " : " + strerror + "\nExiting !\n"
+    print("Unexpected error ! \nError " + str(errno) + " : " + strerror + "\nExiting !\n")
     sys.exit(0)
   else:
-    print "Could not locate binscope at location : %s\n" % binscope_path
-    print "Binscope wasn't installed or the BINSCOPE env variable wasn't set correctly, skipping this check and exiting..."
+    print("Could not locate binscope at location : %s\n" % binscope_path)
+    print("Binscope wasn't installed or the BINSCOPE env variable wasn't set correctly, skipping this check and exiting...")
     sys.exit(0)
 
 proc.wait()
@@ -66,10 +68,10 @@ output = proc.communicate()[0]
 
 # is this a PASS or a FAIL ? 
 if proc.returncode != 0:
-  print "Error count: %d" % proc.returncode
-  print "TEST-UNEXPECTED-FAIL | autobinscope.py | %s is missing a needed Windows protection, such as /GS or ASLR" % binary_path
+  print("Error count: %d" % proc.returncode)
+  print("TEST-UNEXPECTED-FAIL | autobinscope.py | %s is missing a needed Windows protection, such as /GS or ASLR" % binary_path)
   logfile = open(log_file_path, "r")
   for line in logfile:
-    print(line),
+    print((line), end=' ')
 else:
-  print "TEST-PASS | autobinscope.py | %s succeeded" % binary_path
+  print("TEST-PASS | autobinscope.py | %s succeeded" % binary_path)

build/win32/pgomerge.py

@@ -39,6 +39,6 @@ def MergePGOFiles(basename, pgddir, pgcdir):
 
 if __name__ == '__main__':
   if len(sys.argv) != 3:
-      print >>sys.stderr, "Usage: pgomerge.py <binary basename> <dist/bin>"
+      print("Usage: pgomerge.py <binary basename> <dist/bin>", file=sys.stderr)
       sys.exit(1)
   MergePGOFiles(sys.argv[1], os.getcwd(), sys.argv[2])

build/win32/procmem.py

@@ -40,9 +40,9 @@ if __name__ == '__main__':
                                                     0, # no inherit
                                                     int(pid))
         if handle:
-            print "Process %s:" % pid
+            print("Process %s:" % pid)
             vsize, peak_vsize = get_vmsize(handle)
-            print "peak vsize: %d" % peak_vsize
+            print("peak vsize: %d" % peak_vsize)
             ctypes.windll.kernel32.CloseHandle(handle)
         else:
-            print "Couldn't open process %s" % pid
+            print("Couldn't open process %s" % pid)

build/windows_toolchain.py

@@ -200,7 +200,7 @@ def format_manifest(manifest):
 
 def write_zip(zip_path, prefix=None):
     """Write toolchain data to a zip file."""
-    if isinstance(prefix, unicode):
+    if isinstance(prefix, str):
         prefix = prefix.encode('utf-8')
 
     with JarWriter(file=zip_path, optimize=False, compress=5) as zip:
@@ -226,7 +226,7 @@ def write_zip(zip_path, prefix=None):
 
 if __name__ == '__main__':
     if len(sys.argv) != 3:
-        print('usage: %s create-zip <path-prefix>' % sys.argv[0])
+        print(('usage: %s create-zip <path-prefix>' % sys.argv[0]))
         sys.exit(1)
 
     assert sys.argv[1] == 'create-zip'
@@ -244,7 +244,7 @@ if __name__ == '__main__':
         sha256.update(data)
         sha512.update(data)
 
-    print('Hashes of %s (size=%d)' % (destzip, len(data)))
-    print('SHA-1:   %s' % sha1.hexdigest())
-    print('SHA-256: %s' % sha256.hexdigest())
-    print('SHA-512: %s' % sha512.hexdigest())
+    print(('Hashes of %s (size=%d)' % (destzip, len(data))))
+    print(('SHA-1:   %s' % sha1.hexdigest()))
+    print(('SHA-256: %s' % sha256.hexdigest()))
+    print(('SHA-512: %s' % sha512.hexdigest()))

client.mk

@@ -66,7 +66,7 @@ endif
 
 SH := /bin/sh
 PERL ?= perl
-PYTHON ?= $(shell which python2.7 > /dev/null 2>&1 && echo python2.7 || echo python)
+PYTHON ?= $(shell which python3 > /dev/null 2>&1 && echo python3 || echo python)
 
 CONFIG_GUESS_SCRIPT := $(wildcard $(TOPSRCDIR)/build/autoconf/config.guess)
 ifdef CONFIG_GUESS_SCRIPT

client.py

@@ -22,7 +22,7 @@ if topsrcdir == '':
     topsrcdir = '.'
 
 def check_call_noisy(cmd, *args, **kwargs):
-    print "Executing command:", cmd
+    print("Executing command:", cmd)
     check_call(cmd, *args, **kwargs)
 
 def do_hg_pull(dir, repository, hg):
@@ -67,22 +67,22 @@ def do_cvs_export(modules, tag, cvsroot, cvs):
         cvs_module = module_tuple[1]
         fullpath = os.path.join(topsrcdir, module)
         if os.path.exists(fullpath):
-            print "Removing '%s'" % fullpath
+            print("Removing '%s'" % fullpath)
             shutil.rmtree(fullpath)
 
         (parent, leaf) = os.path.split(module)
-        print "CVS export begin: " + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
+        print("CVS export begin: " + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"))
         check_call_noisy([cvs, '-d', cvsroot,
                           'export', '-r', tag, '-d', leaf, cvs_module],
                          cwd=os.path.join(topsrcdir, parent))
-        print "CVS export end: " + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
+        print("CVS export end: " + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"))
 
 def toggle_trailing_blank_line(depname):
   """If the trailing line is empty, then we'll delete it.
   Otherwise we'll add a blank line."""
   lines = open(depname, "r").readlines()
   if not lines:
-      print >>sys.stderr, "unexpected short file"
+      print("unexpected short file", file=sys.stderr)
       return
 
   if not lines[-1].strip():
@@ -95,7 +95,7 @@ def toggle_trailing_blank_line(depname):
 def get_trailing_blank_line_state(depname):
   lines = open(depname, "r").readlines()
   if not lines:
-      print >>sys.stderr, "unexpected short file"
+      print("unexpected short file", file=sys.stderr)
       return "no blank line"
 
   if not lines[-1].strip():
@@ -104,18 +104,18 @@ def get_trailing_blank_line_state(depname):
       return "no blank line"
 
 def update_nspr_or_nss(tag, depfile, destination, hgpath):
-  print "reverting to HG version of %s to get its blank line state" % depfile
+  print("reverting to HG version of %s to get its blank line state" % depfile)
   check_call_noisy([options.hg, 'revert', depfile])
   old_state = get_trailing_blank_line_state(depfile)
-  print "old state of %s is: %s" % (depfile, old_state)
+  print("old state of %s is: %s" % (depfile, old_state))
   do_hg_replace(destination, hgpath, tag, HG_EXCLUSIONS, options.hg)
   new_state = get_trailing_blank_line_state(depfile)
-  print "new state of %s is: %s" % (depfile, new_state)
+  print("new state of %s is: %s" % (depfile, new_state))
   if old_state == new_state:
-    print "toggling blank line in: ", depfile
+    print("toggling blank line in: ", depfile)
     toggle_trailing_blank_line(depfile)
   tag_file = destination + "/TAG-INFO"
-  print >>file(tag_file, "w"), tag
+  print(tag, file=file(tag_file, "w"))
 
 o = OptionParser(usage="client.py [options] update_nspr tagname | update_nss tagname | update_libffi tagname")
 o.add_option("--skip-mozilla", dest="skip_mozilla",
@@ -139,7 +139,7 @@ except IndexError:
     sys.exit(2)
 
 if action in ('checkout', 'co'):
-    print >>sys.stderr, "Warning: client.py checkout is obsolete."
+    print("Warning: client.py checkout is obsolete.", file=sys.stderr)
     pass
 elif action in ('update_nspr'):
     tag, = args[1:]

config/MozZipFile.py

@@ -46,7 +46,7 @@ class ZipFile(zipfile.ZipFile):
                               date_time=time.localtime(time.time()))
       zinfo.compress_type = self.compression
       # Add some standard UNIX file access permissions (-rw-r--r--).
-      zinfo.external_attr = (0x81a4 & 0xFFFF) << 16L
+      zinfo.external_attr = (0x81a4 & 0xFFFF) << 16
     else:
       zinfo = zinfo_or_arcname
 
@@ -58,7 +58,7 @@ class ZipFile(zipfile.ZipFile):
     # as the old, reuse the existing entry.
 
     doSeek = False # store if we need to seek to the eof after overwriting
-    if self.NameToInfo.has_key(zinfo.filename):
+    if zinfo.filename in self.NameToInfo:
       # Find the last ZipInfo with our name.
       # Last, because that's catching multiple overwrites
       i = len(self.filelist)
@@ -109,14 +109,14 @@ class ZipFile(zipfile.ZipFile):
       # adjust file mode if we originally just wrote, now we rewrite
       self.fp.close()
       self.fp = open(self.filename, 'r+b')
-    all = map(lambda zi: (zi, True), self.filelist) + \
-        map(lambda zi: (zi, False), self._remove)
+    all = [(zi, True) for zi in self.filelist] + \
+        [(zi, False) for zi in self._remove]
     all.sort(lambda l, r: cmp(l[0].header_offset, r[0].header_offset))
     # empty _remove for multiple closes
     self._remove = []
 
     lengths = [all[i+1][0].header_offset - all[i][0].header_offset
-               for i in xrange(len(all)-1)]
+               for i in range(len(all)-1)]
     lengths.append(self.end - all[-1][0].header_offset)
     to_pos = 0
     for (zi, keep), length in zip(all, lengths):

config/action_cl.cpp

@@ -0,0 +1,354 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * C++ replacement for mozbuild/action/cl.py that wraps cl.exe on Windows.
+ * This is a Windows-only host tool, built well before XPCOM.
+ */
+#include <windows.h>
+#include <string> // We don't use XPCOM string classes here.
+#include <vector>
+#include <fstream>
+#include <algorithm>
+#pragma warning(disable:4530)
+using namespace std;
+
+
+/* Make passed argv into a command line for cl.exe */
+static wstring quoteArg(const wstring& aCmdArgv) {
+  bool addQuotes = aCmdArgv.find_first_of(L" \t\"") != wstring::npos;
+
+  if (!addQuotes) {
+    return aCmdArgv;
+  }
+
+  wstring out = L"\"";
+  size_t backslash = 0;
+
+  for (wchar_t ch : aCmdArgv) {
+    if (ch == L'\\') {
+      backslash++;
+    } else if (ch == L'"') {
+      // Escape all backslashes before a quote
+      out.append(backslash * 2, L'\\');
+      backslash = 0;
+      out += L"\\\"";
+    } else {
+      out.append(backslash, L'\\');
+      backslash = 0;
+      out += ch;
+    }
+  }
+
+  out.append(backslash * 2, L'\\');
+  out += L"\"";
+
+  return out;
+}
+
+/* We need to detect the showIncludes prefix to extract raw paths for
+   dep files, and drop the lines from display to avoid console spam. */
+static string showIncludesPrefixDetection(const string& aLine) {
+  // Look for absolute Windows paths
+  size_t pos = string::npos;
+
+  // Drive letter: C:\...
+  for (size_t i = 0; i + 2 < aLine.size(); i++) {
+    if (isalpha((unsigned char)aLine[i]) &&
+        aLine[i + 1] == ':' &&
+        (aLine[i + 2] == '\\' || aLine[i + 2] == '/')) {
+      pos = i;
+      break;
+    }
+  }
+
+  // UNC path: \\server\share
+  if (pos == string::npos) {
+    if (aLine.size() >= 2 && aLine[0] == '\\' && aLine[1] == '\\') {
+      pos = 0;
+    }
+  }
+
+  if (pos == string::npos) {
+    return "";
+  }
+
+  // Prefix is everything before the path
+  return aLine.substr(0, pos);
+}
+
+/* Extract the header path from an include line using the known prefix */
+static string extractHeaderPath(const string& aLine, const string& aPrefix) {
+  if (aLine.rfind(aPrefix, 0) != 0) {
+    return "";
+  }
+
+  // Strip prefix
+  string path = aLine.substr(aPrefix.size());
+
+  // Trim leading whitespace
+  while (!path.empty() && (path[0] == ' ' || path[0] == '\t')) {
+    path.erase(0, 1);
+  }
+
+  // Remove trailing newline
+  if (!path.empty() && (path.back() == '\n' || path.back() == '\r')) {
+    path.pop_back();
+  }
+
+  return path;
+}
+
+
+/* Sanitize a dependency path for Makefile depfiles.
+   Performs:
+   1. Strip trailing CR/LF
+   2. Strip leading MSVC indentation
+   3. Normalize slashes to '/'
+   4. Escape spaces and backslashes for Make */
+static string sanitizeMakeDeps(string aLine) {
+  // 1. Strip trailing CR/LF
+  while (!aLine.empty() && (aLine.back() == '\r' || aLine.back() == '\n')) {
+    aLine.pop_back();
+  }
+
+  // 2. Strip leading indentation (MSVC indents include lines)
+  while (!aLine.empty() && (aLine.front() == ' ' || aLine.front() == '\t')) {
+    aLine.erase(aLine.begin());
+  }
+
+  // 3. Normalize slashes
+  for (char& ch : aLine) {
+    if (ch == '\\') {
+      ch = '/';
+    }
+  }
+
+  // 4. Escape for Make
+  string out;
+  out.reserve(aLine.size() * 2);
+
+  for (char ch : aLine) {
+    if (ch == ' ') {
+      out += "\\ ";
+    } else if (ch == '\\') {
+      // After normalization this shouldn't occur, but safe anyway
+      out += "\\\\";
+    } else {
+      out += ch;
+    }
+  }
+
+  return out;
+}
+
+/* MSYS1 does not use wide APIs, so ensure we get the narrow version here. */
+static string getWinEnvVar(const char* aName) {
+  char buffer[32768];
+  DWORD length = GetEnvironmentVariableA(aName, buffer, sizeof(buffer));
+
+  if (length == 0 || length >= sizeof(buffer)) {
+    return string();
+  }
+
+  return string(buffer, length);
+}
+
+/* Exclude system headers by ignoring anything not in our directories.
+ * We don't know where people will put their objdir, so it has to be
+ * tracked separately.
+ */
+static bool isProjectDep(const string& aPath,
+                         const string& aTopSrcDir,
+                         const string& aTopObjDir) {
+  // We need forward slashes for our build system.
+  auto norm = [](string aStr) {
+    replace(aStr.begin(), aStr.end(), '\\', '/');
+    return aStr;
+  };
+
+  string path = norm(aPath);
+  string topSrcDir = norm(aTopSrcDir);
+  string topObjDir = norm(aTopObjDir);
+
+  // Must be absolute or relative under one of the two roots
+  return (path.rfind(topSrcDir, 0) == 0) ||
+         (path.rfind(topObjDir, 0) == 0);
+}
+
+/* Read stdout from the compiler pipe line-by-line, filtering include
+   lines and collecting dependency paths. */
+static vector<string> processOutput(HANDLE aReadPipe) {
+  string lineBuf;
+  vector<string> deps;
+  string topSrcDir = getWinEnvVar("CL_TOPSRCDIR");
+  string topObjDir = getWinEnvVar("CL_TOPOBJDIR");
+  char buffer[4096];
+  DWORD bytesRead;
+
+  string includePrefix;
+  bool prefixFound = false;
+
+  while (ReadFile(aReadPipe, buffer, sizeof(buffer), &bytesRead, nullptr)) {
+    lineBuf.append(buffer, bytesRead);
+
+    size_t pos;
+    while ((pos = lineBuf.find('\n')) != string::npos) {
+      string line = lineBuf.substr(0, pos + 1);
+      lineBuf.erase(0, pos + 1);
+
+      if (!prefixFound) {
+        string maybe = showIncludesPrefixDetection(line);
+        if (!maybe.empty()) {
+          includePrefix = maybe;
+          prefixFound = true;
+        }
+      }
+
+      // If prefix is known and this line starts with it, it's an include line
+      if (prefixFound && line.rfind(includePrefix, 0) == 0) {
+        string header = extractHeaderPath(line, includePrefix);
+        if (!header.empty()) {
+          header = sanitizeMakeDeps(header);
+          if (isProjectDep(header, topSrcDir, topObjDir)) {
+            deps.push_back(header);
+          }
+        }
+        // This is an include line, don't print it
+        continue;
+      }
+
+      // Otherwise, just print
+      fwrite(line.data(), 1, line.size(), stdout);
+    }
+  }
+
+  if (!lineBuf.empty()) {
+    fwrite(lineBuf.data(), 1, lineBuf.size(), stdout);
+  }
+
+  return deps;
+}
+
+/* Convert a UTF-16 wide string to a UTF-8 narrow string. */
+static string wideToUtf8(const wstring& aWide) {
+  if (aWide.empty()) {
+    return {};
+  }
+
+  int len = WideCharToMultiByte(CP_UTF8, 0, aWide.c_str(), -1, nullptr, 0, nullptr, nullptr);
+
+  string out(len - 1, '\0');
+
+  WideCharToMultiByte(CP_UTF8, 0, aWide.c_str(), -1, &out[0], len - 1, nullptr, nullptr);
+
+  return out;
+}
+
+/* Compute the .pp depfile path next to the given object file. */
+static string makeDepfilePath(const string& aObjUtf8) {
+  size_t pos = aObjUtf8.find_last_of("/\\");
+  string dir = (pos == string::npos) ? "" : aObjUtf8.substr(0, pos + 1);
+  string base = (pos == string::npos) ? aObjUtf8 : aObjUtf8.substr(pos + 1);
+
+  return dir + ".deps/" + base + ".pp";
+}
+
+/* Write a Make-style dependency file for the given object and dep list. */
+static void writeDepfile(const wstring& aObjFile, const vector<string>& aDeps) {
+  string objUtf8 = wideToUtf8(aObjFile);
+  string ppName = makeDepfilePath(objUtf8);
+
+  ofstream out(ppName);
+  out << objUtf8 << ":";
+
+  for (auto& dep : aDeps) {
+    out << " " << dep;
+  }
+
+  out << "\r\n";
+}
+
+int wmain(int argc, wchar_t** argv) {
+  if (argc < 2) {
+    fwprintf(stderr, L"Usage: action_cl.exe <compiler> [args...]\n");
+    return 1;
+  }
+
+  const wchar_t* compiler = argv[1];
+
+  // Build command line
+  wstring cmd = quoteArg(compiler);
+  wstring objFile;
+  wstring sourceFile;
+  cmd += L" -showIncludes";
+
+  for (int i = 2; i < argc; i++) {
+    cmd += L" ";
+    cmd += quoteArg(argv[i]);
+  }
+
+  vector<wchar_t> cmdLine(cmd.begin(), cmd.end());
+  cmdLine.push_back(L'\0');
+
+  for (int i = 2; i < argc; i++) {
+    wstring arg = argv[i];
+
+    if (arg.rfind(L"/Fo", 0) == 0 || arg.rfind(L"-Fo", 0) == 0) {
+      objFile = arg.substr(3); // strip /Fo or -Fo
+    }
+
+    if (!arg.empty() && arg[0] != L'-' && arg[0] != L'/') {
+      sourceFile = arg; // last non-flag argument is the source file
+    }
+  }
+
+  // Create pipe for capturing stdout
+  HANDLE readPipe, writePipe;
+  SECURITY_ATTRIBUTES sa = { sizeof(sa), nullptr, TRUE };
+
+  if (!CreatePipe(&readPipe, &writePipe, &sa, 0)) {
+    fwprintf(stderr, L"Failed to create pipe\n");
+    return 1;
+  }
+
+  SetHandleInformation(readPipe, HANDLE_FLAG_INHERIT, 0);
+
+  STARTUPINFOW si = { sizeof(si) };
+  si.dwFlags = STARTF_USESTDHANDLES;
+  si.hStdOutput = writePipe;
+  si.hStdError  = GetStdHandle(STD_ERROR_HANDLE);
+  si.hStdInput  = GetStdHandle(STD_INPUT_HANDLE);
+  PROCESS_INFORMATION pi;
+
+  BOOL ok = CreateProcessW(nullptr, cmdLine.data(), nullptr, nullptr, TRUE, 0, nullptr, nullptr, &si, &pi);
+
+  CloseHandle(writePipe);
+
+  if (!ok) {
+    fwprintf(stderr, L"Failed to launch compiler: %ls (error %lu)\n", compiler, GetLastError());
+    CloseHandle(readPipe);
+    return 1;
+  }
+
+  // Read compiler output and collect dependencies
+  auto deps = processOutput(readPipe);
+  CloseHandle(readPipe);
+
+  WaitForSingleObject(pi.hProcess, INFINITE);
+
+  DWORD exitCode = 0;
+  GetExitCodeProcess(pi.hProcess, &exitCode);
+
+  // Write the depfile only if we know the output object path
+  if (!objFile.empty()) {
+    string srcUtf8 = wideToUtf8(sourceFile);
+    deps.insert(deps.begin(), sanitizeMakeDeps(srcUtf8));
+    writeDepfile(objFile, deps);
+  }
+
+  CloseHandle(pi.hProcess);
+  CloseHandle(pi.hThread);
+
+  return exitCode;
+}

config/check_spidermonkey_style.py

@@ -275,7 +275,7 @@ def check_style():
         edges[inclname] = set()
 
     # Process all the JS files.
-    for filename in js_names.keys():
+    for filename in list(js_names.keys()):
         inclname = js_names[filename]
         file_kind = FileKind.get(filename)
         if file_kind == FileKind.C or file_kind == FileKind.CPP or \

config/config.mk

@@ -120,8 +120,8 @@ CONFIG_TOOLS	= $(MOZ_BUILD_ROOT)/config
 AUTOCONF_TOOLS	= $(MOZILLA_DIR)/build/autoconf
 
 ifdef _MSC_VER
-CC_WRAPPER ?= $(call py_action,cl)
-CXX_WRAPPER ?= $(call py_action,cl)
+CC_WRAPPER ?= $(DEPTH)/config/action_cl$(BIN_SUFFIX)
+CXX_WRAPPER ?= $(DEPTH)/config/action_cl$(BIN_SUFFIX)
 endif # _MSC_VER
 
 CC := $(CC_WRAPPER) $(CC)
@@ -562,7 +562,8 @@ endif
 
 PLY_INCLUDE = -I$(MOZILLA_DIR)/other-licenses/ply
 
-export CL_INCLUDES_PREFIX
+export CL_TOPOBJDIR = $(MOZ_BUILD_ROOT)
+export CL_TOPSRCDIR = $(EXTERNAL_SOURCE_DIR)
 # Make sure that the build system can handle non-ASCII characters
 # in environment variables to prevent it from breking silently on
 # non-English systems.

config/createprecomplete.py

@@ -56,10 +56,12 @@ def generate_precomplete(root_path):
     precomplete_file = open(precomplete_file_path, "wb")
     rel_file_path_list, rel_dir_path_list = get_build_entries(root_path)
     for rel_file_path in rel_file_path_list:
-        precomplete_file.writelines("remove \""+rel_file_path+"\"\n")
+        line = "remove \""+rel_file_path+"\"\n"
+        precomplete_file.write(line.encode('utf-8'))
 
     for rel_dir_path in rel_dir_path_list:
-        precomplete_file.writelines("rmdir \""+rel_dir_path+"\"\n")
+        line = "rmdir \""+rel_dir_path+"\"\n"
+        precomplete_file.write(line.encode('utf-8'))
 
     precomplete_file.close()
 

config/expandlibs.py

@@ -36,7 +36,7 @@ def ensureParentDir(file):
     if dir and not os.path.exists(dir):
         try:
             os.makedirs(dir)
-        except OSError, error:
+        except OSError as error:
             if error.errno != errno.EEXIST:
                 raise
 
@@ -140,4 +140,4 @@ class ExpandArgs(list):
         return [relativize(arg)]
 
 if __name__ == '__main__':
-    print " ".join(ExpandArgs(sys.argv[1:]))
+    print(" ".join(ExpandArgs(sys.argv[1:])))

config/expandlibs_exec.py

@@ -305,11 +305,11 @@ class SectionFinder(object):
         return syms
 
 def print_command(out, args):
-    print >>out, "Executing: " + " ".join(args)
+    print("Executing: " + " ".join(args), file=out)
     for tmp in [f for f in args.tmp if os.path.isfile(f)]:
-        print >>out, tmp + ":"
+        print(tmp + ":", file=out)
         with open(tmp) as file:
-            print >>out, "".join(["    " + l for l in file.readlines()])
+            print("".join(["    " + l for l in file.readlines()]), file=out)
     out.flush()
 
 def main(args, proc_callback=None):
@@ -339,14 +339,14 @@ def main(args, proc_callback=None):
             proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
             if proc_callback:
                 proc_callback(proc)
-        except Exception, e:
-            print >>sys.stderr, 'error: Launching', args, ':', e
+        except Exception as e:
+            print('error: Launching', args, ':', e, file=sys.stderr)
             raise e
         (stdout, stderr) = proc.communicate()
         if proc.returncode and not options.verbose:
             print_command(sys.stderr, args)
-        sys.stderr.write(stdout)
-        sys.stderr.flush()
+        sys.stderr.buffer.write(stdout)
+        sys.stderr.buffer.flush()
         if proc.returncode:
             return proc.returncode
         return 0

config/expandlibs_gen.py

@@ -38,4 +38,4 @@ if __name__ == '__main__':
 
     ensureParentDir(options.output)
     with open(options.output, 'w') as outfile:
-        print >>outfile, generate(args)
+        print(generate(args), file=outfile)

config/external/ffi/moz.build

@@ -101,6 +101,8 @@ else:
             ASFLAGS += ['-no-integrated-as']
     elif CONFIG['FFI_TARGET'] == 'AARCH64':
         ffi_srcs = ('sysv.S', 'ffi.c')
+    elif CONFIG['FFI_TARGET'] == 'LOONGARCH64':
+        ffi_srcs = ('ffi.c', 'sysv.S')
     elif CONFIG['FFI_TARGET'] == 'X86':
         ffi_srcs = ('ffi.c', 'sysv.S', 'win32.S')
     elif CONFIG['FFI_TARGET'] == 'X86_64':

config/find_OOM_errors.py

@@ -95,12 +95,12 @@ def count_lines():
   """Keep track of the amount of times individual lines occur, in order to
      prioritize the errors which occur most frequently."""
   counts = {}
-  for string,count in blacklist.items():
+  for string,count in list(blacklist.items()):
     for line in string.split("\n"):
       counts[line] = counts.get(line, 0) + count
 
   lines = []
-  for k,v in counts.items():
+  for k,v in list(counts.items()):
     lines.append("{0:6}: {1}".format(v, k))
 
   lines.sort()

config/link.py

@@ -18,7 +18,7 @@ def periodically_print_status(proc):
         time.sleep(0.5)
         idleTime += 0.5
         if idleTime > 20 * 60:
-          print "Still linking, 20 minutes passed..."
+          print("Still linking, 20 minutes passed...")
           sys.stdout.flush()
           idleTime = 0
 
@@ -42,6 +42,6 @@ def wrap_linker(args):
 
 if __name__ == "__main__":
     if len(sys.argv) < 2:
-        print >>sys.stderr, "Usage: link.py <commandline>"
+        print("Usage: link.py <commandline>", file=sys.stderr)
         sys.exit(1)
     sys.exit(wrap_linker(sys.argv[1:]))

config/milestone.txt

@@ -8,4 +8,4 @@
 # Referenced by milestone.py.
 #--------------------------------------------------------
 
-6.8.1
+6.9.0

config/moz.build

@@ -23,11 +23,19 @@ if CONFIG['HOST_OS_ARCH'] != 'WINNT':
         'nsinstall.c',
         'pathsub.c',
     ]
+
     # stdc++compat depends on config/export, so avoid a circular
     # dependency added by HostProgram depending on stdc++compat,
     # while the program here is in C.
     HostProgram('nsinstall_real', c_only=True)
 
+if CONFIG['HOST_OS_ARCH'] == 'WINNT':
+    HOST_SOURCES +=[
+        'action_cl.cpp',
+    ]
+
+    HostProgram('action_cl')
+
 if CONFIG['GKMEDIAS_SHARED_LIBRARY']:
     DEFINES['GKMEDIAS_SHARED_LIBRARY'] = True
 

config/mozunit.py

@@ -5,7 +5,7 @@
 from unittest import TextTestRunner as _TestRunner, TestResult as _TestResult
 import unittest
 import inspect
-from StringIO import StringIO
+from io import StringIO
 import os
 import sys
 
@@ -140,7 +140,7 @@ class MockedOpen(object):
     '''
     def __init__(self, files = {}):
         self.files = {}
-        for name, content in files.iteritems():
+        for name, content in files.items():
             self.files[normcase(os.path.abspath(name))] = content
 
     def __call__(self, name, mode = 'r'):
@@ -158,19 +158,19 @@ class MockedOpen(object):
         return file
 
     def __enter__(self):
-        import __builtin__
-        self.open = __builtin__.open
+        import builtins
+        self.open = builtins.open
         self._orig_path_exists = os.path.exists
         self._orig_path_isdir = os.path.isdir
         self._orig_path_isfile = os.path.isfile
-        __builtin__.open = self
+        builtins.open = self
         os.path.exists = self._wrapped_exists
         os.path.isdir = self._wrapped_isdir
         os.path.isfile = self._wrapped_isfile
 
     def __exit__(self, type, value, traceback):
-        import __builtin__
-        __builtin__.open = self.open
+        import builtins
+        builtins.open = self.open
         os.path.exists = self._orig_path_exists
         os.path.isdir = self._orig_path_isdir
         os.path.isfile = self._orig_path_isfile

config/nsinstall.py

@@ -149,7 +149,7 @@ def _nsinstall_internal(argv):
 
 # nsinstall as a native command is always UTF-8
 def nsinstall(argv):
-  return _nsinstall_internal([unicode(arg, "utf-8") for arg in argv])
+  return _nsinstall_internal([str(arg, "utf-8") for arg in argv])
 
 if __name__ == '__main__':
   # sys.argv corrupts characters outside the system code page on Windows
@@ -173,10 +173,7 @@ if __name__ == '__main__':
     # The first argv will be "python", the second will be the .py file
     argv = argv_arr[1:argc.value]
   else:
-    # For consistency, do it on Unix as well
-    if sys.stdin.encoding is not None:
-      argv = [unicode(arg, sys.stdin.encoding) for arg in sys.argv]
-    else:
-      argv = [unicode(arg) for arg in sys.argv]
+    # On Py3, we shouldn't do this for non-Windows. sys.argv is UTF-8 (or ASCII on old Unix).
+    argv = sys.argv
 
   sys.exit(_nsinstall_internal(argv[1:]))

config/printconfigsetting.py

@@ -5,27 +5,27 @@
 import configobj
 import sys
 import re
-from StringIO import StringIO
+from io import StringIO
 
 try:
     (file, section, key) = sys.argv[1:]
 except ValueError:
-    print "Usage: printconfigsetting.py <file> <section> <setting>"
+    print("Usage: printconfigsetting.py <file> <section> <setting>")
     sys.exit(1)
 
 with open(file) as fh:
-    content = re.sub('^\s*;', '#', fh.read(), flags=re.M)
+    content = re.sub(r'^\s*;', '#', fh.read(), flags=re.M)
 
 c = configobj.ConfigObj(StringIO(content))
 
 try:
     s = c[section]
 except KeyError:
-    print >>sys.stderr, "Section [%s] not found." % section
+    print("Section [%s] not found." % section, file=sys.stderr)
     sys.exit(1)
 
 try:
-    print s[key]
+    print(s[key])
 except KeyError:
-    print >>sys.stderr, "Key %s not found." % key
+    print("Key %s not found." % key, file=sys.stderr)
     sys.exit(1)

config/pythonpath.py

@@ -9,7 +9,7 @@ Run a python script, adding extra directories to the python path.
 
 def main(args):
     def usage():
-        print >>sys.stderr, "pythonpath.py -I directory script.py [args...]"
+        print("pythonpath.py -I directory script.py [args...]", file=sys.stderr)
         sys.exit(150)
 
     paths = []
@@ -45,7 +45,7 @@ def main(args):
     frozenglobals['__name__'] = '__main__'
     frozenglobals['__file__'] = script
 
-    execfile(script, frozenglobals)
+    exec(compile(open(script, "rb").read(), script, 'exec'), frozenglobals)
 
 # Freeze scope here ... why this makes things work I have no idea ...
 frozenglobals = globals()

config/rebuild_check.py

@@ -18,7 +18,7 @@ def rebuild_check(args):
     deps = args[1:]
     t = mtime(target)
     if t < 0:
-        print target
+        print(target)
         return
 
     newer = []
@@ -31,13 +31,13 @@ def rebuild_check(args):
             newer.append(dep)
 
     if newer and removed:
-        print 'Rebuilding %s because %s changed and %s was removed' % (target, ', '.join(newer), ', '.join(removed))
+        print('Rebuilding %s because %s changed and %s was removed' % (target, ', '.join(newer), ', '.join(removed)))
     elif newer:
-        print 'Rebuilding %s because %s changed' % (target, ', '.join(newer))
+        print('Rebuilding %s because %s changed' % (target, ', '.join(newer)))
     elif removed:
-        print 'Rebuilding %s because %s was removed' % (target, ', '.join(removed))
+        print('Rebuilding %s because %s was removed' % (target, ', '.join(removed)))
     else:
-        print 'Rebuilding %s for an unknown reason' % target
+        print('Rebuilding %s for an unknown reason' % target)
 
 if __name__ == '__main__':
     import sys

config/system-headers

@@ -335,8 +335,6 @@ curses.h
 cxxabi.h
 DateTimeUtils.h
 dbus/dbus.h
-dbus/dbus-glib.h
-dbus/dbus-glib-lowlevel.h
 ddeml.h
 Debug.h
 dem.h

config/tests/unit-expandlibs.py

@@ -7,7 +7,7 @@ from tempfile import mkdtemp
 from shutil import rmtree
 import mozunit
 
-from UserString import UserString
+from collections import UserString
 # Create a controlled configuration for use by expandlibs
 config_win = {
     'AR': 'lib',
@@ -120,8 +120,7 @@ class ReplicateTests(type):
             del dict[name]
         return type.__new__(cls, clsName, bases, dict)
 
-class TestCaseWithTmpDir(unittest.TestCase):
-    __metaclass__ = ReplicateTests
+class TestCaseWithTmpDir(unittest.TestCase, metaclass=ReplicateTests):
     def init(self):
         self.tmpdir = os.path.abspath(mkdtemp(dir=os.curdir))
 

config/tests/unit-nsinstall.py

@@ -28,9 +28,9 @@ class TestNsinstall(unittest.TestCase):
         # Unicode strings means non-ASCII children can be deleted properly on
         # Windows
         if sys.stdin.encoding is None:
-            tmpdir = unicode(self.tmpdir)
+            tmpdir = str(self.tmpdir)
         else:
-            tmpdir = unicode(self.tmpdir, sys.stdin.encoding)
+            tmpdir = str(self.tmpdir, sys.stdin.encoding)
         rmtree(tmpdir)
 
     # utility methods for tests
@@ -50,14 +50,14 @@ class TestNsinstall(unittest.TestCase):
         "Test nsinstall -D <dir>"
         testdir = os.path.join(self.tmpdir, "test")
         self.assertEqual(nsinstall(["-D", testdir]), 0)
-        self.assert_(os.path.isdir(testdir))
+        self.assertTrue(os.path.isdir(testdir))
 
     def test_nsinstall_basic(self):
         "Test nsinstall <file> <dir>"
         testfile = self.touch("testfile")
         testdir = self.mkdirs("testdir")
         self.assertEqual(nsinstall([testfile, testdir]), 0)
-        self.assert_(os.path.isfile(os.path.join(testdir, "testfile")))
+        self.assertTrue(os.path.isfile(os.path.join(testdir, "testfile")))
 
     def test_nsinstall_basic_recursive(self):
         "Test nsinstall <dir> <dest dir>"
@@ -76,12 +76,12 @@ class TestNsinstall(unittest.TestCase):
                                     '-X', Xdir]), 0)
 
         testdir = os.path.join(destdir, "sourcedir")
-        self.assert_(os.path.isdir(testdir))
-        self.assert_(os.path.isfile(os.path.join(testdir, "testfile")))
-        self.assert_(not os.path.exists(os.path.join(testdir, "Xfile")))
-        self.assert_(os.path.isdir(os.path.join(testdir, "copieddir")))
-        self.assert_(os.path.isfile(os.path.join(testdir, "copieddir", "testfile2")))
-        self.assert_(not os.path.exists(os.path.join(testdir, "Xdir")))
+        self.assertTrue(os.path.isdir(testdir))
+        self.assertTrue(os.path.isfile(os.path.join(testdir, "testfile")))
+        self.assertTrue(not os.path.exists(os.path.join(testdir, "Xfile")))
+        self.assertTrue(os.path.isdir(os.path.join(testdir, "copieddir")))
+        self.assertTrue(os.path.isfile(os.path.join(testdir, "copieddir", "testfile2")))
+        self.assertTrue(not os.path.exists(os.path.join(testdir, "Xdir")))
 
     def test_nsinstall_multiple(self):
         "Test nsinstall <three files> <dest dir>"
@@ -91,7 +91,7 @@ class TestNsinstall(unittest.TestCase):
         testdir = self.mkdirs("testdir")
         self.assertEqual(nsinstall(testfiles + [testdir]), 0)
         for f in testfiles:
-            self.assert_(os.path.isfile(os.path.join(testdir,
+            self.assertTrue(os.path.isfile(os.path.join(testdir,
                                                      os.path.basename(f))))
 
     def test_nsinstall_dir_exists(self):
@@ -99,7 +99,7 @@ class TestNsinstall(unittest.TestCase):
         srcdir = self.mkdirs("test")
         destdir = self.mkdirs("testdir/test")
         self.assertEqual(nsinstall([srcdir, os.path.dirname(destdir)]), 0)
-        self.assert_(os.path.isdir(destdir))
+        self.assertTrue(os.path.isdir(destdir))
 
     def test_nsinstall_t(self):
         "Test that nsinstall -t works (preserve timestamp)"
@@ -110,7 +110,7 @@ class TestNsinstall(unittest.TestCase):
         os.utime(testfile, (t, t))
         self.assertEqual(nsinstall(["-t", testfile, testdir]), 0)
         destfile = os.path.join(testdir, "testfile")
-        self.assert_(os.path.isfile(destfile))
+        self.assertTrue(os.path.isfile(destfile))
         self.assertEqual(os.stat(testfile).st_mtime,
                          os.stat(destfile).st_mtime)
 
@@ -125,7 +125,7 @@ class TestNsinstall(unittest.TestCase):
             self.assertEqual(nsinstall(["-m", "{0:04o}"
                                         .format(mode), testfile, testdir]), 0)
             destfile = os.path.join(testdir, "testfile")
-            self.assert_(os.path.isfile(destfile))
+            self.assertTrue(os.path.isfile(destfile))
             self.assertEqual(os.stat(testfile).st_mode,
                              os.stat(destfile).st_mode)
 
@@ -136,25 +136,25 @@ class TestNsinstall(unittest.TestCase):
         testdir = self.mkdirs("testdir")
         destdir = os.path.join(testdir, "subdir")
         self.assertEqual(nsinstall(["-d", testfile, destdir]), 0)
-        self.assert_(os.path.isdir(os.path.join(destdir, "testfile")))
+        self.assertTrue(os.path.isdir(os.path.join(destdir, "testfile")))
 
     if RUN_NON_ASCII_TESTS:
         def test_nsinstall_non_ascii(self):
             "Test that nsinstall handles non-ASCII files"
-            filename = u"\u2325\u3452\u2415\u5081"
+            filename = "\u2325\u3452\u2415\u5081"
             testfile = self.touch(filename)
-            testdir = self.mkdirs(u"\u4241\u1D04\u1414")
+            testdir = self.mkdirs("\u4241\u1D04\u1414")
             self.assertEqual(nsinstall([testfile.encode("utf-8"),
                                         testdir.encode("utf-8")]), 0)
 
             destfile = os.path.join(testdir, filename)
-            self.assert_(os.path.isfile(destfile))
+            self.assertTrue(os.path.isfile(destfile))
 
         def test_nsinstall_non_ascii_subprocess(self):
             "Test that nsinstall as a subprocess handles non-ASCII files"
-            filename = u"\u2325\u3452\u2415\u5081"
+            filename = "\u2325\u3452\u2415\u5081"
             testfile = self.touch(filename)
-            testdir = self.mkdirs(u"\u4241\u1D04\u1414")
+            testdir = self.mkdirs("\u4241\u1D04\u1414")
             # We don't use subprocess because it can't handle Unicode on
             # Windows <http://bugs.python.org/issue1759845>. mozprocess calls
             # CreateProcessW directly so it's perfect.
@@ -166,7 +166,7 @@ class TestNsinstall(unittest.TestCase):
 
             self.assertEqual(rv, 0)
             destfile = os.path.join(testdir, filename)
-            self.assert_(os.path.isfile(destfile))
+            self.assertTrue(os.path.isfile(destfile))
 
     #TODO: implement -R, -l, -L and test them!
 

config/tests/unitMozZipFile.py

@@ -11,6 +11,7 @@ import sys
 import random
 import copy
 from string import letters
+from functools import reduce
 
 '''
 Test case infrastructure for MozZipFile.
@@ -38,7 +39,7 @@ leafs = (
   'firstdir/oneleaf',
   'seconddir/twoleaf',
   'thirddir/with/sub/threeleaf')
-_lengths = map(lambda n: n * 64, [16, 64, 80])
+_lengths = [n * 64 for n in [16, 64, 80]]
 lengths = 3
 writes = 5
 
@@ -71,7 +72,7 @@ def getid(descs):
 def getContent(length):
   'Get pseudo random content of given length.'
   rv = [None] * length
-  for i in xrange(length):
+  for i in range(length):
     rv[i] = random.choice(letters)
   return ''.join(rv)
 
@@ -133,13 +134,13 @@ class TestExtensiveStored(unittest.TestCase):
   def _verifyZip(self):
     zf = zipfile.ZipFile(self.f)
     badEntry = zf.testzip()
-    self.failIf(badEntry, badEntry)
+    self.assertFalse(badEntry, badEntry)
     zlist = zf.namelist()
     zlist.sort()
-    vlist = self.ref.keys()
+    vlist = list(self.ref.keys())
     vlist.sort()
     self.assertEqual(zlist, vlist)
-    for leaf, content in self.ref.iteritems():
+    for leaf, content in self.ref.items():
       zcontent = zf.read(leaf)
       self.assertEqual(content, zcontent)
   
@@ -158,16 +159,16 @@ class TestExtensiveStored(unittest.TestCase):
     open(self.leaf('stage', leaf), 'w').write(content)
 
 # all leafs in all lengths
-atomics = list(prod(xrange(len(leafs)), xrange(lengths)))
+atomics = list(prod(range(len(leafs)), range(lengths)))
 
 # populate TestExtensiveStore with testcases
-for w in xrange(writes):
+for w in range(writes):
   # Don't iterate over all files for the the first n passes,
   # those are redundant as long as w < lengths.
   # There are symmetries in the trailing end, too, but I don't know
   # how to reduce those out right now.
-  nonatomics = [list(prod(range(min(i,len(leafs))), xrange(lengths)))
-                for i in xrange(1, w+1)] + [atomics]
+  nonatomics = [list(prod(list(range(min(i,len(leafs)))), range(lengths)))
+                for i in range(1, w+1)] + [atomics]
   for descs in prod(*nonatomics):
     suffix = getid(descs)
     dicts = [dict(leaf=leaf, length=length) for leaf, length in descs]
@@ -181,9 +182,9 @@ for w in xrange(writes):
 # and then write all atomics again.
 # This should catch more or less all artifacts generated
 # by the final ordering step when closing the jar.
-files = [list(prod([i], xrange(lengths))) for i in xrange(len(leafs))]
+files = [list(prod([i], range(lengths))) for i in range(len(leafs))]
 allfiles = reduce(lambda l,r:l+r,
-                  [list(prod(*files[:(i+1)])) for i in xrange(len(leafs))])
+                  [list(prod(*files[:(i+1)])) for i in range(len(leafs))])
 
 for first in allfiles:
   testbasename = 'test{0}_'.format(getid(first))

configure.in

@@ -22,4 +22,4 @@ SRCDIR=$(dirname $0)
 TOPSRCDIR="$SRCDIR"
 export OLD_CONFIGURE="$SRCDIR"/old-configure
 
-which python2.7 > /dev/null && exec python2.7 "$TOPSRCDIR/configure.py" "$@" || exec python "$TOPSRCDIR/configure.py" "$@"
+which python3 > /dev/null && exec python3 "$TOPSRCDIR/configure.py" "$@" || exec python "$TOPSRCDIR/configure.py" "$@"

configure.py

@@ -10,13 +10,11 @@ import subprocess
 import sys
 import textwrap
 
-
 base_dir = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(base_dir, 'python', 'mozbuild'))
 from mozbuild.configure import ConfigureSandbox
 from mozbuild.util import (
     indented_repr,
-    encode,
 )
 
 
@@ -36,20 +34,24 @@ def config_status(config):
     # Ideally, all the backend and frontend code would handle the booleans, but
     # there are so many things involved, that it's easier to keep config.status
     # untouched for now.
-    def sanitized_bools(v):
+    def sanitized_values(v):
         if v is True:
             return '1'
         if v is False:
             return ''
+        if isinstance(v, dict):
+            return {key: sanitized_values(value) for key, value in v.items()}
+        if isinstance(v, (list, tuple)):
+            return [sanitized_values(value) for value in v]
         return v
 
     sanitized_config = {}
     sanitized_config['substs'] = {
-        k: sanitized_bools(v) for k, v in config.iteritems()
-        if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR')
+        key: sanitized_values(value) for key, value in config.items()
+        if key not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR')
     }
     sanitized_config['defines'] = {
-        k: sanitized_bools(v) for k, v in config['DEFINES'].iteritems()
+        key: sanitized_values(value) for key, value in config['DEFINES'].items()
     }
     sanitized_config['non_global_defines'] = config['non_global_defines']
     sanitized_config['topsrcdir'] = config['TOPSRCDIR']
@@ -60,19 +62,16 @@ def config_status(config):
     # here, when we're able to skip configure tests/use cached results/not rely
     # on autoconf.
     print("Creating config.status", file=sys.stderr)
-    encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
-    with codecs.open('config.status', 'w', encoding) as fh:
+    with codecs.open('config.status', 'w', encoding='utf-8', errors='replace') as fh:
         fh.write(textwrap.dedent('''\
             #!%(python)s
             # coding=%(encoding)s
             from __future__ import unicode_literals
-            from mozbuild.util import encode
-            encoding = '%(encoding)s'
-        ''') % {'python': config['PYTHON'], 'encoding': encoding})
+        ''') % {'python': config['PYTHON'], 'encoding': 'utf-8'})
         # A lot of the build backend code is currently expecting byte
         # strings and breaks in subtle ways with unicode strings. (bug 1296508)
-        for k, v in sanitized_config.iteritems():
-            fh.write('%s = encode(%s, encoding)\n' % (k, indented_repr(v)))
+        for k, v in sanitized_config.items():
+            fh.write('%s = %s\n' % (k, indented_repr(v)))
         fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
                  "'non_global_defines', 'substs', 'mozconfig']")
 
@@ -88,18 +87,14 @@ def config_status(config):
     # executable permissions.
     os.chmod('config.status', 0o755)
     if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'):
-        os.environ[b'WRITE_MOZINFO'] = b'1'
+        os.environ['WRITE_MOZINFO'] = '1'
         from mozbuild.config_status import config_status
 
         # Some values in sanitized_config also have more complex types, such as
-        # EnumString, which using when calling config_status would currently
-        # break the build, as well as making it inconsistent with re-running
-        # config.status. Fortunately, EnumString derives from unicode, so it's
-        # covered by converting unicode strings.
-
-        # A lot of the build backend code is currently expecting byte strings
-        # and breaks in subtle ways with unicode strings.
-        return config_status(args=[], **encode(sanitized_config, encoding))
+        # EnumString and tuples. Converting string subclasses to plain strings
+        # and tuples to lists keeps this code path consistent with the
+        # generated config.status file, which serializes iterables as lists.
+        return config_status(args=[], **sanitized_config)
     return 0
 
 

devtools/shared/css/generated/mach_commands.py

@@ -97,4 +97,4 @@ class MachCommands(MachCommandBase):
         with open(destination_path, 'wb') as destination:
             destination.write(preamble + contents)
 
-        print('The database was successfully generated at ' + destination_path)
+        print(('The database was successfully generated at ' + destination_path))

dom/apps/tests/create_test_receipts.py

@@ -152,12 +152,12 @@ receipt_with_wrong_typ = {
   "reissue": "http://mochi.test:8888/reissue/5169314356"
 }
 
-print("let valid_receipt1 = \"" + jwt.encode(receipt1, "") + "\";\n")
-print("let valid_receipt2 = \"" + jwt.encode(receipt2, "") + "\";\n")
-print("let receipt_without_typ = \"" + jwt.encode(receipt_without_typ, "") + "\";\n")
-print("let receipt_without_product = \"" + jwt.encode(receipt_without_product, "") + "\";\n")
-print("let receipt_without_user = \"" + jwt.encode(receipt_without_user, "") + "\";\n")
-print("let receipt_without_iss = \"" + jwt.encode(receipt_without_iss, "") + "\";\n")
-print("let receipt_without_nbf = \"" + jwt.encode(receipt_without_nbf, "") + "\";\n")
-print("let receipt_without_iat = \"" + jwt.encode(receipt_without_iat, "") + "\";\n")
-print("let receipt_with_wrong_typ = \"" + jwt.encode(receipt_with_wrong_typ, "") + "\";\n")
+print(("let valid_receipt1 = \"" + jwt.encode(receipt1, "") + "\";\n"))
+print(("let valid_receipt2 = \"" + jwt.encode(receipt2, "") + "\";\n"))
+print(("let receipt_without_typ = \"" + jwt.encode(receipt_without_typ, "") + "\";\n"))
+print(("let receipt_without_product = \"" + jwt.encode(receipt_without_product, "") + "\";\n"))
+print(("let receipt_without_user = \"" + jwt.encode(receipt_without_user, "") + "\";\n"))
+print(("let receipt_without_iss = \"" + jwt.encode(receipt_without_iss, "") + "\";\n"))
+print(("let receipt_without_nbf = \"" + jwt.encode(receipt_without_nbf, "") + "\";\n"))
+print(("let receipt_without_iat = \"" + jwt.encode(receipt_without_iat, "") + "\";\n"))
+print(("let receipt_with_wrong_typ = \"" + jwt.encode(receipt_with_wrong_typ, "") + "\";\n"))

dom/base/Element.cpp

@@ -304,12 +304,18 @@ Element::TabIndex()
 }
 
 void
-Element::Focus(mozilla::ErrorResult& aError)
+Element::Focus(const FocusOptions& aOptions, ErrorResult& aError)
 {
   nsCOMPtr<nsIDOMElement> domElement = do_QueryInterface(this);
   nsIFocusManager* fm = nsFocusManager::GetFocusManager();
+  // Also other browsers seem to have the hack to not re-focus (and flush) when
+  // the element is already focused.
+  // Until https://github.com/whatwg/html/issues/4512 is clarified, we'll
+  // maintain interoperatibility by not re-focusing, independent of aOptions.
+  // I.e., `focus({ preventScroll: true})` followed by `focus( { preventScroll:
+  // false })` won't re-focus.
   if (fm && domElement) {
-    aError = fm->SetFocus(domElement, 0);
+    aError = fm->SetFocus(domElement, nsFocusManager::FocusOptionsToFocusManagerFlags(aOptions));
   }
 }
 
@@ -2326,6 +2332,7 @@ Element::MaybeCheckSameAttrVal(int32_t aNamespaceID,
                                bool* aOldValueSet)
 {
   bool modification = false;
+  CustomElementData* customElementData = GetCustomElementData();
   *aHasListeners = aNotify &&
     nsContentUtils::HasMutationListeners(this,
                                          NS_EVENT_BITS_MUTATION_ATTRMODIFIED,
@@ -2341,13 +2348,14 @@ Element::MaybeCheckSameAttrVal(int32_t aNamespaceID,
   if (*aHasListeners || aNotify) {
     BorrowedAttrInfo info(GetAttrInfo(aNamespaceID, aName));
     if (info.mValue) {
-      // Check whether the old value is the same as the new one.  Note that we
-      // only need to actually _get_ the old value if we have listeners or
-      // if the element is a custom element (because it may have an
-      // attribute changed callback).
-      if (*aHasListeners || GetCustomElementData()) {
-        // Need to store the old value.
-        //
+      bool valueMatches = aValue.EqualsAsStrings(*info.mValue);
+      if (valueMatches && aPrefix == info.mName->GetPrefix()) {
+        return true;
+      }
+
+      // Need to store the old value if listeners are present or this is a
+      // custom element that may run an attribute-changed callback.
+      if (*aHasListeners || customElementData) {
         // If the current attribute value contains a pointer to some other data
         // structure that gets updated in the process of setting the attribute
         // we'll no longer have the old value of the attribute. Therefore, we
@@ -2358,10 +2366,7 @@ Element::MaybeCheckSameAttrVal(int32_t aNamespaceID,
         aOldValue.SetToSerialized(*info.mValue);
         *aOldValueSet = true;
       }
-      bool valueMatches = aValue.EqualsAsStrings(*info.mValue);
-      if (valueMatches && aPrefix == info.mName->GetPrefix()) {
-        return true;
-      }
+
       modification = true;
     }
   }

dom/base/Element.h

@@ -216,7 +216,7 @@ public:
   /**
    * Make focus on this element.
    */
-  virtual void Focus(mozilla::ErrorResult& aError);
+  virtual void Focus(const FocusOptions& aOptions, ErrorResult& aError);
 
   /**
    * Show blur and clear focus.

dom/base/FileReader.cpp

@@ -395,8 +395,14 @@ FileReader::ReadFileContent(Blob& aBlob,
     return;
   }
 
+  CheckedInt<size_t> size(mTotal);
+  if (!size.isValid()) {
+    aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
+    return;
+  }
+
   if (mDataFormat == FILE_AS_ARRAYBUFFER) {
-    mFileData = js_pod_malloc<char>(mTotal);
+    mFileData = js_pod_malloc<char>(size.value());
     if (!mFileData) {
       NS_WARNING("Preallocation failed for ReadFileData");
       aRv.Throw(NS_ERROR_OUT_OF_MEMORY);

dom/base/ImageEncoder.cpp

@@ -400,6 +400,7 @@ ImageEncoder::ExtractDataInternal(const nsAString& aType,
       layers::PlanarYCbCrImage* ycbcrImage = static_cast<layers::PlanarYCbCrImage*> (aImage);
       gfxImageFormat format = SurfaceFormat::A8R8G8B8_UINT32;
       uint32_t stride = GetAlignedStride<16>(aSize.width, 4);
+      auto size = stride * aSize.height;
       size_t length = BufferSizeFromStrideAndHeight(stride, aSize.height);
       if (length == 0) {
         return NS_ERROR_INVALID_ARG;
@@ -413,10 +414,10 @@ ImageEncoder::ExtractDataInternal(const nsAString& aType,
                         stride);
 
       rv = aEncoder->InitFromData(data.Elements(),
-                                  aSize.width * aSize.height * 4,
+                                  size,
                                   aSize.width,
                                   aSize.height,
-                                  aSize.width * 4,
+                                  stride,
                                   imgIEncoder::INPUT_FORMAT_HOSTARGB,
                                   aOptions);
     } else {
@@ -433,10 +434,10 @@ ImageEncoder::ExtractDataInternal(const nsAString& aType,
         return NS_ERROR_INVALID_ARG;
       }
       rv = aEncoder->InitFromData(map.mData,
-                                  aSize.width * aSize.height * 4,
+                                  map.mStride * aSize.height,
                                   aSize.width,
                                   aSize.height,
-                                  aSize.width * 4,
+                                  map.mStride,
                                   imgIEncoder::INPUT_FORMAT_HOSTARGB,
                                   aOptions);
       dataSurface->Unmap();
@@ -467,10 +468,10 @@ ImageEncoder::ExtractDataInternal(const nsAString& aType,
       return NS_ERROR_INVALID_ARG;
     }
     rv = aEncoder->InitFromData(map.mData,
-                                aSize.width * aSize.height * 4,
+                                map.mStride * aSize.height,
                                 aSize.width,
                                 aSize.height,
-                                aSize.width * 4,
+                                map.mStride,
                                 imgIEncoder::INPUT_FORMAT_HOSTARGB,
                                 aOptions);
     emptyCanvas->Unmap();

dom/base/nsContentList.cpp

@@ -523,7 +523,8 @@ nsContentList::NamedItem(const nsAString& aName, bool aDoFlush)
 }
 
 void
-nsContentList::GetSupportedNames(nsTArray<nsString>& aNames)
+nsContentList::GetSupportedNames(nsTArray<nsString>& aNames,
+                                 FilterElementWithName aFilter)
 {
   BringSelfUpToDate(true);
 
@@ -546,7 +547,8 @@ nsContentList::GetSupportedNames(nsTArray<nsString>& aNames)
       // Note: nsINode::HasName means the name is exposed on the document,
       // which is false for options, so we don't check it here.
       const nsAttrValue* val = el->GetParsedAttr(nsGkAtoms::name);
-      if (val && val->Type() == nsAttrValue::eAtom) {
+      if (val && val->Type() == nsAttrValue::eAtom &&
+          (!aFilter || aFilter(el))) {
         nsIAtom* name = val->GetAtomValue();
         MOZ_ASSERT(name != nsGkAtoms::_empty,
                    "Empty names don't get atomized");

dom/base/nsContentList.h

@@ -295,7 +295,9 @@ public:
     aFound = !!item;
     return item;
   }
-  virtual void GetSupportedNames(nsTArray<nsString>& aNames) override;
+  virtual void GetSupportedNames(nsTArray<nsString>& aNames) override {
+    GetSupportedNames(aNames, nullptr);
+  }
 
   // nsContentList public methods
   uint32_t Length(bool aDoFlush);
@@ -303,6 +305,12 @@ public:
   mozilla::dom::Element*
   NamedItem(const nsAString& aName, bool aDoFlush);
 
+  // Used by HTMLAllCollection to limit the elements whose name attribute is
+  // considered. The filter MUST NOT cause any flushes.
+  using FilterElementWithName = bool (*)(nsIContent*);
+  void GetSupportedNames(nsTArray<nsString>& aNames,
+                         FilterElementWithName aFilter);
+
   // nsIMutationObserver
   NS_DECL_NSIMUTATIONOBSERVER_ATTRIBUTECHANGED
   NS_DECL_NSIMUTATIONOBSERVER_CONTENTAPPENDED

dom/base/nsContentUtils.cpp

@@ -2259,6 +2259,31 @@ nsContentUtils::IsCallerContentXBL()
     return xpc::IsContentXBLScope(c);
 }
 
+bool
+nsContentUtils::IsSystemCaller(JSContext* aCx)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  // This is similar to what SubjectPrincipal() does, except we do in fact
+  // assume that we're in a compartment here; anyone who calls this function in
+  // situations where that's not the case is doing it wrong.
+  JSCompartment *compartment = js::GetContextCompartment(aCx);
+  MOZ_ASSERT(compartment);
+
+  JSPrincipals *principals = JS_GetCompartmentPrincipals(compartment);
+  return nsJSPrincipals::get(principals) == sSystemPrincipal;
+}
+
+bool
+nsContentUtils::ThreadsafeIsSystemCaller(JSContext* aCx)
+{
+  if (NS_IsMainThread()) {
+    return IsSystemCaller(aCx);
+  }
+
+  return workers::GetWorkerPrivateFromContext(aCx)->UsesSystemPrincipal();
+}
+
 // static
 bool
 nsContentUtils::LookupBindingMember(JSContext* aCx, nsIContent *aContent,
@@ -7336,6 +7361,17 @@ nsContentUtils::HasDistributedChildren(nsIContent* aContent)
   return false;
 }
 
+// static
+bool nsContentUtils::IsNodeInEditableRegion(nsINode* aNode) {
+  while (aNode) {
+    if (aNode->IsEditable()) {
+      return true;
+    }
+    aNode = aNode->GetParent();
+  }
+  return false;
+}
+
 // static
 bool
 nsContentUtils::IsForbiddenRequestHeader(const nsACString& aHeader,

dom/base/nsContentUtils.h

@@ -202,6 +202,23 @@ public:
   static bool     ThreadsafeIsCallerChrome();
   static bool     IsCallerContentXBL();
 
+  // The APIs for checking whether the caller is system (in the sense of system
+  // principal) should only be used when the JSContext is known to accurately
+  // represent the caller.  In practice, that means you should only use them in
+  // two situations at the moment:
+  //
+  // 1) Functions used in WebIDL Func annotations.
+  // 2) Bindings code.
+  //
+  // Use pretty much anywhere else is almost certainly wrong and should be
+  // replaced with [NeedsCallerType] annotations in bindings.
+
+  // Check whether the caller is system if you know you're on the main thread.
+  static bool IsSystemCaller(JSContext* aCx);
+
+  // Check whether the caller is system if you might be on a worker thread.
+  static bool ThreadsafeIsSystemCaller(JSContext* aCx);
+
   // In the traditional Gecko architecture, both C++ code and untrusted JS code
   // needed to rely on the same XPCOM method/getter/setter to get work done.
   // This required lots of security checks in the various exposed methods, which
@@ -2550,6 +2567,13 @@ public:
    */
   static bool GPCEnabled();
 
+  /**
+   * Returns whether a node has an editable ancestor.
+   *
+   * @param aNode The node to test.
+   */
+  static bool IsNodeInEditableRegion(nsINode* aNode);
+
   /**
    * Returns a LogModule that dump calls from content script are logged to.
    * This can be enabled with the 'Dump' module, and is useful for synchronizing

dom/base/nsFocusManager.cpp

@@ -48,6 +48,7 @@
 
 #include "mozilla/ContentEvents.h"
 #include "mozilla/dom/Element.h"
+#include "mozilla/dom/ElementBinding.h"
 #include "mozilla/dom/ShadowRoot.h"
 #include "mozilla/dom/HTMLInputElement.h"
 #include "mozilla/dom/HTMLSlotElement.h"
@@ -2999,6 +3000,11 @@ nsFocusManager::DetermineElementToMoveFocus(nsPIDOMWindowOuter* aWindow,
   return NS_OK;
 }
 
+uint32_t nsFocusManager::FocusOptionsToFocusManagerFlags(
+    const mozilla::dom::FocusOptions& aOptions) {
+  return aOptions.mPreventScroll ? nsIFocusManager::FLAG_NOSCROLL : 0;
+}
+
 static bool
 IsHostOrSlot(nsIContent* aContent)
 {

dom/base/nsFocusManager.h

@@ -26,6 +26,7 @@ class nsIMessageBroadcaster;
 
 namespace mozilla {
 namespace dom {
+struct FocusOptions;
 class TabParent;
 }
 }
@@ -110,6 +111,9 @@ public:
   static nsIContent* GetFocusedDescendant(nsPIDOMWindowOuter* aWindow, bool aDeep,
                                           nsPIDOMWindowOuter** aFocusedWindow);
 
+  static uint32_t FocusOptionsToFocusManagerFlags(
+      const mozilla::dom::FocusOptions& aOptions);
+
   /**
    * Returns the content node that focus will be redirected to if aContent was
    * focused. This is used for the special case of certain XUL elements such

dom/base/nsGkAtomList.h

@@ -1004,6 +1004,7 @@ GK_ATOM(parameter, "parameter")
 GK_ATOM(parent, "parent")
 GK_ATOM(parentfocused, "parentfocused")
 GK_ATOM(parsetype, "parsetype")
+GK_ATOM(part, "part")
 GK_ATOM(password, "password")
 GK_ATOM(pattern, "pattern")
 GK_ATOM(patternSeparator, "pattern-separator")

dom/base/nsGlobalWindow.cpp

@@ -7395,9 +7395,10 @@ nsGlobalWindow::Confirm(const nsAString& aMessage,
 
 already_AddRefed<Promise>
 nsGlobalWindow::Fetch(const RequestOrUSVString& aInput,
-                      const RequestInit& aInit, ErrorResult& aRv)
+                      const RequestInit& aInit,
+                      CallerType aCallerType, ErrorResult& aRv)
 {
-  return FetchRequest(this, aInput, aInit, aRv);
+  return FetchRequest(this, aInput, aInit, aCallerType, aRv);
 }
 
 void

dom/base/nsGlobalWindow.h

@@ -929,6 +929,7 @@ public:
   already_AddRefed<mozilla::dom::cache::CacheStorage> GetCaches(mozilla::ErrorResult& aRv);
   already_AddRefed<mozilla::dom::Promise> Fetch(const mozilla::dom::RequestOrUSVString& aInput,
                                                 const mozilla::dom::RequestInit& aInit,
+                                                mozilla::dom::CallerType aCallerType,
                                                 mozilla::ErrorResult& aRv);
   void PrintOuter(mozilla::ErrorResult& aError);
   void Print(mozilla::ErrorResult& aError);

dom/base/nsIImageLoadingContent.idl

@@ -37,7 +37,7 @@ native Visibility(mozilla::Visibility);
  * observers to check which request they are getting notifications for.
  *
  * Please make sure to update the MozImageLoadingContent WebIDL
- * interface to mirror this interface when changing it.
+ * mixin to mirror this interface when changing it.
  */
 
 [scriptable, builtinclass, uuid(0357123d-9224-4d12-a47e-868c32689777)]

dom/base/nsINode.cpp

@@ -16,6 +16,7 @@
 #include "mozilla/CORSMode.h"
 #include "mozilla/EventDispatcher.h"
 #include "mozilla/EventListenerManager.h"
+#include "mozilla/HTMLEditor.h"
 #include "mozilla/InternalMutationEvent.h"
 #include "mozilla/Likely.h"
 #include "mozilla/MemoryReporting.h"

dom/base/nsWrapperCache.cpp

@@ -33,17 +33,29 @@ nsWrapperCache::HoldJSObjects(void* aScriptObjectHolder,
   }
 }
 
+static inline bool
+IsNurseryWrapper(JSObject* aWrapper) {
+  return aWrapper && !JS::ObjectIsTenured(aWrapper);
+}
+
 void
-nsWrapperCache::SetWrapperJSObject(JSObject* aWrapper)
-{
-  mWrapper = aWrapper;
+nsWrapperCache::SetWrapperJSObject(JSObject* aNewWrapper) {
+  JSObject* oldWrapper = mWrapper;
+  mWrapper = aNewWrapper;
   UnsetWrapperFlags(kWrapperFlagsMask & ~WRAPPER_IS_NOT_DOM_BINDING);
 
-  if (aWrapper && !JS::ObjectIsTenured(aWrapper)) {
+  if (IsNurseryWrapper(aNewWrapper) && !IsNurseryWrapper(oldWrapper)) {
     CycleCollectedJSContext::Get()->NurseryWrapperAdded(this);
   }
 }
 
+void nsWrapperCache::ClearWrapperOnWrapFailure() {
+  if (IsNurseryWrapper(mWrapper)) {
+    CycleCollectedJSContext::Get()->NurseryWrapperRemovedSlow(this);
+  }
+  ClearWrapper();
+}
+
 void
 nsWrapperCache::ReleaseWrapper(void* aScriptObjectHolder)
 {

dom/base/nsWrapperCache.h

@@ -131,6 +131,8 @@ public:
     SetWrapperJSObject(nullptr);
   }
 
+  void ClearWrapperOnWrapFailure();
+
   /**
    * Update the wrapper if the object it contains is moved.
    *

dom/base/test/file_websocket_wsh.py

@@ -72,7 +72,7 @@ def web_socket_transfer_data(request):
     msgutil.send_message(request, resp.decode('utf-8'))
   elif request.ws_protocol == "test-12":
     msg = msgutil.receive_message(request)
-    if msg ==  u'a\ufffdb':
+    if msg ==  'a\ufffdb':
       # converted unpaired surrogate in UTF-16 to UTF-8 OK
       msgutil.send_message(request, "SUCCESS")
     else:

dom/base/test/websocket_hybi/file_binary-frames_wsh.py

@@ -15,4 +15,4 @@ def web_socket_transfer_data(request):
 
 
 def all_distinct_bytes():
-    return ''.join([chr(i) for i in xrange(256)])
+    return ''.join([chr(i) for i in range(256)])

dom/base/test/websocket_hybi/file_check-binary-messages_wsh.py

@@ -18,4 +18,4 @@ def web_socket_transfer_data(request):
 
 
 def all_distinct_bytes():
-    return ''.join([chr(i) for i in xrange(256)])
+    return ''.join([chr(i) for i in range(256)])

dom/bindings/Codegen.py

@@ -13,6 +13,7 @@ import functools
 
 from WebIDL import BuiltinTypes, IDLBuiltinType, IDLNullValue, IDLSequenceType, IDLType, IDLAttribute, IDLInterfaceMember, IDLUndefinedValue, IDLEmptySequenceValue, IDLDictionary
 from Configuration import NoSuchDescriptorError, getTypesFromDescriptor, getTypesFromDictionary, getTypesFromCallback, getAllTypes, Descriptor, MemberIsUnforgeable, iteratorNativeType
+from functools import reduce
 
 AUTOGENERATED_WARNING_COMMENT = \
     "/* THIS FILE IS AUTOGENERATED BY Codegen.py - DO NOT EDIT */\n\n"
@@ -1117,13 +1118,12 @@ class CGHeaders(CGWrapper):
 
         declareIncludes = set(declareIncludes)
 
-        def addHeadersForType((t, dictionary)):
+        def addHeadersForType(t_dictionary):
             """
             Add the relevant headers for this type.  We use dictionary, if
             passed, to decide what to do with interface types.
             """
-            # Dictionaries have members that need to be actually
-            # declared, not just forward-declared.
+            t, dictionary = t_dictionary
             if dictionary:
                 headerSet = declareIncludes
             else:
@@ -1197,9 +1197,9 @@ class CGHeaders(CGWrapper):
                 # parametrized over, if needed.
                 addHeadersForType((t.inner, dictionary))
 
-        map(addHeadersForType,
+        list(map(addHeadersForType,
             getAllTypes(descriptors + callbackDescriptors, dictionaries,
-                        callbacks))
+                        callbacks)))
 
         # Now make sure we're not trying to include the header from inside itself
         declareIncludes.discard(prefix + ".h")
@@ -2068,8 +2068,7 @@ class MemberCondition:
         if nonExposedGlobals:
             # Nonempty set
             self.nonExposedGlobals = " | ".join(
-                map(lambda g: "GlobalNames::%s" % g,
-                    sorted(nonExposedGlobals)))
+                ["GlobalNames::%s" % g for g in sorted(nonExposedGlobals)])
         else:
             self.nonExposedGlobals = "0"
 
@@ -3711,7 +3710,7 @@ class CGWrapWithCacheMethod(CGAbstractMethod):
         failureCode = dedent(
             """
             aCache->ReleaseWrapper(aObject);
-            aCache->ClearWrapper();
+            aCache->ClearWrapperOnWrapFailure();
             return false;
             """)
 
@@ -4464,9 +4463,6 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
                                 isOptional=False,
                                 invalidEnumValueFatal=True,
                                 defaultValue=None,
-                                treatNullAs="Default",
-                                isEnforceRange=False,
-                                isClamp=False,
                                 isNullOrUndefined=False,
                                 exceptionCode=None,
                                 lenientFloatCode=None,
@@ -4544,6 +4540,12 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
     # And we can't both be an object and be null or undefined
     assert not isDefinitelyObject or not isNullOrUndefined
 
+    # Types can also have extended attributes, copy them over
+    isClamp = type.clamp
+    isEnforceRange = type.enforceRange
+    if type.treatNullAsEmpty:
+        treatNullAs = "EmptyString"
+
     # If exceptionCode is not set, we'll just rethrow the exception we got.
     # Note that we can't just set failureCode to exceptionCode, because setting
     # failureCode will prevent pending exceptions from being set in cases when
@@ -4677,7 +4679,7 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
                                         declArgs=declArgs)
 
     def incrementNestingLevel():
-        if nestingLevel is "":
+        if nestingLevel == "":
             return 1
         return nestingLevel + 1
 
@@ -4999,7 +5001,7 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
         memberTypes = type.flatMemberTypes
         names = []
 
-        interfaceMemberTypes = filter(lambda t: t.isNonCallbackInterface(), memberTypes)
+        interfaceMemberTypes = [t for t in memberTypes if t.isNonCallbackInterface()]
         if len(interfaceMemberTypes) > 0:
             interfaceObject = []
             for memberType in interfaceMemberTypes:
@@ -5013,7 +5015,7 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
         else:
             interfaceObject = None
 
-        sequenceObjectMemberTypes = filter(lambda t: t.isSequence(), memberTypes)
+        sequenceObjectMemberTypes = [t for t in memberTypes if t.isSequence()]
         if len(sequenceObjectMemberTypes) > 0:
             assert len(sequenceObjectMemberTypes) == 1
             name = getUnionMemberName(sequenceObjectMemberTypes[0])
@@ -5024,7 +5026,7 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
         else:
             sequenceObject = None
 
-        dateObjectMemberTypes = filter(lambda t: t.isDate(), memberTypes)
+        dateObjectMemberTypes = [t for t in memberTypes if t.isDate()]
         if len(dateObjectMemberTypes) > 0:
             assert len(dateObjectMemberTypes) == 1
             memberType = dateObjectMemberTypes[0]
@@ -5036,7 +5038,7 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
         else:
             dateObject = None
 
-        callbackMemberTypes = filter(lambda t: t.isCallback() or t.isCallbackInterface(), memberTypes)
+        callbackMemberTypes = [t for t in memberTypes if t.isCallback() or t.isCallbackInterface()]
         if len(callbackMemberTypes) > 0:
             assert len(callbackMemberTypes) == 1
             memberType = callbackMemberTypes[0]
@@ -5048,7 +5050,7 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
         else:
             callbackObject = None
 
-        dictionaryMemberTypes = filter(lambda t: t.isDictionary(), memberTypes)
+        dictionaryMemberTypes = [t for t in memberTypes if t.isDictionary()]
         if len(dictionaryMemberTypes) > 0:
             assert len(dictionaryMemberTypes) == 1
             name = getUnionMemberName(dictionaryMemberTypes[0])
@@ -5059,7 +5061,7 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
         else:
             setDictionary = None
 
-        recordMemberTypes = filter(lambda t: t.isRecord(), memberTypes)
+        recordMemberTypes = [t for t in memberTypes if t.isRecord()]
         if len(recordMemberTypes) > 0:
             assert len(recordMemberTypes) == 1
             name = getUnionMemberName(recordMemberTypes[0])
@@ -5070,7 +5072,7 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
         else:
             recordObject = None
 
-        objectMemberTypes = filter(lambda t: t.isObject(), memberTypes)
+        objectMemberTypes = [t for t in memberTypes if t.isObject()]
         if len(objectMemberTypes) > 0:
             assert len(objectMemberTypes) == 1
             # Very important to NOT construct a temporary Rooted here, since the
@@ -5135,9 +5137,9 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
                 return CGGeneric("done = (failed = !%s.TrySetTo%s(cx, ${val}, tryNext)) || !tryNext;\n"
                                  "break;\n" % (unionArgumentObj, name))
             other = CGList([])
-            stringConversion = map(getStringOrPrimitiveConversion, stringTypes)
-            numericConversion = map(getStringOrPrimitiveConversion, numericTypes)
-            booleanConversion = map(getStringOrPrimitiveConversion, booleanTypes)
+            stringConversion = list(map(getStringOrPrimitiveConversion, stringTypes))
+            numericConversion = list(map(getStringOrPrimitiveConversion, numericTypes))
+            booleanConversion = list(map(getStringOrPrimitiveConversion, booleanTypes))
             if stringConversion:
                 if booleanConversion:
                     other.append(CGIfWrapper(booleanConversion[0],
@@ -5648,6 +5650,10 @@ def getJSToNativeConversionInfo(type, descriptorProvider, failureCode=None,
             undefinedBehavior = "eNull"
         else:
             undefinedBehavior = "eStringify"
+            if type.treatNullAsEmpty:
+                treatNullAs = "EmptyString"
+            else:
+                treatNullAs = "Default"
         nullBehavior = treatAs[treatNullAs]
 
         def getConversionCode(varName):
@@ -6297,9 +6303,6 @@ class CGArgumentConverter(CGThing):
                         not self.argument.variadic),
             invalidEnumValueFatal=self.invalidEnumValueFatal,
             defaultValue=self.argument.defaultValue,
-            treatNullAs=self.argument.treatNullAs,
-            isEnforceRange=self.argument.enforceRange,
-            isClamp=self.argument.clamp,
             lenientFloatCode=self.lenientFloatCode,
             isMember="Variadic" if self.argument.variadic else False,
             allowTreatNonCallableAsNull=self.argument.allowTreatNonCallableAsNull(),
@@ -7744,7 +7747,7 @@ class CGPerSignatureCall(CGThing):
         }
         try:
             wrapCode += wrapForType(self.returnType, self.descriptor, resultTemplateValues)
-        except MethodNotNewObjectError, err:
+        except MethodNotNewObjectError as err:
             assert not returnsNewObject
             raise TypeError("%s being returned from non-NewObject method or property %s.%s" %
                             (err.typename,
@@ -8255,7 +8258,7 @@ class CGMethodCall(CGThing):
             # a string overload, then boolean and numeric are conditional, and
             # if not then boolean is conditional if we have a numeric overload.
             def findUniqueSignature(filterLambda):
-                sigs = filter(filterLambda, possibleSignatures)
+                sigs = list(filter(filterLambda, possibleSignatures))
                 assert len(sigs) < 2
                 if len(sigs) > 0:
                     return sigs[0]
@@ -8370,18 +8373,6 @@ class FakeArgument():
         self.variadic = False
         self.defaultValue = None
         self._allowTreatNonCallableAsNull = allowTreatNonCallableAsNull
-        # For FakeArguments generated by maplike/setlike convenience functions,
-        # we won't have an interfaceMember to pass in.
-        if interfaceMember:
-            self.treatNullAs = interfaceMember.treatNullAs
-        else:
-            self.treatNullAs = "Default"
-        if isinstance(interfaceMember, IDLAttribute):
-            self.enforceRange = interfaceMember.enforceRange
-            self.clamp = interfaceMember.clamp
-        else:
-            self.enforceRange = False
-            self.clamp = False
 
         self.identifier = FakeIdentifier(name)
 
@@ -9729,7 +9720,7 @@ class CGEnum(CGThing):
                           name=ENUM_ENTRY_VARIABLE_NAME,
                           count=self.nEnumStrings(),
                           entries=''.join('{"%s", %d},\n' % (val, len(val))
-                                          for val in self.enum.values()))))
+                                          for val in list(self.enum.values())))))
         toJSValue = CGEnumToJSValue(enum)
         self.cgThings = CGList([strings, toJSValue], "\n")
 
@@ -9737,7 +9728,7 @@ class CGEnum(CGThing):
         return self.enum.identifier.name + "Values"
 
     def nEnumStrings(self):
-        return len(self.enum.values()) + 1
+        return len(list(self.enum.values())) + 1
 
     def declare(self):
         decl = fill(
@@ -9748,7 +9739,7 @@ class CGEnum(CGThing):
             };
             """,
             name=self.enum.identifier.name,
-            enums=",\n".join(map(getEnumValueName, self.enum.values())) + ",\n")
+            enums=",\n".join(map(getEnumValueName, list(self.enum.values()))) + ",\n")
         strings = CGNamespace(self.stringsNamespace(),
                               CGGeneric(declare="extern const EnumEntry %s[%d];\n"
                                         % (ENUM_ENTRY_VARIABLE_NAME, self.nEnumStrings())))
@@ -11063,7 +11054,6 @@ class CGProxySpecialOperation(CGPerSignatureCall):
             argument = arguments[1]
             info = getJSToNativeConversionInfo(
                 argument.type, descriptor,
-                treatNullAs=argument.treatNullAs,
                 sourceDescription=("value being assigned to %s setter" %
                                    descriptor.interface.identifier.name))
             if argumentHandleValue is None:
@@ -12694,8 +12684,6 @@ class CGDictionary(CGThing):
              getJSToNativeConversionInfo(
                  member.type,
                  descriptorProvider,
-                 isEnforceRange=member.enforceRange,
-                 isClamp=member.clamp,
                  isMember="Dictionary",
                  isOptional=member.canHaveMissingValue(),
                  defaultValue=member.defaultValue,
@@ -13659,7 +13647,7 @@ class ForwardDeclarationBuilder:
         if self.decls:
             decls.append(CGList([CGClassForwardDeclare(cname, isStruct)
                                  for cname, isStruct in sorted(self.decls)]))
-        for namespace, child in sorted(self.children.iteritems()):
+        for namespace, child in sorted(self.children.items()):
             decls.append(CGNamespace(namespace, child._build(atTopLevel=False), declareOnly=True))
 
         cg = CGList(decls, "\n")
@@ -14009,10 +13997,10 @@ class CGBindingRoot(CGThing):
 
         # Add header includes.
         bindingHeaders = [header
-                          for header, include in bindingHeaders.iteritems()
+                          for header, include in bindingHeaders.items()
                           if include]
         bindingDeclareHeaders = [header
-                                 for header, include in bindingDeclareHeaders.iteritems()
+                                 for header, include in bindingDeclareHeaders.items()
                                  if include]
 
         curr = CGHeaders(descriptors,
@@ -14620,7 +14608,7 @@ class CGBindingImplClass(CGClass):
                                (returnType, args),
                                descriptor.getExtendedAttributes(op)))
         # Sort things by name so we get stable ordering in the output.
-        ops = descriptor.operations.items()
+        ops = list(descriptor.operations.items())
         ops.sort(key=lambda x: x[0])
         for name, op in ops:
             appendSpecialOperation(name, op)
@@ -15575,7 +15563,6 @@ class CGCallbackInterface(CGCallback):
 
 class FakeMember():
     def __init__(self, name=None):
-        self.treatNullAs = "Default"
         if name is not None:
             self.identifier = FakeIdentifier(name)
 
@@ -16970,7 +16957,7 @@ class GlobalGenRoots():
     @staticmethod
     def UnionConversions(config):
         unionTypes = []
-        for l in config.unionsPerFilename.itervalues():
+        for l in config.unionsPerFilename.values():
             unionTypes.extend(l)
         unionTypes.sort(key=lambda u: u.name)
         headers, unions = UnionConversions(unionTypes,
@@ -17087,7 +17074,7 @@ class CGEventMethod(CGNativeMember):
                 # -3 on the left to ignore the type, bubbles, and cancelable parameters
                 # -1 on the right to ignore the .trusted property which bleeds through
                 # here because it is [Unforgeable].
-                len(signature[1]) - 3 == len(filter(lambda x: x.isAttr(), iface.members)) - 1):
+                len(signature[1]) - 3 == len([x for x in iface.members if x.isAttr()]) - 1):
                 allowed = True
                 self.isInit = True
 

dom/bindings/Configuration.py

@@ -28,7 +28,7 @@ class Configuration(DescriptorProvider):
 
         # Read the configuration file.
         glbl = {}
-        execfile(filename, glbl)
+        exec(compile(open(filename, "rb").read(), filename, 'exec'), glbl)
         config = glbl['DOMInterfaces']
 
         # Build descriptors for all the interfaces we have in the parse data.
@@ -45,7 +45,8 @@ class Configuration(DescriptorProvider):
                 # Our build system doesn't support dep build involving
                 # addition/removal of "implements" statements that appear in a
                 # different .webidl file than their LHS interface.  Make sure we
-                # don't have any of those.
+                # don't have any of those.  See similar block below for partial
+                # interfaces!
                 #
                 # But whitelist a RHS that is LegacyQueryInterface,
                 # since people shouldn't be adding any of those.
@@ -66,6 +67,33 @@ class Configuration(DescriptorProvider):
             if not thing.isInterface() and not thing.isNamespace():
                 continue
             iface = thing
+            # Our build system doesn't support dep builds involving
+            # addition/removal of partial interfaces that appear in a different
+            # .webidl file than the interface they are extending.  Make sure we
+            # don't have any of those.  See similar block above for "implements"
+            # statements!
+            if not iface.isExternal():
+                for partialIface in iface.getPartials():
+                    if (partialIface.filename() != iface.filename() and
+                        # Unfortunately, NavigatorProperty does exactly the
+                        # thing we're trying to prevent here.  I'm not sure how
+                        # to deal with that, short of effectively requiring a
+                        # clobber when NavigatorProperty is added/removed and
+                        # whitelisting the things it outputs here as
+                        # restrictively as I can.
+                        (partialIface.identifier.name != "Navigator" or
+                         len(partialIface.members) != 1 or
+                         partialIface.members[0].location != partialIface.location or
+                         partialIface.members[0].identifier.location.filename() !=
+                           "<builtin>")):
+                        raise TypeError(
+                            "The binding build system doesn't really support "
+                            "partial interfaces which don't appear in the "
+                            "file in which the interface they are extending is "
+                            "defined.  Don't do this.\n"
+                            "%s\n"
+                            "%s" %
+                            (partialIface.location, iface.location))
             self.interfaces[iface.identifier.name] = iface
             if iface.identifier.name not in config:
                 # Completely skip consequential interfaces with no descriptor
@@ -87,7 +115,7 @@ class Configuration(DescriptorProvider):
             self.descriptorsByName[desc.interface.identifier.name] = desc
 
         # Keep the descriptor list sorted for determinism.
-        self.descriptors.sort(lambda x, y: cmp(x.name, y.name))
+        self.descriptors.sort(key=lambda x: x.name)
 
 
         self.descriptorsByFile = {}
@@ -149,8 +177,7 @@ class Configuration(DescriptorProvider):
                                 # unions for the file where we previously found
                                 # them.
                                 unionsForFilename = self.unionsPerFilename[f]
-                                unionsForFilename = filter(lambda u: u.name != t.name,
-                                                           unionsForFilename)
+                                unionsForFilename = [u for u in unionsForFilename if u.name != t.name]
                                 if len(unionsForFilename) == 0:
                                     del self.unionsPerFilename[f]
                                 else:
@@ -170,7 +197,7 @@ class Configuration(DescriptorProvider):
         # Collect up our filters, because we may have a webIDLFile filter that
         # we always want to apply first.
         tofilter = []
-        for key, val in filters.iteritems():
+        for key, val in filters.items():
             if key == 'webIDLFile':
                 # Special-case this part to make it fast, since most of our
                 # getDescriptors calls are conditioned on a webIDLFile.  We may
@@ -210,17 +237,17 @@ class Configuration(DescriptorProvider):
                 getter = (lambda attrName: lambda x: getattr(x, attrName))(key)
             tofilter.append((getter, val))
         for f in tofilter:
-            curr = filter(lambda x: f[0](x) == f[1], curr)
+            curr = [x for x in curr if f[0](x) == f[1]]
         return curr
 
     def getEnums(self, webIDLFile):
-        return filter(lambda e: e.filename() == webIDLFile, self.enums)
+        return [e for e in self.enums if e.filename() == webIDLFile]
 
     def getDictionaries(self, webIDLFile):
-        return filter(lambda d: d.filename() == webIDLFile, self.dictionaries)
+        return [d for d in self.dictionaries if d.filename() == webIDLFile]
 
     def getCallbacks(self, webIDLFile):
-        return filter(lambda c: c.filename() == webIDLFile, self.callbacks)
+        return [c for c in self.callbacks if c.filename() == webIDLFile]
 
     def getDescriptor(self, interfaceName):
         """
@@ -491,7 +518,7 @@ class Descriptor(DescriptorProvider):
                 if config == '*':
                     iface = self.interface
                     while iface:
-                        add('all', map(lambda m: m.name, iface.members), attribute)
+                        add('all', [m.name for m in iface.members], attribute)
                         iface = iface.parent
                 else:
                     add('all', [config], attribute)
@@ -543,7 +570,7 @@ class Descriptor(DescriptorProvider):
 
     @property
     def prototypeNameChain(self):
-        return map(lambda p: self.getDescriptor(p).name, self.prototypeChain)
+        return [self.getDescriptor(p).name for p in self.prototypeChain]
 
     @property
     def parentPrototypeName(self):

dom/bindings/ErrorResult.h

@@ -164,6 +164,7 @@ public:
 
   void Throw(nsresult rv) {
     MOZ_ASSERT(NS_FAILED(rv), "Please don't try throwing success");
+    ClearUnionData();
     AssignErrorCode(rv);
   }
 
@@ -324,6 +325,7 @@ public:
   // Throw() here because people can easily pass success codes to
   // this.
   void operator=(nsresult rv) {
+    ClearUnionData();
     AssignErrorCode(rv);
   }
 
@@ -393,6 +395,7 @@ private:
   }
 
   void AssignErrorCode(nsresult aRv) {
+    MOZ_ASSERT(mUnionState == HasNothing);
     MOZ_ASSERT(aRv != NS_ERROR_TYPE_ERR, "Use ThrowTypeError()");
     MOZ_ASSERT(aRv != NS_ERROR_RANGE_ERR, "Use ThrowRangeError()");
     MOZ_ASSERT(!IsErrorWithMessage(), "Don't overwrite errors with message");

dom/bindings/GenerateCSS2PropertiesWebIDL.py

@@ -13,7 +13,7 @@ from mozbuild import shellutil
 # (whether camelCase, _underscorePrefixed, etc.) and the given array of
 # extended attributes.
 def generateLine(propName, extendedAttrs):
-    return "  [%s] attribute DOMString %s;\n" % (", ".join(extendedAttrs),
+    return "  [%s] attribute [TreatNullAs=EmptyString] DOMString %s;\n" % (", ".join(extendedAttrs),
                                                  propName)
 def generate(output, idlFilename, preprocessorHeader):
     print(idlFilename)
@@ -30,8 +30,8 @@ def generate(output, idlFilename, preprocessorHeader):
             continue
         # Unfortunately, even some of the getters here are fallible
         # (e.g. on nsComputedDOMStyle).
-        extendedAttrs = ["Throws", "TreatNullAs=EmptyString"]
-        if pref is not "":
+        extendedAttrs = ["Throws"]
+        if pref != "":
             extendedAttrs.append('Pref="%s"' % pref)
 
         # webkit properties get a capitalized "WebkitFoo" accessor (added here)

dom/bindings/mozwebidlcodegen/__init__.py

@@ -11,6 +11,7 @@ import errno
 import hashlib
 import json
 import logging
+import io
 import os
 import sys
 
@@ -99,7 +100,7 @@ class WebIDLCodegenManagerState(dict):
         self['version'] = state['version']
         self['global_depends'] = state['global_depends']
 
-        for k, v in state['webidls'].items():
+        for k, v in list(state['webidls'].items()):
             self['webidls'][k] = v
 
             # Sets are converted to lists for serialization because JSON
@@ -111,12 +112,12 @@ class WebIDLCodegenManagerState(dict):
         """Dump serialized state to a file handle."""
         normalized = deepcopy(self)
 
-        for k, v in self['webidls'].items():
+        for k, v in list(self['webidls'].items()):
             # Convert sets to lists because JSON doesn't support sets.
             normalized['webidls'][k]['outputs'] = sorted(v['outputs'])
             normalized['webidls'][k]['inputs'] = sorted(v['inputs'])
-
-        json.dump(normalized, fh, sort_keys=True)
+        fh_text = io.TextIOWrapper(fh, encoding='utf-8', errors='replace')
+        json.dump(normalized, fh_text, sort_keys=True)
 
 
 class WebIDLCodegenManager(LoggingMixin):
@@ -199,7 +200,7 @@ class WebIDLCodegenManager(LoggingMixin):
         self._state = WebIDLCodegenManagerState()
 
         if os.path.exists(state_path):
-            with open(state_path, 'rb') as fh:
+            with open(state_path, 'r', encoding='utf-8') as fh:
                 try:
                     self._state = WebIDLCodegenManagerState(fh=fh)
                 except Exception as e:
@@ -270,7 +271,7 @@ class WebIDLCodegenManager(LoggingMixin):
         # Generate bindings from .webidl files.
         for filename in sorted(changed_inputs):
             basename = mozpath.basename(filename)
-            print basename
+            print(basename)
             sys.stdout.flush()
             result.inputs.add(filename)
             written, deps = self._generate_build_files_for_webidl(filename)
@@ -296,7 +297,7 @@ class WebIDLCodegenManager(LoggingMixin):
         if self._make_deps_path:
             mk = Makefile()
             codegen_rule = mk.create_rule([self._make_deps_target])
-            codegen_rule.add_dependencies(global_hashes.keys())
+            codegen_rule.add_dependencies(list(global_hashes.keys()))
             codegen_rule.add_dependencies(self._input_paths)
 
             with FileAvoidWrite(self._make_deps_path) as fh:
@@ -314,7 +315,7 @@ class WebIDLCodegenManager(LoggingMixin):
 
         example_paths = self._example_paths(interface)
         for path in example_paths:
-            print "Generating %s" % path
+            print("Generating %s" % path)
 
         return self._maybe_write_codegen(root, *example_paths)
 
@@ -333,7 +334,7 @@ class WebIDLCodegenManager(LoggingMixin):
             with open(path, 'rb') as fh:
                 data = fh.read()
                 hashes[path] = hashlib.sha1(data).hexdigest()
-                parser.parse(data, path)
+                parser.parse(data.decode('utf-8'), path)
 
         self._parser_results = parser.finish()
         self._config = Configuration(self._config_path, self._parser_results,
@@ -382,7 +383,7 @@ class WebIDLCodegenManager(LoggingMixin):
 
         # Now we move on to the input files.
         old_hashes = {v['filename']: v['sha1']
-                      for v in self._state['webidls'].values()}
+                      for v in list(self._state['webidls'].values())}
 
         old_filenames = set(old_hashes.keys())
         new_filenames = self._input_paths
@@ -403,7 +404,7 @@ class WebIDLCodegenManager(LoggingMixin):
         # Inherit dependencies from previous run. The full set of dependencies
         # is associated with each record, so we don't need to perform any fancy
         # graph traversal.
-        for v in self._state['webidls'].values():
+        for v in list(self._state['webidls'].values()):
             if any(dep for dep in v['inputs'] if dep in changed_inputs):
                 changed_inputs.add(v['filename'])
 
@@ -512,7 +513,7 @@ class WebIDLCodegenManager(LoggingMixin):
             return True, current_hashes
 
         # Compare hashes.
-        for f, sha1 in current_hashes.items():
+        for f, sha1 in list(current_hashes.items()):
             if sha1 != self._state['global_depends'][f]:
                 return True, current_hashes
 
@@ -550,7 +551,8 @@ def create_build_system_manager(topsrcdir, topobjdir, dist_dir):
     src_dir = os.path.join(topsrcdir, 'dom', 'bindings')
     obj_dir = os.path.join(topobjdir, 'dom', 'bindings')
 
-    with open(os.path.join(obj_dir, 'file-lists.json'), 'rb') as fh:
+    with open(os.path.join(obj_dir, 'file-lists.json'), 'r',
+              encoding='utf-8', errors='replace') as fh:
         files = json.load(fh)
 
     inputs = (files['webidls'], files['exported_stems'],