Python 3 Support

.travis.yml

@@ -1,6 +1,7 @@
 language: python
 python:
-  - "2.7"
+  - "2.7"    
+  - "3.6"
 addons:
   apt:
     packages:

mx_benchmark.py

@@ -39,6 +39,7 @@
 from collections import OrderedDict
 
 import mx
+from mx_portable import _long
 
 _bm_suites = {}
 _benchmark_executor = None
@@ -377,7 +378,7 @@ def bm_suite_valid_keys():
 
 def vm_registries():
     res = set()
-    for bm_suite in _bm_suites.itervalues():
+    for bm_suite in _bm_suites.values():
         if isinstance(bm_suite, VmBenchmarkSuite):
             res.add(bm_suite.get_vm_registry())
     return res
@@ -471,7 +472,7 @@ def parse(self, text):
         varpat = re.compile(r"\$([a-zA-Z_][0-9a-zA-Z_]*)")
         for iteration, m in enumerate(self.parseResults(text)):
             datapoint = {}
-            for key, value in self.replacement.iteritems():
+            for key, value in self.replacement.items():
                 inst = value
                 if isinstance(inst, tuple):
                     v, vtype = inst
@@ -488,8 +489,8 @@ def var(name):
                         inst = str(v)
                     elif vtype is int:
                         inst = int(v)
-                    elif vtype is long:
-                        inst = long(v)
+                    elif vtype is _long:
+                        inst = _long(v)
                     elif vtype is float:
                         inst = float(v)
                     elif vtype is bool:
@@ -498,7 +499,7 @@ def var(name):
                         inst = vtype(v)
                     else:
                         raise RuntimeError("Cannot handle object '{0}' of expected type {1}".format(v, vtype))
-                if not isinstance(inst, (str, int, long, float, bool)):
+                if not isinstance(inst, (str, int, _long, float, bool)):
                     raise RuntimeError("Object '{0}' has unknown type: {1}".format(inst, type(inst)))
                 datapoint[key] = inst
             datapoints.append(datapoint)
@@ -680,9 +681,9 @@ def parse(self, text):
 
                 if "params" in result:
                     # add all parameter as a single string
-                    d["extra.jmh.params"] = ", ".join(["=".join(kv) for kv in result["params"].iteritems()])
+                    d["extra.jmh.params"] = ", ".join(["=".join(kv) for kv in result["params"].items()])
                     # and also the individual values
-                    for k, v in result["params"].iteritems():
+                    for k, v in result["params"].items():
                         d["extra.jmh.param." + k] = str(v)
 
                 for k in self.getExtraJmhKeys():
@@ -1370,7 +1371,7 @@ class JMHRunnerBenchmarkSuite(JMHBenchmarkSuiteBase): #pylint: disable=too-many-
 
     def benchmarkList(self, bmSuiteArgs):
         """Return all different JMH versions found."""
-        return list(JMHRunnerBenchmarkSuite.get_jmh_projects_dict().iterkeys())
+        return list(JMHRunnerBenchmarkSuite.get_jmh_projects_dict().keys())
 
     def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
         if benchmarks is None:
@@ -1394,7 +1395,7 @@ def get_jmh_projects_dict():
         jmhProjects = {}
         projects = mx.projects_opt_limit_to_suites()
         if mx.primary_suite() == mx._mx_suite:
-            projects = [p for p in mx._projects.itervalues() if p.suite == mx._mx_suite]
+            projects = [p for p in mx._projects.values() if p.suite == mx._mx_suite]
         for p in projects:
             for x in p.deps:
                 if x.name.startswith('JMH'):
@@ -1825,7 +1826,7 @@ def benchmark(self, mxBenchmarkArgs, bmSuiteArgs):
                         vmregToSuites.setdefault(vmreg, []).append(bm_suite_name)
                     else:
                         noVmRegSuites.append(bm_suite_name)
-                for vmreg, bm_suite_names in vmregToSuites.iteritems():
+                for vmreg, bm_suite_names in vmregToSuites.items():
                     print("\nThe following {} benchmark suites are available:\n".format(vmreg.vm_type_name))
                     for name in bm_suite_names:
                         print("  " + name)
@@ -1838,7 +1839,7 @@ def benchmark(self, mxBenchmarkArgs, bmSuiteArgs):
 
         if mxBenchmarkArgs.help or mxBenchmarkArgs.benchmark is None:
             parser.print_help()
-            for key, entry in parsers.iteritems():
+            for key, entry in parsers.items():
                 if mxBenchmarkArgs.benchmark is None or key in suite.parserNames():
                     print(entry.description)
                     entry.parser.print_help()

mx_benchplot.py

@@ -365,7 +365,7 @@ def extract_results(files, names, last_n=None, selected_benchmarks=None):
                     higher = entry['metric.better'] == 'higher'
                     result[benchmark] = {'scores': [score], 'higher': higher, 'name': name}
 
-        for _, entry in result.iteritems():
+        for _, entry in result.items():
             if last_n and len(entry['scores']) > abs(last_n):
                 if last_n < 0:
                     entry['trimmed_scores'] = entry['scores'][:-last_n]
@@ -379,7 +379,7 @@ def extract_results(files, names, last_n=None, selected_benchmarks=None):
 
         # Compute a variance value.  This is a percentage variance relative to the average score
         # which is easier to interpret than a raw number.
-        for _, entry in result.iteritems():
+        for _, entry in result.items():
             variance = 0
             for score in entry['scores']:
                 variance = variance + (score - entry['score']) * (score - entry['score'])

mx_compat.py

@@ -26,7 +26,7 @@
 
 from __future__ import print_function
 
-import sys, inspect, re, types, bisect
+import sys, inspect, re, bisect
 from collections import OrderedDict
 from os.path import join
 import mx
@@ -395,13 +395,13 @@ def check_checkstyle_config(self):
 
 def minVersion():
     _ensureCompatLoaded()
-    return _versionsMap.keys()[0]
+    return list(_versionsMap)[0]
 
 def getMxCompatibility(version):
     """:rtype: MxCompatibility500"""
     if version < minVersion():  # ensures compat loaded
         return None
-    keys = _versionsMap.keys()
+    keys = list(_versionsMap.keys())
     return _versionsMap[keys[bisect.bisect_right(keys, version)-1]]
 
 _versionsMap = OrderedDict()
@@ -411,12 +411,12 @@ def _ensureCompatLoaded():
 
         def flattenClassTree(tree):
             root = tree[0][0]
-            assert isinstance(root, types.TypeType), root
+            assert isinstance(root, type), root
             yield root
             if len(tree) > 1:
                 assert len(tree) == 2
                 rest = tree[1]
-                assert isinstance(rest, types.ListType), rest
+                assert isinstance(rest, list), rest
                 for c in flattenClassTree(rest):
                     yield c
 

mx_downstream.py

@@ -30,7 +30,7 @@
 
 from os.path import join, exists, isabs, basename
 from argparse import ArgumentParser
-from urlparse import urlparse
+from mx_portable import urllib_parse
 import os
 import mx
 import mx_urlrewrites
@@ -117,7 +117,7 @@ def ignore_output_root(d, names):
     targetDir = None
     for repoUrl in repoUrls:
         # Deduce a target name from the target URL
-        url = urlparse(repoUrl)
+        url = urllib_parse.urlparse(repoUrl)
         targetName = url.path
         if targetName.rfind('/') != -1:
             targetName = targetName[targetName.rfind('/') + 1:]

mx_gate.py

@@ -36,6 +36,8 @@
 import mx
 import sys
 
+from mx_portable import _basestring
+
 """
 Predefined Task tags.
 """
@@ -73,7 +75,7 @@ class Task:
 
     def tag_matches(self, _tags):
         for t in _tags:
-            assert isinstance(t, basestring), '{} is not a string and thus not a valid tag'.format(t)
+            assert isinstance(t, _basestring), '{} is not a string and thus not a valid tag'.format(t)
             if t in Task.tags:
                 if t not in Task.tags_range:
                     # no range restriction
@@ -457,6 +459,7 @@ def _run_gate(cleanArgs, args, tasks):
         if t:
             mx.command_function('version')(['--oneline'])
             mx.command_function('sversions')([])
+            mx.log("Python version: {}".format(sys.version_info))
 
     with Task('JDKReleaseInfo', tasks, tags=[Tags.always]) as t:
         if t:

mx_javamodules.py

@@ -30,7 +30,7 @@
 import re
 import zipfile
 import pickle
-import StringIO
+from mx_portable import StringIO, _cmp, _viewkeys
 import shutil
 import itertools
 from os.path import join, exists, dirname, basename
@@ -39,6 +39,7 @@
 from zipfile import ZipFile
 
 import mx
+from mx_portable import _basestring
 
 
 class JavaModuleDescriptor(object):
@@ -66,7 +67,7 @@ def __init__(self, name, exports, requires, uses, provides, packages=None, conce
         self.concealedRequires = concealedRequires if concealedRequires else {}
         self.uses = frozenset(uses)
         self.provides = provides
-        exportedPackages = frozenset(exports.viewkeys())
+        exportedPackages = frozenset(_viewkeys(exports))
         self.packages = exportedPackages if packages is None else frozenset(packages)
         assert len(exports) == 0 or exportedPackages.issubset(self.packages), exportedPackages - self.packages
         self.conceals = self.packages - exportedPackages
@@ -83,7 +84,7 @@ def __repr__(self):
 
     def __cmp__(self, other):
         assert isinstance(other, JavaModuleDescriptor)
-        return cmp(self.name, other.name)
+        return _cmp(self.name, other.name)
 
     @staticmethod
     def load(dist, jdk, fatalIfNotCreated=True):
@@ -145,17 +146,17 @@ def as_module_info(self):
         """
         Gets this module descriptor expressed as the contents of a ``module-info.java`` file.
         """
-        out = StringIO.StringIO()
+        out = StringIO()
         print('module ' + self.name + ' {', file=out)
-        for dependency, modifiers in sorted(self.requires.iteritems()):
+        for dependency, modifiers in sorted(self.requires.items()):
             modifiers_string = (' '.join(sorted(modifiers)) + ' ') if len(modifiers) != 0 else ''
             print('    requires ' + modifiers_string + dependency + ';', file=out)
-        for source, targets in sorted(self.exports.iteritems()):
+        for source, targets in sorted(self.exports.items()):
             targets_string = (' to ' + ', '.join(sorted(targets))) if len(targets) != 0 else ''
             print('    exports ' + source + targets_string + ';', file=out)
         for use in sorted(self.uses):
             print('    uses ' + use + ';', file=out)
-        for service, providers in sorted(self.provides.iteritems()):
+        for service, providers in sorted(self.provides.items()):
             print('    provides ' + service + ' with ' + ', '.join((p for p in providers)) + ';', file=out)
         for pkg in sorted(self.conceals):
             print('    // conceals: ' + pkg, file=out)
@@ -166,7 +167,7 @@ def as_module_info(self):
         if self.modulepath:
             print('    // modulepath: ' + ', '.join([jmd.name for jmd in self.modulepath]), file=out)
         if self.concealedRequires:
-            for dependency, packages in sorted(self.concealedRequires.iteritems()):
+            for dependency, packages in sorted(self.concealedRequires.items()):
                 for package in sorted(packages):
                     print('    // concealed-requires: ' + dependency + '/' + package, file=out)
         print('}', file=out)
@@ -365,7 +366,7 @@ def is_valid_module_name(name):
             provides.setdefault(service, []).extend(providers)
         else:
             mx.abort('Cannot parse module descriptor line: ' + str(parts))
-    packages.update(exports.viewkeys())
+    packages.update(_viewkeys(exports))
 
     if save:
         try:
@@ -603,7 +604,7 @@ def make_java_module(dist, jdk):
                 javacCmd.append('--upgrade-module-path')
                 javacCmd.append(os.pathsep.join(upgrademodulepathJars))
             if concealedRequires:
-                for module, packages_ in concealedRequires.iteritems():
+                for module, packages_ in concealedRequires.items():
                     for package in packages_:
                         javacCmd.append('--add-exports=' + module + '/' + package + '=' + moduleName)
             # https://blogs.oracle.com/darcy/new-javac-warning-for-setting-an-older-source-without-bootclasspath
@@ -659,10 +660,10 @@ def lookup_module(name):
     def add_transitive(mod):
         if mod not in transitive_closure:
             transitive_closure.add(mod)
-            for name in mod.requires.iterkeys():
+            for name in mod.requires.keys():
                 add_transitive(lookup_module(name))
     for root in roots:
-        if isinstance(root, basestring):
+        if isinstance(root, _basestring):
             root = lookup_module(root)
         add_transitive(root)
     return transitive_closure

mx_native.py

@@ -148,7 +148,7 @@ def _ninja_deps(cls):  # pylint: disable=no-self-argument
             Ninja.binary = mx.join(dep.get_path(False), 'ninja')
 
         try:
-            import ninja_syntax  # pylint: disable=unused-variable
+            import ninja_syntax  # pylint: disable=unused-variable, unused-import
         except ImportError:
             def raise_(e):
                 raise e

mx_portable.py

@@ -0,0 +1,59 @@
+import sys
+import itertools
+import subprocess
+
+if sys.version_info[0] < 3:
+    from StringIO import StringIO                   #pylint: disable=unused-import
+    import __builtin__ as builtins                  #pylint: disable=unused-import
+    import urllib2                                  #pylint: disable=unused-import
+    urllib_request = urllib2
+    urllib_error = urllib2
+    del urllib2
+    import urlparse as urllib_parse                 #pylint: disable=unused-import
+
+    _filter = itertools.ifilter
+    _cmp = cmp                                      #pylint: disable=undefined-variable
+    _raw_input = raw_input                          #pylint: disable=undefined-variable
+    _unicode = unicode                              #pylint: disable=undefined-variable
+    _long = long                                    #pylint: disable=undefined-variable
+    _basestring = basestring                        #pylint: disable=undefined-variable
+
+    def _py3_decode(x):
+        return x
+    def _py3_encode(x):
+        return x
+
+    def _func_code(f):
+        return f.func_code
+
+    def _viewkeys(dictionary):
+        return dictionary.viewkeys()
+else:
+    from io import StringIO                         #pylint: disable=unused-import
+    import builtins                                 #pylint: disable=unused-import
+    import urllib.request as urllib_request         #pylint: disable=unused-import,no-name-in-module
+    import urllib.error as urllib_error             #pylint: disable=unused-import,no-name-in-module
+    import urllib.parse as urllib_parse             #pylint: disable=unused-import,no-name-in-module
+
+    _filter = filter
+    def _cmp(a, b):
+        return (a > b) - (a < b)
+
+    _raw_input = input
+    _unicode = str
+    _long = int
+    _basestring = str
+
+    def _py3_decode(x):
+        return x.decode()
+    def _py3_encode(x):
+        return x.encode()
+
+    def _func_code(f):
+        return f.__code__
+
+    def _viewkeys(dictionary):
+        return dictionary.keys()
+
+def _check_output(*args, **kwargs):
+    return _py3_decode(subprocess.check_output(*args, **kwargs))

mx_unittest.py

@@ -187,14 +187,14 @@ def _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whit
                 mx.abort("Method specification is class#method: " + tests[0])
             t, method = words
 
-            for c, p in candidates.iteritems():
+            for c, p in candidates.items():
                 # prefer exact matches first
                 if t == c:
                     found = True
                     classes.append(c)
                     depsContainingTests.add(p)
             if not found:
-                for c, p in candidates.iteritems():
+                for c, p in candidates.items():
                     if t in c:
                         found = True
                         classes.append(c)
@@ -209,7 +209,7 @@ def _run_tests(args, harness, vmLauncher, annotations, testfile, blacklist, whit
             for t in tests:
                 if '#' in t:
                     mx.abort('Method specifications can only be used in a single test: ' + t)
-                for c, p in candidates.iteritems():
+                for c, p in candidates.items():
                     if t in c:
                         found = True
                         classes.append(c)