[GR-12625] add cpython stats to running unittests

PullRequest: graalpython/284

Author: cosminbasca

graalpython/com.oracle.graal.python.test/src/python_unittests.py

@@ -62,17 +62,18 @@
 CSV_RESULTS_NAME = "{}.csv".format(_BASE_NAME)
 HTML_RESULTS_NAME = "{}.html".format(_BASE_NAME)
 
+HR = "".join(['-' for _ in range(120)])
 
 PTRN_ERROR = re.compile(r'^(?P<error>[A-Z][a-z][a-zA-Z]+):(?P<message>.*)$')
-PTRN_UNITTEST = re.compile(r'^#### running: graalpython/lib-python/3/test/(?P<unittest>.*)$')
+PTRN_UNITTEST = re.compile(r'^#### running: graalpython/lib-python/3/test/(?P<unittest>[\w.]+).*$', re.DOTALL)
 PTRN_NUM_TESTS = re.compile(r'^Ran (?P<num_tests>\d+) test.*$')
 PTRN_FAILED = re.compile(
     r'^FAILED \((failures=(?P<failures>\d+))?(, )?(errors=(?P<errors>\d+))?(, )?(skipped=(?P<skipped>\d+))?\)$')
 PTRN_OK = re.compile(
     r'^OK \((failures=(?P<failures>\d+))?(, )?(errors=(?P<errors>\d+))?(, )?(skipped=(?P<skipped>\d+))?\)$')
 PTRN_JAVA_EXCEPTION = re.compile(r'^(?P<exception>com\.oracle\.[^:]*):(?P<message>.*)')
 PTRN_MODULE_NOT_FOUND = re.compile(r'.*ModuleNotFound: \'(?P<module>.*)\'\..*', re.DOTALL)
-
+PTRN_IMPORT_ERROR = re.compile(r".*cannot import name \'(?P<module>.*)\'.*", re.DOTALL)
 
 # ----------------------------------------------------------------------------------------------------------------------
 #
@@ -126,8 +127,11 @@ def scp(results_file_path, destination_path, destination_name=None):
     return _run_cmd(cmd)[0]
 
 
-def _run_unittest(test_path):
-    cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path, "-v"]
+def _run_unittest(test_path, with_cpython=False):
+    if with_cpython:
+        cmd = ["python3", test_path, "-v"]
+    else:
+        cmd = ["mx", "python3", "--python.CatchAllExceptions=true", test_path, "-v"]
     success, output = _run_cmd(cmd)
     output = '''
 ##############################################################
@@ -139,7 +143,7 @@ def _run_unittest(test_path):
 TIMEOUT = 60 * 20  # 20 mins per unittest wait time max ...
 
 
-def run_unittests(unittests, timeout):
+def run_unittests(unittests, timeout, with_cpython=False):
     assert isinstance(unittests, (list, tuple))
     num_unittests = len(unittests)
     log("[EXEC] running {} unittests ... ", num_unittests)
@@ -148,7 +152,7 @@ def run_unittests(unittests, timeout):
 
     pool = Pool()
     for ut in unittests:
-        results.append(pool.apply_async(_run_unittest, args=(ut, )))
+        results.append(pool.apply_async(_run_unittest, args=(ut, with_cpython)))
     pool.close()
 
     log("[INFO] collect results ... ")
@@ -163,11 +167,12 @@ def run_unittests(unittests, timeout):
             timed_out.append(unittests[i])
         log("[PROGRESS] {} / {}: \t {}%", i+1, num_unittests, int(((i+1) * 100.0) / num_unittests))
 
-    log("".join(['-' for i in range(120)]))
-    for t in timed_out:
-        log("[TIMEOUT] skipped: {}", t)
-    log("".join(['-' for i in range(120)]))
-    log("[STATS] processed {} out of {} unitttests", num_unittests - len(timed_out), num_unittests)
+    if timed_out:
+        log(HR)
+        for t in timed_out:
+            log("[TIMEOUT] skipped: {}", t)
+        log(HR)
+    log("[STATS] processed {} out of {} unittests", num_unittests - len(timed_out), num_unittests)
     pool.terminate()
     pool.join()
     return out
@@ -279,11 +284,14 @@ def process_output(output_lines):
 # python  error processing
 #
 # ----------------------------------------------------------------------------------------------------------------------
-def process_errors(unittests, error_messages, error=None, msg_processor=None):
+def process_errors(unittests, error_messages, err=None, msg_processor=None):
+    if isinstance(err, str):
+        err = {err,}
+
     def _err_filter(item):
-        if not error:
+        if not err:
             return True
-        return item[0] == error
+        return item[0] in err
 
     def _processor(msg):
         if not msg_processor:
@@ -304,6 +312,11 @@ def get_missing_module(msg):
     return match.group('module') if match else None
 
 
+def get_cannot_import_module(msg):
+    match = re.match(PTRN_IMPORT_ERROR, msg)
+    return match.group('module') if match else None
+
+
 # ----------------------------------------------------------------------------------------------------------------------
 #
 # csv reporting
@@ -317,6 +330,11 @@ class Col(object):
     NUM_SKIPPED = 'num_skipped'
     NUM_PASSES = 'num_passes'
     PYTHON_ERRORS = 'python_errors'
+    CPY_NUM_TESTS = 'cpy_num_tests'
+    CPY_NUM_FAILS = 'cpy_num_fails'
+    CPY_NUM_ERRORS = 'cpy_num_errors'
+    CPY_NUM_SKIPPED = 'cpy_num_skipped'
+    CPY_NUM_PASSES = 'cpy_num_passes'
 
 
 CSV_HEADER = [
@@ -326,6 +344,11 @@ class Col(object):
     Col.NUM_ERRORS,
     Col.NUM_SKIPPED,
     Col.NUM_PASSES,
+    Col.CPY_NUM_TESTS,
+    Col.CPY_NUM_FAILS,
+    Col.CPY_NUM_ERRORS,
+    Col.CPY_NUM_SKIPPED,
+    Col.CPY_NUM_PASSES,
     Col.PYTHON_ERRORS
 ]
 
@@ -350,7 +373,7 @@ def save_as_txt(report_path, results):
         return output
 
 
-def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
+def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats, cpy_stats=None):
     rows = []
     with open(report_path, 'w') as CSV:
         totals = {
@@ -365,18 +388,27 @@ def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
 
         for unittest in unittests:
             unittest_stats = stats[unittest]
+            cpy_unittest_stats = cpy_stats[unittest] if cpy_stats else None
             unittest_errmsg = error_messages[unittest]
             if not unittest_errmsg:
                 unittest_errmsg = java_exceptions[unittest]
 
             rows.append({
                 Col.UNITTEST: unittest,
+                # graalpython stats
                 Col.NUM_TESTS: unittest_stats.num_tests,
                 Col.NUM_FAILS: unittest_stats.num_fails,
                 Col.NUM_ERRORS: unittest_stats.num_errors,
                 Col.NUM_SKIPPED: unittest_stats.num_skipped,
                 Col.NUM_PASSES: unittest_stats.num_passes,
-                Col.PYTHON_ERRORS: dumps(list(unittest_errmsg))
+                # cpython stats
+                Col.CPY_NUM_TESTS: cpy_unittest_stats.num_tests if cpy_unittest_stats else None,
+                Col.CPY_NUM_FAILS: cpy_unittest_stats.num_fails if cpy_unittest_stats else None,
+                Col.CPY_NUM_ERRORS: cpy_unittest_stats.num_errors if cpy_unittest_stats else None,
+                Col.CPY_NUM_SKIPPED: cpy_unittest_stats.num_skipped if cpy_unittest_stats else None,
+                Col.CPY_NUM_PASSES: cpy_unittest_stats.num_passes if cpy_unittest_stats else None,
+                # errors
+                Col.PYTHON_ERRORS: dumps(list(unittest_errmsg)),
             })
 
             # update totals that ran in some way
@@ -499,7 +531,7 @@ def save_as_csv(report_path, unittests, error_messages, java_exceptions, stats):
 '''
 
 
-def save_as_html(report_name, rows, totals, missing_modules, java_issues, current_date):
+def save_as_html(report_name, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date):
     def grid(*components):
         def _fmt(cmp):
             if isinstance(cmp, tuple):
@@ -601,6 +633,11 @@ def format_val(row, k):
         for cnt, name in sorted(((cnt, name) for name, cnt in missing_modules.items()), reverse=True)
     ])
 
+    cannot_import_modules_info = ul('modules which could not be imported', [
+        '<b>{}</b>&nbsp;<span class="text-muted">could not be imported by {} unittests</span>'.format(name, cnt)
+        for cnt, name in sorted(((cnt, name) for name, cnt in cannot_import_modules.items()), reverse=True)
+    ])
+
     java_issues_info = ul('Java issues', [
         '<b>{}</b>&nbsp;<span class="text-muted">caused by {} unittests</span>'.format(name, cnt)
         for cnt, name in sorted(((cnt, name) for name, cnt in java_issues.items()), reverse=True)
@@ -619,7 +656,10 @@ def format_val(row, k):
 
     table_stats = table('stats', CSV_HEADER, rows)
 
-    content = ' <br> '.join([total_stats_info, table_stats, missing_modules_info, java_issues_info])
+    content = ' <br> '.join([total_stats_info, table_stats,
+                             missing_modules_info,
+                             cannot_import_modules_info,
+                             java_issues_info])
 
     report = HTML_TEMPLATE.format(
         title='GraalPython Unittests Stats',
@@ -679,24 +719,37 @@ def _fmt(t):
         skip_tests = set([_fmt(test) for test in flags.skip_tests.split(",")]) if flags.skip_tests else None
         unittests = get_unittests(flags.tests_path, limit=flags.limit, skip_tests=skip_tests)
 
-    results = run_unittests(unittests, flags.timeout)
+    # get cpython stats
+    log(HR)
+    log("[INFO] get cpython stats")
+    cpy_results = run_unittests(unittests, 60 * 5, with_cpython=True)
+    cpy_stats = process_output('\n'.join(cpy_results))[-1]
+
+    # get graalpython stats
+    log(HR)
+    log("[INFO] get graalpython stats")
+    results = run_unittests(unittests, flags.timeout, with_cpython=False)
     txt_report_path = file_name(TXT_RESULTS_NAME, current_date)
     output = save_as_txt(txt_report_path, results)
 
     unittests, error_messages, java_exceptions, stats = process_output(output)
 
     csv_report_path = file_name(CSV_RESULTS_NAME, current_date)
-    rows, totals = save_as_csv(csv_report_path, unittests, error_messages, java_exceptions, stats)
+    rows, totals = save_as_csv(csv_report_path, unittests, error_messages, java_exceptions, stats, cpy_stats=cpy_stats)
 
-    missing_modules = process_errors(unittests, error_messages, error='ModuleNotFoundError',
+    missing_modules = process_errors(unittests, error_messages, 'ModuleNotFoundError',
                                      msg_processor=get_missing_module)
     log("[MISSING MODULES] \n{}", pformat(dict(missing_modules)))
 
+    cannot_import_modules = process_errors(unittests, error_messages, err='ImportError',
+                                           msg_processor=get_cannot_import_module)
+    log("[CANNOT IMPORT MODULES] \n{}", pformat(dict(cannot_import_modules)))
+
     java_issues = process_errors(unittests, java_exceptions)
     log("[JAVA ISSUES] \n{}", pformat(dict(java_issues)))
 
     html_report_path = file_name(HTML_RESULTS_NAME, current_date)
-    save_as_html(html_report_path, rows, totals, missing_modules, java_issues, current_date)
+    save_as_html(html_report_path, rows, totals, missing_modules, cannot_import_modules, java_issues, current_date)
 
     if flags.path:
         log("[SAVE] saving results to {} ... ", flags.path)

mx.graalpython/mx_graalpython_benchmark.py

@@ -122,52 +122,60 @@ def run(self, cwd, args):
         return ret_code, out.data
 
 
-class CPythonVm(AbstractPythonVm):
-    PYTHON_INTERPRETER = "python3"
-
-    def __init__(self, config_name, options=None, virtualenv=None, no_warmup=False):
-        super(CPythonVm, self).__init__(config_name, options)
-        self._virtualenv = virtualenv
-        self._no_warmup = no_warmup
-
-    @property
-    def interpreter(self):
-        if self._virtualenv:
-            return os.path.join(self._virtualenv, CPythonVm.PYTHON_INTERPRETER)
-        return CPythonVm.PYTHON_INTERPRETER
+class AbstractPythonIterationsControlVm(AbstractPythonVm):
+    __metaclass__ = ABCMeta
 
-    def name(self):
-        return VM_NAME_CPYTHON
+    def __init__(self, config_name, options=None, iterations=None):
+        super(AbstractPythonIterationsControlVm, self).__init__(config_name, options)
+        self._iterations = iterations
 
-    @staticmethod
-    def remove_warmup_runs(args):
+    def _override_iterations_args(self, args):
         _args = []
         i = 0
         while i < len(args):
             arg = args[i]
             _args.append(arg)
             if arg == '-i':
-                _args.append('0')
+                _args.append(str(self._iterations))
                 i += 1
             i += 1
         return _args
 
     def run(self, cwd, args):
-        if self._no_warmup:
-            args = CPythonVm.remove_warmup_runs(args)
-        return super(CPythonVm, self).run(cwd, args)
+        if isinstance(self._iterations, (int, long)):
+            args = self._override_iterations_args(args)
+        return super(AbstractPythonIterationsControlVm, self).run(cwd, args)
 
 
-class PyPyVm(AbstractPythonVm):
-    def __init__(self, config_name, options=None):
-        super(PyPyVm, self).__init__(config_name, options=options)
+class CPythonVm(AbstractPythonIterationsControlVm):
+    PYTHON_INTERPRETER = "python3"
+
+    def __init__(self, config_name, options=None, virtualenv=None, iterations=False):
+        super(CPythonVm, self).__init__(config_name, options=options, iterations=iterations)
+        self._virtualenv = virtualenv
+
+    @property
+    def interpreter(self):
+        if self._virtualenv:
+            return os.path.join(self._virtualenv, CPythonVm.PYTHON_INTERPRETER)
+        return CPythonVm.PYTHON_INTERPRETER
+
+    def name(self):
+        return VM_NAME_CPYTHON
+
+
+class PyPyVm(AbstractPythonIterationsControlVm):
+    PYPY_INTERPRETER = "pypy3"
+
+    def __init__(self, config_name, options=None, iterations=False):
+        super(PyPyVm, self).__init__(config_name, options=options, iterations=iterations)
 
     @property
     def interpreter(self):
         home = mx.get_env(ENV_PYPY_HOME)
         if not home:
             mx.abort("{} is not set!".format(ENV_PYPY_HOME))
-        return join(home, 'bin', 'pypy3')
+        return join(home, 'bin', PyPyVm.PYPY_INTERPRETER)
 
     def name(self):
         return VM_NAME_PYPY