mx python3-unittest: add -o option (filter only specified tests)

add missing header

Author: cosminbasca

graalpython/com.oracle.graal.python.test/src/python_unittests.py

@@ -1,3 +1,40 @@
+# Copyright (c) 2018, Oracle and/or its affiliates.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or data
+# (collectively the "Software"), free of charge and under any and all copyright
+# rights in the Software, and any and all patent rights owned or freely
+# licensable by each licensor hereunder covering either (i) the unmodified
+# Software as contributed to or provided by such licensor, or (ii) the Larger
+# Works (as defined below), to deal in both
+#
+# (a) the Software, and
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+#     one is included with the Software (each a "Larger Work" to which the
+#     Software is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
 import csv
 import os
 import re
@@ -228,9 +265,22 @@ class Col(object):
 ]
 
 
+class Stat(object):
+    # unittest level aggregates
+    UT_TOTAL = "ut_total"  # all the unittests
+    UT_RUNS = 'ut_runs'  # all unittests which could run
+    UT_PASS = 'ut_pass'  # all unittests which pass
+    UT_PERCENT_RUNS = "ut_percent_runs"  # all unittests which could run even with failures (percent)
+    UT_PERCENT_PASS = "ut_percent_pass"  # all unittests which could run with no failures (percent)
+    # test level aggregates
+    TEST_RUNS = "test_runs"  # total number of tests that could be loaded and run even with failures
+    TEST_PASS = "test_pass"  # number of tests which ran
+    TEST_PERCENT_PASS = "test_percent_pass"  # percentage of tests which pass from all running tests (all unittests)
+
+
 def save_as_csv(report_path, unittests, error_messages, stats, current_date):
     rows = []
-    with open(file_name(CSV_RESULTS_NAME, current_date), 'w') as CSV:
+    with open(report_path, 'w') as CSV:
         totals = {
             Col.NUM_TESTS: 0,
             Col.NUM_FAILS: 0,
@@ -266,16 +316,23 @@ def save_as_csv(report_path, unittests, error_messages, stats, current_date):
             else:
                 total_not_run_at_all += 1
 
-        _all_runs = len(unittests)-total_not_run_at_all
-        _all_total = len(unittests)
-        _percent_all_runs = float(_all_runs) / float(_all_total) * 100.0
-        _percent_all_full_passes = float(total_pass_all) / float(_all_total) * 100.0
+        # unittest stats
+        totals[Stat.UT_TOTAL] = len(unittests)
+        totals[Stat.UT_RUNS] = len(unittests) - total_not_run_at_all
+        totals[Stat.UT_PASS] = total_pass_all
+        totals[Stat.UT_PERCENT_RUNS] = float(totals[Stat.UT_RUNS]) / float(totals[Stat.UT_TOTAL]) * 100.0
+        totals[Stat.UT_PERCENT_PASS] = float(totals[Stat.UT_PASS]) / float(totals[Stat.UT_TOTAL]) * 100.0
+        # test stats
+        totals[Stat.TEST_RUNS] = totals[Col.NUM_TESTS]
+        totals[Stat.TEST_PASS] = totals[Col.NUM_PASSES]
+        totals[Stat.TEST_PERCENT_PASS] = float(totals[Stat.TEST_PASS]) / float(totals[Stat.TEST_RUNS]) * 100.0 \
+            if totals[Stat.TEST_RUNS] else 0
 
-        _test_runs = totals[Col.NUM_PASSES]
-        _test_total = totals[Col.NUM_TESTS]
-        _percent_test_runs = float(_test_runs) / float(_test_total) * 100.0 if _test_total else 0
-
-        rows.append({
+        writer = csv.DictWriter(CSV, fieldnames=CSV_HEADER)
+        writer.writeheader()
+        for row in rows:
+            writer.writerow(row)
+        writer.writerow({
             Col.UNITTEST: 'TOTAL',
             Col.NUM_TESTS: totals[Col.NUM_TESTS],
             Col.NUM_FAILS: totals[Col.NUM_FAILS],
@@ -284,16 +341,13 @@ def save_as_csv(report_path, unittests, error_messages, stats, current_date):
             Col.NUM_PASSES: totals[Col.NUM_PASSES],
             Col.PYTHON_ERRORS: 'Could run {0}/{1} unittests ({2:.2f}%). Unittests which pass completely: {3:.2f}%. '
                                'Of the ones which ran, could run: {4}/{5} tests ({6:.2f}%)'.format(
-                _all_runs, _all_total, _percent_all_runs, _percent_all_full_passes,
-                _test_runs, _test_total, _percent_test_runs)
+                totals[Stat.UT_RUNS], totals[Stat.UT_TOTAL],
+                totals[Stat.UT_PERCENT_RUNS], totals[Stat.UT_PERCENT_PASS],
+                totals[Stat.TEST_PASS], totals[Stat.TEST_PASS],
+                totals[Stat.TEST_PERCENT_PASS])
         })
 
-        writer = csv.DictWriter(CSV, fieldnames=CSV_HEADER)
-        writer.writeheader()
-        for row in rows:
-            writer.writerow(row)
-
-        return rows
+        return rows, totals
 
 
 HTML_TEMPLATE = '''
@@ -370,35 +424,88 @@ def save_as_csv(report_path, unittests, error_messages, stats, current_date):
 '''
 
 
-def save_as_html(report_name, rows, missing_modules, current_date):
+def save_as_html(report_name, rows, totals, missing_modules, current_date):
+    def grid(*components):
+        def _fmt(cmp):
+            if isinstance(cmp, tuple):
+                return '<div class="col-sm-{}">{}</div>'.format(cmp[1], cmp[0])
+            return '<div class="col-sm">{}</div>'.format(cmp)
+        return '''
+        <div class="container" style="width: 100%;">
+          <div class="row">
+            {}
+          </div>
+        </div>
+        '''.format('\n'.join([_fmt(cmp) for cmp in components]))
+
+    def progress_bar(value, color='success'):
+        if 0.0 <= value <= 1.0:
+            value = 100 * value
+        return '''
+        <div class="progress">
+          <div class="progress-bar progress-bar-{color}" role="progressbar" aria-valuenow="{value}"
+            aria-valuemin="0" aria-valuemax="100" style="width:{value}%">
+            {value:.2f}% Complete
+          </div>
+        </div>
+        '''.format(color=color, value=value)
+
+    def fluid_div(title, div_content):
+        return '''
+        <div class="container-fluid">
+            <div class="panel panel-default">
+                <div class="panel-heading clickable">
+                    <h3 class="panel-title"><i class="fa fa-minus-square-o" aria-hidden="true">&nbsp;&nbsp;</i>{title}</h3>
+                </div>
+                {content}
+            </div>
+        </div>
+        '''.format(title=title, content=div_content)
+
+    def ul(title, items):
+        return fluid_div(title, '<ul class="list-group">{}</ul>'.format('\n'.join([
+            '<li class="list-group-item">{}</span></li>'.format(itm) for itm in items
+        ])))
+
     def table(tid, tcols, trows):
-        thead = '''
+        _thead = '''
             <tr class="text-align: right;">
                 <th data-orderable="false">&nbsp;</th>
                 {columns}
             </tr>
             '''.format(columns='\n'.join(['<th>{}</th>'.format(c) for c in tcols]))
 
-        format_val = lambda row, k: '<code>{}</code>'.format(row[k]) if k == Col.PYTHON_ERRORS else row[k]
+        def format_val(row, k):
+            value = row[k]
+            if k == Col.PYTHON_ERRORS:
+                return '<code class="h6">{}</code>'.format(value)
+            elif k == Col.UNITTEST:
+                _class = "text-info"
+            elif k == Col.NUM_PASSES and value > 0:
+                _class = "text-success"
+            elif k in [Col.NUM_ERRORS, Col.NUM_FAILS] and value > 0:
+                _class = "text-danger"
+            elif k == Col.NUM_SKIPPED and value > 0:
+                _class = "text-warning"
+            elif k == Col.NUM_TESTS:
+                _class = "text-dark"
+            else:
+                _class = "text-danger" if value < 0 else "text-muted"
+            return '<span class="{} h6"><b>{}</b></span>'.format(_class, value)
 
-        tbody = '\n'.join([
+        _tbody = '\n'.join([
                 '<tr class="{cls}"><td>{i}</td>{vals}</tr>'.format(
                     cls='info' if i % 2 == 0 else '', i=i,
                     vals=' '.join(['<td>{}</td>'.format(format_val(row, k)) for k in tcols]))
                 for i, row in enumerate(trows)])
 
-        return '''
-        <div class="container-fluid">
-            <div class="panel panel-default">
-                <div class="panel-heading clickable">
-                    <h3 class="panel-title"><i class="fa fa-minus-square-o" aria-hidden="true">&nbsp;&nbsp;</i>{title}</h3>
-                </div>
-                <table id="{tid}" class="table {tclass}" cellspacing="0" width="100%">
-                    <thead>{thead}</thead><tbody>{tbody}</tbody>
-                </table>
-            </div>
-        </div>
-        '''.format(title='unittest run statistics', tid=tid, tclass='', thead=thead, tbody=tbody)
+        _table = '''
+        <table id="{tid}" class="table {tclass}" cellspacing="0" width="100%">
+            <thead>{thead}</thead><tbody>{tbody}</tbody>
+        </table>
+        '''.format(tid=tid, tclass='', thead=_thead, tbody=_tbody)
+
+        return fluid_div('<b>cPython unittests</b> run statistics', _table)
 
     scripts = '''
         <script src="https:////cdn.datatables.net/{datatables_version}/js/jquery.dataTables.min.js"></script>
@@ -414,21 +521,25 @@ def table(tid, tcols, trows):
         </script>
         '''
 
-    modules_info = '''
-        <div class="container-fluid">
-            <div class="panel panel-default">
-                <div class="panel-heading clickable">
-                    <h3 class="panel-title"><i class="fa fa-minus-square-o" aria-hidden="true">&nbsp;&nbsp;</i>{title}</h3>
-                </div>
-                <ul class="list-group">{content}</ul>
-            </div>
-        </div>
-    '''.format(title='missing modules', content='\n'.join([
-        '<li class="list-group-item"><b>{}</b>&nbsp;<span class="text-muted">count: {}</span></li>'.format(name, cnt)
+    missing_modules_info = ul('missing modules', [
+        '<b>{}</b>&nbsp;<span class="text-muted">count: {}</span>'.format(name, cnt)
         for cnt, name in sorted(((cnt, name) for name, cnt in missing_modules.items()), reverse=True)
-    ]))
+    ])
+
+    total_stats_info = ul("<b>Summary</b>", [
+        grid('<b># total</b> unittests: {}'.format(totals[Stat.UT_TOTAL])),
+        grid((progress_bar(totals[Stat.UT_PERCENT_RUNS], color="info"), 3),
+             '<b># unittest</b> which run: {}'.format(totals[Stat.UT_RUNS])),
+        grid((progress_bar(totals[Stat.UT_PERCENT_PASS], color="success"), 3),
+             '<b># unittest</b> which pass: {}'.format(totals[Stat.UT_PASS])),
+        grid('<b># tests</b> which run: {}'.format(totals[Stat.TEST_RUNS])),
+        grid((progress_bar(totals[Stat.TEST_PERCENT_PASS], color="info"), 3),
+             '<b># tests</b> which pass: {}'.format(totals[Stat.TEST_PASS])),
+    ])
 
-    content = modules_info + table('stats', CSV_HEADER, rows)
+    table_stats = table('stats', CSV_HEADER, rows)
+
+    content = ' <br> '.join([total_stats_info, table_stats, missing_modules_info])
 
     report = HTML_TEMPLATE.format(
         title='GraalPython Unittests Stats',
@@ -459,6 +570,7 @@ def main(prog, args):
     parser.add_argument("-v", "--verbose", help="Verbose output.", action="store_true")
     parser.add_argument("-l", "--limit", help="Limit the number of unittests to run.", default=None, type=int)
     parser.add_argument("-t", "--tests_path", help="Unittests path.", default=PATH_UNITTESTS)
+    parser.add_argument("-o", "--only_tests", help="Run only these unittests (comma sep values).", default=None)
     parser.add_argument("path", help="Path to store the csv output and logs to.", nargs='?', default=None)
 
     global flags
@@ -473,18 +585,26 @@ def main(prog, args):
     else:
         log("[INFO] results will not be saved remotely")
 
-    unittests = get_unittests(flags.tests_path, limit=flags.limit)
+    if flags.only_tests:
+        def _fmt(t):
+            t = t.strip()
+            return os.path.join(flags.tests_path, t if t.endswith(".py") else t + ".py")
+        only_tests = set([_fmt(test) for test in flags.only_tests.split(",")])
+        unittests = [t for t in get_unittests(flags.tests_path) if t in only_tests]
+    else:
+        unittests = get_unittests(flags.tests_path, limit=flags.limit)
+
     results = run_unittests(unittests)
     unittests, error_messages, stats = process_output('\n'.join(results))
 
     csv_report_path = file_name(CSV_RESULTS_NAME, current_date)
-    rows = save_as_csv(csv_report_path, unittests, error_messages, stats, current_date)
+    rows, totals = save_as_csv(csv_report_path, unittests, error_messages, stats, current_date)
 
     missing_modules = process_errors(unittests, error_messages, 'ModuleNotFoundError', get_missing_module)
     log("[MISSING MODULES] \n{}", pformat(dict(missing_modules)))
 
     html_report_path = file_name(HTML_RESULTS_NAME, current_date)
-    save_as_html(html_report_path, rows, missing_modules, current_date)
+    save_as_html(html_report_path, rows, totals, missing_modules, current_date)
 
     if flags.path:
         log("[SAVE] saving results to {} ... ", flags.path)