[GR-29724] Add interpreter benchmarks.

steve-s · steve-s · commit 035e76e6fd40 · 2021-03-03T17:01:06.000Z
PullRequest: graalpython/1645
diff --git a/ci.jsonnet b/ci.jsonnet
@@ -1 +1 @@
-{ "overlay": "69d605b7f6f27a86c9e0e8e6c0991a61dbf09d3f" }
+{ "overlay": "96dbc8ee7d775b1067f7462f80c50274f11045f0" }
diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/meso/raytrace-simple.py b/graalpython/com.oracle.graal.python.benchmarks/python/meso/raytrace-simple.py
@@ -365,5 +365,5 @@ def _main(width=400, height=400):
     s.addObject(Halfspace(Point(0,0,0), Vector.UP), CheckerboardSurface())
     s.render(c)
 
-def __benchmark__():
-    _main()
+def __benchmark__(*args):
+    _main(*args)
diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/builtin-len.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/builtin-len.py
@@ -33,13 +33,13 @@ def call_len(num, ll):
     return length
 
 
-def measure():
+def measure(num=500000000):
     ll = [x * 2 for x in range(1000)]
-    length = call_len(500000000, ll)  # 1000000000
+    length = call_len(num, ll)  # 1000000000
 
     print("Final length ", length)
 
 
 
 def __benchmark__(*args):
-    measure()
+    measure(*args)
diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-expression-sized.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-expression-sized.py
@@ -45,11 +45,11 @@ def call_generator_localvar(num, iteration):
     return item
 
 
-def measure(num):
-    last_item = call_generator(num, 10000)
+def measure(num, iteration):
+    last_item = call_generator(num, iteration)
 
     print("Last item ", last_item)
 
 
-def __benchmark__(num):
-    measure(num)
+def __benchmark__(num, it=10000):
+    measure(num, it)
diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-notaligned-sized.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-notaligned-sized.py
@@ -40,11 +40,11 @@ def call_generator(num, iteration):
     return item
 
 
-def measure(num):
-    last_item = call_generator(num, 10000)
+def measure(num, iteration):
+    last_item = call_generator(num, iteration)
 
     print("Last item ", last_item)
 
 
-def __benchmark__(num):
-    measure(num)
+def __benchmark__(num, it=10000):
+    measure(num, it)
diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-sized.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/generator-sized.py
@@ -39,11 +39,11 @@ def call_generator(num, iteration):
     return item
 
 
-def measure(num):
-    last_item = call_generator(num, 10000)
+def measure(num, iteration):
+    last_item = call_generator(num, iteration)
 
     print("Last item ", last_item)
 
 
-def __benchmark__(num):
-    measure(num)
+def __benchmark__(num, it=10000):
+    measure(num, it)
diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/genexp-builtin-call-sized.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/genexp-builtin-call-sized.py
@@ -55,10 +55,10 @@ def call_generator_localvar(num, iteration):
     return item
 
 
-def measure(num):
-    last_item = call_generator(num, 10000)
+def measure(num, iteration):
+    last_item = call_generator(num, iteration)
     print("Last item ", last_item)
 
 
-def __benchmark__(num=1000):
-    measure(num)
+def __benchmark__(num=1000, it=10000):
+    measure(num, it)
diff --git a/graalpython/com.oracle.graal.python.benchmarks/python/micro/special-len.py b/graalpython/com.oracle.graal.python.benchmarks/python/micro/special-len.py
@@ -47,12 +47,12 @@ def do_compute(num):
     return sum_
 
 
-def measure(num):
+def measure(num, iteration):
     for run in range(num):
-        sum_ = do_compute(20000)  # 10000
+        sum_ = do_compute(iteration)  # 10000
 
     print("sum", sum_)
 
 
-def __benchmark__(num=5):
-    measure(num)
+def __benchmark__(num=5, iteration=20000):
+    measure(num, iteration)
diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py
@@ -27,14 +27,15 @@
 import contextlib
 import datetime
 import glob
-import itertools
 import json
 import os
 import pathlib
 import platform
 import re
 import shlex
 import shutil
+
+import itertools
 import sys
 
 HPY_IMPORT_ORPHAN_BRANCH_NAME = "hpy-import"
@@ -57,12 +58,14 @@
 import mx_graalpython_bisect
 from mx_gate import Task
 from mx_graalpython_bench_param import PATH_MESO, BENCHMARKS, WARMUP_BENCHMARKS, JBENCHMARKS, PARSER_BENCHMARKS
-from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, GraalPythonVm, \
+from mx_graalpython_benchmark import PythonBenchmarkSuite, python_vm_registry, CPythonVm, PyPyVm, JythonVm, \
+    GraalPythonVm, \
     CONFIGURATION_DEFAULT, CONFIGURATION_SANDBOXED, CONFIGURATION_NATIVE, \
     CONFIGURATION_DEFAULT_MULTI, CONFIGURATION_SANDBOXED_MULTI, CONFIGURATION_NATIVE_MULTI, \
     CONFIGURATION_DEFAULT_MULTI_TIER, CONFIGURATION_NATIVE_MULTI_TIER, \
-    PythonInteropBenchmarkSuite, PythonVmWarmupBenchmarkSuite, PythonParserBenchmarkSuite
-
+    PythonInteropBenchmarkSuite, PythonVmWarmupBenchmarkSuite, PythonParserBenchmarkSuite, \
+    CONFIGURATION_INTERPRETER, CONFIGURATION_INTERPRETER_MULTI, CONFIGURATION_NATIVE_INTERPRETER, \
+    CONFIGURATION_NATIVE_INTERPRETER_MULTI
 
 if not sys.modules.get("__main__"):
     # workaround for pdb++
@@ -1490,9 +1493,15 @@ def _register_vms(namespace):
 
     # graalpython
     python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT), SUITE, 10)
+    python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_INTERPRETER, extra_polyglot_args=[
+        '--experimental-options', '--engine.Compilation=false'
+    ]), SUITE, 10)
     python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT_MULTI, extra_polyglot_args=[
         '--experimental-options', '-multi-context',
     ]), SUITE, 10)
+    python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_INTERPRETER_MULTI, extra_polyglot_args=[
+        '--experimental-options', '-multi-context', '--engine.Compilation=false'
+    ]), SUITE, 10)
     python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_DEFAULT_MULTI_TIER, extra_polyglot_args=[
         '--experimental-options', '--engine.MultiTier=true',
     ]), SUITE, 10)
@@ -1501,12 +1510,17 @@ def _register_vms(namespace):
     ]), SUITE, 10)
     python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE, extra_polyglot_args=[
     ]), SUITE, 10)
+    python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_INTERPRETER, extra_polyglot_args=[
+        '--experimental-options', '--engine.Compilation=false']), SUITE, 10)
     python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_SANDBOXED_MULTI, extra_polyglot_args=[
         '--experimental-options', '-multi-context', '--llvm.managed',
     ]), SUITE, 10)
     python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_MULTI, extra_polyglot_args=[
         '--experimental-options', '-multi-context',
     ]), SUITE, 10)
+    python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_INTERPRETER_MULTI, extra_polyglot_args=[
+        '--experimental-options', '-multi-context', '--engine.Compilation=false',
+    ]), SUITE, 10)
     python_vm_registry.add_vm(GraalPythonVm(config_name=CONFIGURATION_NATIVE_MULTI_TIER, extra_polyglot_args=[
         '--experimental-options', '--engine.MultiTier=true',
     ]), SUITE, 10)
diff --git a/mx.graalpython/mx_graalpython_bench_param.py b/mx.graalpython/mx_graalpython_bench_param.py
@@ -52,6 +52,8 @@
 ITER_20 = ['-i', '20']
 ITER_15 = ['-i', '15']
 ITER_10 = ['-i', '10']
+ITER_6 = ['-i', '6']
+WARMUP_2 = ['-w', '2']
 
 MICRO_BENCHMARKS = {
     'arith-binop': ITER_10 + ['5'],
@@ -101,6 +103,52 @@
     'tuple-indexing-from-literal': ITER_10 + ['10000000'],
 }
 
+MICRO_BENCHMARKS_SMALL = {
+    'arith-modulo-sized': ITER_6 + WARMUP_2 + ['1'],
+    'attribute-access-polymorphic': ITER_6 + WARMUP_2 + ['20'],
+    'attribute-access': ITER_6 + WARMUP_2 + ['100'],
+    'attribute-access-super': ITER_6 + WARMUP_2 + ['40'],
+    'attribute-bool': ITER_6 + WARMUP_2 + ['2'],
+    'boolean-logic-sized': ITER_6 + WARMUP_2 + ['10'],
+    'builtin-len-tuple-sized': ITER_6 + WARMUP_2 + ['10_000_000'],
+    'builtin-len': ITER_6 + WARMUP_2 + ['2_500_000'],
+    'class-access': ITER_6 + WARMUP_2 + ['40'],
+    'call-method-polymorphic': ITER_6 + WARMUP_2 + ['10'],
+    'for-range': ITER_6 + WARMUP_2 + WARMUP_2 + ['50'],
+    'function-call-sized': ITER_6 + WARMUP_2 + ['2_000_000'],
+    'generator-expression-sized': ITER_6 + WARMUP_2 + ['3000', '500'],
+    'generator-notaligned-sized': ITER_6 + WARMUP_2 + ['3000', '500'],
+    'generator-sized': ITER_6 + WARMUP_2 + ['3000', '500'],
+    'genexp-builtin-call-sized': ITER_6 + WARMUP_2 + ['3000', '500'],
+    'list-comp': ITER_6 + WARMUP_2 + ['25'],
+    'list-indexing': ITER_6 + WARMUP_2 + ['10_000'],
+    'list-indexing-from-constructor': ITER_6 + WARMUP_2 + ['250_000'],
+    'list-indexing-from-literal': ITER_6 + WARMUP_2 + ['250_000'],
+    'list-iterating-explicit': ITER_6 + WARMUP_2 + ['10_000'],
+    'list-iterating': ITER_6 + WARMUP_2 + ['25_000'],
+    'list-iterating-obj-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
+    'list-constructions-sized': ITER_6 + WARMUP_2 + ['500'],
+    'dict-getitem-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
+    'math-sqrt': ITER_6 + WARMUP_2 + ['20_000_000'],
+    'object-allocate': ITER_6 + WARMUP_2 + ['50'],
+    'object-layout-change': ITER_6 + WARMUP_2 + ['10_000'],
+    'special-add-int-sized': ITER_6 + WARMUP_2 + ['1_000'],
+    'special-add-sized': ITER_6 + WARMUP_2 + ['1_000'],
+    'special-len': ITER_6 + WARMUP_2 + ['1', '1_000'],
+    'member-access': ITER_6 + WARMUP_2 + ['10'],
+    'magic-bool-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
+    'magic-iter': ITER_6 + WARMUP_2 + ['250000'],
+    'call-classmethod-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
+    'mmap-anonymous-sized': ITER_6 + WARMUP_2 + ['1_000'],
+    'mmap-file': ITER_6 + WARMUP_2 + ['100'],
+    'generate-functions-sized': ITER_6 + WARMUP_2 + ['1_000_000'],
+    'try-except-simple': ITER_6 + WARMUP_2 + ['2_500_000'],
+    'try-except-store-simple': ITER_6 + WARMUP_2 + ['2_000_000'],
+    'try-except-store-two-types': ITER_6 + WARMUP_2 + ['1_000_000'],
+    'try-except-two-types': ITER_6 + WARMUP_2 + ['1_000_000'],
+    'tuple-indexing-from-constructor': ITER_6 + WARMUP_2 + ['250_000'],
+    'tuple-indexing-from-literal': ITER_6 + WARMUP_2 + ['400_000'],
+}
 
 def _pickling_benchmarks(module='pickle'):
     return {
@@ -179,6 +227,15 @@ def _pickling_benchmarks(module='pickle'):
 }
 
 
+MESO_BENCHMARKS_SMALL = {
+    'pads-eratosthenes': ITER_10 + WARMUP_2 + ['2000'],
+    'richards3': ITER_10 + WARMUP_2 + ['2'],
+    'chaos': ITER_10 + WARMUP_2 + ['2'],
+    'image-magix': ITER_10 + WARMUP_2 + ['100'],
+    'raytrace-simple': ITER_10 + WARMUP_2 + ['110', '110'],
+}
+
+
 MACRO_BENCHMARKS = {
     'gcbench': ITER_10 + ['10'],
 }
@@ -242,6 +299,8 @@ def _pickling_benchmarks(module='pickle'):
     "meso": [PATH_MESO, MESO_BENCHMARKS],
     "macro": [PATH_MACRO, MACRO_BENCHMARKS],
     "interop": [PATH_INTEROP, INTEROP_BENCHMARKS],
+    "micro-small": [PATH_MICRO, MICRO_BENCHMARKS_SMALL],
+    "meso-small": [PATH_MESO, MESO_BENCHMARKS_SMALL],
 }
 
 WARMUP_BENCHMARKS = {
diff --git a/mx.graalpython/mx_graalpython_benchmark.py b/mx.graalpython/mx_graalpython_benchmark.py
@@ -58,7 +58,11 @@
 SUBGROUP_GRAAL_PYTHON = "graalpython"
 PYTHON_VM_REGISTRY_NAME = "Python"
 CONFIGURATION_DEFAULT = "default"
+CONFIGURATION_INTERPRETER = "interpreter"
+CONFIGURATION_NATIVE_INTERPRETER = "native-interpreter"
 CONFIGURATION_DEFAULT_MULTI = "default-multi"
+CONFIGURATION_INTERPRETER_MULTI = "interpreter-multi"
+CONFIGURATION_NATIVE_INTERPRETER_MULTI = "native-interpreter-multi"
 CONFIGURATION_DEFAULT_MULTI_TIER = "default-multi-tier"
 CONFIGURATION_NATIVE = "native"
 CONFIGURATION_NATIVE_MULTI = "native-multi"
@@ -334,7 +338,7 @@ def run(self, cwd, args):
                 if len(cp) > 0:
                     extra_polyglot_args.append("--vm.classpath="+":".join(cp))
 
-                return host_vm.run_lang('graalpython', extra_polyglot_args + args, cwd)
+                return host_vm.run_launcher('graalpython', extra_polyglot_args + args, cwd)
 
         # Otherwise, we're running from the source tree
         truffle_options = [
@@ -790,7 +794,7 @@ def createCommandLineArgs(self, benchmarks, bmSuiteArgs):
         if mx.suite("tools", fatalIfMissing=False):
             dists.extend(('CHROMEINSPECTOR', 'TRUFFLE_PROFILER'))
         if mx.suite("sulong", fatalIfMissing=False):
-            dists.append('SULONG')
+            dists.append('SULONG_NATIVE')
             if mx.suite("sulong-managed", fatalIfMissing=False):
                 dists.append('SULONG_MANAGED')
 

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-{ "overlay": "69d605b7f6f27a86c9e0e8e6c0991a61dbf09d3f" }`
	`1`	`+{ "overlay": "96dbc8ee7d775b1067f7462f80c50274f11045f0" }`