[GR-19973] Improve list overallocation

PullRequest: graalpython/865

Author: qunaibit

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/SequenceStorageNodes.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -29,6 +29,7 @@
 import com.oracle.graal.python.builtins.objects.code.PCode;
 import com.oracle.graal.python.builtins.objects.frame.PFrame;
 import com.oracle.graal.python.builtins.objects.function.PArguments;
+import com.oracle.graal.python.builtins.objects.iterator.PRangeIterator;
 import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject;
 import com.oracle.graal.python.builtins.objects.type.LazyPythonClass;
 import com.oracle.graal.python.parser.ExecutionCellSlots;
@@ -59,9 +60,11 @@ public final class PGenerator extends PythonBuiltinObject {
     private boolean finished;
     private PCode code;
     private int currentCallTarget;
+    private final Object iterator;
+    private final boolean isPRangeIterator;
 
     public static PGenerator create(LazyPythonClass clazz, String name, RootCallTarget[] callTargets, FrameDescriptor frameDescriptor, Object[] arguments, PCell[] closure,
-                    ExecutionCellSlots cellSlots, int numOfActiveFlags, int numOfGeneratorBlockNode, int numOfGeneratorForNode, PythonObjectFactory factory) {
+                    ExecutionCellSlots cellSlots, int numOfActiveFlags, int numOfGeneratorBlockNode, int numOfGeneratorForNode, PythonObjectFactory factory, Object iterator) {
         /*
          * Setting up the persistent frame in {@link #arguments}.
          */
@@ -90,10 +93,10 @@ public static PGenerator create(LazyPythonClass clazz, String name, RootCallTarg
             generatorFrame.setObject(cellVarSlots[i], new PCell(cellVarAssumptions[i]));
         }
         PArguments.setGeneratorFrameLocals(generatorFrameArguments, factory.createDictLocals(generatorFrame));
-        return new PGenerator(clazz, name, callTargets, frameDescriptor, arguments, closure);
+        return new PGenerator(clazz, name, callTargets, frameDescriptor, arguments, closure, iterator);
     }
 
-    private PGenerator(LazyPythonClass clazz, String name, RootCallTarget[] callTargets, FrameDescriptor frameDescriptor, Object[] arguments, PCell[] closure) {
+    private PGenerator(LazyPythonClass clazz, String name, RootCallTarget[] callTargets, FrameDescriptor frameDescriptor, Object[] arguments, PCell[] closure, Object iterator) {
         super(clazz);
         this.name = name;
         this.callTargets = callTargets;
@@ -102,6 +105,8 @@ private PGenerator(LazyPythonClass clazz, String name, RootCallTarget[] callTarg
         this.arguments = arguments;
         this.closure = closure;
         this.finished = false;
+        this.iterator = iterator;
+        this.isPRangeIterator = iterator != null && iterator instanceof PRangeIterator;
     }
 
     public FrameDescriptor getFrameDescriptor() {
@@ -137,6 +142,14 @@ public PCell[] getClosure() {
         return closure;
     }
 
+    public Object getIterator() {
+        return iterator;
+    }
+
+    public boolean isPRangeIterator() {
+        return isPRangeIterator;
+    }
+
     @Override
     public String toString() {
         return "<generator object '" + name + "' at " + hashCode() + ">";

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/PGenerator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -25,8 +25,10 @@
  */
 package com.oracle.graal.python.builtins.objects.iterator;
 
+import com.oracle.graal.python.builtins.objects.range.PRange;
 import com.oracle.graal.python.builtins.objects.type.LazyPythonClass;
 import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.profiles.ConditionProfile;
 
 public final class PRangeIterator extends PIntegerIterator {
 
@@ -53,6 +55,17 @@ public int getStep() {
         return step;
     }
 
+    public int getLength(ConditionProfile stepProfile, ConditionProfile positveRangeProfile) {
+        if (stepProfile.profile(step > 0)) {
+            if (positveRangeProfile.profile(index >= 0 && index < stop)) {
+                return (stop - index) / step;
+            }
+            return PRange.getLenOfRange(index, stop, step);
+        } else {
+            return PRange.getLenOfRange(stop, index, -step);
+        }
+    }
+
     @Override
     public int next() {
         assert hasNext();

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/iterator/PRangeIterator.java

@@ -64,10 +64,12 @@
 import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.ListGeneralizationNode;
 import com.oracle.graal.python.builtins.objects.function.PArguments;
 import com.oracle.graal.python.builtins.objects.function.PArguments.ThreadState;
+import com.oracle.graal.python.builtins.objects.generator.PGenerator;
 import com.oracle.graal.python.builtins.objects.ints.PInt;
 import com.oracle.graal.python.builtins.objects.iterator.PDoubleSequenceIterator;
 import com.oracle.graal.python.builtins.objects.iterator.PIntegerSequenceIterator;
 import com.oracle.graal.python.builtins.objects.iterator.PLongSequenceIterator;
+import com.oracle.graal.python.builtins.objects.iterator.PRangeIterator;
 import com.oracle.graal.python.builtins.objects.iterator.PSequenceIterator;
 import com.oracle.graal.python.builtins.objects.list.ListBuiltinsFactory.ListReverseNodeFactory;
 import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
@@ -81,21 +83,25 @@
 import com.oracle.graal.python.nodes.builtins.ListNodes.IndexNode;
 import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
 import com.oracle.graal.python.nodes.control.GetIteratorExpressionNode.GetIteratorNode;
+import com.oracle.graal.python.nodes.control.GetNextNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonTernaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.nodes.object.GetLazyClassNode;
+import com.oracle.graal.python.nodes.object.IsBuiltinClassProfile;
 import com.oracle.graal.python.nodes.truffle.PythonArithmeticTypes;
 import com.oracle.graal.python.runtime.PythonCore;
+import com.oracle.graal.python.runtime.exception.PException;
 import com.oracle.graal.python.runtime.exception.PythonErrorType;
 import com.oracle.graal.python.runtime.sequence.PSequence;
 import com.oracle.graal.python.runtime.sequence.storage.DoubleSequenceStorage;
 import com.oracle.graal.python.runtime.sequence.storage.EmptySequenceStorage;
 import com.oracle.graal.python.runtime.sequence.storage.IntSequenceStorage;
 import com.oracle.graal.python.runtime.sequence.storage.LongSequenceStorage;
 import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
+import com.oracle.graal.python.runtime.sequence.storage.SequenceStorageFactory;
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
@@ -211,6 +217,38 @@ PNone listRange(PList list, PRange range) {
             return PNone.NONE;
         }
 
+        @Specialization(guards = "iterable.isPRangeIterator()")
+        PNone listPGenerator(VirtualFrame frame, PList list, PGenerator iterable,
+                        @Cached GetIteratorNode getIteratorNode,
+                        @Cached SequenceStorageNodes.AppendNode appendNode,
+                        @Cached GetNextNode getNextNode,
+                        @Cached IsBuiltinClassProfile errorProfile,
+                        @Cached("createBinaryProfile()") ConditionProfile stepProfile,
+                        @Cached("createBinaryProfile()") ConditionProfile positiveRangeProfile) {
+            clearStorage(list);
+            Object iterObj = getIteratorNode.executeWith(frame, iterable);
+            SequenceStorage storage = EmptySequenceStorage.INSTANCE;
+
+            PRangeIterator range = (PRangeIterator) iterable.getIterator();
+            final int len = range.getLength(stepProfile, positiveRangeProfile);
+            if (len > 0) {
+                Object value = getNextNode.execute(frame, iterObj);
+                storage = SequenceStorageFactory.createStorage(value, len);
+                storage = appendNode.execute(storage, value, ListGeneralizationNode.SUPPLIER);
+                while (true) {
+                    try {
+                        storage = appendNode.execute(storage, getNextNode.execute(frame, iterObj), ListGeneralizationNode.SUPPLIER);
+                    } catch (PException e) {
+                        e.expectStopIteration(errorProfile);
+                        break;
+                    }
+                }
+            }
+
+            list.setSequenceStorage(storage);
+            return PNone.NONE;
+        }
+
         @Specialization(guards = {"!isNoValue(iterable)", "!isString(iterable)"})
         PNone listIterable(VirtualFrame frame, PList list, Object iterable,
                         @Cached("create()") GetIteratorNode getIteratorNode,

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/list/ListBuiltins.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -126,11 +126,13 @@ public RootNode getFunctionRootNode() {
     @Override
     public Object execute(VirtualFrame frame) {
         Object[] arguments;
+        Object iterator = null;
         if (getIterator == null) {
             arguments = PArguments.create(0);
         } else {
             arguments = PArguments.create(1);
-            PArguments.setArgument(arguments, 0, getIterator.execute(frame));
+            iterator = getIterator.execute(frame);
+            PArguments.setArgument(arguments, 0, iterator);
         }
         PArguments.setGlobals(arguments, PArguments.getGlobals(frame));
 
@@ -143,7 +145,7 @@ public Object execute(VirtualFrame frame) {
 
         PCell[] closure = getClosureFromGeneratorOrFunctionLocals(frame);
         return factory.createGenerator(name, callTargets, frameDescriptor, arguments, closure, executionCellSlots,
-                        numOfActiveFlags, numOfGeneratorBlockNode, numOfGeneratorForNode);
+                        numOfActiveFlags, numOfGeneratorBlockNode, numOfGeneratorForNode, iterator);
     }
 
     @Override

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/function/GeneratorExpressionNode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -90,7 +90,7 @@ public Object execute(VirtualFrame frame) {
             callTargets = createYieldTargets(callTarget);
         }
         return factory.createGenerator(getName(), callTargets, frameDescriptor, frame.getArguments(), PArguments.getClosure(frame), cellSlots, numOfActiveFlags, numOfGeneratorBlockNode,
-                        numOfGeneratorForNode);
+                        numOfGeneratorForNode, null);
     }
 
     public static RootCallTarget[] createYieldTargets(RootCallTarget callTarget) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/generator/GeneratorFunctionRootNode.java

@@ -531,9 +531,9 @@ public PDictView createDictItemsView(PHashingCollection dict) {
      */
 
     public PGenerator createGenerator(String name, RootCallTarget[] callTargets, FrameDescriptor frameDescriptor, Object[] arguments, PCell[] closure, ExecutionCellSlots cellSlots,
-                    int numOfActiveFlags, int numOfGeneratorBlockNode, int numOfGeneratorForNode) {
+                    int numOfActiveFlags, int numOfGeneratorBlockNode, int numOfGeneratorForNode, Object iterator) {
         return trace(PGenerator.create(PythonBuiltinClassType.PGenerator, name, callTargets, frameDescriptor, arguments, closure, cellSlots, numOfActiveFlags, numOfGeneratorBlockNode,
-                        numOfGeneratorForNode, this));
+                        numOfGeneratorForNode, this, iterator));
     }
 
     public PGeneratorFunction createGeneratorFunction(String name, String enclosingClassName, PCode code, PythonObject globals, PCell[] closure, Object[] defaultValues,

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/object/PythonObjectFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -27,6 +27,10 @@
 
 public abstract class BasicSequenceStorage extends SequenceStorage {
 
+    public BasicSequenceStorage(ListStorageType type) {
+        super(type);
+    }
+
     // nominated storage length
     protected int length;
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/sequence/storage/BasicSequenceStorage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -32,6 +32,7 @@ public final class BoolSequenceStorage extends TypedSequenceStorage {
     private boolean[] values;
 
     public BoolSequenceStorage() {
+        super(ListStorageType.Boolean);
         values = new boolean[]{};
     }
 
@@ -40,12 +41,14 @@ public BoolSequenceStorage(boolean[] elements) {
     }
 
     public BoolSequenceStorage(boolean[] elements, int length) {
+        super(ListStorageType.Boolean);
         this.values = elements;
         this.capacity = values.length;
         this.length = length;
     }
 
     public BoolSequenceStorage(int capacity) {
+        super(ListStorageType.Boolean);
         this.values = new boolean[capacity];
         this.capacity = capacity;
         this.length = 0;
@@ -259,9 +262,4 @@ public Object getCopyOfInternalArrayObject() {
     public void setInternalArrayObject(Object arrayObject) {
         this.values = (boolean[]) arrayObject;
     }
-
-    @Override
-    public ListStorageType getElementType() {
-        return ListStorageType.Boolean;
-    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/sequence/storage/BoolSequenceStorage.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -44,12 +44,14 @@ public ByteSequenceStorage(byte[] elements) {
     }
 
     public ByteSequenceStorage(byte[] elements, int length) {
+        super(ListStorageType.Byte);
         this.values = elements;
         this.capacity = values.length;
         this.length = length;
     }
 
     public ByteSequenceStorage(int capacity) {
+        super(ListStorageType.Byte);
         this.values = new byte[capacity];
         this.capacity = capacity;
         this.length = 0;
@@ -366,9 +368,4 @@ public Object getCopyOfInternalArrayObject() {
     public void setInternalArrayObject(Object arrayObject) {
         this.values = (byte[]) arrayObject;
     }
-
-    @Override
-    public ListStorageType getElementType() {
-        return ListStorageType.Byte;
-    }
 }