Use SourceRange instead of start/end offset

Author: otethal

graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py

@@ -102,7 +102,16 @@ public void atomString7() throws Exception {
     @Test
     public void atomFString() throws Exception {
         checkTreeResult("f'a{b!r}'");
-        checkError("f'a{b!g}'", "Generic[0:4]:f-string: invalid conversion character: expected 's', 'r', or 'a'");
+        checkError("f'a{b!g}'", "Generic[2:9]:f-string: invalid conversion character: expected 's', 'r', or 'a'");
+    }
+
+    @Test
+    public void atomFStringMultiline() throws Exception {
+        checkTreeResult("f\"\"\"First line.{\n" +
+                        "   v1\n" +
+                        "+\n" +
+                        "v2}\n" +
+                        "Another line.\"\"\"");
     }
 
     @Test

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/AtomsTests.java

@@ -54,6 +54,7 @@
 import java.util.Arrays;
 import java.util.List;
 
+import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
 import org.junit.Rule;
 import org.junit.rules.TestName;
 
@@ -97,7 +98,7 @@ public ModTy parse(String src, String moduleName, int mode) {
 
         FExprParser fexpParser = new FExprParser() {
             @Override
-            public ExprTy parse(String code) {
+            public ExprTy parse(String code, SourceRange sourceRange) {
                 ParserTokenizer tok = new ParserTokenizer(code);
                 return (ExprTy) new Parser(tok, factory, this).parse(InputType.FSTRING);
             }
@@ -203,13 +204,13 @@ public void checkError(String source, String... expectedErrors) {
         ArrayList<String> errors = new ArrayList<>();
         ParserErrorCallback errorCb = new ParserErrorCallback() {
             @Override
-            public void onError(ParserErrorCallback.ErrorType type, int start, int end, String message) {
-                errors.add(String.format("%s[%d:%d]:%s", type.name(), start, end, message));
+            public void onError(ParserErrorCallback.ErrorType type, SourceRange sourceRange, String message) {
+                errors.add(String.format("%s[%d:%d]:%s", type.name(), sourceRange.startOffset, sourceRange.endOffset, message));
             }
         };
         FExprParser fexpParser = new FExprParser() {
             @Override
-            public ExprTy parse(String code) {
+            public ExprTy parse(String code, SourceRange range) {
                 ParserTokenizer tok = new ParserTokenizer(code);
                 return (ExprTy) new Parser(tok, factory, this, errorCb).parse(InputType.FSTRING);
             }

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/ParserTestBase.java

@@ -59,8 +59,8 @@ private String indent() {
 
     private static String addHeader(SSTNode node) {
         StringBuilder sb = new StringBuilder();
-        sb.append(node.getClass().getSimpleName()).append("[").append(node.getStartOffset());
-        sb.append(", ").append(node.getEndOffset()).append("]");
+        sb.append(node.getClass().getSimpleName()).append("[").append(node.getSourceRange().startOffset);
+        sb.append(", ").append(node.getSourceRange().endOffset).append("]");
         return sb.toString();
     }
 
@@ -224,8 +224,8 @@ public String visit(ExprTy.Compare node) {
     @Override
     public String visit(ExprTy.Constant node) {
         StringBuilder sb = new StringBuilder();
-        sb.append(node.kind).append("[").append(node.getStartOffset());
-        sb.append(", ").append(node.getEndOffset()).append("]");
+        sb.append(node.kind).append("[").append(node.getSourceRange().startOffset);
+        sb.append(", ").append(node.getSourceRange().endOffset).append("]");
         sb.append(" Value: ");
         switch (node.kind) {
             case LONG:

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/pegparser/sst/SSTTreePrinterVisitor.java

@@ -470,8 +470,8 @@ private void checkTokensFromTestDataFile() throws Exception {
                     sb.append("exact_type:").append(getCPythonValueOfTokenType(token.type));
                     sb.append(" (").append(token.typeName()).append(") ");
                 }
-                sb.append("start:[").append(token.startLine).append(", ").append(token.startColumn).append("] ");
-                sb.append("end:[").append(token.endLine).append(", ").append(token.endColumn).append("] ");
+                sb.append("start:[").append(token.sourceRange.startLine).append(", ").append(token.sourceRange.startColumn).append("] ");
+                sb.append("end:[").append(token.sourceRange.endLine).append(", ").append(token.sourceRange.endColumn).append("] ");
                 sb.append("string:'").append(tokenizer.getTokenString(token)).append("'");
                 String goldenToken = goldenResult.get(goldenResultIndex);
                 assertEquals("Code: '" + line + "'", goldenToken, sb.toString());

graalpython/com.oracle.graal.python.pegparser.test/src/com/oracle/graal/python/tokenizer/TokenizerTest.java

@@ -0,0 +1,13 @@
+Module[0, 45]
+    JoinedStr[0, 45]
+         Values:
+            RAW[4, 15] Value: First line.
+            FormattedValue[16, 27]
+                Value: 
+                    BinOp[5, 12]
+                        Op: ADD
+                        LHS: Name[5, 7] Value: "v1"
+                        RHS: Name[10, 12] Value: "v2"
+                Conversion: NONE
+            RAW[28, 42] Value: 
+Another line.

graalpython/com.oracle.graal.python.pegparser.test/testData/parser/goldenFiles/AtomsTests/atomFStringMultiline.tast

@@ -52,6 +52,7 @@
 import com.oracle.graal.python.pegparser.sst.KeywordTy;
 import com.oracle.graal.python.pegparser.sst.SSTNode;
 import com.oracle.graal.python.pegparser.sst.StmtTy;
+import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
 import com.oracle.graal.python.pegparser.tokenizer.Token;
 
 /**
@@ -170,7 +171,7 @@ private void resetParserState() {
 
     /**
      * Get position in the tokenizer.
-     * 
+     *
      * @return the position in tokenizer.
      */
     public int mark() {
@@ -179,7 +180,7 @@ public int mark() {
 
     /**
      * Reset position in the tokenizer
-     * 
+     *
      * @param position where the tokenizer should set the current position
      */
     public void reset(int position) {
@@ -189,7 +190,7 @@ public void reset(int position) {
     /**
      * Is the expected token on the current position in tokenizer? If there is the expected token,
      * then the current position in tokenizer is changed to the next token.
-     * 
+     *
      * @param tokenKind - the token kind that is expected on the current position
      * @return The expected token or null if the token on the current position is not the expected
      *         one.
@@ -205,7 +206,7 @@ public Token expect(int tokenKind) {
     /**
      * Is the expected token on the current position in tokenizer? If there is the expected token,
      * then the current position in tokenizer is changed to the next token.
-     * 
+     *
      * @param text - the token on the current position has to have this text
      * @return The expected token or null if the token on the current position is not the expected
      *         one.
@@ -258,7 +259,7 @@ public Token getAndInitializeToken() {
         Token token = tokenizer.getToken();
         while (token.type == Token.Kind.TYPE_IGNORE) {
             String tag = getText(token);
-            comments.put(token.startLine, tag);
+            comments.put(token.sourceRange.startLine, tag);
             pos++;
             token = tokenizer.getToken();
         }
@@ -288,14 +289,14 @@ public Token getLastNonWhitespaceToken() {
     public ExprTy.Name name_token() {
         Token t = expect(Token.Kind.NAME);
         if (t != null) {
-            return factory.createVariable(getText(t), t.startOffset, t.endOffset);
+            return factory.createVariable(getText(t), t.sourceRange);
         } else {
             return null;
         }
     }
 
     /**
-     * 
+     *
      * @return flags that influence parsing.
      */
     public int getFlags() {
@@ -325,7 +326,7 @@ protected ExprTy.Name expect_SOFT_KEYWORD(String keyword) {
         Token t = tokenizer.peekToken();
         if (t.type == Token.Kind.NAME && getText(t).equals(keyword)) {
             tokenizer.getToken();
-            return factory.createVariable(getText(t), t.startOffset, t.endOffset);
+            return factory.createVariable(getText(t), t.sourceRange);
         }
         return null;
     }
@@ -350,7 +351,7 @@ public Token string_token() {
     public ExprTy number_token() {
         Token t = expect(Token.Kind.NUMBER);
         if (t != null) {
-            return factory.createNumber(getText(t), t.startOffset, t.endOffset);
+            return factory.createNumber(getText(t), t.sourceRange);
         } else {
             return null;
         }
@@ -374,7 +375,7 @@ public ExprTy.Name name_from_token(Token t) {
             return null;
         }
         String id = getText(t);
-        return factory.createVariable(id, t.startOffset, t.endOffset);
+        return factory.createVariable(id, t.sourceRange);
     }
 
     /**
@@ -401,7 +402,7 @@ public ExprTy.Name dummyName(@SuppressWarnings("unused") Object... args) {
         if (cachedDummyName != null) {
             return cachedDummyName;
         }
-        cachedDummyName = factory.createVariable("", 0, 0);
+        cachedDummyName = factory.createVariable("", new SourceRange(0, 0, 0, 0, 0, 0));
         return cachedDummyName;
     }
 
@@ -410,7 +411,7 @@ public ExprTy.Name dummyName(@SuppressWarnings("unused") Object... args) {
      */
     public SSTNode joinNamesWithDot(ExprTy a, ExprTy b) {
         String id = ((ExprTy.Name) a).id + "." + ((ExprTy.Name) b).id;
-        return factory.createVariable(id, a.getStartOffset(), b.getEndOffset());
+        return factory.createVariable(id, a.getSourceRange().withEnd(b.getSourceRange()));
     }
 
     /**
@@ -460,23 +461,21 @@ public ExprTy[] appendToEnd(ExprTy[] seq, ExprTy element) {
     public SSTNode concatenateStrings(Token[] tokens) {
         int n = tokens.length;
         String[] values = new String[n];
-        Token t = tokens[0];
-        int startOffset = t.startOffset;
-        values[0] = getText(t);
-        for (int i = 1; i < n; i++) {
-            t = tokens[i];
+        SourceRange[] sourceRanges = new SourceRange[n];
+        for (int i = 0; i < n; i++) {
+            Token t = tokens[i];
             values[i] = getText(t);
+            sourceRanges[i] = t.sourceRange;
         }
-        int endOffset = t.endOffset;
-        return factory.createString(values, startOffset, endOffset, fexprParser, errorCb);
+        return factory.createString(values, sourceRanges, fexprParser, errorCb);
     }
 
     /**
      * _PyPegen_check_barry_as_flufl
      */
     public boolean checkBarryAsFlufl(Token token) {
         if ((flags & PARSE_BARRY_AS_BDFL) != 0 && !getText(token).equals("<>")) {
-            errorCb.onError(token.startOffset, token.endOffset, BARRY_AS_BDFL);
+            errorCb.onError(token.sourceRange, BARRY_AS_BDFL);
             return true;
         }
         return false;
@@ -762,9 +761,9 @@ static String[] extractNames(ExprTy[] seq) {
     /**
      * _PyPegen_collect_call_seqs
      */
-    final ExprTy collectCallSequences(ExprTy[] a, KeywordOrStarred[] b, int startOffset, int endOffset) {
+    final ExprTy collectCallSequences(ExprTy[] a, KeywordOrStarred[] b, SourceRange sourceRange) {
         if (b == null) {
-            return factory.createCall(dummyName(), a, EMPTY_KWDS, startOffset, endOffset);
+            return factory.createCall(dummyName(), a, EMPTY_KWDS, sourceRange);
         } else {
             ExprTy[] starred = extractStarredExpressions(b);
             ExprTy[] args;
@@ -774,7 +773,7 @@ final ExprTy collectCallSequences(ExprTy[] a, KeywordOrStarred[] b, int startOff
             } else {
                 args = a;
             }
-            return factory.createCall(dummyName(), args, deleteStarredExpressions(b), startOffset, endOffset);
+            return factory.createCall(dummyName(), args, deleteStarredExpressions(b), sourceRange);
         }
     }
 
@@ -844,7 +843,7 @@ SSTNode raiseSyntaxErrorInvalidTarget(TargetsType type, ExprTy expr) {
     SSTNode raiseSyntaxError(String msg, Object... arguments) {
         errorIndicator = true;
         Token errorToken = tokenizer.peekToken();
-        errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.startOffset, errorToken.endOffset, msg, arguments);
+        errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.sourceRange, msg, arguments);
         return null;
     }
 
@@ -853,7 +852,7 @@ SSTNode raiseSyntaxError(String msg, Object... arguments) {
      */
     SSTNode raiseSyntaxErrorKnownLocation(Token errorToken, String msg, Object... argument) {
         errorIndicator = true;
-        errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.startOffset, errorToken.endOffset, msg, argument);
+        errorCb.onError(ParserErrorCallback.ErrorType.Syntax, errorToken.sourceRange, msg, argument);
         return null;
     }
 
@@ -862,7 +861,7 @@ SSTNode raiseSyntaxErrorKnownLocation(Token errorToken, String msg, Object... ar
      */
     SSTNode raiseSyntaxErrorKnownLocation(SSTNode where, String msg, Object... argument) {
         errorIndicator = true;
-        errorCb.onError(ParserErrorCallback.ErrorType.Syntax, where.getStartOffset(), where.getEndOffset(), msg, argument);
+        errorCb.onError(ParserErrorCallback.ErrorType.Syntax, where.getSourceRange(), msg, argument);
         return null;
     }
 
@@ -871,7 +870,7 @@ SSTNode raiseSyntaxErrorKnownLocation(SSTNode where, String msg, Object... argum
      */
     SSTNode raiseErrorKnownLocation(ParserErrorCallback.ErrorType typeError, SSTNode where, String msg, Object... argument) {
         errorIndicator = true;
-        errorCb.onError(typeError, where.getStartOffset(), where.getEndOffset(), msg, argument);
+        errorCb.onError(typeError, where.getSourceRange(), msg, argument);
         return null;
     }
 

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java

@@ -41,34 +41,30 @@
 
 package com.oracle.graal.python.pegparser;
 
+import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
+
 import java.util.ArrayList;
 import java.util.List;
 
 public class DefaultParserErrorCallback implements ParserErrorCallback {
 
     public static class Error {
         private final ParserErrorCallback.ErrorType type;
-        private int startOffset;
-        private int endOffset;
+        private SourceRange sourceRange;
         private final String message;
 
-        public Error(ErrorType type, int startOffset, int endOffset, String message) {
+        public Error(ErrorType type, SourceRange sourceRange, String message) {
             this.type = type;
-            this.startOffset = startOffset;
-            this.endOffset = endOffset;
+            this.sourceRange = sourceRange;
             this.message = message;
         }
 
         public ErrorType getType() {
             return type;
         }
 
-        public int getStartOffset() {
-            return startOffset;
-        }
-
-        public int getEndOffset() {
-            return endOffset;
+        public SourceRange getSourceRange() {
+            return sourceRange;
         }
 
         public String getMessage() {
@@ -88,8 +84,8 @@ public boolean hasErrors() {
     }
 
     @Override
-    public void onError(ErrorType errorType, int startOffset, int endOffset, String message) {
-        errors.add(new Error(errorType, startOffset, endOffset, message));
+    public void onError(ErrorType errorType, SourceRange sourceRange, String message) {
+        errors.add(new Error(errorType, sourceRange, message));
     }
 
 }

graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/DefaultParserErrorCallback.java

@@ -41,8 +41,10 @@
 package com.oracle.graal.python.pegparser;
 
 import com.oracle.graal.python.pegparser.sst.ExprTy;
+import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
 
 @FunctionalInterface
 public interface FExprParser {
-    public abstract ExprTy parse(String code);
+
+    ExprTy parse(String code, SourceRange sourceRange);
 }