string and backslash continuation

Author: qunaibit

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/DescriptiveBailErrorListener.java

@@ -41,6 +41,7 @@
 package com.oracle.graal.python.parser.antlr;
 
 import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
@@ -78,7 +79,7 @@ public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
             }
         }
         if (isInteractive(recognizer)) {
-            PIncompleteSourceException handleRecognitionException = handleInteractiveException(recognizer);
+            PIncompleteSourceException handleRecognitionException = handleInteractiveException(recognizer, offendingSymbol);
             if (handleRecognitionException != null) {
                 throw handleRecognitionException;
             }
@@ -96,8 +97,8 @@ private static PIncompleteSourceException handleRecognitionException(IntervalSet
         return null;
     }
 
-    private static PIncompleteSourceException handleInteractiveException(Recognizer<?, ?> recognizer) {
-        if (isOpened(((Python3Parser) recognizer).getTokenStream())) {
+    private static PIncompleteSourceException handleInteractiveException(Recognizer<?, ?> recognizer, Object offendingSymbol) {
+        if (isOpened(((Python3Parser) recognizer).getTokenStream()) || isBackslash(offendingSymbol)) {
             return new PIncompleteSourceException("", null, -1);
         }
         return null;
@@ -130,6 +131,18 @@ private static boolean isOpened(TokenStream input) {
         return lexer.isOpened();
     }
 
+    private static final int BACKSLASH = '\\';
+
+    private static boolean isBackslash(Object offendingSymbol) {
+        if (offendingSymbol instanceof Token) {
+            CharStream cs = ((Token) offendingSymbol).getInputStream();
+            if (cs.size() > 2 && cs.LA(-2) == BACKSLASH) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private static class EmptyRecognitionException extends RecognitionException {
         private static final long serialVersionUID = 1L;
         private Token offendingToken;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.g4

@@ -51,6 +51,9 @@ tokens { INDENT, DEDENT }
   // wether we have expanded EOF to include necessary DEDENTS and a NEWLINE
   private boolean expandedEOF = false;
 
+  private boolean longQuote1 = false; // """
+  private boolean longQuote2 = false; // '''
+
   @Override
   public void emit(Token t) {
     super.setToken(t);
@@ -93,7 +96,19 @@ tokens { INDENT, DEDENT }
   }
 
   public boolean isOpened() {
-	  return opened > 0;
+	  return this.opened > 0 || this.longQuote1 || this.longQuote2;
+  }
+
+  private void usedQuote1() {
+	if (!this.longQuote2){
+		this.longQuote1 = !this.longQuote1;
+	}
+  }
+
+  private void usedQuote2() {
+	if (!this.longQuote1){
+		this.longQuote2 = !this.longQuote2;
+	}
   }
 
   private Token createDedent() {
@@ -1801,6 +1816,8 @@ LEFT_SHIFT_ASSIGN : '<<=';
 RIGHT_SHIFT_ASSIGN : '>>=';
 POWER_ASSIGN : '**=';
 IDIV_ASSIGN : '//=';
+LONG_QUOTES1 : '"""' {usedQuote1();};
+LONG_QUOTES2 : '\'\'\'' {usedQuote2();};
 
 SKIP_
  : ( SPACES | COMMENT | LINE_JOINING ) -> skip

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.interp

@@ -91,10 +91,12 @@ LEFT_SHIFT_ASSIGN=90
 RIGHT_SHIFT_ASSIGN=91
 POWER_ASSIGN=92
 IDIV_ASSIGN=93
-SKIP_=94
-UNKNOWN_CHAR=95
-INDENT=96
-DEDENT=97
+LONG_QUOTES1=94
+LONG_QUOTES2=95
+SKIP_=96
+UNKNOWN_CHAR=97
+INDENT=98
+DEDENT=99
 'def'=2
 'return'=3
 'raise'=4
@@ -177,3 +179,5 @@ DEDENT=97
 '>>='=91
 '**='=92
 '//='=93
+'"""'=94
+'\'\'\''=95

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.tokens

@@ -91,8 +91,10 @@ LEFT_SHIFT_ASSIGN=90
 RIGHT_SHIFT_ASSIGN=91
 POWER_ASSIGN=92
 IDIV_ASSIGN=93
-SKIP_=94
-UNKNOWN_CHAR=95
+LONG_QUOTES1=94
+LONG_QUOTES2=95
+SKIP_=96
+UNKNOWN_CHAR=97
 'def'=2
 'return'=3
 'raise'=4
@@ -175,3 +177,5 @@ UNKNOWN_CHAR=95
 '>>='=91
 '**='=92
 '//='=93
+'"""'=94
+'\'\'\''=95

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3Lexer.interp

@@ -84,8 +84,8 @@ public class Python3Parser extends Parser {
 		EQUALS=74, GT_EQ=75, LT_EQ=76, NOT_EQ_1=77, NOT_EQ_2=78, AT=79, ARROW=80, 
 		ADD_ASSIGN=81, SUB_ASSIGN=82, MULT_ASSIGN=83, AT_ASSIGN=84, DIV_ASSIGN=85, 
 		MOD_ASSIGN=86, AND_ASSIGN=87, OR_ASSIGN=88, XOR_ASSIGN=89, LEFT_SHIFT_ASSIGN=90, 
-		RIGHT_SHIFT_ASSIGN=91, POWER_ASSIGN=92, IDIV_ASSIGN=93, SKIP_=94, UNKNOWN_CHAR=95, 
-		INDENT=96, DEDENT=97;
+		RIGHT_SHIFT_ASSIGN=91, POWER_ASSIGN=92, IDIV_ASSIGN=93, LONG_QUOTES1=94, 
+		LONG_QUOTES2=95, SKIP_=96, UNKNOWN_CHAR=97, INDENT=98, DEDENT=99;
 	public static final int
 		RULE_single_input = 0, RULE_file_input = 1, RULE_eval_input = 2, RULE_decorator = 3, 
 		RULE_decorators = 4, RULE_decorated = 5, RULE_async_funcdef = 6, RULE_funcdef = 7, 
@@ -142,7 +142,7 @@ private static String[] makeLiteralNames() {
 			"'|'", "'^'", "'&'", "'<<'", "'>>'", "'+'", "'-'", "'/'", "'%'", "'//'", 
 			"'~'", "'{'", "'}'", "'<'", "'>'", "'=='", "'>='", "'<='", "'<>'", "'!='", 
 			"'@'", "'->'", "'+='", "'-='", "'*='", "'@='", "'/='", "'%='", "'&='", 
-			"'|='", "'^='", "'<<='", "'>>='", "'**='", "'//='"
+			"'|='", "'^='", "'<<='", "'>>='", "'**='", "'//='", "'\"\"\"'", "'''''"
 		};
 	}
 	private static final String[] _LITERAL_NAMES = makeLiteralNames();
@@ -161,8 +161,8 @@ private static String[] makeSymbolicNames() {
 			"GREATER_THAN", "EQUALS", "GT_EQ", "LT_EQ", "NOT_EQ_1", "NOT_EQ_2", "AT", 
 			"ARROW", "ADD_ASSIGN", "SUB_ASSIGN", "MULT_ASSIGN", "AT_ASSIGN", "DIV_ASSIGN", 
 			"MOD_ASSIGN", "AND_ASSIGN", "OR_ASSIGN", "XOR_ASSIGN", "LEFT_SHIFT_ASSIGN", 
-			"RIGHT_SHIFT_ASSIGN", "POWER_ASSIGN", "IDIV_ASSIGN", "SKIP_", "UNKNOWN_CHAR", 
-			"INDENT", "DEDENT"
+			"RIGHT_SHIFT_ASSIGN", "POWER_ASSIGN", "IDIV_ASSIGN", "LONG_QUOTES1", 
+			"LONG_QUOTES2", "SKIP_", "UNKNOWN_CHAR", "INDENT", "DEDENT"
 		};
 	}
 	private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
@@ -8371,7 +8371,7 @@ public final Yield_exprContext yield_expr() throws RecognitionException {
 	}
 
 	public static final String _serializedATN =
-		"\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3c\u0657\4\2\t\2\4"+
+		"\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3e\u0657\4\2\t\2\4"+
 		"\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
 		"\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
 		"\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -8771,8 +8771,8 @@ public final Yield_exprContext yield_expr() throws RecognitionException {
 		"d\63\2\u03c4\u03c5\b\62\1\2\u03c5\u03c7\3\2\2\2\u03c6\u03be\3\2\2\2\u03c6"+
 		"\u03c2\3\2\2\2\u03c7\u03c8\3\2\2\2\u03c8\u03c9\b\62\1\2\u03c9c\3\2\2\2"+
 		"\u03ca\u03d5\b\63\1\2\u03cb\u03d6\5&\24\2\u03cc\u03cd\7\'\2\2\u03cd\u03cf"+
-		"\7b\2\2\u03ce\u03d0\5$\23\2\u03cf\u03ce\3\2\2\2\u03d0\u03d1\3\2\2\2\u03d1"+
-		"\u03cf\3\2\2\2\u03d1\u03d2\3\2\2\2\u03d2\u03d3\3\2\2\2\u03d3\u03d4\7c"+
+		"\7d\2\2\u03ce\u03d0\5$\23\2\u03cf\u03ce\3\2\2\2\u03d0\u03d1\3\2\2\2\u03d1"+
+		"\u03cf\3\2\2\2\u03d1\u03d2\3\2\2\2\u03d2\u03d3\3\2\2\2\u03d3\u03d4\7e"+
 		"\2\2\u03d4\u03d6\3\2\2\2\u03d5\u03cb\3\2\2\2\u03d5\u03cc\3\2\2\2\u03d6"+
 		"\u03d7\3\2\2\2\u03d7\u03d8\b\63\1\2\u03d8e\3\2\2\2\u03d9\u03da\5n8\2\u03da"+
 		"\u03e1\b\64\1\2\u03db\u03dc\7\r\2\2\u03dc\u03dd\5n8\2\u03dd\u03de\7\17"+