Lex whitespace in range quantifiers

PCRE does not allow whitespace here, instead
treating the sequence as literal if whitespace is
present. However this behavior is quite
unintuitive. Instead, lex whitespace between range
operands.

Author: hamishknight

Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift

@@ -430,7 +430,7 @@ extension Source {
 
       return try src.tryEating { src in
         guard src.tryEat("{"),
-              let range = try src.lexRange(context: context),
+              let range = try src.lexRange(context: context, trivia: &trivia),
               src.tryEat("}")
         else { return nil }
         return range.value
@@ -456,11 +456,17 @@ extension Source {
   ///                  | ExpRange
   ///     ExpRange    -> '..<' <Int> | '...' <Int>
   ///                  | <Int> '..<' <Int> | <Int> '...' <Int>?
-  mutating func lexRange(context: ParsingContext) throws -> Located<Quant.Amount>? {
+  mutating func lexRange(
+    context: ParsingContext, trivia: inout [AST.Trivia]
+  ) throws -> Located<Quant.Amount>? {
     try recordLoc { src in
       try src.tryEating { src in
+        if let t = src.lexWhitespace() { trivia.append(t) }
+
         let lowerOpt = try src.lexNumber()
 
+        if let t = src.lexWhitespace() { trivia.append(t) }
+
         // ',' or '...' or '..<' or nothing
         // TODO: We ought to try and consume whitespace here and emit a
         // diagnostic for the user warning them that it would cause the range to
@@ -480,11 +486,15 @@ extension Source {
           closedRange = nil
         }
 
+        if let t = src.lexWhitespace() { trivia.append(t) }
+
         let upperOpt = try src.lexNumber()?.map { upper in
           // If we have an open range, the upper bound should be adjusted down.
           closedRange == true ? upper : upper - 1
         }
 
+        if let t = src.lexWhitespace() { trivia.append(t) }
+
         switch (lowerOpt, closedRange, upperOpt) {
         case let (l?, nil, nil):
           return .exactly(l)

Tests/RegexTests/MatchTests.swift

@@ -341,8 +341,12 @@ extension RegexTests {
 
     firstMatchTest(
       #"a{1,2}"#, input: "123aaaxyz", match: "aa")
+    firstMatchTest(
+      #"a{ 1 , 2 }"#, input: "123aaaxyz", match: "aa")
     firstMatchTest(
       #"a{,2}"#, input: "123aaaxyz", match: "")
+    firstMatchTest(
+      #"a{ , 2 }"#, input: "123aaaxyz", match: "")
     firstMatchTest(
       #"a{,2}x"#, input: "123aaaxyz", match: "aax")
     firstMatchTest(
@@ -351,6 +355,8 @@ extension RegexTests {
       #"a{2,}"#, input: "123aaaxyz", match: "aaa")
     firstMatchTest(
       #"a{1}"#, input: "123aaaxyz", match: "a")
+    firstMatchTest(
+      #"a{ 1 }"#, input: "123aaaxyz", match: "a")
     firstMatchTest(
       #"a{1,2}?"#, input: "123aaaxyz", match: "a")
     firstMatchTest(

Tests/RegexTests/ParseTests.swift

@@ -832,6 +832,10 @@ extension RegexTests {
       #"a{1,1}"#,
       quantRange(1...1, of: "a"))
 
+    parseTest("x{3, 5}", quantRange(3 ... 5, of: "x"))
+    parseTest("x{ 3 , 5  }", quantRange(3 ... 5, of: "x"))
+    parseTest("x{3 }", exactly(3, of: "x"))
+
     // Make sure ranges get treated as literal if invalid.
     parseTest("{", "{")
     parseTest("{,", concat("{", ","))
@@ -851,11 +855,6 @@ extension RegexTests {
     parseTest("x{+", concat("x", oneOrMore(of: "{")))
     parseTest("x{6,+", concat("x", "{", "6", oneOrMore(of: ",")))
 
-    // TODO: We should emit a diagnostic for this.
-    parseTest("x{3, 5}", concat("x", "{", "3", ",", " ", "5", "}"))
-    parseTest("{3, 5}", concat("{", "3", ",", " ", "5", "}"))
-    parseTest("{3 }", concat("{", "3", " ", "}"))
-
     // MARK: Groups
 
     // Named captures
@@ -1771,10 +1770,10 @@ extension RegexTests {
 
     // PCRE states that whitespace won't be ignored within a range.
     // http://pcre.org/current/doc/html/pcre2api.html#SEC20
-    // TODO: We ought to warn on this, and produce a range anyway.
+    // We however do ignore it.
     parseTest("(?x)a{1, 3}", concat(
       changeMatchingOptions(matchingOptions(adding: .extended)),
-      "a", "{", "1", ",", "3", "}"
+      quantRange(1 ... 3, of: "a")
     ))
 
     // Test that we cover the list of whitespace characters covered by PCRE.
@@ -2695,6 +2694,9 @@ extension RegexTests {
     diagnosticTest("{1,3}", .quantifierRequiresOperand("{1,3}"))
     diagnosticTest("a{3,2}", .invalidQuantifierRange(3, 2))
 
+    diagnosticTest("{3, 5}", .quantifierRequiresOperand("{3, 5}"))
+    diagnosticTest("{3 }", .quantifierRequiresOperand("{3 }"))
+
     // These are not quantifiable.
     diagnosticTest(#"\b?"#, .notQuantifiable)
     diagnosticTest(#"\B*"#, .notQuantifiable)