Allow trivia between character class range operands

Factor out the logic that deals with parsing an
individual character class member, and interleave
`lexTrivia` calls between range operand parsing.

Author: hamishknight

Sources/_RegexParser/Regex/AST/CustomCharClass.swift

@@ -51,11 +51,16 @@ extension AST {
       public var lhs: Atom
       public var dashLoc: SourceLocation
       public var rhs: Atom
+      public var trivia: [AST.Trivia]
 
-      public init(_ lhs: Atom, _ dashLoc: SourceLocation, _ rhs: Atom) {
+      public init(
+        _ lhs: Atom, _ dashLoc: SourceLocation, _ rhs: Atom,
+        trivia: [AST.Trivia]
+      ) {
         self.lhs = lhs
         self.dashLoc = dashLoc
         self.rhs = rhs
+        self.trivia = trivia
       }
     }
     public enum SetOp: String, Hashable {
@@ -95,6 +100,11 @@ extension CustomCC.Member {
     return false
   }
 
+  public var asTrivia: AST.Trivia? {
+    guard case .trivia(let t) = self else { return nil }
+    return t
+  }
+
   public var isSemantic: Bool {
     !isTrivia
   }

Sources/_RegexParser/Regex/Parse/LexicalAnalysis.swift

@@ -2039,20 +2039,11 @@ extension Source {
     return AST.Atom(kind.value, kind.location)
   }
 
-  /// Try to lex the end of a range in a custom character class, which consists
-  /// of a '-' character followed by an atom.
-  mutating func lexCustomCharClassRangeEnd(
-    context: ParsingContext
-  ) throws -> (dashLoc: SourceLocation, AST.Atom)? {
-    // Make sure we don't have a binary operator e.g '--', and the '-' is not
-    // ending the custom character class (in which case it is literal).
-    guard peekCCBinOp() == nil, !starts(with: "-]"),
-          let dash = tryEatWithLoc("-"),
-          let end = try lexAtom(context: context)
-    else {
-      return nil
-    }
-    return (dash, end)
+  /// Try to lex the range operator '-' for a custom character class.
+  mutating func lexCustomCharacterClassRangeOperator() -> SourceLocation? {
+    // Eat a '-', making sure we don't have a binary op such as '--'.
+    guard peekCCBinOp() == nil else { return nil }
+    return tryEatWithLoc("-")
   }
 
   /// Try to consume a newline sequence matching option kind.

Sources/_RegexParser/Regex/Parse/Parse.swift

@@ -495,43 +495,72 @@ extension Parser {
     return CustomCC(start, members, loc(start.location.start))
   }
 
+  mutating func parseCCCMember() throws -> CustomCC.Member? {
+    guard !source.isEmpty && source.peek() != "]" && source.peekCCBinOp() == nil
+    else { return nil }
+
+    // Nested custom character class.
+    if let cccStart = source.lexCustomCCStart() {
+      return .custom(try parseCustomCharacterClass(cccStart))
+    }
+
+    // Quoted sequence.
+    if let quote = try source.lexQuote(context: context) {
+      return .quote(quote)
+    }
+
+    // Lex triva if we're allowed.
+    if let trivia = try source.lexTrivia(context: context) {
+      return .trivia(trivia)
+    }
+
+    if let atom = try source.lexAtom(context: context) {
+      return .atom(atom)
+    }
+    return nil
+  }
+
   mutating func parseCCCMembers(
     into members: inout Array<CustomCC.Member>
   ) throws {
     // Parse members until we see the end of the custom char class or an
     // operator.
-    while !source.isEmpty && source.peek() != "]" &&
-          source.peekCCBinOp() == nil {
-
-      // Nested custom character class.
-      if let cccStart = source.lexCustomCCStart() {
-        members.append(.custom(try parseCustomCharacterClass(cccStart)))
-        continue
-      }
-
-      // Quoted sequence.
-      if let quote = try source.lexQuote(context: context) {
-        members.append(.quote(quote))
-        continue
-      }
-
-      // Lex trivia if we're allowed.
-      if let trivia = try source.lexTrivia(context: context) {
-        members.append(.trivia(trivia))
-        continue
-      }
+    while let member = try parseCCCMember() {
+      members.append(member)
+
+      // If we have an atom, we can try to parse a character class range. Each
+      // time we parse a component of the range, we append to `members` in case
+      // it ends up not being a range, and we bail. If we succeed in parsing, we
+      // remove the intermediate members.
+      if case .atom(let lhs) = member {
+        let membersBeforeRange = members.count - 1
+
+        while let t = try source.lexTrivia(context: context) {
+          members.append(.trivia(t))
+        }
 
-      guard let atom = try source.lexAtom(context: context) else { break }
+        guard let dash = source.lexCustomCharacterClassRangeOperator() else {
+          continue
+        }
+        // If we can't parse a range, '-' becomes literal, e.g `[6-]`.
+        members.append(.atom(.init(.char("-"), dash)))
 
-      // Range between atoms.
-      if let (dashLoc, rhs) =
-          try source.lexCustomCharClassRangeEnd(context: context) {
-        members.append(.range(.init(atom, dashLoc, rhs)))
-        continue
+        while let t = try source.lexTrivia(context: context) {
+          members.append(.trivia(t))
+        }
+        guard let rhs = try parseCCCMember() else { continue }
+        members.append(rhs)
+
+        guard case let .atom(rhs) = rhs else { continue }
+
+        // We've successfully parsed an atom LHS and RHS, so form a range,
+        // collecting the trivia we've parsed, and replacing the members that
+        // would have otherwise been added to the custom character class.
+        let rangeMemberCount = members.count - membersBeforeRange
+        let trivia = members.suffix(rangeMemberCount).compactMap(\.asTrivia)
+        members.removeLast(rangeMemberCount)
+        members.append(.range(.init(lhs, dash, rhs, trivia: trivia)))
       }
-
-      members.append(.atom(atom))
-      continue
     }
   }
 }

Sources/_StringProcessing/Utility/ASTBuilder.swift

@@ -416,7 +416,7 @@ func prop_m(
 func range_m(
   _ lower: AST.Atom, _ upper: AST.Atom
 ) -> AST.CustomCharacterClass.Member {
-  .range(.init(lower, .fake, upper))
+  .range(.init(lower, .fake, upper, trivia: []))
 }
 func range_m(
   _ lower: AST.Atom.Kind, _ upper: AST.Atom.Kind

Tests/RegexTests/MatchTests.swift

@@ -594,6 +594,8 @@ extension RegexTests {
     firstMatchTest("[a-z]", input: "123ABCxyz", match: "x")
     firstMatchTest("[a-z]", input: "123-abcxyz", match: "a")
 
+    firstMatchTest("(?x)[ a - z ]+", input: " 123-abcxyz", match: "abcxyz")
+
     // Character class subtraction
     firstMatchTest("[a-d--a-c]", input: "123abcdxyz", match: "d")
 

Tests/RegexTests/ParseTests.swift

@@ -517,6 +517,8 @@ extension RegexTests {
       "[a-b-c]", charClass(range_m("a", "b"), "-", "c"))
 
     parseTest("[-a-]", charClass("-", "a", "-"))
+    parseTest("[[a]-]", charClass(charClass("a"), "-"))
+    parseTest("[[a]-b]", charClass(charClass("a"), "-", "b"))
 
     parseTest("[a-z]", charClass(range_m("a", "z")))
     parseTest("[a-a]", charClass(range_m("a", "a")))
@@ -679,6 +681,16 @@ extension RegexTests {
       throwsError: .unsupported
     )
 
+    parseTest(#"(?x)[  a -  b  ]"#, concat(
+      changeMatchingOptions(matchingOptions(adding: .extended)),
+      charClass(range_m("a", "b"))
+    ))
+
+    parseTest(#"(?x)[a - b]"#, concat(
+      changeMatchingOptions(matchingOptions(adding: .extended)),
+      charClass(range_m("a", "b"))
+    ))
+
     // MARK: Operators
 
     parseTest(
@@ -2120,6 +2132,17 @@ extension RegexTests {
       /#
       """#, charClass("a", range_m("b", "c"), "d"))
 
+    parseWithDelimitersTest(#"""
+      #/
+      [
+        a # interesting
+        -   #a
+         b
+      ]
+      /#
+      """#, charClass(range_m("a", "b")))
+
+
     // MARK: Delimiter skipping: Make sure we can skip over the ending delimiter
     // if it's clear that it's part of the regex syntax.