From 957fbde3e646200f490fda13d38dff0d05fe179b Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Nov 2019 21:34:15 +0100 Subject: [PATCH] Improve handling of end markers 1. Simplify the logic 2. Make end marker errors more reliable by recognizing end markers in more places 3. Allow nested end marker scopes at same indentation width Example in pos/end-nested.scala ```scala def f[T](x: Option[T]) = x match case Some(y) => case None => end f ``` Here, we should not give a "misaligned endmarker" error, which means we have to skip the nested `match` scope. --- .../dotty/tools/dotc/parsing/Parsers.scala | 39 ++++----- .../dotty/tools/dotc/parsing/Scanners.scala | 81 +++++++++---------- tests/neg/endmarkers.scala | 18 ++--- tests/neg/endmarkers1.scala | 18 +++++ tests/pos/end-nested.scala | 14 ++++ 5 files changed, 93 insertions(+), 77 deletions(-) create mode 100644 tests/neg/endmarkers1.scala create mode 100644 tests/pos/end-nested.scala diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 49763d4bb88c..9d47db6171c1 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1278,15 +1278,8 @@ object Parsers { if silentTemplateIndent && !isNew then in.observeIndented() newLineOptWhenFollowedBy(LBRACE) - def indentRegion[T](tag: EndMarkerTag)(op: => T): T = { - val iw = in.currentRegion.indentWidth - val t = op - in.consumeEndMarker(tag, iw) - t - } - - def indentRegion[T](pid: Tree)(op: => T): T = pid match { - case pid: RefTree => indentRegion(pid.name.toTermName)(op) + def endMarkerScope[T](pid: Tree)(op: => T): T = pid match { + case pid: RefTree => in.endMarkerScope(pid.name.toTermName)(op) case _ => op } @@ -1789,9 +1782,9 @@ object Parsers { def expr1(location: Location.Value = Location.ElseWhere): Tree = in.token match { case IF => - indentRegion(IF) { ifExpr(in.offset, If) } + in.endMarkerScope(IF) { ifExpr(in.offset, If) } case WHILE => - indentRegion(WHILE) { + in.endMarkerScope(WHILE) { atSpan(in.skipToken()) { val cond = condExpr(DO) newLinesOpt() @@ -1826,7 +1819,7 @@ object Parsers { WhileDo(Block(body, cond), Literal(Constant(()))) } case TRY => - indentRegion(TRY) { + in.endMarkerScope(TRY) { val tryOffset = in.offset atSpan(in.skipToken()) { val body = expr() @@ -1961,7 +1954,7 @@ object Parsers { /** `match' (`{' CaseClauses `}' | CaseClause) */ def matchExpr(t: Tree, start: Offset, mkMatch: (Tree, List[CaseDef]) => Match) = - indentRegion(MATCH) { + in.endMarkerScope(MATCH) { atSpan(start, in.skipToken()) { mkMatch(t, casesExpr(caseClause)) } @@ -2157,7 +2150,7 @@ object Parsers { * | ‘new’ TemplateBody */ def newExpr(): Tree = - indentRegion(NEW) { + in.endMarkerScope(NEW) { val start = in.skipToken() def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset)) possibleBracesStart() @@ -2323,7 +2316,7 @@ object Parsers { * {nl} [`yield'] Expr * | `for' Enumerators (`do' Expr | `yield' Expr) */ - def forExpr(): Tree = indentRegion(FOR) { + def forExpr(): Tree = in.endMarkerScope(FOR) { atSpan(in.skipToken()) { var wrappedEnums = true val start = in.offset @@ -3074,7 +3067,7 @@ object Parsers { else emptyType val rhs = if (tpt.isEmpty || in.token == EQUALS) - indentRegion(first) { + endMarkerScope(first) { accept(EQUALS) if (in.token == USCORE && !tpt.isEmpty && mods.is(Mutable) && (lhs.toList forall (_.isInstanceOf[Ident]))) @@ -3166,7 +3159,7 @@ object Parsers { if (in.isScala2CompatMode) newLineOptWhenFollowedBy(LBRACE) val rhs = if (in.token == EQUALS) - indentRegion(name) { + in.endMarkerScope(name) { in.nextToken() subExpr() } @@ -3298,7 +3291,7 @@ object Parsers { } def classDefRest(start: Offset, mods: Modifiers, name: TypeName): TypeDef = - indentRegion(name.toTermName) { + in.endMarkerScope(name.toTermName) { val constr = classConstr(isCaseClass = mods.is(Case)) val templ = templateOpt(constr) finalizeDef(TypeDef(name, templ), mods, start) @@ -3322,7 +3315,7 @@ object Parsers { */ def objectDef(start: Offset, mods: Modifiers): ModuleDef = atSpan(start, nameStart) { val name = ident() - indentRegion(name) { + in.endMarkerScope(name) { val templ = templateOpt(emptyConstructor) finalizeDef(ModuleDef(name, templ), mods, start) } @@ -3332,7 +3325,7 @@ object Parsers { */ def enumDef(start: Offset, mods: Modifiers): TypeDef = atSpan(start, nameStart) { val modulName = ident() - indentRegion(modulName) { + in.endMarkerScope(modulName) { val clsName = modulName.toTypeName val constr = classConstr() val templ = template(constr, isEnum = true) @@ -3412,7 +3405,7 @@ object Parsers { else (EmptyTermName, isIdent(nme.extension)) - val gdef = indentRegion(if name.isEmpty then GIVEN else name) { + val gdef = in.endMarkerScope(if name.isEmpty then GIVEN else name) { if isExtension then if (in.token == COLON) in.nextToken() assert(ident() == nme.extension) @@ -3597,7 +3590,7 @@ object Parsers { */ def packaging(start: Int): Tree = { val pkg = qualId() - indentRegion(pkg) { + endMarkerScope(pkg) { possibleTemplateStart() val stats = inDefScopeBraces(topStatSeq()) makePackaging(start, pkg, stats) @@ -3785,7 +3778,7 @@ object Parsers { else val pkg = qualId() var continue = false - indentRegion(pkg) { + endMarkerScope(pkg) { possibleTemplateStart() if in.token == EOF then ts += makePackaging(start, pkg, List()) diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 90e62f75ee9f..5238dce9b313 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -226,8 +226,8 @@ object Scanners { /** The current region. This is initially an Indented region with indentation width. */ var currentRegion: Region = Indented(IndentWidth.Zero, Set(), EMPTY, null) - /** The end marker that was skipped last */ - val endMarkers = new mutable.ListBuffer[EndMarker] + /** The number of open end marker scopes */ + var openEndMarkers: List[(EndMarkerTag, IndentWidth)] = Nil // Scala 2 compatibility @@ -324,48 +324,47 @@ object Scanners { this.token = token } - /** If this token and the next constitute an end marker, skip them and append a new EndMarker - * value at the end of the endMarkers queue. + /** What can be referred to in an end marker */ + type EndMarkerTag = TermName | Token + + /** Establish a scope for a passible end-marker with given tag, parsed by `op` */ + def endMarkerScope[T](tag: EndMarkerTag)(op: => T): T = + val saved = openEndMarkers + openEndMarkers = (tag, currentRegion.indentWidth) :: openEndMarkers + try op finally openEndMarkers = saved + + /** If this token and the next constitute an end marker, skip them and check they + * align with an opening construct with the same end marker tag. */ - private def handleEndMarkers(width: IndentWidth): Unit = - if (next.token == IDENTIFIER && next.name == nme.end && width == currentRegion.indentWidth) { + protected def skipEndMarker(width: IndentWidth): Unit = + if next.token == IDENTIFIER && next.name == nme.end then val lookahead = LookaheadScanner() - lookahead.nextToken() // skip the `end` + val start = lookahead.offset + + def handle(tag: EndMarkerTag) = + def checkAligned(): Unit = openEndMarkers match + case (etag, ewidth) :: rest if width <= ewidth => + if width < ewidth || tag != etag then + openEndMarkers = rest + checkAligned() + case _ => + lexical.println(i"misaligned end marker $tag, $width, $openEndMarkers") + errorButContinue("misaligned end marker", start) - def handle(tag: EndMarkerTag) = { val skipTo = lookahead.charOffset lookahead.nextToken() - if (lookahead.isAfterLineEnd || lookahead.token == EOF) { - lexical.println(i"produce end marker $tag $width") - endMarkers += EndMarker(tag, width, offset) + if lookahead.isAfterLineEnd || lookahead.token == EOF then + checkAligned() next.token = EMPTY - while (charOffset < skipTo) nextChar() - } - } + while charOffset < skipTo do nextChar() + end handle - lookahead.token match { + lookahead.nextToken() // skip the `end` + lookahead.token match case IDENTIFIER | BACKQUOTED_IDENT => handle(lookahead.name) case IF | WHILE | FOR | MATCH | TRY | NEW | GIVEN => handle(lookahead.token) case _ => - } - } - - /** Consume and cancel the head of the end markers queue if it has the given `tag` and width. - * Flag end markers with higher indent widths as errors. - */ - def consumeEndMarker(tag: EndMarkerTag, width: IndentWidth): Unit = { - lexical.println(i"consume end marker $tag $width") - if (endMarkers.nonEmpty) { - val em = endMarkers.head - if (width <= em.width) { - if (em.tag != tag || em.width != width) { - lexical.println(i"misaligned end marker ${em.tag}, ${em.width} at ${width}") - errorButContinue("misaligned end marker", em.offset) - } - endMarkers.trimStart(1) - } - } - } + end skipEndMarker /** A leading symbolic or backquoted identifier is treated as an infix operator if * - it does not follow a blank line, and @@ -501,6 +500,7 @@ object Scanners { && !isLeadingInfixOperator() then insert(if (pastBlankLine) NEWLINES else NEWLINE, lineOffset) + skipEndMarker(nextWidth) else if indentIsSignificant then if nextWidth < lastWidth || nextWidth == lastWidth && (indentPrefix == MATCH || indentPrefix == CATCH) && token != CASE then @@ -511,7 +511,7 @@ object Scanners { case r: Indented => currentRegion = r.enclosing insert(OUTDENT, offset) - handleEndMarkers(nextWidth) + skipEndMarker(nextWidth) case r: InBraces if !closingRegionTokens.contains(token) => ctx.warning("Line is indented too far to the left, or a `}' is missing", source.atSpan(Span(offset))) @@ -883,6 +883,7 @@ object Scanners { class LookaheadScanner(indent: Boolean = false) extends Scanner(source, offset) { override val indentSyntax = indent + override def skipEndMarker(width: IndentWidth) = () override protected def printState() = { print("la:") super.printState() @@ -1416,16 +1417,6 @@ object Scanners { val Zero = Run(' ', 0) } - /** What can be referred to in an end marker */ - type EndMarkerTag = TermName | Token - - /** A processed end marker - * @param tag The name or token referred to in the marker - * @param width The indentation width where the marker occurred - * @param offset The offset of the `end` - */ - case class EndMarker(tag: EndMarkerTag, width: IndentWidth, offset: Int) - // ------------- keyword configuration ----------------------------------- private val (lastKeywordStart, kwArray) = buildKeywordArray(keywords) diff --git a/tests/neg/endmarkers.scala b/tests/neg/endmarkers.scala index 3f6f41627f31..e8310bc81fd8 100644 --- a/tests/neg/endmarkers.scala +++ b/tests/neg/endmarkers.scala @@ -3,14 +3,14 @@ object Test locally { var x = 0 while x < 10 do x += 1 - end while // error: end of statement expected but while found // error: not found: end - val f = 10 // error: ';' expected, but 'val' found + end while + val f = 10 while x += 1 x < 10 do () - end while // error: misaligned end marker // error: not found : end - } // error: ';' expected, but '}' found + end while // error: misaligned end marker + } def f(x: Int): Int = val y = @@ -50,8 +50,8 @@ class Test2 object x new Test2 { override def foo = 2 - end new // error: end of statement expected but new found // error: not found: end - } // error: ';' expected, but '}' found + end new // error: misaligned end marker + } def bar = 2 end Test2 // error: misaligned end marker end Test2 @@ -59,7 +59,7 @@ end Test2 class Test3 self => def foo = 1 - end Test3 // error: not found: end + end Test3 // error: misaligned end marker import collection.mutable.HashMap @@ -67,7 +67,7 @@ class Coder(words: List[String]) class Foo println() - end Foo // error: not found: end + end Foo // error: misaligned end marker (2 -> "ABC", new ArrowAssoc('3') -> "DEF") @@ -101,4 +101,4 @@ class Coder(words: List[String]) .flatMap { case (digit, str) => str map (ltr => ltr -> digit) } - end Coder // error: The start of this line does not match any of the previous indentation widths. \ No newline at end of file + end Coder // error: misaligned end marker \ No newline at end of file diff --git a/tests/neg/endmarkers1.scala b/tests/neg/endmarkers1.scala new file mode 100644 index 000000000000..ad670862bf3f --- /dev/null +++ b/tests/neg/endmarkers1.scala @@ -0,0 +1,18 @@ + +def f7[T](x: Option[T]) = x match + case Some(y) => + case None => +end if // error: misaligned end marker + +object Test4 with + def f[T](x: Option[T]) = x match + case Some(y) => + case None => + end if // error: misaligned end marker +end Test // error: misaligned end marker + +def f[T](x: Option[T]) = x match + case Some(y) => + case None => "hello" + end f // error: misaligned end marker + diff --git a/tests/pos/end-nested.scala b/tests/pos/end-nested.scala new file mode 100644 index 000000000000..96c25c2b1ea2 --- /dev/null +++ b/tests/pos/end-nested.scala @@ -0,0 +1,14 @@ +def f[T](x: Option[T]) = x match + case Some(y) => + case None => +end f + +object Test with + try List(1, 2, 3) match + case x :: xs => println(x) + case Nil => println("Nil") + catch + case ex: java.io.IOException => println(ex) + case ex: Throwable => throw ex + end try +