diff --git a/compiler/src/dotty/tools/dotc/ast/Trees.scala b/compiler/src/dotty/tools/dotc/ast/Trees.scala index 24bfa98401a7..7cbdb441fdca 100644 --- a/compiler/src/dotty/tools/dotc/ast/Trees.scala +++ b/compiler/src/dotty/tools/dotc/ast/Trees.scala @@ -369,6 +369,8 @@ object Trees { comment.map(putAttachment(DocComment, _)) this } + + def name: Name } /** A ValDef or DefDef tree */ diff --git a/compiler/src/dotty/tools/dotc/core/Names.scala b/compiler/src/dotty/tools/dotc/core/Names.scala index c94043da0719..06d318165d61 100644 --- a/compiler/src/dotty/tools/dotc/core/Names.scala +++ b/compiler/src/dotty/tools/dotc/core/Names.scala @@ -25,8 +25,6 @@ object Names { def toTermName: TermName } - implicit def eqName: Eql[Name, Name] = Eql.derived - /** A common superclass of Name and Symbol. After bootstrap, this should be * just the type alias Name | Symbol */ @@ -37,7 +35,7 @@ object Names { * in a name table. A derived term name adds a tag, and possibly a number * or a further simple name to some other name. */ - abstract class Name extends Designator with PreName { + abstract class Name extends Designator, PreName derives Eql { /** A type for names of the same kind as this name */ type ThisName <: Name diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 9dbb2a7efbb9..751f814d42d4 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -160,17 +160,12 @@ object Parsers { def skipBraces(): Unit = { accept(if (in.token == INDENT) INDENT else LBRACE) var openBraces = 1 - val savedCheckEndMarker = in.checkEndMarker - try - in.checkEndMarker = false - while (in.token != EOF && openBraces > 0) - skipBracesHook() getOrElse { - if (in.token == LBRACE || in.token == INDENT) openBraces += 1 - else if (in.token == RBRACE || in.token == OUTDENT) openBraces -= 1 - in.nextToken() - } - finally - in.checkEndMarker = savedCheckEndMarker + while (in.token != EOF && openBraces > 0) + skipBracesHook() getOrElse { + if (in.token == LBRACE || in.token == INDENT) openBraces += 1 + else if (in.token == RBRACE || in.token == OUTDENT) openBraces -= 1 + in.nextToken() + } } } @@ -368,19 +363,18 @@ object Parsers { def acceptStatSep(): Unit = if in.isNewLine then in.nextToken() else accept(SEMI) - def acceptStatSepUnlessAtEnd(altEnd: Token = EOF): Unit = + def acceptStatSepUnlessAtEnd(stats: ListBuffer[Tree], altEnd: Token = EOF): Unit = in.observeOutdented() - if (!isStatSeqEnd) - in.token match { - case EOF => - case `altEnd` => - case NEWLINE | NEWLINES => in.nextToken() - case SEMI => in.nextToken() - case _ => + in.token match + case SEMI | NEWLINE | NEWLINES => + in.nextToken() + checkEndMarker(stats) + case `altEnd` => + case _ => + if !isStatSeqEnd then syntaxError(i"end of statement expected but ${showToken(in.token)} found") in.nextToken() // needed to ensure progress; otherwise we might cycle forever accept(SEMI) - } def rewriteNotice(additionalOption: String = "") = { val optionStr = if (additionalOption.isEmpty) "" else " " ++ additionalOption @@ -1292,10 +1286,48 @@ object Parsers { else newLineOptWhenFollowedBy(LBRACE) - def endMarkerScope[T](pid: Tree)(op: => T): T = pid match { - case pid: RefTree => in.endMarkerScope(pid.name.toTermName)(op) - case _ => op - } + def checkEndMarker(stats: ListBuffer[Tree]): Unit = + + def matches(stat: Tree): Boolean = stat match + case stat: MemberDef if !stat.name.isEmpty => + if stat.name == nme.CONSTRUCTOR then in.token == THIS + else in.isIdent && in.name == stat.name.toTermName + case ModuleDef(name, Template(_, Nil, _, _)) => + in.token == IDENTIFIER && in.name == nme.extension + case PackageDef(pid: RefTree, _) => + in.isIdent && in.name == pid.name + case PatDef(_, IdPattern(id, _) :: Nil, _, _) => + in.isIdent && in.name == id.name + case stat: MemberDef if stat.mods.is(Given) => in.token == GIVEN + case _: PatDef => in.token == VAL + case _: If => in.token == IF + case _: WhileDo => in.token == WHILE + case _: ParsedTry => in.token == TRY + case _: Match => in.token == MATCH + case _: New => in.token == NEW + case _: (ForYield | ForDo) => in.token == FOR + case _ => false + + if isIdent(nme.end) then + val start = in.offset + val isEndMarker = + val endLine = source.offsetToLine(start) + val lookahead = in.LookaheadScanner() + lookahead.nextToken() + source.offsetToLine(lookahead.offset) == endLine + && endMarkerTokens.contains(in.token) + && { + lookahead.nextToken() + lookahead.token == EOF + || source.offsetToLine(lookahead.offset) > endLine + } + if isEndMarker then + in.nextToken() + if stats.isEmpty || !matches(stats.last) then + syntaxError("misaligned end marker", Span(start, in.lastCharOffset)) + in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion + in.nextToken() + end checkEndMarker /* ------------- TYPES ------------------------------------------------------ */ @@ -1878,15 +1910,13 @@ object Parsers { def expr1(location: Location.Value = Location.ElseWhere): Tree = in.token match case IF => - in.endMarkerScope(IF) { ifExpr(in.offset, If) } + ifExpr(in.offset, If) case WHILE => - in.endMarkerScope(WHILE) { - atSpan(in.skipToken()) { - val cond = condExpr(DO) - newLinesOpt() - val body = subExpr() - WhileDo(cond, body) - } + atSpan(in.skipToken()) { + val cond = condExpr(DO) + newLinesOpt() + val body = subExpr() + WhileDo(cond, body) } case DO => ctx.errorOrMigrationWarning( @@ -1913,40 +1943,38 @@ object Parsers { WhileDo(Block(body, cond), Literal(Constant(()))) } case TRY => - in.endMarkerScope(TRY) { - val tryOffset = in.offset - atSpan(in.skipToken()) { - val body = expr() - val (handler, handlerStart) = - if in.token == CATCH then - val span = in.offset - in.nextToken() - (if in.token == CASE then Match(EmptyTree, caseClause(exprOnly = true) :: Nil) - else subExpr(), - span) - else (EmptyTree, -1) - - handler match { - case Block(Nil, EmptyTree) => - assert(handlerStart != -1) - syntaxError( - EmptyCatchBlock(body), - Span(handlerStart, endOffset(handler)) - ) - case _ => - } - - val finalizer = - if (in.token == FINALLY) { in.nextToken(); subExpr() } - else { - if (handler.isEmpty) warning( - EmptyCatchAndFinallyBlock(body), - source.atSpan(Span(tryOffset, endOffset(body))) - ) - EmptyTree - } - ParsedTry(body, handler, finalizer) + val tryOffset = in.offset + atSpan(in.skipToken()) { + val body = expr() + val (handler, handlerStart) = + if in.token == CATCH then + val span = in.offset + in.nextToken() + (if in.token == CASE then Match(EmptyTree, caseClause(exprOnly = true) :: Nil) + else subExpr(), + span) + else (EmptyTree, -1) + + handler match { + case Block(Nil, EmptyTree) => + assert(handlerStart != -1) + syntaxError( + EmptyCatchBlock(body), + Span(handlerStart, endOffset(handler)) + ) + case _ => } + + val finalizer = + if (in.token == FINALLY) { in.nextToken(); subExpr() } + else { + if (handler.isEmpty) warning( + EmptyCatchAndFinallyBlock(body), + source.atSpan(Span(tryOffset, endOffset(body))) + ) + EmptyTree + } + ParsedTry(body, handler, finalizer) } case THROW => atSpan(in.skipToken()) { Throw(expr()) } @@ -2045,10 +2073,8 @@ object Parsers { /** MatchClause ::= `match' `{' CaseClauses `}' */ def matchClause(t: Tree): Match = - in.endMarkerScope(MATCH) { - atSpan(t.span.start, in.skipToken()) { - Match(t, inBracesOrIndented(caseClauses(() => caseClause()))) - } + atSpan(t.span.start, in.skipToken()) { + Match(t, inBracesOrIndented(caseClauses(() => caseClause()))) } /** `match' `{' TypeCaseClauses `}' @@ -2249,21 +2275,19 @@ object Parsers { * | ‘new’ TemplateBody */ def newExpr(): Tree = - in.endMarkerScope(NEW) { - val start = in.skipToken() - def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset)) - possibleTemplateStart() - val parents = - if in.isNestedStart then Nil - else constrApps(commaOK = false, templateCanFollow = true) - colonAtEOLOpt() - possibleTemplateStart(isNew = true) - parents match { - case parent :: Nil if !in.isNestedStart => - reposition(if (parent.isType) ensureApplied(wrapNew(parent)) else parent) - case _ => - New(reposition(templateBodyOpt(emptyConstructor, parents, Nil))) - } + val start = in.skipToken() + def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset)) + possibleTemplateStart() + val parents = + if in.isNestedStart then Nil + else constrApps(commaOK = false, templateCanFollow = true) + colonAtEOLOpt() + possibleTemplateStart(isNew = true) + parents match { + case parent :: Nil if !in.isNestedStart => + reposition(if (parent.isType) ensureApplied(wrapNew(parent)) else parent) + case _ => + New(reposition(templateBodyOpt(emptyConstructor, parents, Nil))) } /** ExprsInParens ::= ExprInParens {`,' ExprInParens} @@ -2415,7 +2439,7 @@ object Parsers { * {nl} [`yield'] Expr * | `for' Enumerators (`do' Expr | `yield' Expr) */ - def forExpr(): Tree = in.endMarkerScope(FOR) { + def forExpr(): Tree = atSpan(in.skipToken()) { var wrappedEnums = true val start = in.offset @@ -2492,7 +2516,6 @@ object Parsers { ForDo(enums, expr()) } } - } /** CaseClauses ::= CaseClause {CaseClause} * TypeCaseClauses ::= TypeCaseClause {TypeCaseClause} @@ -3175,15 +3198,13 @@ object Parsers { else emptyType val rhs = if tpt.isEmpty || in.token == EQUALS then - endMarkerScope(first) { - accept(EQUALS) - subExpr() match - case rhs0 @ Ident(name) if placeholderParams.nonEmpty && name == placeholderParams.head.name - && !tpt.isEmpty && mods.is(Mutable) && lhs.forall(_.isInstanceOf[Ident]) => - placeholderParams = placeholderParams.tail - atSpan(rhs0.span) { Ident(nme.WILDCARD) } - case rhs0 => rhs0 - } + accept(EQUALS) + subExpr() match + case rhs0 @ Ident(name) if placeholderParams.nonEmpty && name == placeholderParams.head.name + && !tpt.isEmpty && mods.is(Mutable) && lhs.forall(_.isInstanceOf[Ident]) => + placeholderParams = placeholderParams.tail + atSpan(rhs0.span) { Ident(nme.WILDCARD) } + case rhs0 => rhs0 else EmptyTree lhs match { case IdPattern(id, t) :: Nil if t.isEmpty => @@ -3286,10 +3307,8 @@ object Parsers { if (migrateTo3) newLineOptWhenFollowedBy(LBRACE) val rhs = if (in.token == EQUALS) - in.endMarkerScope(name) { - in.nextToken() - subExpr() - } + in.nextToken() + subExpr() else if (!tpt.isEmpty) EmptyTree else if (scala2ProcedureSyntax(": Unit")) { @@ -3418,11 +3437,9 @@ object Parsers { } def classDefRest(start: Offset, mods: Modifiers, name: TypeName): TypeDef = - in.endMarkerScope(name.toTermName) { - val constr = classConstr(isCaseClass = mods.is(Case)) - val templ = templateOpt(constr) - finalizeDef(TypeDef(name, templ), mods, start) - } + val constr = classConstr(isCaseClass = mods.is(Case)) + val templ = templateOpt(constr) + finalizeDef(TypeDef(name, templ), mods, start) /** ClassConstr ::= [ClsTypeParamClause] [ConstrMods] ClsParamClauses */ @@ -3442,10 +3459,8 @@ object Parsers { */ def objectDef(start: Offset, mods: Modifiers): ModuleDef = atSpan(start, nameStart) { val name = ident() - in.endMarkerScope(name) { - val templ = templateOpt(emptyConstructor) - finalizeDef(ModuleDef(name, templ), mods, start) - } + val templ = templateOpt(emptyConstructor) + finalizeDef(ModuleDef(name, templ), mods, start) } private def checkAccessOnly(mods: Modifiers, where: String): Modifiers = @@ -3459,12 +3474,10 @@ object Parsers { def enumDef(start: Offset, mods: Modifiers): TypeDef = atSpan(start, nameStart) { val mods1 = checkAccessOnly(mods, "definitions") val modulName = ident() - in.endMarkerScope(modulName) { - val clsName = modulName.toTypeName - val constr = classConstr() - val templ = template(constr, isEnum = true) - finalizeDef(TypeDef(clsName, templ), mods1, start) - } + val clsName = modulName.toTypeName + val constr = classConstr() + val templ = template(constr, isEnum = true) + finalizeDef(TypeDef(clsName, templ), mods1, start) } /** EnumCase = `case' (id ClassConstr [`extends' ConstrApps] | ids) @@ -3532,7 +3545,7 @@ object Parsers { val nameStart = in.offset val name = if isIdent && hasGivenSig then ident() else EmptyTermName - val gdef = in.endMarkerScope(if name.isEmpty then GIVEN else name) { + val gdef = val tparams = typeParamClauseOpt(ParamOwner.Def) newLineOpt() val vparamss = @@ -3565,7 +3578,7 @@ object Parsers { val templ = templateBodyOpt(makeConstructor(tparams1, vparamss1), parents, Nil) if tparams.isEmpty && vparamss.isEmpty then ModuleDef(name, templ) else TypeDef(name.toTypeName, templ) - } + end gdef finalizeDef(gdef, mods1, start) } @@ -3575,23 +3588,21 @@ object Parsers { in.nextToken() val nameOffset = in.offset val name = if isIdent && !isIdent(nme.on) then ident() else EmptyTermName - in.endMarkerScope(if name.isEmpty then nme.extension else name) { - val (tparams, vparamss, extensionFlag) = - if isIdent(nme.on) then - in.nextToken() - val tparams = typeParamClauseOpt(ParamOwner.Def) - val extParams = paramClause(0, prefix = true) - val givenParamss = paramClauses(givenOnly = true) - (tparams, extParams :: givenParamss, Extension) - else - (Nil, Nil, EmptyFlags) - possibleTemplateStart() - if !in.isNestedStart then syntaxError("Extension without extension methods") - val templ = templateBodyOpt(makeConstructor(tparams, vparamss), Nil, Nil) - templ.body.foreach(checkExtensionMethod(tparams, vparamss, _)) - val edef = atSpan(start, nameOffset, in.offset)(ModuleDef(name, templ)) - finalizeDef(edef, addFlag(mods, Given | extensionFlag), start) - } + val (tparams, vparamss, extensionFlag) = + if isIdent(nme.on) then + in.nextToken() + val tparams = typeParamClauseOpt(ParamOwner.Def) + val extParams = paramClause(0, prefix = true) + val givenParamss = paramClauses(givenOnly = true) + (tparams, extParams :: givenParamss, Extension) + else + (Nil, Nil, EmptyFlags) + possibleTemplateStart() + if !in.isNestedStart then syntaxError("Extension without extension methods") + val templ = templateBodyOpt(makeConstructor(tparams, vparamss), Nil, Nil) + templ.body.foreach(checkExtensionMethod(tparams, vparamss, _)) + val edef = atSpan(start, nameOffset, in.offset)(ModuleDef(name, templ)) + finalizeDef(edef, addFlag(mods, Given | extensionFlag), start) /* -------- TEMPLATES ------------------------------------------- */ @@ -3695,14 +3706,11 @@ object Parsers { /** Packaging ::= package QualId [nl] `{' TopStatSeq `}' */ - def packaging(start: Int): Tree = { + def packaging(start: Int): Tree = val pkg = qualId() - endMarkerScope(pkg) { - possibleTemplateStart() - val stats = inDefScopeBraces(topStatSeq()) - makePackaging(start, pkg, stats) - } - } + possibleTemplateStart() + val stats = inDefScopeBraces(topStatSeq()) + makePackaging(start, pkg, stats) /** TopStatSeq ::= TopStat {semi TopStat} * TopStat ::= Import @@ -3735,7 +3743,7 @@ object Parsers { syntaxErrorOrIncomplete(OnlyCaseClassOrCaseObjectAllowed()) else syntaxErrorOrIncomplete(ExpectedToplevelDef()) - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(stats) } stats.toList } @@ -3769,7 +3777,7 @@ object Parsers { } else { stats += first - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(stats) } } var exitOnError = false @@ -3787,7 +3795,7 @@ object Parsers { exitOnError = mustStartStat syntaxErrorOrIncomplete("illegal start of definition") } - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(stats) } (self, if (stats.isEmpty) List(EmptyTree) else stats.toList) } @@ -3825,7 +3833,7 @@ object Parsers { "illegal start of declaration" + (if (inFunReturnType) " (possible cause: missing `=` in front of current method body)" else "")) - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(stats) } stats.toList } @@ -3864,7 +3872,7 @@ object Parsers { exitOnError = mustStartStat syntaxErrorOrIncomplete(IllegalStartOfStatement(isModifier)) } - acceptStatSepUnlessAtEnd(CASE) + acceptStatSepUnlessAtEnd(stats, CASE) } stats.toList } @@ -3882,26 +3890,24 @@ object Parsers { in.nextToken() ts += objectDef(start, Modifiers(Package)) if (in.token != EOF) { - acceptStatSep() + acceptStatSepUnlessAtEnd(ts) ts ++= topStatSeq() } } else val pkg = qualId() var continue = false - endMarkerScope(pkg) { - possibleTemplateStart() - if in.token == EOF then - ts += makePackaging(start, pkg, List()) - else if in.isNestedStart then - ts += inDefScopeBraces(makePackaging(start, pkg, topStatSeq())) - continue = true - else - acceptStatSep() - ts += makePackaging(start, pkg, topstats()) - } + possibleTemplateStart() + if in.token == EOF then + ts += makePackaging(start, pkg, List()) + else if in.isNestedStart then + ts += inDefScopeBraces(makePackaging(start, pkg, topStatSeq())) + continue = true + else + acceptStatSep() + ts += makePackaging(start, pkg, topstats()) if continue then - acceptStatSepUnlessAtEnd() + acceptStatSepUnlessAtEnd(ts) ts ++= topStatSeq() } else diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 4182c8795b2a..80d341eb5cce 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -89,9 +89,6 @@ object Scanners { /** the base of a number */ var base: Int = 0 - /** Set to false to disable end marker alignment checks, used for outline parsing. */ - var checkEndMarker: Boolean = true - def copyFrom(td: TokenData): Unit = { this.token = td.token this.offset = td.offset @@ -181,6 +178,8 @@ object Scanners { /** A switch whether operators at the start of lines can be infix operators */ private[Scanners] var allowLeadingInfixOperators = true + var debugTokenStream = false + val rewrite = ctx.settings.rewrite.value.isDefined val oldSyntax = ctx.settings.oldSyntax.value val newSyntax = ctx.settings.newSyntax.value @@ -265,9 +264,6 @@ object Scanners { /** The current region. This is initially an Indented region with indentation width. */ var currentRegion: Region = Indented(IndentWidth.Zero, Set(), EMPTY, null) - /** The number of open end marker scopes */ - var openEndMarkers: List[(EndMarkerTag, IndentWidth)] = Nil - // Get next token ------------------------------------------------------------ /** Are we directly in a multiline string interpolation expression? @@ -344,11 +340,11 @@ object Scanners { if (isAfterLineEnd) handleNewLine(lastToken) postProcessToken() - //printState() + printState() } - protected def printState() = - print("[" + show + "]") + final def printState() = + if debugTokenStream then print("[" + show + "]") /** Insert `token` at assumed `offset` in front of current one. */ def insert(token: Token, offset: Int) = { @@ -358,51 +354,6 @@ object Scanners { this.token = token } - /** What can be referred to in an end marker */ - type EndMarkerTag = TermName | Token - - /** Establish a scope for a passible end-marker with given tag, parsed by `op` */ - def endMarkerScope[T](tag: EndMarkerTag)(op: => T): T = - val saved = openEndMarkers - openEndMarkers = (tag, currentRegion.indentWidth) :: openEndMarkers - try op - finally openEndMarkers = saved - - /** If this token and the next constitute an end marker, skip them and check they - * align with an opening construct with the same end marker tag, - * unless `checkEndMarker` is false. - */ - protected def skipEndMarker(width: IndentWidth): Unit = - if next.token == IDENTIFIER && next.name == nme.end then - val lookahead = LookaheadScanner() - val start = lookahead.offset - - def handle(tag: EndMarkerTag) = - def checkAligned(): Unit = openEndMarkers match - case (etag, ewidth) :: rest if width <= ewidth => - if width < ewidth || tag != etag then - openEndMarkers = rest - checkAligned() - case _ => - if checkEndMarker - lexical.println(i"misaligned end marker $tag, $width, $openEndMarkers") - errorButContinue("misaligned end marker", start) - - val skipTo = lookahead.charOffset - lookahead.nextToken() - if lookahead.isAfterLineEnd || lookahead.token == EOF then - checkAligned() - next.token = EMPTY - while charOffset < skipTo do nextChar() - end handle - - lookahead.nextToken() // skip the `end` - lookahead.token match - case IDENTIFIER | BACKQUOTED_IDENT => handle(lookahead.name) - case IF | WHILE | FOR | MATCH | TRY | NEW | GIVEN => handle(lookahead.token) - case _ => - end skipEndMarker - /** A leading symbolic or backquoted identifier is treated as an infix operator if * - it does not follow a blank line, and * - it is followed on the same line by at least one ' ' @@ -535,7 +486,6 @@ object Scanners { && !isLeadingInfixOperator() then insert(if (pastBlankLine) NEWLINES else NEWLINE, lineOffset) - skipEndMarker(nextWidth) else if indentIsSignificant then if nextWidth < lastWidth || nextWidth == lastWidth && (indentPrefix == MATCH || indentPrefix == CATCH) && token != CASE then @@ -546,7 +496,6 @@ object Scanners { case r: Indented => currentRegion = r.enclosing insert(OUTDENT, offset) - skipEndMarker(nextWidth) case r: InBraces if !closingRegionTokens.contains(token) => ctx.warning("Line is indented too far to the left, or a `}` is missing", source.atSpan(Span(offset))) @@ -934,13 +883,7 @@ object Scanners { // Lookahead --------------------------------------------------------------- - class LookaheadScanner() extends Scanner(source, offset) { - override def skipEndMarker(width: IndentWidth) = () - override protected def printState() = { - print("la:") - super.printState() - } - } + class LookaheadScanner() extends Scanner(source, offset) /** Skip matching pairs of `(...)` or `[...]` parentheses. * @pre The current token is `(` or `[` @@ -1343,8 +1286,8 @@ object Scanners { def show: String = token match { case IDENTIFIER | BACKQUOTED_IDENT => s"id($name)" case CHARLIT => s"char($strVal)" - case INTLIT => s"int($strVal, $base)" - case LONGLIT => s"long($strVal, $base)" + case INTLIT => s"int($strVal, base = $base)" + case LONGLIT => s"long($strVal, base = $base)" case FLOATLIT => s"float($strVal)" case DOUBLELIT => s"double($strVal)" case STRINGLIT => s"string($strVal)" diff --git a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala index 389407497ea9..b7e5501f1289 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala @@ -256,8 +256,7 @@ object Tokens extends TokensCommon { final val canStartStatTokens3: TokenSet = canStartExprTokens3 | mustStartStatTokens | BitSet( AT, CASE) - final val canEndStatTokens: TokenSet = atomicExprTokens | BitSet( - TYPE, RPAREN, RBRACE, RBRACKET, OUTDENT) // TODO: remove GIVEN once old import syntax is dropped + final val canEndStatTokens: TokenSet = atomicExprTokens | BitSet(TYPE, RPAREN, RBRACE, RBRACKET, OUTDENT) /** Tokens that stop a lookahead scan search for a `<-`, `then`, or `do`. * Used for disambiguating between old and new syntax. @@ -282,5 +281,7 @@ object Tokens extends TokensCommon { final val scala3keywords = BitSet(ENUM, ERASED, GIVEN) + final val endMarkerTokens = identifierTokens | BitSet(IF, WHILE, FOR, MATCH, TRY, NEW, GIVEN, VAL, THIS) + final val softModifierNames = Set(nme.inline, nme.opaque, nme.open, nme.transparent) } diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index f795781b05d9..56c6f5716b6d 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -50,9 +50,7 @@ object SyntaxHighlighting { else highlightRange(span.start, span.end, color) - val scanner = new Scanner(source) { - override protected def printState() = () - } + val scanner = new Scanner(source) while (scanner.token != EOF) { val start = scanner.offset val token = scanner.token diff --git a/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala b/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala index bcff04f10249..99be4cc440f8 100644 --- a/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala +++ b/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala @@ -53,6 +53,7 @@ class FrontEnd extends Phase { if (unit.isJava) new JavaParser(unit.source).parse() else { val p = new Parser(unit.source) + // p.in.debugTokenStream = true val tree = p.parse() if (p.firstXmlPos.exists && !firstXmlPos.exists) firstXmlPos = p.firstXmlPos diff --git a/compiler/src/dotty/tools/dotc/typer/Nullables.scala b/compiler/src/dotty/tools/dotc/typer/Nullables.scala index d4f062b88a47..ad64a8f41f89 100644 --- a/compiler/src/dotty/tools/dotc/typer/Nullables.scala +++ b/compiler/src/dotty/tools/dotc/typer/Nullables.scala @@ -189,7 +189,7 @@ object Nullables: val mutables = infos.foldLeft(Set[TermRef]())((ms, info) => ms.union(info.asserted.filter(_.symbol.is(Mutable)))) infos.extendWith(NotNullInfo(Set(), mutables)) - // end notNullInfoOps + end notNullInfoOps extension refOps on (ref: TermRef): @@ -244,6 +244,7 @@ object Nullables: refSym.is(Mutable) // if it is immutable, we don't need to check the rest conditions && refOwner.isTerm && recur(ctx.owner) + end refOps extension treeOps on (tree: Tree): @@ -334,6 +335,7 @@ object Nullables: traverseChildren(tree) tree.computeNullable() }.traverse(tree) + end treeOps extension assignOps on (tree: Assign): def computeAssignNullable()(using Context): tree.type = tree.lhs match @@ -351,6 +353,7 @@ object Nullables: tree.withNotNullInfo(NotNullInfo(Set(ref), Set())) else tree case _ => tree + end assignOps private val analyzedOps = Set(nme.EQ, nme.NE, nme.eq, nme.ne, nme.ZAND, nme.ZOR, nme.UNARY_!) @@ -444,13 +447,16 @@ object Nullables: * xs = xs.next */ def whileContext(whileSpan: Span)(using Context): Context = + def isRetracted(ref: TermRef): Boolean = val sym = ref.symbol sym.span.exists && assignmentSpans.getOrElse(sym.span.start, Nil).exists(whileSpan.contains(_)) && ctx.notNullInfos.impliesNotNull(ref) + val retractedVars = ctx.notNullInfos.flatMap(_.asserted.filter(isRetracted)).toSet ctx.addNotNullInfo(NotNullInfo(Set(), retractedVars)) + end whileContext /** Post process all arguments to by-name parameters by removing any not-null * info that was used when typing them. Concretely: diff --git a/docs/docs/internals/syntax.md b/docs/docs/internals/syntax.md index 96d0506c06d6..4b3867148b62 100644 --- a/docs/docs/internals/syntax.md +++ b/docs/docs/internals/syntax.md @@ -104,9 +104,8 @@ yield ### Soft keywords ``` -as derives extension inline on opaque open transparent -using -* + - +as derives end extension inline on opaque open transparent using +* + - ``` ## Context-free Syntax @@ -241,6 +240,7 @@ BlockStat ::= Import | {Annotation [nl]} [‘implicit’ | ‘lazy’] Def | {Annotation [nl]} {LocalModifier} TmplDef | Expr1 + | EndMarker ForExpr ::= ‘for’ (‘(’ Enumerators ‘)’ | ‘{’ Enumerators ‘}’) ForYield(enums, expr) {nl} [‘yield’] Expr @@ -344,6 +344,10 @@ ImportSelectors ::= id [‘=>’ id | ‘=>’ ‘_’] [‘,’ ImportSelect WildCardSelector ::= ‘given’ (‘_' | InfixType) | ‘_' Export ::= ‘export’ [‘given’] ImportExpr {‘,’ ImportExpr} + +EndMarker ::= ‘end’ EndMarkerTag -- when followed by EOL +EndMarkerTag ::= id | ‘if’ | ‘while’ | ‘for’ | ‘match’ | ‘try’ + | ‘new’ | ‘this’ | ‘given’ | ‘extension’ | ‘val’ ``` ### Declarations and Definitions @@ -406,6 +410,7 @@ TemplateStat ::= Import | {Annotation [nl]} {Modifier} Def | {Annotation [nl]} {Modifier} Dcl | Expr1 + | EndMarker | SelfType ::= id [‘:’ InfixType] ‘=>’ ValDef(_, name, tpt, _) | ‘this’ ‘:’ InfixType ‘=>’ @@ -422,6 +427,7 @@ TopStat ::= Import | {Annotation [nl]} {Modifier} Def | Packaging | PackageObject + | EndMarker | Packaging ::= ‘package’ QualId [nl | colonEol] ‘{’ TopStatSeq ‘}’ Package(qid, stats) PackageObject ::= ‘package’ ‘object’ ObjectDef object with package in mods. diff --git a/docs/docs/reference/other-new-features/indentation.md b/docs/docs/reference/other-new-features/indentation.md index ce41be1f63cc..d3b5db2a38fc 100644 --- a/docs/docs/reference/other-new-features/indentation.md +++ b/docs/docs/reference/other-new-features/indentation.md @@ -187,28 +187,102 @@ println(".") Indentation-based syntax has many advantages over other conventions. But one possible problem is that it makes it hard to discern when a large indentation region ends, since there is no specific token that delineates the end. Braces are not much better since a brace by itself also contains no information about what region is closed. -To solve this problem, Scala 3 offers an optional `end` marker. Example +To solve this problem, Scala 3 offers an optional `end` marker. Example: ```scala def largeMethod(...) = - ... - if ... then ... - else - ... // a large block - end if - ... // more code + ... + if ... then ... + else + ... // a large block + end if + ... // more code end largeMethod ``` -An `end` marker consists of the identifier `end` which follows an `` token, and is in turn followed on the same line by exactly one other token, which is either an identifier or one of the reserved words +An `end` marker consists of the identifier `end` and a follow-on specifier token that together constitute all the tokes of a line. Possible specifier tokens are +identifiers or one of the following keywords ```scala -if while for match try new given extension +if while for match try new this val given +``` +End markers are allowed in statement sequences. The specifier token `s` of an end marker must correspond to the statement that precedes it. This means: + + - If the statement defines a member `x` then `s` must be the same identifier `x`. + - If the statement defines a constructor then `s` must be `this`. + - If the statement defines an anonymous given, then `s` must be `given`. + - If the statement defines an anonymous extension, then `s` must be `extension`. + - If the statement defines an anonymous class, then `s` must be `new`. + - If the statement is a `val` definition binding a pattern, then `s` must be `val`. + - If the statement is a package clause that refers to package `p`, then `s` must be the same identifier `p`. + - If the statement is an `if`, `while`, `for`, `try`, or `match` statement, then `s` must be that same token. + +For instance, the following end markers are all legal: + ```scala + package p1.p2: + + abstract class C(): + + def this(x: Int) = + this() + if x > 0 then + val a :: b = + x :: Nil + end val + var y = + x + end y + while y > 0 do + println(y) + y -= 1 + end while + try + x match + case 0 => println("0") + case _ => + end match + finally + println("done") + end try + end if + end this + + def f: String + end C + + object C: + given C = + new C: + def f = "!" + end f + end new + end given + end C + + extension on (x: C): + def ff: String = x.f ++ x.f + end extension + + end p2 ``` -If `end` is followed by a reserved word, the compiler checks that the marker closes an indentation region belonging to a construct that starts with the reserved word. If it is followed by an identifier _id_, the compiler checks that the marker closes a definition -that defines _id_ or a package clause that refers to _id_. -`end` itself is a soft keyword. It is only treated as an `end` marker if it -occurs at the start of a line and is followed by an identifier or one of the reserved words above. +#### When to Use End Markers + +It is recommended that `end` markers are used for code where the extent of an indentation region is not immediately apparent "at a glance". People will have different preferences what this means, but one can nevertheless give some guidelines that stem from experience. An end marker makes sense if + + - the construct contains blank lines, or + - the construct is long, say 15-20 lines or more, + - the construct ends heavily indented, say 4 indentation levels or more. -It is recommended that `end` markers are used for code where the extent of an indentation region is not immediately apparent "at a glance". Typically this is the case if an indentation region spans 20 lines or more. +If none of these criteria apply, it's often better to not use an end marker since the code will be just as clear and more concise. If there are several ending regions that satisfy one of the criteria above, we usually need an end marker only for the outermost closed reason. So cascades of end markers as in the example above are usually better avoided. + +#### Syntax + +``` +EndMarker ::= ‘end’ EndMarkerTag -- when followed by EOL +EndMarkerTag ::= id | ‘if’ | ‘while’ | ‘for’ | ‘match’ | ‘try’ + | ‘new’ | ‘this’ | ‘given’ | ‘extension’ | ‘val’ +BlockStat ::= ... | EndMarker +TemplateStat ::= ... | EndMarker +TopStat ::= ... | EndMarker +``` ### Example diff --git a/docs/docs/reference/overview.md b/docs/docs/reference/overview.md index 5d63e07ff236..d333a71e8933 100644 --- a/docs/docs/reference/overview.md +++ b/docs/docs/reference/overview.md @@ -33,7 +33,7 @@ These constructs replace existing constructs with the aim of making the language - [Trait Parameters](other-new-features/trait-parameters.md) replace [early initializers](dropped-features/early-initializers.md) with a more generally useful construct. - [Given Instances](contextual/delegates.md) replace implicit objects and defs, focussing on intent over mechanism. - - [Given Clauses](contextual/given-clauses.md) replace implicit parameters, avoiding their ambiguities. + - [Using Clauses](contextual/using-clauses.md) replace implicit parameters, avoiding their ambiguities. - [Extension Methods](contextual/extension-methods.md) replace implicit classes with a clearer and simpler mechanism. - [Opaque Type Aliases](other-new-features/opaques.md) replace most uses of value classes while guaranteeing absence of boxing. diff --git a/tests/neg/endmarkers.scala b/tests/neg/endmarkers.scala index 71d080a0c379..5c7c1c686cd0 100644 --- a/tests/neg/endmarkers.scala +++ b/tests/neg/endmarkers.scala @@ -9,7 +9,7 @@ object Test: x += 1 x < 10 do () - end while // error: misaligned end marker + end while // warning: line indented too far to the left } def f(x: Int): Int = diff --git a/tests/neg/endmarkers1.scala b/tests/neg/endmarkers1.scala index 9de2f9746522..b2b746c57371 100644 --- a/tests/neg/endmarkers1.scala +++ b/tests/neg/endmarkers1.scala @@ -14,5 +14,5 @@ end Test // error: misaligned end marker def f[T](x: Option[T]) = x match case Some(y) => case None => "hello" - end f // error: misaligned end marker + end f // error: The start of this line does not match any of the previous indentation widths. diff --git a/tests/neg/i8731.scala b/tests/neg/i8731.scala new file mode 100644 index 000000000000..bda0a4e7325e --- /dev/null +++ b/tests/neg/i8731.scala @@ -0,0 +1,29 @@ +object test: + + 3 match + case 3 => ??? + end match + case _ => () // error: missing parameter type + end match + + if 3 == 3 + () + end if + else // error: illegal start of definition + () + end if + + class Test { + val test = 3 + end Test // error: misaligned end marker + } + + while + 3 == 3 + end while // error: `do` expected + do () + + for + a <- Seq() + end for // error: `yield` or `do` expected + do () \ No newline at end of file diff --git a/tests/pos/endmarkers.scala b/tests/pos/endmarkers.scala new file mode 100644 index 000000000000..f889e7d63f12 --- /dev/null +++ b/tests/pos/endmarkers.scala @@ -0,0 +1,58 @@ +trait T: + object O: + def foo = + 1 + end foo + def bar = 2 + end bar + end O +end T + +object T +end T + +package p1.p2: + + abstract class C(): + + def this(x: Int) = + this() + if x > 0 then + val a :: b = + x :: Nil + end val + var y = + x + end y + while y > 0 do + println(y) + y -= 1 + end while + try + x match + case 0 => println("0") + case _ => + end match + finally + println("done") + end try + end if + end this + + def f: String + end C + + object C: + given C = + new C: + def f = "!" + end f + end new + end given + end C + + extension on (x: C): + def ff: String = x.f + end extension + +end p2 \ No newline at end of file