Skip to content

Port text block support #14882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 15, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions compiler/src/dotty/tools/dotc/config/CommandLineParser.scala
Original file line number Diff line number Diff line change
Expand Up @@ -95,19 +95,17 @@ object CommandLineParser:

def tokenize(line: String): List[String] = tokenize(line, x => throw new ParseException(x))

/**
* Expands all arguments starting with @ to the contents of the
* file named like each argument.
/** Expands all arguments starting with @ to the contents of the file named like each argument.
*/
def expandArg(arg: String): List[String] =
def stripComment(s: String) = s takeWhile (_ != '#')
val path = Paths.get(arg stripPrefix "@")
if (!Files.exists(path))
val path = Paths.get(arg.stripPrefix("@"))
if !Files.exists(path) then
System.err.nn.println(s"Argument file ${path.nn.getFileName} could not be found")
Nil
else
val lines = Files.readAllLines(path).nn // default to UTF-8 encoding
val params = lines.asScala map stripComment mkString " "
def stripComment(s: String) = s.indexOf('#') match { case -1 => s case i => s.substring(0, i) }
val lines = Files.readAllLines(path).nn
val params = lines.asScala.map(stripComment).filter(!_.nn.isEmpty).mkString(" ")
tokenize(params)

class ParseException(msg: String) extends RuntimeException(msg)
246 changes: 197 additions & 49 deletions compiler/src/dotty/tools/dotc/parsing/JavaScanners.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import core.Names.SimpleName
import Scanners._
import util.SourceFile
import JavaTokens._
import scala.annotation.{ switch, tailrec }
import scala.annotation.{switch, tailrec}
import util.Chars._
import PartialFunction.cond

object JavaScanners {

Expand All @@ -31,23 +32,29 @@ object JavaScanners {
// Get next token ------------------------------------------------------------

def nextToken(): Unit =
if (next.token == EMPTY) {
if next.token == EMPTY then
lastOffset = lastCharOffset
fetchToken()
}
else {
this copyFrom next
else
this.copyFrom(next)
next.token = EMPTY
}

def lookaheadToken: Int = {
prev copyFrom this
nextToken()
def lookaheadToken: Int =
lookAhead()
val t = token
next copyFrom this
this copyFrom prev
reset()
t
}

def lookAhead() =
prev.copyFrom(this)
nextToken()

def reset() =
next.copyFrom(this)
this.copyFrom(prev)

class LookaheadScanner extends JavaScanner(source, startFrom = charOffset - 1):
override protected def initialize(): Unit = nextChar()

/** read next token
*/
Expand Down Expand Up @@ -93,15 +100,23 @@ object JavaScanners {

case '\"' =>
nextChar()
while (ch != '\"' && (isUnicodeEscape || ch != CR && ch != LF && ch != SU))
getlitch()
if (ch == '\"') {
token = STRINGLIT
setStrVal()
nextChar()
}
if ch != '\"' then // "..." non-empty string literal
while ch != '\"' && (isUnicodeEscape || ch != CR && ch != LF && ch != SU) do
getlitch()
if ch == '\"' then
token = STRINGLIT
setStrVal()
nextChar()
else
error("unclosed string literal")
else
error("unclosed string literal")
nextChar()
if ch != '\"' then // "" empty string literal
token = STRINGLIT
setStrVal()
else
nextChar()
getTextBlock()

case '\'' =>
nextChar()
Expand Down Expand Up @@ -399,46 +414,177 @@ object JavaScanners {

// Literals -----------------------------------------------------------------

/** read next character in character or string literal:
/** Read next character in character or string literal.
*/
protected def getlitch(): Unit =
if (ch == '\\') {
protected def getlitch(): Unit = getlitch(scanOnly = false, inTextBlock = false)

/** Read next character in character or string literal.
*
* @param scanOnly skip emitting errors or adding to the literal buffer
* @param inTextBlock is this for a text block?
*/
def getlitch(scanOnly: Boolean, inTextBlock: Boolean): Unit =
def octal: Char =
val leadch: Char = ch
var oct: Int = digit2int(ch, 8)
nextChar()
if ('0' <= ch && ch <= '7') {
val leadch: Char = ch
var oct: Int = digit2int(ch, 8)
oct = oct * 8 + digit2int(ch, 8)
nextChar()
if ('0' <= ch && ch <= '7') {
if (leadch <= '3' && '0' <= ch && ch <= '7') {
oct = oct * 8 + digit2int(ch, 8)
nextChar()
if (leadch <= '3' && '0' <= ch && ch <= '7') {
oct = oct * 8 + digit2int(ch, 8)
nextChar()
}
}
oct.asInstanceOf[Char]
end octal
def greatEscape: Char =
nextChar()
if '0' <= ch && ch <= '7' then octal
else
val x = ch match
case 'b' => '\b'
case 's' => ' '
case 't' => '\t'
case 'n' => '\n'
case 'f' => '\f'
case 'r' => '\r'
case '\"' => '\"'
case '\'' => '\''
case '\\' => '\\'
case CR | LF if inTextBlock =>
if !scanOnly then nextChar()
0
case _ =>
if !scanOnly then error("invalid escape character", charOffset - 1)
ch
if x != 0 then nextChar()
x
end greatEscape

// begin getlitch
val c: Char =
if ch == '\\' then greatEscape
else
val res = ch
nextChar()
res
if c != 0 && !scanOnly then putChar(c)
end getlitch

/** Read a triple-quote delimited text block, starting after the first three double quotes.
*/
private def getTextBlock(): Unit = {
// Open delimiter is followed by optional space, then a newline
while (ch == ' ' || ch == '\t' || ch == FF) {
nextChar()
}
if (ch != LF && ch != CR) { // CR-LF is already normalized into LF by `JavaCharArrayReader`
error("illegal text block open delimiter sequence, missing line terminator")
return
}
nextChar()

/* Do a lookahead scan over the full text block to:
* - compute common white space prefix
* - find the offset where the text block ends
*/
var commonWhiteSpacePrefix = Int.MaxValue
var blockEndOffset = 0
var blockClosed = false
var lineWhiteSpacePrefix = 0
var lineIsOnlyWhitespace = true
val in = LookaheadScanner()
while (!blockClosed && (isUnicodeEscape || ch != SU)) {
if (in.ch == '\"') { // Potential end of the block
in.nextChar()
if (in.ch == '\"') {
in.nextChar()
if (in.ch == '\"') {
blockClosed = true
commonWhiteSpacePrefix = commonWhiteSpacePrefix min lineWhiteSpacePrefix
blockEndOffset = in.charOffset - 2
}
}
putChar(oct.asInstanceOf[Char])

// Not the end of the block - just a single or double " character
if (!blockClosed) {
lineIsOnlyWhitespace = false
}
} else if (in.ch == CR || in.ch == LF) { // new line in the block
in.nextChar()
if (!lineIsOnlyWhitespace) {
commonWhiteSpacePrefix = commonWhiteSpacePrefix min lineWhiteSpacePrefix
}
lineWhiteSpacePrefix = 0
lineIsOnlyWhitespace = true
} else if (lineIsOnlyWhitespace && Character.isWhitespace(in.ch)) { // extend white space prefix
in.nextChar()
lineWhiteSpacePrefix += 1
} else {
lineIsOnlyWhitespace = false
in.getlitch(scanOnly = true, inTextBlock = true)
}
else {
ch match {
case 'b' => putChar('\b')
case 't' => putChar('\t')
case 'n' => putChar('\n')
case 'f' => putChar('\f')
case 'r' => putChar('\r')
case '\"' => putChar('\"')
case '\'' => putChar('\'')
case '\\' => putChar('\\')
case _ =>
error("invalid escape character", charOffset - 1)
putChar(ch)
}

// Bail out if the block never did have an end
if (!blockClosed) {
error("unclosed text block")
return
}

// Second pass: construct the literal string value this time
while (charOffset < blockEndOffset) {
// Drop the line's leading whitespace
var remainingPrefix = commonWhiteSpacePrefix
while (remainingPrefix > 0 && ch != CR && ch != LF && charOffset < blockEndOffset) {
nextChar()
remainingPrefix -= 1
}

var trailingWhitespaceLength = 0
var escapedNewline = false // Does the line end with `\`?
while (ch != CR && ch != LF && charOffset < blockEndOffset && !escapedNewline) {
if (Character.isWhitespace(ch)) {
trailingWhitespaceLength += 1
} else {
trailingWhitespaceLength = 0
}

// Detect if the line is about to end with `\`
if ch == '\\' && cond(lookaheadChar()) { case CR | LF => true } then
escapedNewline = true

getlitch(scanOnly = false, inTextBlock = true)
}

// Remove the last N characters from the buffer */
def popNChars(n: Int): Unit =
if n > 0 then
val text = litBuf.toString
litBuf.clear()
val trimmed = text.substring(0, text.length - (n min text.length))
trimmed.nn.foreach(litBuf.append)

// Drop the line's trailing whitespace
popNChars(trailingWhitespaceLength)

// Normalize line terminators
if ((ch == CR || ch == LF) && !escapedNewline) {
nextChar()
putChar('\n')
}
}
else {
putChar(ch)
nextChar()
}

token = STRINGLIT
setStrVal()

// Trailing """
nextChar()
nextChar()
nextChar()
}
end getTextBlock

/** read fractional part and exponent of floating point number
* if one is present.
Expand Down Expand Up @@ -585,8 +731,10 @@ object JavaScanners {
}

/* Initialization: read first char, then first token */
nextChar()
nextToken()
protected def initialize(): Unit =
nextChar()
nextToken()
initialize()
}

private val (lastKeywordStart, kwArray) = buildKeywordArray(keywords)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class PatmatExhaustivityTest {
val options = List("-pagewidth", "80", "-color:never", "-Ystop-after:explicitSelf", "-classpath", TestConfiguration.basicClasspath)

private def compile(files: List[JPath]): Seq[String] = {
val opts = toolArgsFor(files)
val opts = toolArgsFor(files).get(ToolName.Scalac).getOrElse(Nil)
val stringBuffer = new StringWriter()
val printWriter = new PrintWriter(stringBuffer)
val reporter = TestReporter.simplifiedReporter(printWriter)
Expand Down
2 changes: 1 addition & 1 deletion compiler/test/dotty/tools/repl/ReplTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ extends ReplDriver(options, new PrintStream(out, true, StandardCharsets.UTF_8.na

val expectedOutput = lines.filter(nonBlank)
val actualOutput = {
val opts = toolArgsParse(lines.take(1))
val opts = toolArgsFor(ToolName.Scalac)(lines.take(1))
val (optsLine, inputLines) = if opts.isEmpty then ("", lines) else (lines.head, lines.drop(1))
resetToInitial(opts)

Expand Down
Loading