Proper UTF-8 encoding and line endings on Windows

compiler/src/dotty/tools/dotc/decompiler/DecompilationPrinter.scala

@@ -3,6 +3,8 @@ package decompiler
 
 import java.io.{OutputStream, PrintStream}
 
+import scala.io.Codec
+
 import dotty.tools.dotc.core.Contexts._
 import dotty.tools.dotc.core.Phases.Phase
 import dotty.tools.dotc.core.tasty.TastyPrinter
@@ -24,8 +26,9 @@ class DecompilationPrinter extends Phase {
       var os: OutputStream = null
       var ps: PrintStream = null
       try {
+        implicit val codec = Codec.UTF8
         os = File(outputDir.fileNamed("decompiled.scala").path).outputStream(append = true)
-        ps = new PrintStream(os)
+        ps = new PrintStream(os, /*autoFlush*/false, "UTF-8")
         printToOutput(ps)
       } finally {
         if (os ne null) os.close()

compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala

@@ -15,6 +15,9 @@ import scala.annotation.switch
 import scala.collection.mutable
 
 trait MessageRendering {
+
+  private final val EOL: String = sys.props("line.separator")
+
   /** Remove ANSI coloring from `str`, useful for getting real length of
     * strings
     *
@@ -101,7 +104,7 @@ trait MessageRendering {
 
     msg.linesIterator
       .map { line => " " * (offset - 1) + "|" + padding + line}
-      .mkString(sys.props("line.separator"))
+      .mkString(EOL)
   }
 
   /** The separator between errors containing the source file and error type
@@ -132,21 +135,21 @@ trait MessageRendering {
            |${Blue("Explanation")}
            |${Blue("===========")}"""
     )
-    sb.append('\n').append(m.explanation)
-    if (m.explanation.lastOption != Some('\n')) sb.append('\n')
+    sb.append(EOL).append(m.explanation)
+    if (m.explanation.lastOption != Some(EOL)) sb.append(EOL)
     sb.toString
   }
 
   /** The whole message rendered from `msg` */
   def messageAndPos(msg: Message, pos: SourcePosition, diagnosticLevel: String)(implicit ctx: Context): String = {
     val sb = mutable.StringBuilder.newBuilder
     val posString = posStr(pos, diagnosticLevel, msg)
-    if (posString.nonEmpty) sb.append(posString).append('\n')
+    if (posString.nonEmpty) sb.append(posString).append(EOL)
     if (pos.exists) {
       val (srcBefore, srcAfter, offset) = sourceLines(pos)
       val marker = columnMarker(pos, offset)
       val err = errorMsg(pos, msg.msg, offset)
-      sb.append((srcBefore ::: marker :: err :: outer(pos, " " * (offset - 1)) ::: srcAfter).mkString("\n"))
+      sb.append((srcBefore ::: marker :: err :: outer(pos, " " * (offset - 1)) ::: srcAfter).mkString(EOL))
     } else sb.append(msg.msg)
     sb.toString
   }

compiler/src/dotty/tools/repl/ReplDriver.scala

@@ -1,6 +1,6 @@
 package dotty.tools.repl
 
-import java.io.PrintStream
+import java.io.{File => JFile, PrintStream}
 
 import dotty.tools.dotc.ast.Trees._
 import dotty.tools.dotc.ast.{tpd, untpd}
@@ -324,9 +324,9 @@ class ReplDriver(settings: Array[String],
       state
 
     case Load(path) =>
-      val file = new java.io.File(path)
+      val file = new JFile(path)
       if (file.exists) {
-        val contents = scala.io.Source.fromFile(file).mkString
+        val contents = scala.io.Source.fromFile(file, "UTF-8").mkString
         run(contents)
       }
       else {

compiler/test/dotty/tools/repl/ReplCompilerTests.scala

@@ -5,6 +5,8 @@ import org.junit.{Ignore, Test}
 
 class ReplCompilerTests extends ReplTest {
 
+  private final val EOL: String = sys.props("line.separator")
+
   @Test def compileSingle = fromInitialState { implicit state =>
     run("def foo: 1 = 1")
     assertEquals("def foo: Int(1)", storedOutput().trim)
@@ -51,7 +53,7 @@ class ReplCompilerTests extends ReplTest {
       "val res1: Int = 20"
     )
 
-    assertEquals(expected, storedOutput().split("\n").toList)
+    assertEquals(expected, storedOutput().split(EOL).toList)
   }
 
   @Test def testImportMutable =
@@ -122,6 +124,6 @@ class ReplCompilerTests extends ReplTest {
     )
 
     run(source)
-    assertEquals(expected, storedOutput().split("\n").toList)
+    assertEquals(expected, storedOutput().split(EOL).toList)
   }
 }

compiler/test/dotty/tools/repl/ScriptedTests.scala

@@ -14,6 +14,8 @@ import dotc.reporting.MessageRendering
 /** Runs all tests contained in `compiler/test-resources/repl/` */
 class ScriptedTests extends ReplTest with MessageRendering {
 
+  private final val EOL: String = sys.props("line.separator")
+
   private def scripts(path: String): Array[JFile] = {
     val dir = new JFile(getClass.getResource(path).getPath)
     assert(dir.exists && dir.isDirectory, "Couldn't load scripts dir")
@@ -22,7 +24,7 @@ class ScriptedTests extends ReplTest with MessageRendering {
 
   private def testFile(f: JFile): Unit = {
     val prompt = "scala>"
-    val lines = Source.fromFile(f).getLines().buffered
+    val lines = Source.fromFile(f, "UTF-8").getLines().buffered
 
     assert(lines.head.startsWith(prompt),
       s"""Each file has to start with the prompt: "$prompt"""")
@@ -44,7 +46,7 @@ class ScriptedTests extends ReplTest with MessageRendering {
     def evaluate(state: State, input: String, prompt: String) =
       try {
         val nstate = run(input.drop(prompt.length))(state)
-        val out = input + "\n" + storedOutput()
+        val out = input + EOL + storedOutput()
         (out, nstate)
       }
       catch {
@@ -60,7 +62,7 @@ class ScriptedTests extends ReplTest with MessageRendering {
       }
 
     val expectedOutput =
-      Source.fromFile(f).getLines().flatMap(filterEmpties).mkString("\n")
+      Source.fromFile(f, "UTF-8").getLines().flatMap(filterEmpties).mkString(EOL)
     val actualOutput = {
       resetToInitial()
       val inputRes = extractInputs(prompt)
@@ -70,7 +72,7 @@ class ScriptedTests extends ReplTest with MessageRendering {
         buf.append(out)
         nstate
       }
-      buf.flatMap(filterEmpties).mkString("\n")
+      buf.flatMap(filterEmpties).mkString(EOL)
     }
 
     if (expectedOutput != actualOutput) {

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -187,6 +187,8 @@ trait ParallelTesting extends RunnerOrchestration { self =>
     protected final val realStdout = System.out
     protected final val realStderr = System.err
 
+    protected final val EOL: String = sys.props("line.separator")
+
     /** A runnable that logs its contents in a buffer */
     trait LoggedRunnable extends Runnable {
       /** Instances of `LoggedRunnable` implement this method instead of the
@@ -535,16 +537,16 @@ trait ParallelTesting extends RunnerOrchestration { self =>
                     val ignoredFilePathLine = "/** Decompiled from"
                     val stripTrailingWhitespaces = "(.*\\S|)\\s+".r
                     val output = Source.fromFile(outDir.getParent + "_decompiled" + JFile.separator + outDir.getName
-                      + JFile.separator + "decompiled.scala").getLines().map {line =>
+                      + JFile.separator + "decompiled.scala", "UTF-8").getLines().map {line =>
                       stripTrailingWhitespaces.unapplySeq(line).map(_.head).getOrElse(line)
                     }.toList
 
-                    val check: String = Source.fromFile(checkFile).getLines().filter(!_.startsWith(ignoredFilePathLine))
-                      .mkString("\n")
+                    val check: String = Source.fromFile(checkFile, "UTF-8").getLines().filter(!_.startsWith(ignoredFilePathLine))
+                      .mkString(EOL)
 
-                    if (output.filter(!_.startsWith(ignoredFilePathLine)).mkString("\n") != check) {
+                    if (output.filter(!_.startsWith(ignoredFilePathLine)).mkString(EOL) != check) {
                       val outFile = dotty.tools.io.File(checkFile.toPath).addExtension(".out")
-                      outFile.writeAll(output.mkString("\n"))
+                      outFile.writeAll(output.mkString(EOL))
                       val msg =
                         s"""Output differed for test $name, use the following command to see the diff:
                            |  > diff $checkFile $outFile
@@ -617,7 +619,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
         case Success(_) if !checkFile.isDefined || !checkFile.get.exists => // success!
         case Success(output) => {
           val outputLines = output.linesIterator.toArray :+ DiffUtil.EOF
-          val checkLines: Array[String] = Source.fromFile(checkFile.get).getLines().toArray :+ DiffUtil.EOF
+          val checkLines: Array[String] = Source.fromFile(checkFile.get, "UTF-8").getLines().toArray :+ DiffUtil.EOF
           val sourceTitle = testSource.title
 
           def linesMatch =
@@ -726,7 +728,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
           val errorMap = new HashMap[String, Integer]()
           var expectedErrors = 0
           files.filter(_.getName.endsWith(".scala")).foreach { file =>
-            Source.fromFile(file).getLines().zipWithIndex.foreach { case (line, lineNbr) =>
+            Source.fromFile(file, "UTF-8").getLines().zipWithIndex.foreach { case (line, lineNbr) =>
               val errors = line.sliding("// error".length).count(_.mkString == "// error")
               if (errors > 0)
                 errorMap.put(s"${file.getAbsolutePath}:${lineNbr}", errors)