Update-check option for partest, added "do not edit" header

Jenkins uses US-ASCII charset, added tool to detect when invalid chars are used and fixed files.

Author: vsalvis

src/dotty/tools/dotc/core/tasty/TastyFormat.scala

@@ -70,7 +70,7 @@ Standard-Section: "ASTs" TopLevelStat*
 
   Term          = Path
                   Application
-                  IDENT                 NameRef Type     // used when ident’s type is not a TermRef
+                  IDENT                 NameRef Type     // used when ident's type is not a TermRef
                   SELECT                possiblySigned_NameRef qual_Term
                   NEW                   cls_Type
                   SUPER          Length this_Term mixinTrait_Type?
@@ -169,8 +169,8 @@ Standard-Section: "ASTs" TopLevelStat*
                   LABEL                           // method generated as a label
                   FIELDaccessor               // getter or setter
                   CASEaccessor              // getter for case class param
-                  COVARIANT                     // type param marked “+”
-                  CONTRAVARIANT             // type param marked “-”
+                  COVARIANT                     // type param marked "+"
+                  CONTRAVARIANT             // type param marked "-"
                   SCALA2X                           // Imported from Scala2.x
                   DEFAULTparameterized  // Method with default params
                   INSUPERCALL           // defined in the argument of a constructor supercall

src/dotty/tools/dotc/transform/PatternMatcher.scala

@@ -680,7 +680,7 @@ class PatternMatcher extends MiniPhaseTransform with DenotTransformer {thisTrans
       }
     }
 
-    /** implements the run-time aspects of (§8.2) (typedPattern has already done the necessary type transformations)
+    /** implements the run-time aspects of (P8.2) (typedPattern has already done the necessary type transformations)
       *
       * Type patterns consist of types, type variables, and wildcards. A type pattern T is of one of the following forms:
         - A reference to a class C, p.C, or T#C.
@@ -699,7 +699,7 @@ class PatternMatcher extends MiniPhaseTransform with DenotTransformer {thisTrans
 
         - A parameterized type pattern T[a1,...,an], where the ai are type variable patterns or wildcards _.
           This type pattern matches all values which match T for some arbitrary instantiation of the type variables and wildcards.
-          The bounds or alias type of these type variable are determined as described in (§8.3).
+          The bounds or alias type of these type variable are determined as described in (P8.3).
 
         - A parameterized type pattern scala.Array[T1], where T1 is a type pattern. // TODO
           This type pattern matches any non-null instance of type scala.Array[U1], where U1 is a type matched by T1.
@@ -1284,28 +1284,28 @@ class PatternMatcher extends MiniPhaseTransform with DenotTransformer {thisTrans
 
     // Some notes from the specification
 
-    /*A constructor pattern is of the form c(p1, ..., pn) where n ≥ 0.
+    /*A constructor pattern is of the form c(p1, ..., pn) where n >= 0.
       It consists of a stable identifier c, followed by element patterns p1, ..., pn.
-      The constructor c is a simple or qualified name which denotes a case class (§5.3.2).
+      The constructor c is a simple or qualified name which denotes a case class (P5.3.2).
 
       If the case class is monomorphic, then it must conform to the expected type of the pattern,
-      and the formal parameter types of x’s primary constructor (§5.3) are taken as the expected
+      and the formal parameter types of x's primary constructor (P5.3) are taken as the expected
       types of the element patterns p1, ..., pn.
 
       If the case class is polymorphic, then its type parameters are instantiated so that the
       instantiation of c conforms to the expected type of the pattern.
-      The instantiated formal parameter types of c’s primary constructor are then taken as the
+      The instantiated formal parameter types of c's primary constructor are then taken as the
       expected types of the component patterns p1, ..., pn.
 
       The pattern matches all objects created from constructor invocations c(v1, ..., vn)
       where each element pattern pi matches the corresponding value vi .
-      A special case arises when c’s formal parameter types end in a repeated parameter.
-      This is further discussed in (§8.1.9).
+      A special case arises when c's formal parameter types end in a repeated parameter.
+      This is further discussed in (P8.1.9).
     **/
 
     /* A typed pattern x : T consists of a pattern variable x and a type pattern T.
        The type of x is the type pattern T, where each type variable and wildcard is replaced by a fresh, unknown type.
-       This pattern matches any value matched by the type pattern T (§8.2); it binds the variable name to that value.
+       This pattern matches any value matched by the type pattern T (P8.2); it binds the variable name to that value.
     */
 
     /* A pattern binder x@p consists of a pattern variable x and a pattern p.
@@ -1319,8 +1319,8 @@ class PatternMatcher extends MiniPhaseTransform with DenotTransformer {thisTrans
          A literal pattern L matches any value that is equal (in terms of ==) to the literal L.
          The type of L must conform to the expected type of the pattern.
 
-       8.1.5 Stable Identifier Patterns  (a stable identifier r (see §3.1))
-         The pattern matches any value v such that r == v (§12.1).
+       8.1.5 Stable Identifier Patterns  (a stable identifier r (see P3.1))
+         The pattern matches any value v such that r == v (P12.1).
          The type of r must conform to the expected type of the pattern.
     */
 

test/dotty/partest/DPConsoleRunner.scala

@@ -39,13 +39,13 @@ object DPConsoleRunner {
 
 // console runner has a suite runner which creates a test runner for each test
 class DPConsoleRunner(args: String, extraJars: List[String]) extends ConsoleRunner(args) {
-  println("ConsoleRunner options: " + args)
 
   override val suiteRunner = new DPSuiteRunner (
     testSourcePath = optSourcePath getOrElse DPConfig.testRoot,
     fileManager = new DottyFileManager(extraJars),
     updateCheck = optUpdateCheck,
-    failed = optFailed)
+    failed = optFailed,
+    consoleArgs = args)
 
   override def run = {}
   def runPartest = super.run
@@ -62,6 +62,7 @@ class DPSuiteRunner(testSourcePath: String, // relative path, like "files", or "
   fileManager: DottyFileManager,
   updateCheck: Boolean,
   failed: Boolean,
+  consoleArgs: String,
   javaCmdPath: String = PartestDefaults.javaCmd,
   javacCmdPath: String = PartestDefaults.javacCmd,
   scalacExtraArgs: Seq[String] = Seq.empty)
@@ -76,9 +77,11 @@ extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPat
   override def banner: String = {
     s"""|Welcome to Partest for Dotty! Partest version: ${Properties.versionNumberString}
         |Compiler under test: dotty.tools.dotc.Bench or dotty.tools.dotc.Main
-        |Test root: ${PathSettings.srcDir}${File.separator}
+        |Generated test sources: ${PathSettings.srcDir}${File.separator}
         |Test directories: ${DPConfig.testDirs.toList.mkString(", ")}
+        |Debugging: failed tests have compiler output in test-kind.clog, run output in test-kind.log, class files in test-kind.obj
         |Parallel: ${DPConfig.runTestsInParallel}
+        |Options: (use partest --help for usage information) ${consoleArgs}
     """.stripMargin
   }
 
@@ -92,12 +95,13 @@ extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPat
         // Parts of test output might end up in the wrong file or get lost.
         Console.out.flush
         Console.err.flush
-        val clog = SFile(runner.logFile).changeExtension("clog")
+        val clog = runner.cLogFile
         val stream = new PrintStream(new FileOutputStream(clog.jfile), true)
         val result = Console.withOut(stream)({ Console.withErr(stream)({
-          runner.run()
+          val res = runner.run()
           Console.err.flush
           Console.out.flush
+          res
         })})
         result match {
           // Append compiler output to transcript if compilation failed,
@@ -117,6 +121,8 @@ extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPat
 }
 
 class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runner(testFile, suiteRunner) {
+  val cLogFile = SFile(logFile).changeExtension("clog")
+
   // override to provide DottyCompiler
   override def newCompiler = new dotty.partest.DPDirectCompiler(this)
 
@@ -199,6 +205,38 @@ class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runn
     }
   }
 
+  // override to change check file updating to original file, not generated
+  override def diffIsOk: Boolean = {
+    // always normalize the log first
+    normalizeLog()
+    val diff = currentDiff
+    // if diff is not empty, is update needed?
+    val updating: Option[Boolean] = (
+      if (diff == "") None
+      else Some(suiteRunner.updateCheck)
+    )
+    pushTranscript(s"diff $logFile $checkFile")
+    nextTestAction(updating) {
+      case Some(true) =>
+        val origCheck = SFile(checkFile.changeExtension("checksrc").fileLines(1))
+        NestUI.echo("Updating original checkfile " + origCheck)
+        origCheck writeAll file2String(logFile)
+        genUpdated()
+      case Some(false) =>
+        // Get a word-highlighted diff from git if we can find it
+        val bestDiff = if (updating.isEmpty) "" else {
+          if (checkFile.canRead)
+            gitDiff(logFile, checkFile) getOrElse {
+              s"diff $logFile $checkFile\n$diff"
+            }
+          else diff
+        }
+        pushTranscript(bestDiff)
+        genFail("output differs")
+      case None        => genPass()  // redundant default case
+    } getOrElse true
+  }
+
   // override because Dotty currently doesn't handle separate compilation well,
   // so we ignore groups (tests suffixed with _1 and _2)
   override def groupedFiles(sources: List[File]): List[List[File]] = {
@@ -223,4 +261,10 @@ class DPTestRunner(testFile: File, suiteRunner: DPSuiteRunner) extends nest.Runn
   // override to add dotty and scala jars to classpath
   override def extraClasspath = suiteRunner.fileManager.asInstanceOf[DottyFileManager].extraJarList ::: super.extraClasspath
 
+  // override to keep class files if failed and delete clog if ok
+  override def cleanup = if (lastState.isOk) {
+    logFile.delete
+    cLogFile.delete
+    Directory(outDir).deleteRecursively
+  }
 }

test/partest

@@ -1,12 +1,4 @@
 #!/usr/bin/env bash
-# partest error message references partest script to update check files, but
-# doesn't work for dotty because we don't know where tests came from.
+# partest error message references partest script to update check files
 
-if [ $1='--update-check' ];
-then
-  echo """ERROR: Since dotty partest runs on generated files, please update the check
-files in the original location (run tests) or update the expected error count
-(neg tests) in the test file."
-else
-  echo "This script doesn't launch partest, please use sbt partest instead."
-fi
+exec sbt "partest-only $*"

test/test/CompilerTest.scala

@@ -101,7 +101,7 @@ abstract class CompilerTest extends DottyTest {
       }
     }
   }
-  def runFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0, 
+  def runFile(prefix: String, fileName: String, args: List[String] = Nil, xerrors: Int = 0,
       extension: String = ".scala")(implicit defaultOptions: List[String]): Unit =
     compileFile(prefix, fileName, args, xerrors, extension, true)
 
@@ -131,7 +131,7 @@ abstract class CompilerTest extends DottyTest {
         val firstDest = Directory(DPConfig.testRoot + JFile.separator + kind + JFile.separator + dirName)
         computeDestAndCopyFiles(sourceDir, firstDest, kind, flags, xerrors.toString)
         if (deep == "deep")
-          deleteDir(sourceDir)
+          Directory(sourceDir).deleteRecursively
       } else {
         throw new java.io.FileNotFoundException(s"Unable to locate test dir $prefix$dirName")
       }
@@ -172,7 +172,7 @@ abstract class CompilerTest extends DottyTest {
         recCopyFiles(jfile, destDir / jfile.getName)
       })
       compileDir(DPConfig.testRoot + JFile.separator, testName, args, xerrors)
-      deleteDir(destDir.jfile)
+      destDir.deleteRecursively
     }
   }
 
@@ -186,14 +186,14 @@ abstract class CompilerTest extends DottyTest {
   }
 
   // In particular, don't copy flags from scalac tests
-  private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java", "check")
+  private val extensionsToCopy = scala.collection.immutable.HashSet("scala", "java")
 
   /** Determines what kind of test to run. */
   private def testKind(prefixDir: String, xerrors: Int, runTest: Boolean) = {
     if (runTest) "run"
     else if (xerrors > 0) "neg"
     else if (prefixDir.endsWith("run" + JFile.separator)) {
-      log("WARNING: test is being run as pos test despite being in a run directory. " + 
+      log("WARNING: test is being run as pos test despite being in a run directory. " +
         "Use runFile/runDir instead of compileFile/compileDir to do a run test")
       "pos"
     } else "pos"
@@ -239,10 +239,12 @@ abstract class CompilerTest extends DottyTest {
     if (nerr != "0")
       dest.changeExtension("nerr").createFile(true).writeAll(nerr)
     sourceFile.changeExtension("check").ifFile({ check =>
-      if (kind == "run")
+      if (kind == "run") {
         FileManager.copyFile(check.jfile, dest.changeExtension("check").jfile)
-      else
+        dest.changeExtension("checksrc").createFile(true).writeAll("check file generated from source:\n" + check.toString)
+      } else {
         log(s"WARNING: ignoring $check for test kind $kind")
+      }
     })
 
   }
@@ -253,7 +255,18 @@ abstract class CompilerTest extends DottyTest {
     processFileDir(sourceFile, { sf =>
       if (extensionsToCopy.contains(sf.extension)) {
         dest.parent.jfile.mkdirs
-        FileManager.copyFile(sourceFile.jfile, dest.jfile)
+        val contents = try sf.slurp catch {
+          case e: java.nio.charset.MalformedInputException =>
+            println("ERROR: illegal character detected for file " + sf + " (charset " + sf.creationCodec + ")")
+            checkEncoding(sf)
+            throw e
+        }
+        dest.toFile.writeAll("/* ==========================================\n",
+                             " * ========= AUTOMATICALLY GENERATED ========\n",
+                             " * ========= DO NOT EDIT THIS FILE ==========\n",
+                             " * ==========================================\n",
+                             " * Original: " + sf.toString + " */\n\n",
+                             contents)
       } else {
         log(s"WARNING: ignoring $sf")
       }
@@ -326,16 +339,27 @@ abstract class CompilerTest extends DottyTest {
     destDir.jfile
   }
 
-  /** Recursively deletes directories (and files). */
-  private def deleteDir(dir: JFile): Unit = {
-    val children = dir.listFiles
-    if (children != null)
-      children.foreach(deleteDir(_))
-    dir.delete
-  }
-
   /** Write either to console (JUnit) or log file (partest). */
   private def log(msg: String) = logFile.map(_.appendAll(msg + "\n")).getOrElse(println(msg))
+
+  /** Jenkins uses different Character encoding, check if this is the problem. */
+  private def checkEncoding(sf: SFile) = {
+    val reader1 = sf.reader(scala.io.Codec.UTF8) // local machine
+    val reader2 = sf.reader(scala.io.Codec(java.nio.charset.Charset.forName("US-ASCII"))) // jenkins
+    try {
+      var read1, read2, count, diffcount = 0
+      while ((read1 > -1 || read2 > -1) && diffcount < 30) {
+        read1 = reader1.read(); read2 = reader2.read()
+        count += 1
+        if (read1 != read2) {
+          println("difference at character " + count  + ": UTF8: " + read1.toChar + ", US-ASCII (jenkins): " + read2.toChar)
+          diffcount += 1
+        }
+      }
+    } finally {
+      reader1.close; reader2.close
+    }
+  }
 }
 
 object CompilerTest extends App {