From c15097213682b91a653cdd91f328fa45fcb72401 Mon Sep 17 00:00:00 2001 From: Phil Date: Mon, 13 Dec 2021 19:28:23 -0700 Subject: [PATCH 01/15] refactor common code from scripting tests; invalid tests fail by default --- .../tools/scripting/BashScriptsTests.scala | 138 ++--------- .../tools/scripting/ClasspathTests.scala | 115 +-------- .../dotty/tools/scripting/ScriptTestEnv.scala | 225 ++++++++++++++++++ .../tools/scripting/ScriptingTests.scala | 78 +++--- 4 files changed, 283 insertions(+), 273 deletions(-) create mode 100644 compiler/test/dotty/tools/scripting/ScriptTestEnv.scala diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 759430a26817..0e7f7238616f 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -2,29 +2,28 @@ package dotty package tools package scripting -import java.io.File -import java.nio.file.{Path, Paths, Files} -import scala.sys.process._ - import org.junit.Test import org.junit.Assert.assertEquals import vulpix.TestConfiguration -import dotty.tools.dotc.config.Properties._ +import ScriptTestEnv.* /** Verifies correct handling of command line arguments by `dist/bin/scala` and `dist/bin/scalac`. * +. arguments following a script path must be treated as script arguments * +. preserve script command line arguments. + * +. prevent SCALA_OPTS in build environment from infecting tests, via 'SCALA_OPTS= ' prefix + * +. test scripts must not throw execptions or exit with nonzero. */ class BashScriptsTests: // classpath tests managed by scripting.ClasspathTests.scala def testFiles = scripts("/scripting") + lazy val argsfile = createArgsFile() // avoid problems caused by drive letter printf("osname[%s]\n", osname) - printf("using JAVA_HOME=%s\n", javaHome) - printf("using SCALA_HOME=%s\n", scalaHome) - printf("first 5 PATH entries:\n%s\n", pathEntries.take(5).mkString("\n")) + printf("using JAVA_HOME=%s\n", envJavaHome) + printf("using SCALA_HOME=%s\n", envScalaHome) + printf("first 5 PATH entries:\n%s\n", adjustedPathEntries.take(5).mkString("\n")) printf("scala path: [%s]\n", scalaPath) printf("scalac path: [%s]\n", scalacPath) @@ -44,9 +43,9 @@ class BashScriptsTests: /* verify `dist/bin/scalac` non-interference with command line args following script name */ @Test def verifyScalacArgs = - val commandline = (Seq(scalacPath, "-script", showArgsScript) ++ testScriptArgs).mkString(" ") + val commandline = (Seq("SCALA_OPTS= ", scalacPath, "-script", showArgsScript) ++ testScriptArgs).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - if validTest then + if verifyValid(validTest) then var fail = false printf("\n") for (line, expect) <- stdout zip expectedOutput do @@ -59,33 +58,33 @@ class BashScriptsTests: /* verify `dist/bin/scala` with -J setting */ @Test def verifyScJProperty = - val commandline = Seq(scalaPath, "-J-Dkey=World", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") + val commandline = Seq("SCALA_OPTS= ", scalaPath, "-J-Dkey=World", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) assertEquals(stdout.mkString("/n"), "Hello World") /* verify `dist/bin/scala` with -J setting */ @Test def verifyScalaJProperty = - val commandline = Seq(scalaPath, "-J-Dkey=World3", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") + val commandline = Seq("SCALA_OPTS= ", scalaPath, "-J-Dkey=World3", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) assertEquals(stdout.mkString("/n"), "Hello World3") /* verify `dist/bin/scala` with -D setting */ @Test def verifyScDProperty = - val commandline = Seq(scalaPath, "-Dkey=World3", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") + val commandline = Seq("SCALA_OPTS= ", scalaPath, "-Dkey=World3", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) assertEquals(stdout.mkString("/n"), "Hello World3") /* verify `dist/bin/scala` with -D setting */ @Test def verifyScalaDProperty = - val commandline = Seq(scalaPath, "-Dkey=World4", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") + val commandline = Seq("SCALA_OPTS= ", scalaPath, "-Dkey=World4", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) assertEquals(stdout.mkString("/n"), "Hello World4") /* verify `dist/bin/scala` with -D setting */ @Test def saveAndRunWithDProperty = - val commandline = Seq(scalaPath, "-save", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") + val commandline = Seq("SCALA_OPTS= ", scalaPath, "-save", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") val (_, _, _, _) = bashCommand(commandline) - val commandline2 = Seq(scalaPath, "-Dkey=World5", testFiles.find(_.getName == "envtest.jar").get.absPath).mkString(" ") + val commandline2 = Seq("SCALA_OPTS= ", scalaPath, "-Dkey=World5", testFiles.find(_.getName == "envtest.jar").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline2) assertEquals(stdout.mkString("/n"), "Hello World5") @@ -93,7 +92,7 @@ class BashScriptsTests: @Test def verifyScalaArgs = val commandline = (Seq("SCALA_OPTS= ", scalaPath, showArgsScript) ++ testScriptArgs).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - if validTest then + if verifyValid(validTest) then var fail = false printf("\n") var mismatches = List.empty[(String, String)] @@ -115,7 +114,7 @@ class BashScriptsTests: printf("===> verify valid system property script.path is reported by script [%s]\n", scriptFile.getName) printf("calling scriptFile: %s\n", scriptFile) val (validTest, exitCode, stdout, stderr) = bashCommand(scriptFile.absPath) - if validTest then + if verifyValid(validTest) then stdout.foreach { printf("stdout: [%s]\n", _) } stderr.foreach { printf("stderr: [%s]\n", _) } val valid = stdout.exists { _.endsWith(expected) } @@ -131,7 +130,7 @@ class BashScriptsTests: val envPairs = List(("SCALA_OPTS", s"@$argsfile")) val (validTest, exitCode, stdout, stderr) = bashCommand(scriptFile.absPath, envPairs) printf("stdout: %s\n", stdout.mkString("\n","\n","")) - if validTest then + if verifyValid(validTest) then val expected = s"${workingDirectory.norm}" val output = stdout.find( _.trim.startsWith("cwd") ).getOrElse("").dropWhile(_!=' ').trim printf("output [%s]\n", output) @@ -140,104 +139,3 @@ class BashScriptsTests: if valid then printf(s"\n===> success: classpath begins with %s, as reported by [%s]\n", workingDirectory, scriptFile.getName) assert(valid, s"script ${scriptFile.absPath} did not report valid java.class.path first entry") - def existingPath: String = envOrElse("PATH", "").norm - def adjustedPath = s"$javaHome/bin$psep$scalaHome/bin$psep$existingPath" - def pathEntries = adjustedPath.split(psep).toList - - lazy val argsfile = createArgsFile() // avoid problems caused by drive letter - def createArgsFile(): String = - val utfCharset = java.nio.charset.StandardCharsets.UTF_8.name - val path = Files.createTempFile("scriptingTest", ".args") - val text = s"-classpath ${workingDirectory.absPath}" - Files.write(path, text.getBytes(utfCharset)) - path.toFile.getAbsolutePath.norm - - def fixHome(s: String): String = - s.startsWith("~") match { - case false => s - case true => s.replaceFirst("~", userHome) - } - - extension(s: String) { - def toPath: Path = Paths.get(fixHome(s)) // .toAbsolutePath - def toFile: File = s.toPath.toFile - def absPath: String = s.toFile.absPath - def norm: String = s.replace('\\', '/') // bash expects forward slash - def isFile: Boolean = s.toFile.isFile - def exists: Boolean = s.toPath.toFile.exists - def name: String = s.toFile.getName - def dropExtension: String = s.reverse.dropWhile(_ != '.').drop(1).reverse - def parent(up: Int): String = s.norm.split("/").reverse.drop(up).reverse.mkString("/") - } - - extension(p: Path) { - def listFiles: Seq[File] = p.toFile.listFiles.toList - def norm: String = p.normalize.toString.replace('\\', '/') - def name: String = p.toFile.getName - } - - extension(f: File) { - def name = f.getName - def norm: String = f.toPath.normalize.norm - def absPath: String = f.getAbsolutePath.norm - } - - lazy val psep: String = propOrElse("path.separator", "") - lazy val osname = propOrElse("os.name", "").toLowerCase - - lazy val scalacPath = s"$workingDirectory/dist/target/pack/bin/scalac".norm - lazy val scalaPath = s"$workingDirectory/dist/target/pack/bin/scala".norm - - // use optional working directory TEST_CWD, if defined - lazy val workingDirectory: String = envOrElse("TEST_CWD", userDir) - - // use optional TEST_BASH if defined, otherwise, bash must be in PATH - lazy val bashExe: String = envOrElse("TEST_BASH", whichBash) - - // test env SCALA_HOME is: - // dist/target/pack, if present - // else, SCALA_HOME if defined - // else, not defined - lazy val scalaHome = - if scalacPath.isFile then scalacPath.replaceAll("/bin/scalac", "") - else envOrElse("SCALA_HOME", "").norm - - lazy val javaHome = whichJava.parent(2) - - lazy val testEnvPairs = List( - ("JAVA_HOME", javaHome), - ("SCALA_HOME", scalaHome), - ("PATH", adjustedPath), - ).filter { case (name, valu) => valu.nonEmpty } - - lazy val whichBash: String = whichExe("bash") - lazy val whichJava: String = whichExe("java") - - def whichExe(basename: String): String = - val exeName = if (osname.toLowerCase.startsWith("windows")) s"$basename.exe" else basename - which(exeName) - - def bashCommand(cmdstr: String, additionalEnvPairs: List[(String, String)] = Nil): (Boolean, Int, Seq[String], Seq[String]) = { - var (stdout, stderr) = (List.empty[String], List.empty[String]) - if bashExe.toFile.exists then - val cmd = Seq(bashExe, "-c", cmdstr) - val envPairs = testEnvPairs ++ additionalEnvPairs - val proc = Process(cmd, None, envPairs *) - val exitVal = proc ! ProcessLogger ( - (out: String) => stdout ::= out, - (err: String) => stderr ::= err - ) - val validTest = exitVal == 0 && ! stderr.exists(_.contains("Permission denied")) - if ! validTest then - printf("\nunable to execute script, return value is %d\n", exitVal) - stderr.foreach { System.err.printf("stderr [%s]\n", _) } - (validTest, exitVal, stdout.reverse, stderr.reverse) - else - (false, -1, Nil, Nil) - } - - def execCmd(command: String, options: String *): Seq[String] = - val cmd = (command :: options.toList).toSeq - for { - line <- Process(cmd).lazyLines_! - } yield line diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index d755789543a3..976ca0172f20 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -2,25 +2,12 @@ package dotty package tools package scripting -import java.io.File -import java.nio.file.{Files, Paths, Path} - import org.junit.Test - import vulpix.TestConfiguration - -import scala.sys.process._ -import scala.jdk.CollectionConverters._ -import dotty.tools.dotc.config.Properties._ +import ScriptTestEnv.* /** Test java command line generated by bin/scala and bin/scalac */ class ClasspathTests: - val packBinDir = "dist/target/pack/bin" - val packLibDir = "dist/target/pack/lib" - - def exists(scriptPath: Path): Boolean = Files.exists(scriptPath) - def packBinScalaExists:Boolean = exists(Paths.get(s"$packBinDir/scala")) - /* * verify classpath reported by called script. */ @@ -64,7 +51,7 @@ class ClasspathTests: */ @Test def unglobClasspathVerifyTest = { val testScriptName = "unglobClasspath.sc" - val testScript = scripts("/scripting").find { _.getName.matches(testScriptName) } match + val testScript = scripts("/scripting").find { _.name.matches(testScriptName) } match case None => sys.error(s"test script not found: ${testScriptName}") case Some(file) => file @@ -81,105 +68,9 @@ class ClasspathTests: // test script reports the classpath it sees val scriptOutput = exec(cmd:_*) val scriptCp = findTaggedLine("unglobbed classpath", scriptOutput) + printf("%s\n", scriptCp) val classpathJars = scriptCp.split(psep).map { _.getName }.sorted.distinct //classpathJars.foreach { printf("%s\n", _) } assert(classpathJars.size > 1) } - -//////////////// end of tests //////////////// -lazy val cwd = Paths.get(".").toAbsolutePath -lazy val wildcardEntry = "dist/target/pack/lib/*" - -def listJars(dir: String) = - val packlibDir = Paths.get(dir).toFile - if packlibDir.isDirectory then - packlibDir.listFiles.toList.map { _.getName }.filter { _.endsWith(".jar") } - else - Nil - -import scala.jdk.CollectionConverters._ -lazy val env:Map[String, String] = System.getenv.asScala.toMap - -// script output expected as ": " -def findTaggedLine(tag: String, lines: Seq[String]): String = - lines.find { _.startsWith(tag) } match - case None => - lines.foreach { System.err.printf("line[%s]\n", _) } - sys.error(s"no $tag: found in script output") - case Some(cwd) => cwd.dropWhile( _ != ' ').trim // discard tag - -def exec(cmd: String *): Seq[String] = Process(cmd).lazyLines_!.toList - -def which(str:String) = - var out = "" - path.find { entry => - val it = Paths.get(entry).toAbsolutePath - it.toFile.isDirectory && { - var testpath = s"$it/$str".norm - val test = Paths.get(testpath) - if test.toFile.exists then - out = testpath - true - else - val test = Paths.get(s"$it/$str.exe".norm) - if test.toFile.exists then - out = testpath - true - else - false - } - } - - out - -def bashExe = which("bash") -def unameExe = which("uname") -def path = envOrElse("PATH", "").split(psep).toList -def psep = sys.props("path.separator") - -def cygwin = ostype == "cygwin" -def mingw = ostype == "mingw" -def msys = ostype == "msys" -def winshell = cygwin || mingw || msys - -def ostypeFull = if unameExe.nonEmpty then exec(unameExe).mkString else "" -def ostype = ostypeFull.toLowerCase.takeWhile{ cc => cc >= 'a' && cc <='z' || cc >= 'A' && cc <= 'Z' } - -extension(p:Path) - def relpath: Path = cwd.relativize(p) - def norm: String = p.toString.replace('\\', '/') - -extension(path: String) - def getName: String = norm.replaceAll(".*/", "") - - // Normalize path separator, convert relative path to absolute - def norm: String = - path.replace('\\', '/') match { - case s if s.secondChar == ":" => s // .drop(2) // path without drive letter - case s if s.startsWith("./") => s.drop(2) - case s => s - } - - def parent: String = norm.replaceAll("/[^/]*$", "") - - // convert to absolute path relative to cwd. - def absPath: String = norm match - case str if str.isAbsolute => norm - case _ => Paths.get(userDir, norm).toString.norm - - def isDir: Boolean = Files.isDirectory(Paths.get(path)) - - def toUrl: String = Paths.get(absPath).toUri.toURL.toString - - // Treat norm paths with a leading '/' as absolute. - // Windows java.io.File#isAbsolute treats them as relative. - def isAbsolute = path.norm.startsWith("/") || (isWin && path.secondChar == ":") - def secondChar: String = path.take(2).drop(1).mkString("") - -extension (str: String) def dropExtension = - str.reverse.dropWhile(_ != '.').drop(1).reverse - -//extension(f: File) def absPath = -//f.getAbsolutePath.replace('\\', '/') - diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala new file mode 100644 index 000000000000..b6b428288699 --- /dev/null +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -0,0 +1,225 @@ + +package dotty +package tools +package scripting + +import java.io.File +import java.nio.file.{Path, Paths, Files} + +import dotty.tools.dotc.config.Properties.* + +import scala.sys.process.* +import scala.jdk.CollectionConverters.* + +/** + */ +object ScriptTestEnv { + def osname: String = sys.props("os.name").toLowerCase + def psep: String = sys.props("path.separator") + def userDir: String = sys.props("user.dir").norm + + def whichJava: String = whichExe("java") + def whichBash: String = whichExe("bash") + + def workingDirectory: String = envOrElse("TEST_CWD", userDir) // optional working directory TEST_CWD + + def envPath: String = envOrElse("PATH", "").norm + def adjustedPath: String = s"$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath" + def adjustedPathEntries: List[String] = adjustedPath.split(psep).toList + def envPathEntries: List[String] = envPath.split(psep).toList + + def bashExe: String = envOrElse("TEST_BASH", whichBash) + + def unameExe = which("uname") + def ostypeFull = if unameExe.nonEmpty then exec(unameExe).mkString else "" + def ostype = ostypeFull.toLowerCase.takeWhile{ cc => cc >= 'a' && cc <='z' || cc >= 'A' && cc <= 'Z' } + + def cygwin = ostype == "cygwin" + def mingw = ostype == "mingw" + def msys = ostype == "msys" + def winshell = cygwin || mingw || msys + + def which(str: String) = + var out = "" + // use of adjusted path here would result in circular reference + envPathEntries.find { entry => + val it = Paths.get(entry).toAbsolutePath + it.toFile.isDirectory && { + var testpath = s"$it/$str".norm + val test = Paths.get(testpath) + if test.toFile.exists then + out = testpath + true + else + val test = Paths.get(s"$it/$str.exe".norm) + if test.toFile.exists then + out = testpath + true + else + false + } + } + out + + def whichExe(basename: String): String = + val exeName = if (osname.toLowerCase.startsWith("windows")) s"$basename.exe" else basename + which(exeName) + + def bashCommand(cmdstr: String, additionalEnvPairs: List[(String, String)] = Nil): (Boolean, Int, Seq[String], Seq[String]) = { + var (stdout, stderr) = (List.empty[String], List.empty[String]) + if bashExe.toFile.exists then + val cmd = Seq(bashExe, "-c", cmdstr) + val envPairs = testEnvPairs ++ additionalEnvPairs + val proc = Process(cmd, None, envPairs *) + val exitVal = proc ! ProcessLogger ( + (out: String) => stdout ::= out, + (err: String) => stderr ::= err + ) + // val validTest = exitVal == 0 && ! stderr.exists(_.contains("Permission denied")) + val validTest = !stderr.exists(_.contains("Permission denied")) + if ! validTest then + printf("\nunable to execute script, return value is %d\n", exitVal) + stderr.foreach { System.err.printf("stderr [%s]\n", _) } + + (validTest, exitVal, stdout.reverse, stderr.reverse) + else + (false, -1, Nil, Nil) + } + + def execCmd(command: String, options: String *): Seq[String] = + val cmd = (command :: options.toList).toSeq + for { + line <- Process(cmd).lazyLines_! + } yield line + + + def packBinDir = "dist/target/pack/bin" + def packLibDir = "dist/target/pack/lib" + def packBinScalaExists: Boolean = Files.exists(Paths.get(s"$packBinDir/scala")) + + def listJars(dir: String) = + val packlibDir = Paths.get(dir).toFile + if packlibDir.isDirectory then + packlibDir.listFiles.toList.map { _.getName }.filter { _.endsWith(".jar") } + else + Nil + + // script output expected as ": " + def findTaggedLine(tag: String, lines: Seq[String]): String = + lines.find { _.startsWith(tag) } match + case None => + lines.foreach { System.err.printf("line[%s]\n", _) } + sys.error(s"no $tag: found in script output") + case Some(cwd) => cwd.dropWhile( _ != ' ').trim // discard tag + + def exec(cmd: String *): Seq[String] = Process(cmd).lazyLines_!.toList + + def script2jar(scriptFile: File) = + val jarName = s"${scriptFile.getName.dropExtension}.jar" + File(scriptFile.getParent, jarName) + + def showScriptUnderTest(scriptFile: File): Unit = + printf("===> test script name [%s]\n", scriptFile.getName) + + def callExecutableJar(script: File, jar: File, scriptArgs: Array[String] = Array.empty[String]) = { + import scala.sys.process._ + val cmd = Array("java", s"-Dscript.path=${script.getName}", "-jar", jar.absPath) + ++ scriptArgs + Process(cmd).lazyLines_!.foreach { println } + } + + //////////////////////////////////////////////////////////////////////////////// + + def createArgsFile(): String = + val utfCharset = java.nio.charset.StandardCharsets.UTF_8.name + val path = Files.createTempFile("scriptingTest", ".args") + val text = s"-classpath ${workingDirectory.absPath}" + Files.write(path, text.getBytes(utfCharset)) + path.toFile.getAbsolutePath.norm + + def fixHome(s: String): String = + s.startsWith("~") match { + case false => s + case true => s.replaceFirst("~", userHome) + } + + extension(s: String) { + def norm: String = s.replace('\\', '/') // bash expects forward slash + def noDrive = if s.secondChar == ":" then s.drop(2).norm else s.norm + def toPath: Path = Paths.get(fixHome(s.noDrive)) // .toAbsolutePath + def toFile: File = new File(s) + def absPath: String = s.toFile.absPath + def isFile: Boolean = s.toFile.isFile + def isDirectory: Boolean = s.toFile.isDirectory + def exists: Boolean = s.toFile.exists + def name: String = s.toFile.getName + def getName: String = s.toFile.getName + def dropExtension: String = s.reverse.dropWhile(_ != '.').drop(1).reverse + def parent(up: Int): String = s.norm.split("/").reverse.drop(up).reverse.mkString("/") + def secondChar: String = s.take(2).drop(1).mkString("") + } + + extension(p: Path) { + def norm: String = p.normalize.toString.replace('\\', '/') + + def noDrive = p.norm match { + case str if str.drop(1).take(1) == ":" => str.drop(2) + case str => str + } + def name: String = p.toFile.getName + def relpath: Path = cwd.relativize(p) + def files: Seq[File] = p.toFile.files + def parent: String = norm.replaceAll("/[^/]*$", "") + + // convert to absolute path relative to cwd. + def absPath: String = if (p.isAbsolute) p.norm else Paths.get(userDir, p.norm).toString.norm + + def isDir: Boolean = Files.isDirectory(p) + + def toUrl: String = Paths.get(absPath).toUri.toURL.toString + + // Treat norm paths with a leading '/' as absolute (Windows java.io.File#isAbsolute treats them as relative) + def isAbsolute = p.norm.startsWith("/") || (isWin && p.norm.secondChar == ":") + } + + extension(f: File) { + def name = f.getName + def norm: String = f.toPath.normalize.norm + def absPath: String = f.getAbsolutePath.norm + def files: Seq[File] = f.listFiles.toList + } + + lazy val cwd: Path = Paths.get(".").toAbsolutePath + + lazy val scalacPath = s"$workingDirectory/dist/target/pack/bin/scalac".norm + lazy val scalaPath = s"$workingDirectory/dist/target/pack/bin/scala".norm + + // use optional TEST_BASH if defined, otherwise, bash must be in PATH + + // envScalaHome is: + // dist/target/pack, if present + // else, SCALA_HOME if defined + // else, not defined + lazy val envScalaHome = + if scalacPath.isFile then scalacPath.replaceAll("/bin/scalac", "") + else envOrElse("SCALA_HOME", "").norm + + lazy val envJavaHome: String = envOrElse("JAVA_HOME", whichJava.parent(2)).norm + + lazy val testEnvPairs = List( + ("JAVA_HOME", envJavaHome), + ("SCALA_HOME", envScalaHome), + ("PATH", adjustedPath), + ).filter { case (name, valu) => valu.nonEmpty } + + // if unable to execute bash commands, this prevents invalid tests from failing + lazy val passInvalidTests = envOrElse("PASS_INVALID_TESTS", "").nonEmpty + + def verifyValid(validTest: Boolean): Boolean = + // !validTest implies unable to execute scripts via bash (e.g., permissions, or bash not found, etc.) + if !validTest && !passInvalidTests then + assert(validTest == true, s"unable to call script via bash -c") + + validTest + +} diff --git a/compiler/test/dotty/tools/scripting/ScriptingTests.scala b/compiler/test/dotty/tools/scripting/ScriptingTests.scala index 922564d4ff58..7ddab2b55424 100644 --- a/compiler/test/dotty/tools/scripting/ScriptingTests.scala +++ b/compiler/test/dotty/tools/scripting/ScriptingTests.scala @@ -8,49 +8,13 @@ import java.nio.file.Path import org.junit.Test import vulpix.TestConfiguration - +import ScriptTestEnv.* /** Runs all tests contained in `compiler/test-resources/scripting/` */ class ScriptingTests: - extension (str: String) def dropExtension = - str.reverse.dropWhile(_ != '.').drop(1).reverse - - extension(f: File) def absPath = - f.getAbsolutePath.replace('\\', '/') - // classpath tests managed by scripting.ClasspathTests.scala def testFiles = scripts("/scripting").filter { ! _.getName.toLowerCase.contains("classpath") } - def script2jar(scriptFile: File) = - val jarName = s"${scriptFile.getName.dropExtension}.jar" - File(scriptFile.getParent, jarName) - - def showScriptUnderTest(scriptFile: File): Unit = - printf("===> test script name [%s]\n", scriptFile.getName) - - val argss: Map[String, Array[String]] = ( - for - argFile <- testFiles - if argFile.getName.endsWith(".args") - name = argFile.getName.dropExtension - scriptArgs = readLines(argFile).toArray - yield name -> scriptArgs).toMap - - def scalaFilesWithArgs(extension: String) = ( - for - scriptFile <- testFiles - if scriptFile.getName.endsWith(extension) - name = scriptFile.getName.dropExtension - scriptArgs = argss.getOrElse(name, Array.empty[String]) - yield scriptFile -> scriptArgs).toList.sortBy { (file, args) => file.getName } - - def callExecutableJar(script: File, jar: File, scriptArgs: Array[String] = Array.empty[String]) = { - import scala.sys.process._ - val cmd = Array("java", s"-Dscript.path=${script.getName}", "-jar", jar.absPath) - ++ scriptArgs - Process(cmd).lazyLines_!.foreach { println } - } - /* * Call .scala scripts without -save option, verify no jar created */ @@ -71,7 +35,7 @@ class ScriptingTests: printf("mainClass from ScriptingDriver: %s\n", mainClass) true // call compiled script main method } - assert(! unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}") + assert( !unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}" ) /* * Call .sc scripts without -save option, verify no jar created @@ -89,7 +53,7 @@ class ScriptingTests: ) ++ scriptArgs Main.main(mainArgs) - assert(! unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}") + assert( !unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}") /* * Call .sc scripts with -save option, verify jar is created. @@ -125,7 +89,7 @@ class ScriptingTests: // verify main method not called when false is returned printf("testing script compile, with no call to script main method.\n") touchedFile.delete - assert(!touchedFile.exists, s"unable to delete ${touchedFile}") + assert( !touchedFile.exists, s"unable to delete ${touchedFile}" ) ScriptingDriver( compilerArgs = Array("-classpath", TestConfiguration.basicClasspath), scriptFile = scriptFile, @@ -174,12 +138,44 @@ class ScriptingTests: assert(expectedJar.exists, s"unable to create executable jar [$expectedJar]") touchedFile.delete - assert(!touchedFile.exists, s"unable to delete ${touchedFile}") + assert( !touchedFile.exists, s"unable to delete ${touchedFile}" ) printf("calling executable jar %s\n", expectedJar) callExecutableJar(scriptFile, expectedJar) if touchedFile.exists then printf("success: executable jar created file %s\n", touchedFile) assert( touchedFile.exists, s"expected to find file ${touchedFile}" ) +/////////////////////////////////// def touchFileScript = testFiles.find(_.getName == "touchFile.sc").get + def touchedFile = File("touchedFile.out") + + def script2jar(scriptFile: File) = + val jarName = s"${scriptFile.getName.dropExtension}.jar" + File(scriptFile.getParent, jarName) + + def showScriptUnderTest(scriptFile: File): Unit = + printf("===> test script name [%s]\n", scriptFile.getName) + + def argss: Map[String, Array[String]] = ( + for + argFile <- testFiles + if argFile.getName.endsWith(".args") + name = argFile.getName.dropExtension + scriptArgs = readLines(argFile).toArray + yield name -> scriptArgs).toMap + + def scalaFilesWithArgs(extension: String) = ( + for + scriptFile <- testFiles + if scriptFile.getName.endsWith(extension) + name = scriptFile.getName.dropExtension + scriptArgs = argss.getOrElse(name, Array.empty[String]) + yield scriptFile -> scriptArgs).toList.sortBy { (file, args) => file.getName } + + def callExecutableJar(script: File, jar: File, scriptArgs: Array[String] = Array.empty[String]) = { + import scala.sys.process._ + val cmd = Array("java", s"-Dscript.path=${script.getName}", "-jar", jar.absPath) + ++ scriptArgs + Process(cmd).lazyLines_!.foreach { println } + } From 1da66ba74e27ceae3ca8473f6210ba0b8fe7f3f7 Mon Sep 17 00:00:00 2001 From: Phil Date: Mon, 13 Dec 2021 19:57:27 -0700 Subject: [PATCH 02/15] dump stdout/stderr if BashScriptsTests.verifyScalaOpts fails --- .../test/dotty/tools/scripting/BashScriptsTests.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 0e7f7238616f..08c3349f387b 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -132,10 +132,13 @@ class BashScriptsTests: printf("stdout: %s\n", stdout.mkString("\n","\n","")) if verifyValid(validTest) then val expected = s"${workingDirectory.norm}" - val output = stdout.find( _.trim.startsWith("cwd") ).getOrElse("").dropWhile(_!=' ').trim - printf("output [%s]\n", output) + val cwdline = stdout.find( _.trim.startsWith("cwd") ).getOrElse("") + printf("cwdline [%s]\n", cwdline) printf("expected[%s]\n", expected) - val valid = output.startsWith(expected) + val valid = cwdline.endsWith(expected) + if (!valid) then + stdout.foreach { printf("stdout[%s]\n", _) } + stderr.foreach { printf("stderr[%s]\n", _) } if valid then printf(s"\n===> success: classpath begins with %s, as reported by [%s]\n", workingDirectory, scriptFile.getName) assert(valid, s"script ${scriptFile.absPath} did not report valid java.class.path first entry") From 05b3fd760974a62d3b1ce94854c6f9136f41891f Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 14 Dec 2021 11:54:02 -0700 Subject: [PATCH 03/15] fix for failing cygwin tests for #14106 --- .../scripting/classpathReport.sc | 2 +- .../scripting/unglobClasspath.sc | 2 +- .../tools/scripting/BashScriptsTests.scala | 25 ++++++---- .../tools/scripting/ClasspathTests.scala | 46 +++++++++++++++---- .../dotty/tools/scripting/ScriptTestEnv.scala | 34 +++++++++----- dist/bin/scala | 10 ++++ 6 files changed, 87 insertions(+), 32 deletions(-) diff --git a/compiler/test-resources/scripting/classpathReport.sc b/compiler/test-resources/scripting/classpathReport.sc index 2a240d2fc3a0..a9eacbbba1f7 100755 --- a/compiler/test-resources/scripting/classpathReport.sc +++ b/compiler/test-resources/scripting/classpathReport.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env -S bin/scala -classpath 'dist/target/pack/lib/*' +#!bin/scala -classpath 'dist/target/pack/lib/*' import java.nio.file.Paths diff --git a/compiler/test-resources/scripting/unglobClasspath.sc b/compiler/test-resources/scripting/unglobClasspath.sc index d05ff78435b3..796697cdedf2 100755 --- a/compiler/test-resources/scripting/unglobClasspath.sc +++ b/compiler/test-resources/scripting/unglobClasspath.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env -S bin/scala -classpath 'dist/target/pack/lib/*' +#!bin/scala -classpath 'dist/target/pack/lib/*' // won't compile unless the hashbang line sets classpath import org.jline.terminal.Terminal diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 08c3349f387b..d671de2a85a1 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -2,7 +2,7 @@ package dotty package tools package scripting -import org.junit.Test +import org.junit.{Test, AfterClass} import org.junit.Assert.assertEquals import vulpix.TestConfiguration @@ -15,10 +15,20 @@ import ScriptTestEnv.* * +. prevent SCALA_OPTS in build environment from infecting tests, via 'SCALA_OPTS= ' prefix * +. test scripts must not throw execptions or exit with nonzero. */ +object BashScriptsTests: + lazy val argsfile = createArgsFile() // avoid problems caused by drive letter + + @AfterClass def cleanup: Unit = { + val af = argsfile.toFile + if (af.exists) { + af.delete() + } + } + class BashScriptsTests: + import BashScriptsTests.* // classpath tests managed by scripting.ClasspathTests.scala def testFiles = scripts("/scripting") - lazy val argsfile = createArgsFile() // avoid problems caused by drive letter printf("osname[%s]\n", osname) printf("using JAVA_HOME=%s\n", envJavaHome) @@ -56,31 +66,31 @@ class BashScriptsTests: if fail then assert(stdout == expectedOutput) - /* verify `dist/bin/scala` with -J setting */ + /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.sc */ @Test def verifyScJProperty = val commandline = Seq("SCALA_OPTS= ", scalaPath, "-J-Dkey=World", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) assertEquals(stdout.mkString("/n"), "Hello World") - /* verify `dist/bin/scala` with -J setting */ + /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.scala */ @Test def verifyScalaJProperty = val commandline = Seq("SCALA_OPTS= ", scalaPath, "-J-Dkey=World3", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) assertEquals(stdout.mkString("/n"), "Hello World3") - /* verify `dist/bin/scala` with -D setting */ + /* verify that `dist/bin/scala` can set system properties via -D for envtest.sc */ @Test def verifyScDProperty = val commandline = Seq("SCALA_OPTS= ", scalaPath, "-Dkey=World3", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) assertEquals(stdout.mkString("/n"), "Hello World3") - /* verify `dist/bin/scala` with -D setting */ + /* verify that `dist/bin/scala` can set system properties via -D for envtest.scala */ @Test def verifyScalaDProperty = val commandline = Seq("SCALA_OPTS= ", scalaPath, "-Dkey=World4", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) assertEquals(stdout.mkString("/n"), "Hello World4") - /* verify `dist/bin/scala` with -D setting */ + /* verify that `dist/bin/scala` can set system properties via -D when executing compiled script via -jar envtest.jar */ @Test def saveAndRunWithDProperty = val commandline = Seq("SCALA_OPTS= ", scalaPath, "-save", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") val (_, _, _, _) = bashCommand(commandline) @@ -95,7 +105,6 @@ class BashScriptsTests: if verifyValid(validTest) then var fail = false printf("\n") - var mismatches = List.empty[(String, String)] for (line, expect) <- stdout zip expectedOutput do printf("expected: %-17s\nactual : %s\n", expect, line) if line != expect then diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index 976ca0172f20..16a1dec61382 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -2,15 +2,24 @@ package dotty package tools package scripting -import org.junit.Test +import java.io.File +import java.nio.file.Path + +import org.junit.{Test, Ignore, AfterClass} import vulpix.TestConfiguration import ScriptTestEnv.* /** Test java command line generated by bin/scala and bin/scalac */ + class ClasspathTests: /* + * Test disabled (temporarily). * verify classpath reported by called script. + * This need to be reconceptualized. + * System property "java.class.path" does not necessarily contain the actual runtime path, + * So this test can fail even when the classpath is correct. */ + @Ignore @Test def hashbangClasspathVerifyTest = { // only interested in classpath test scripts val testScriptName = "classpathReport.sc" @@ -28,21 +37,38 @@ class ClasspathTests: cmd.foreach { printf("[%s]\n", _) } - // test script reports the classpath it sees - val scriptOutput = exec(cmd:_*) - val scriptCwd = findTaggedLine("cwd", scriptOutput) + // classpathReport.sc is expected to produce two lines: + // cwd: + // classpath: + + val scriptOutput: Seq[String] = exec(cmd:_*) + val scriptCwd: String = findTaggedLine("cwd", scriptOutput) // the value tagged "cwd: " printf("script ran in directory [%s]\n", scriptCwd) - val scriptCp = findTaggedLine("classpath", scriptOutput) + val scriptCp = findTaggedLine("classpath", scriptOutput) // the value tagged "classpath: " - val hashbangClasspathJars = scriptCp.split(psep).map { _.getName }.sorted.distinct - val packlibJars = listJars(s"$scriptCwd/$packLibDir").sorted.distinct + // convert scriptCp to a list of files + val hashbangJars: List[File] = scriptCp.split(psep).map { _.toFile }.toList + val hashbangClasspathJars = hashbangJars.map { _.name }.sorted.distinct // get jar basenames, remove duplicates + val packlibDir = s"$scriptCwd/$packLibDir" // classpathReport.sc specifies a wildcard classpath in this directory + val packlibJars: List[File] = listJars(packlibDir) // classpath entries expected to have been reported by the script printf("%d jar files in dist/target/pack/lib\n", packlibJars.size) printf("%d test script jars in classpath\n", hashbangClasspathJars.size) - // verify that the classpath set in the hashbang line is effective - if hashbangClasspathJars.size != packlibJars.size then - printf("hashbangClasspathJars: %s\n", hashbangClasspathJars.mkString("\n ", "\n ", "")) + val (diff: Set[File], msg: String) = if (packlibJars.size > hashbangClasspathJars.size) { + (packlibJars.toSet -- hashbangJars.toSet , "only in packlib classpath") + } else { + (hashbangJars.toSet -- packlibJars.toSet , "only in hashbang classpath") + } + // verify that the script hasbang classpath setting was effective at supplementing the classpath + // (a minimal subset of jars below dist/target/pack/lib are always be in the classpath) + val missingClasspathEntries = if hashbangClasspathJars.size != packlibJars.size then + printf("packlib dir [%s]\n", packlibDir) + printf("hashbangClasspathJars: %s\n", hashbangJars.map { _.relpath.norm }.mkString("\n ", "\n ", "")) + printf("# %s\n", msg) + diff.foreach { (f: File) => printf(" %s\n", f.relpath.norm) } + else + Set.empty[String] assert(hashbangClasspathJars.size == packlibJars.size) } diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index b6b428288699..59e3828a8d21 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -1,4 +1,3 @@ - package dotty package tools package scripting @@ -12,6 +11,10 @@ import scala.sys.process.* import scala.jdk.CollectionConverters.* /** + * Common Code for supporting scripting tests. + * To override the path to the bash executable, set TEST_BASH= + * To specify where `dist/target/pack/bin` resides, set TEST_CWD= + * Test scripts run in a bash env, so paths are converted to forward slash via .norm. */ object ScriptTestEnv { def osname: String = sys.props("os.name").toLowerCase @@ -21,7 +24,7 @@ object ScriptTestEnv { def whichJava: String = whichExe("java") def whichBash: String = whichExe("bash") - def workingDirectory: String = envOrElse("TEST_CWD", userDir) // optional working directory TEST_CWD + def workingDirectory: String = envOrElse("TEST_CWD", userDir).norm // optional working directory TEST_CWD def envPath: String = envOrElse("PATH", "").norm def adjustedPath: String = s"$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath" @@ -43,7 +46,7 @@ object ScriptTestEnv { var out = "" // use of adjusted path here would result in circular reference envPathEntries.find { entry => - val it = Paths.get(entry).toAbsolutePath + val it = Paths.get(entry).toAbsolutePath.normalize it.toFile.isDirectory && { var testpath = s"$it/$str".norm val test = Paths.get(testpath) @@ -65,6 +68,12 @@ object ScriptTestEnv { val exeName = if (osname.toLowerCase.startsWith("windows")) s"$basename.exe" else basename which(exeName) + /* returned values are: + * validTest: Boolean - false if permissions problems occur, true otherwise + * exitVal: Int - the conventional return error code, where zero implies "no errors". + * stdout: Seq[String] - the lines captured from STDOUT + * stderr: Seq[String] - the lines captured from STDERR + */ def bashCommand(cmdstr: String, additionalEnvPairs: List[(String, String)] = Nil): (Boolean, Int, Seq[String], Seq[String]) = { var (stdout, stderr) = (List.empty[String], List.empty[String]) if bashExe.toFile.exists then @@ -75,7 +84,7 @@ object ScriptTestEnv { (out: String) => stdout ::= out, (err: String) => stderr ::= err ) - // val validTest = exitVal == 0 && ! stderr.exists(_.contains("Permission denied")) + // a misconfigured environment (e.g., script is not executable) can prevent script execution val validTest = !stderr.exists(_.contains("Permission denied")) if ! validTest then printf("\nunable to execute script, return value is %d\n", exitVal) @@ -97,10 +106,10 @@ object ScriptTestEnv { def packLibDir = "dist/target/pack/lib" def packBinScalaExists: Boolean = Files.exists(Paths.get(s"$packBinDir/scala")) - def listJars(dir: String) = + def listJars(dir: String): List[File] = val packlibDir = Paths.get(dir).toFile if packlibDir.isDirectory then - packlibDir.listFiles.toList.map { _.getName }.filter { _.endsWith(".jar") } + packlibDir.listFiles.toList.filter { _.getName.endsWith(".jar") } else Nil @@ -139,8 +148,8 @@ object ScriptTestEnv { def fixHome(s: String): String = s.startsWith("~") match { - case false => s - case true => s.replaceFirst("~", userHome) + case false => s + case true => s.replaceFirst("~", userHome) } extension(s: String) { @@ -163,8 +172,8 @@ object ScriptTestEnv { def norm: String = p.normalize.toString.replace('\\', '/') def noDrive = p.norm match { - case str if str.drop(1).take(1) == ":" => str.drop(2) - case str => str + case str if str.drop(1).take(1) == ":" => str.drop(2) + case str => str } def name: String = p.toFile.getName def relpath: Path = cwd.relativize(p) @@ -186,10 +195,12 @@ object ScriptTestEnv { def name = f.getName def norm: String = f.toPath.normalize.norm def absPath: String = f.getAbsolutePath.norm + def relpath: Path = f.toPath.relpath def files: Seq[File] = f.listFiles.toList + def parentDir: Path = f.toPath.getParent } - lazy val cwd: Path = Paths.get(".").toAbsolutePath + lazy val cwd: Path = Paths.get(".").toAbsolutePath.normalize lazy val scalacPath = s"$workingDirectory/dist/target/pack/bin/scalac".norm lazy val scalaPath = s"$workingDirectory/dist/target/pack/bin/scala".norm @@ -221,5 +232,4 @@ object ScriptTestEnv { assert(validTest == true, s"unable to call script via bash -c") validTest - } diff --git a/dist/bin/scala b/dist/bin/scala index 8a87d6d0d8a0..b3116b2706b3 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -45,6 +45,16 @@ while [[ $# -gt 0 ]]; do addScala "$1" shift ;; + -classpath*) + if [ "$1" != "${1##* }" ]; then + # hashbang-combined args "-classpath 'lib/*'" + A=$1 ; shift # consume $1 before adding its substrings back + set -- $A "$@" # split $1 on whitespace and put it back + else + addScala "$1" + shift + fi + ;; *) addScala "$1" shift From 0f058e162ad9020992c6064ac532584ef549f2b6 Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 14 Dec 2021 14:56:22 -0700 Subject: [PATCH 04/15] normalize scala and scalac paths; set proper SHELLOPTS in cygwin bashCommandline env --- .../tools/scripting/BashScriptsTests.scala | 1 + .../dotty/tools/scripting/ScriptTestEnv.scala | 44 ++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index d671de2a85a1..ddf125a4e16c 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -31,6 +31,7 @@ class BashScriptsTests: def testFiles = scripts("/scripting") printf("osname[%s]\n", osname) + printf("uname[%s]\n", ostypeFull) printf("using JAVA_HOME=%s\n", envJavaHome) printf("using SCALA_HOME=%s\n", envScalaHome) printf("first 5 PATH entries:\n%s\n", adjustedPathEntries.take(5).mkString("\n")) diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index 59e3828a8d21..749d8d2b4713 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -40,11 +40,11 @@ object ScriptTestEnv { def cygwin = ostype == "cygwin" def mingw = ostype == "mingw" def msys = ostype == "msys" - def winshell = cygwin || mingw || msys + def winshell: Boolean = cygwin || mingw || msys def which(str: String) = var out = "" - // use of adjusted path here would result in circular reference + // must not use adjusted path here! (causes recursive call / stack overflow) envPathEntries.find { entry => val it = Paths.get(entry).toAbsolutePath.normalize it.toFile.isDirectory && { @@ -176,12 +176,12 @@ object ScriptTestEnv { case str => str } def name: String = p.toFile.getName - def relpath: Path = cwd.relativize(p) + def relpath: Path = cwd.relativize(p).normalize def files: Seq[File] = p.toFile.files def parent: String = norm.replaceAll("/[^/]*$", "") // convert to absolute path relative to cwd. - def absPath: String = if (p.isAbsolute) p.norm else Paths.get(userDir, p.norm).toString.norm + def absPath: String = if (p.isAbsolute) p.norm else Paths.get(userDir, p.norm).norm def isDir: Boolean = Files.isDirectory(p) @@ -202,8 +202,8 @@ object ScriptTestEnv { lazy val cwd: Path = Paths.get(".").toAbsolutePath.normalize - lazy val scalacPath = s"$workingDirectory/dist/target/pack/bin/scalac".norm - lazy val scalaPath = s"$workingDirectory/dist/target/pack/bin/scala".norm + lazy val scalacPath = s"$workingDirectory/dist/target/pack/bin/scalac".toPath.normalize.norm + lazy val scalaPath = s"$workingDirectory/dist/target/pack/bin/scala".toPath.normalize.norm // use optional TEST_BASH if defined, otherwise, bash must be in PATH @@ -216,12 +216,32 @@ object ScriptTestEnv { else envOrElse("SCALA_HOME", "").norm lazy val envJavaHome: String = envOrElse("JAVA_HOME", whichJava.parent(2)).norm - - lazy val testEnvPairs = List( - ("JAVA_HOME", envJavaHome), - ("SCALA_HOME", envScalaHome), - ("PATH", adjustedPath), - ).filter { case (name, valu) => valu.nonEmpty } + lazy val cyghome = envOrElse("CYGWIN", "") + lazy val msyshome = envOrElse("MSYS", "") + + // remove xtrace, if present, add :igncr: if not present + lazy val shellopts: String = { + val value: String = envOrElse("SHELLOPTS", "braceexpand:hashall:igncr:ignoreeof:monitor:vi") + val list: List[String] = value.split(":").toList + "igncr" :: list.filter { + case "igncr" | "xtrace" => false + case _ => true + } + }.mkString(":") + + lazy val testEnvPairs = { + val pairs = List( + ("JAVA_HOME", envJavaHome), + ("SCALA_HOME", envScalaHome), + ("PATH", adjustedPath), + ("CYGWIN", cyghome), + ("MSYS", msyshome), + ("SHELLOPTS", shellopts), + ).filter { case (name, valu) => valu.nonEmpty } + for (k, v) <- pairs do + printf("%s : %s\n", k ,v) + pairs + } // if unable to execute bash commands, this prevents invalid tests from failing lazy val passInvalidTests = envOrElse("PASS_INVALID_TESTS", "").nonEmpty From 621bbec2469fa1e073ea0cf2506952157114e738 Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 14 Dec 2021 16:35:53 -0700 Subject: [PATCH 05/15] improved detection of scalacPath and scalaPath; additional logging --- .../tools/scripting/ClasspathTests.scala | 2 +- .../dotty/tools/scripting/ScriptTestEnv.scala | 53 +++++++++++++++---- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index 16a1dec61382..767c30e60ad6 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -86,7 +86,7 @@ class ClasspathTests: printf("bash is [%s]\n", bashExe) if packBinScalaExists then - val bashCmdline = s"SCALA_OPTS= $relpath" + val bashCmdline = s"set +x ; SCALA_OPTS= $relpath" val cmd = Array(bashExe, "-c", bashCmdline) cmd.foreach { printf("[%s]\n", _) } diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index 749d8d2b4713..0dcf66beaed1 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -24,12 +24,20 @@ object ScriptTestEnv { def whichJava: String = whichExe("java") def whichBash: String = whichExe("bash") - def workingDirectory: String = envOrElse("TEST_CWD", userDir).norm // optional working directory TEST_CWD + lazy val workingDirectory: String = { + val dirstr = envOrElse("TEST_CWD", userDir).norm // optional working directory TEST_CWD + printf("working directory is [%s]\n", dirstr) + val test = Paths.get(s"$dirstr/dist/target/pack/bin").normalize + if !test.isDirectory then + printf("warning: not found below working directory: %s\n", test.norm) + dirstr + } - def envPath: String = envOrElse("PATH", "").norm - def adjustedPath: String = s"$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath" - def adjustedPathEntries: List[String] = adjustedPath.split(psep).toList - def envPathEntries: List[String] = envPath.split(psep).toList + def envPath: String = envOrElse("PATH", "") + // remove duplicate entries in path + def adjustedPathEntries: List[String] = s"dist/target/pack/bin$psep$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath".norm.split(psep).toList.distinct + def adjustedPath: String = adjustedPathEntries.mkString(psep) + def envPathEntries: List[String] = envPath.split(psep).toList.distinct def bashExe: String = envOrElse("TEST_BASH", whichBash) @@ -184,6 +192,8 @@ object ScriptTestEnv { def absPath: String = if (p.isAbsolute) p.norm else Paths.get(userDir, p.norm).norm def isDir: Boolean = Files.isDirectory(p) + def isDirectory: Boolean = p.toFile.isDirectory + def isFile: Boolean = p.toFile.isFile def toUrl: String = Paths.get(absPath).toUri.toURL.toString @@ -202,8 +212,18 @@ object ScriptTestEnv { lazy val cwd: Path = Paths.get(".").toAbsolutePath.normalize - lazy val scalacPath = s"$workingDirectory/dist/target/pack/bin/scalac".toPath.normalize.norm - lazy val scalaPath = s"$workingDirectory/dist/target/pack/bin/scala".toPath.normalize.norm + lazy val (scalacPath: String, scalaPath: String) = { + val scalac = s"$workingDirectory/dist/target/pack/bin/scalac".toPath.normalize + val scala = s"$workingDirectory/dist/target/pack/bin/scala".toPath.normalize + if (scalac.isFile){ + (scalac.norm, scala.norm) + } else { + val s1 = findFile("scalac").replaceAll("/bin/scalac$", "") + val s2 = findFile("scala").replaceAll("/bin/scala$", "") + (s1.norm, s2.norm) + } + } + // use optional TEST_BASH if defined, otherwise, bash must be in PATH @@ -212,8 +232,19 @@ object ScriptTestEnv { // else, SCALA_HOME if defined // else, not defined lazy val envScalaHome = + printf("scalacPath: %s\n", scalacPath.norm) if scalacPath.isFile then scalacPath.replaceAll("/bin/scalac", "") - else envOrElse("SCALA_HOME", "").norm + else envOrElse("SCALA_HOME", "not-found").norm + + lazy val fallbackScalaHome = { + findFile("scalac").replaceAll("/bin/scalac$", "") + } + def findFile(name: String) = { + val (valid, err, stdout, stderr) = bashCommand(s"/usr/bin/find . -name $name -type f") + stdout.foreach { printf("out[%s]\n", _) } + stderr.foreach { printf("err[%s]\n", _) } + stdout.take(1).mkString("").norm + } lazy val envJavaHome: String = envOrElse("JAVA_HOME", whichJava.parent(2)).norm lazy val cyghome = envOrElse("CYGWIN", "") @@ -223,10 +254,14 @@ object ScriptTestEnv { lazy val shellopts: String = { val value: String = envOrElse("SHELLOPTS", "braceexpand:hashall:igncr:ignoreeof:monitor:vi") val list: List[String] = value.split(":").toList - "igncr" :: list.filter { + val minlist = list.filter { case "igncr" | "xtrace" => false case _ => true } + if isWin then + "igncr" :: minlist + else + minlist }.mkString(":") lazy val testEnvPairs = { From 34c739ef865199e8e0006b69b420d407a6555eac Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 15 Dec 2021 05:45:57 -0700 Subject: [PATCH 06/15] print warnings; remove unused code --- .../dotty/tools/scripting/ScriptTestEnv.scala | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index 0dcf66beaed1..5715e0092aaf 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -20,16 +20,24 @@ object ScriptTestEnv { def osname: String = sys.props("os.name").toLowerCase def psep: String = sys.props("path.separator") def userDir: String = sys.props("user.dir").norm + def testCwd = envOrElse("TEST_CWD", "").norm // optional working directory TEST_CWD def whichJava: String = whichExe("java") def whichBash: String = whichExe("bash") lazy val workingDirectory: String = { - val dirstr = envOrElse("TEST_CWD", userDir).norm // optional working directory TEST_CWD - printf("working directory is [%s]\n", dirstr) + val dirstr = if testCwd.nonEmpty then + printf("TEST_CWD set to [%s]\n", testCwd) + testCwd + else + userDir // userDir, if TEST_CWD not set + + // issue warning if things don't look right val test = Paths.get(s"$dirstr/dist/target/pack/bin").normalize if !test.isDirectory then printf("warning: not found below working directory: %s\n", test.norm) + + printf("working directory is [%s]\n", dirstr) dirstr } @@ -215,13 +223,7 @@ object ScriptTestEnv { lazy val (scalacPath: String, scalaPath: String) = { val scalac = s"$workingDirectory/dist/target/pack/bin/scalac".toPath.normalize val scala = s"$workingDirectory/dist/target/pack/bin/scala".toPath.normalize - if (scalac.isFile){ - (scalac.norm, scala.norm) - } else { - val s1 = findFile("scalac").replaceAll("/bin/scalac$", "") - val s2 = findFile("scala").replaceAll("/bin/scala$", "") - (s1.norm, s2.norm) - } + (scalac.norm, scala.norm) } @@ -236,16 +238,6 @@ object ScriptTestEnv { if scalacPath.isFile then scalacPath.replaceAll("/bin/scalac", "") else envOrElse("SCALA_HOME", "not-found").norm - lazy val fallbackScalaHome = { - findFile("scalac").replaceAll("/bin/scalac$", "") - } - def findFile(name: String) = { - val (valid, err, stdout, stderr) = bashCommand(s"/usr/bin/find . -name $name -type f") - stdout.foreach { printf("out[%s]\n", _) } - stderr.foreach { printf("err[%s]\n", _) } - stdout.take(1).mkString("").norm - } - lazy val envJavaHome: String = envOrElse("JAVA_HOME", whichJava.parent(2)).norm lazy val cyghome = envOrElse("CYGWIN", "") lazy val msyshome = envOrElse("MSYS", "") From 7eacdf26a05f306034bdb37118346ca92dd11270 Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 15 Dec 2021 08:32:36 -0700 Subject: [PATCH 07/15] strip ansi colors from bash command line output, to fix windows tests --- .../test/dotty/tools/scripting/BashScriptsTests.scala | 3 ++- compiler/test/dotty/tools/scripting/ScriptTestEnv.scala | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index ddf125a4e16c..5fddb5f0099b 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -142,7 +142,8 @@ class BashScriptsTests: printf("stdout: %s\n", stdout.mkString("\n","\n","")) if verifyValid(validTest) then val expected = s"${workingDirectory.norm}" - val cwdline = stdout.find( _.trim.startsWith("cwd") ).getOrElse("") + // stdout might be polluted with an ANSI color prefix, so be careful + val cwdline = stdout.find( _.trim.matches(".*cwd: .*") ).getOrElse("") printf("cwdline [%s]\n", cwdline) printf("expected[%s]\n", expected) val valid = cwdline.endsWith(expected) diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index 5715e0092aaf..1e964f0d7194 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -131,12 +131,18 @@ object ScriptTestEnv { // script output expected as ": " def findTaggedLine(tag: String, lines: Seq[String]): String = - lines.find { _.startsWith(tag) } match + lines.map { stripColors(_) }.find { _.startsWith(tag) } match case None => lines.foreach { System.err.printf("line[%s]\n", _) } sys.error(s"no $tag: found in script output") case Some(cwd) => cwd.dropWhile( _ != ' ').trim // discard tag + def stripColors(line:String): String = + // ESC has be seen in the wild replaced by "\u2190" + // Also, BOM marker appears as  + lazy val colorsRegex = "(\u001b|\u2190)\\[[0-9;]*m|".r + colorsRegex.replaceAllIn(line,"") + def exec(cmd: String *): Seq[String] = Process(cmd).lazyLines_!.toList def script2jar(scriptFile: File) = From 4fa4e9729dfe7b7fd262510cbde2c3f274c50dfe Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 15 Dec 2021 09:52:21 -0700 Subject: [PATCH 08/15] dist/pack before sbt test in test_windows_full and test_non_bootstrapped --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8a2c1728496d..1e9a145127bf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -66,7 +66,7 @@ jobs: - name: Test run: | - ./project/scripts/sbt ";compile ;test" + ./project/scripts/sbt ";dist/pack; compile ;test" ./project/scripts/cmdTests test: @@ -181,7 +181,7 @@ jobs: uses: actions/checkout@v2 - name: Test - run: sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test" + run: sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test" shell: cmd - name: Scala.js Test From 4303a16cb4b8bcec848e9c24abe6d1e709f009a6 Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 15 Dec 2021 14:11:40 -0700 Subject: [PATCH 09/15] squeeze redundancy from env-var-setting tests, add more log messages --- .../tools/scripting/BashScriptsTests.scala | 59 +++++++++++++++---- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 5fddb5f0099b..ee4ba162e204 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -67,37 +67,72 @@ class BashScriptsTests: if fail then assert(stdout == expectedOutput) + def testFile(name: String): String = + val file = testFiles.find(_.getName == name) match { + case Some(f) => + val ff = f.absPath + printf("test file [%s] is [%s]\n", name, ff) + ff + case None => + printf("test file [%s] not found!\n", name) + name.absPath + } + file + + lazy val Seq(envtestSc, envtestScala) = + Seq("envtest.sc", "envtest.scala").map { testFile(_) } + +/* + def testParameters(tag: String, script: String, keyPfx: String) = + val keyArg = s"$keyPfx=$tag" + (tag, script, keyArg) +*/ + + def buildCmd(tag: String, script: String, keyPre: String): String = + val keyArg = s"$keyPre=$tag" + printf("pass tag [%s] via [%s] to script [%s]\n", tag, keyArg, script) + val cmd: String = Seq("SCALA_OPTS= ", scalaPath, keyArg, script).mkString(" ") + printf("cmd: [%s]\n", cmd) + cmd + /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.sc */ @Test def verifyScJProperty = - val commandline = Seq("SCALA_OPTS= ", scalaPath, "-J-Dkey=World", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") + val tag = "World1" + val commandline = buildCmd(tag, envtestSc, s"-J-Dkey") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(stdout.mkString("/n"), "Hello World") + assertEquals( s"Hello $tag", stdout.mkString("/n")) /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.scala */ @Test def verifyScalaJProperty = - val commandline = Seq("SCALA_OPTS= ", scalaPath, "-J-Dkey=World3", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") + val tag = "World2" + val commandline = buildCmd(tag, envtestScala, s"-J-Dkey") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(stdout.mkString("/n"), "Hello World3") + assertEquals(s"Hello $tag", stdout.mkString("/n")) /* verify that `dist/bin/scala` can set system properties via -D for envtest.sc */ @Test def verifyScDProperty = - val commandline = Seq("SCALA_OPTS= ", scalaPath, "-Dkey=World3", testFiles.find(_.getName == "envtest.sc").get.absPath).mkString(" ") + val tag = "World3" + val commandline = buildCmd(tag, envtestSc, s"-Dkey") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(stdout.mkString("/n"), "Hello World3") + assertEquals(s"Hello $tag", stdout.mkString("/n")) /* verify that `dist/bin/scala` can set system properties via -D for envtest.scala */ @Test def verifyScalaDProperty = - val commandline = Seq("SCALA_OPTS= ", scalaPath, "-Dkey=World4", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") + val tag = "World4" + val commandline = buildCmd(tag, envtestScala, s"-Dkey") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(stdout.mkString("/n"), "Hello World4") + assertEquals(s"Hello $tag", stdout.mkString("/n")) /* verify that `dist/bin/scala` can set system properties via -D when executing compiled script via -jar envtest.jar */ @Test def saveAndRunWithDProperty = - val commandline = Seq("SCALA_OPTS= ", scalaPath, "-save", testFiles.find(_.getName == "envtest.scala").get.absPath).mkString(" ") - val (_, _, _, _) = bashCommand(commandline) - val commandline2 = Seq("SCALA_OPTS= ", scalaPath, "-Dkey=World5", testFiles.find(_.getName == "envtest.jar").get.absPath).mkString(" ") + val commandline = Seq("SCALA_OPTS= ", scalaPath, "-save", envtestScala).mkString(" ") + val (_, _, _, _) = bashCommand(commandline) // compile jar, discard output + + val tag = "World5" + val envtestJar = testFile("envtest.jar") // jar is created by the previous bashCommand() + val commandline2 = Seq("SCALA_OPTS= ", scalaPath, s"-Dkey=$tag", envtestJar).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline2) - assertEquals(stdout.mkString("/n"), "Hello World5") + assertEquals(s"Hello $tag", stdout.mkString("/n")) /* verify `dist/bin/scala` non-interference with command line args following script name */ @Test def verifyScalaArgs = From 3ff19fd36b168cdb5b670d1726cf9d5e72d35c53 Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 15 Dec 2021 18:28:26 -0700 Subject: [PATCH 10/15] further reduced redundancy; additional log messages --- .../tools/scripting/BashScriptsTests.scala | 96 +++++++++---------- .../dotty/tools/scripting/ScriptTestEnv.scala | 14 ++- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index ee4ba162e204..929019b66464 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -17,6 +17,7 @@ import ScriptTestEnv.* */ object BashScriptsTests: lazy val argsfile = createArgsFile() // avoid problems caused by drive letter + lazy val testFiles = scripts("/scripting") @AfterClass def cleanup: Unit = { val af = argsfile.toFile @@ -24,12 +25,6 @@ object BashScriptsTests: af.delete() } } - -class BashScriptsTests: - import BashScriptsTests.* - // classpath tests managed by scripting.ClasspathTests.scala - def testFiles = scripts("/scripting") - printf("osname[%s]\n", osname) printf("uname[%s]\n", ostypeFull) printf("using JAVA_HOME=%s\n", envJavaHome) @@ -38,7 +33,7 @@ class BashScriptsTests: printf("scala path: [%s]\n", scalaPath) printf("scalac path: [%s]\n", scalacPath) - lazy val expectedOutput = List( + val expectedOutput = List( "arg 0:[a]", "arg 1:[b]", "arg 2:[c]", @@ -47,26 +42,11 @@ class BashScriptsTests: "arg 5:[-script]", "arg 6:[-debug]", ) - lazy val testScriptArgs = Seq( + val testScriptArgs = Seq( "a", "b", "c", "-repl", "-run", "-script", "-debug" ) val showArgsScript = testFiles.find(_.getName == "showArgs.sc").get.absPath - /* verify `dist/bin/scalac` non-interference with command line args following script name */ - @Test def verifyScalacArgs = - val commandline = (Seq("SCALA_OPTS= ", scalacPath, "-script", showArgsScript) ++ testScriptArgs).mkString(" ") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - if verifyValid(validTest) then - var fail = false - printf("\n") - for (line, expect) <- stdout zip expectedOutput do - printf("expected: %-17s\nactual : %s\n", expect, line) - if line != expect then - fail = true - - if fail then - assert(stdout == expectedOutput) - def testFile(name: String): String = val file = testFiles.find(_.getName == name) match { case Some(f) => @@ -79,61 +59,81 @@ class BashScriptsTests: } file - lazy val Seq(envtestSc, envtestScala) = - Seq("envtest.sc", "envtest.scala").map { testFile(_) } - -/* - def testParameters(tag: String, script: String, keyPfx: String) = - val keyArg = s"$keyPfx=$tag" - (tag, script, keyArg) -*/ + val Seq(envtestSc, envtestScala) = Seq("envtest.sc", "envtest.scala").map { testFile(_) } - def buildCmd(tag: String, script: String, keyPre: String): String = + // create command line with given options, execute specified script, return stdout + def callScript(tag: String, script: String, keyPre: String): String = val keyArg = s"$keyPre=$tag" printf("pass tag [%s] via [%s] to script [%s]\n", tag, keyArg, script) val cmd: String = Seq("SCALA_OPTS= ", scalaPath, keyArg, script).mkString(" ") printf("cmd: [%s]\n", cmd) - cmd + val (validTest, exitCode, stdout, stderr) = bashCommand(cmd) + stderr.filter { !_.contains("Inappropriate ioctl") }.foreach { System.err.printf("stderr [%s]\n", _) } + stdout.mkString("\n") + +class BashScriptsTests: + import BashScriptsTests.* + // classpath tests managed by scripting.ClasspathTests.scala + + ////////////////////////// begin tests ////////////////////// + /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.sc */ @Test def verifyScJProperty = val tag = "World1" - val commandline = buildCmd(tag, envtestSc, s"-J-Dkey") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals( s"Hello $tag", stdout.mkString("/n")) + val stdout = callScript(tag, envtestSc, s"-J-Dkey") + assertEquals( s"Hello $tag", stdout) /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.scala */ @Test def verifyScalaJProperty = val tag = "World2" - val commandline = buildCmd(tag, envtestScala, s"-J-Dkey") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(s"Hello $tag", stdout.mkString("/n")) + val stdout = callScript(tag, envtestScala, s"-J-Dkey") + assertEquals(s"Hello $tag", stdout) /* verify that `dist/bin/scala` can set system properties via -D for envtest.sc */ @Test def verifyScDProperty = val tag = "World3" - val commandline = buildCmd(tag, envtestSc, s"-Dkey") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(s"Hello $tag", stdout.mkString("/n")) + val stdout = callScript(tag, envtestSc, s"-Dkey") + assertEquals(s"Hello $tag", stdout) /* verify that `dist/bin/scala` can set system properties via -D for envtest.scala */ @Test def verifyScalaDProperty = val tag = "World4" - val commandline = buildCmd(tag, envtestScala, s"-Dkey") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) - assertEquals(s"Hello $tag", stdout.mkString("/n")) + val stdout = callScript(tag, envtestScala, s"-Dkey") + assertEquals(s"Hello $tag", stdout) /* verify that `dist/bin/scala` can set system properties via -D when executing compiled script via -jar envtest.jar */ @Test def saveAndRunWithDProperty = val commandline = Seq("SCALA_OPTS= ", scalaPath, "-save", envtestScala).mkString(" ") val (_, _, _, _) = bashCommand(commandline) // compile jar, discard output + val testJar = testFile("envtest.jar") // jar is created by the previous bashCommand() + if (testJar.isFile){ + printf("compiled envtest.scala to %s\n", testJar.norm) + } else { + sys.error(s"error: unable to compile envtest.scala to ${testJar.norm}") + } val tag = "World5" - val envtestJar = testFile("envtest.jar") // jar is created by the previous bashCommand() - val commandline2 = Seq("SCALA_OPTS= ", scalaPath, s"-Dkey=$tag", envtestJar).mkString(" ") - val (validTest, exitCode, stdout, stderr) = bashCommand(commandline2) + val commandline2 = Seq("SCALA_OPTS= ", scalaPath.relpath, s"-Dkey=$tag", testJar.relpath) + printf("cmd[%s]\n", commandline2.mkString(" ")) + val (validTest, exitCode, stdout, stderr) = bashCommand(commandline2.mkString(" ")) assertEquals(s"Hello $tag", stdout.mkString("/n")) + /* verify `dist/bin/scalac` non-interference with command line args following script name */ + @Test def verifyScalacArgs = + val commandline = (Seq("SCALA_OPTS= ", scalacPath, "-script", showArgsScript) ++ testScriptArgs).mkString(" ") + val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) + if verifyValid(validTest) then + var fail = false + printf("\n") + for (line, expect) <- stdout zip expectedOutput do + printf("expected: %-17s\nactual : %s\n", expect, line) + if line != expect then + fail = true + + if fail then + assert(stdout == expectedOutput) + /* verify `dist/bin/scala` non-interference with command line args following script name */ @Test def verifyScalaArgs = val commandline = (Seq("SCALA_OPTS= ", scalaPath, showArgsScript) ++ testScriptArgs).mkString(" ") diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index 1e964f0d7194..5d58091f2433 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -43,7 +43,8 @@ object ScriptTestEnv { def envPath: String = envOrElse("PATH", "") // remove duplicate entries in path - def adjustedPathEntries: List[String] = s"dist/target/pack/bin$psep$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath".norm.split(psep).toList.distinct + def supplementedPath: String = s"dist/target/pack/bin$psep$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath".norm + def adjustedPathEntries: List[String] = supplementedPath.norm.split(psep).toList.distinct def adjustedPath: String = adjustedPathEntries.mkString(psep) def envPathEntries: List[String] = envPath.split(psep).toList.distinct @@ -93,6 +94,8 @@ object ScriptTestEnv { def bashCommand(cmdstr: String, additionalEnvPairs: List[(String, String)] = Nil): (Boolean, Int, Seq[String], Seq[String]) = { var (stdout, stderr) = (List.empty[String], List.empty[String]) if bashExe.toFile.exists then + def q = "\"" + printf("bashCmd: %s -c %s\n", bashExe, s"$q$cmdstr$q") val cmd = Seq(bashExe, "-c", cmdstr) val envPairs = testEnvPairs ++ additionalEnvPairs val proc = Process(cmd, None, envPairs *) @@ -177,9 +180,10 @@ object ScriptTestEnv { extension(s: String) { def norm: String = s.replace('\\', '/') // bash expects forward slash def noDrive = if s.secondChar == ":" then s.drop(2).norm else s.norm - def toPath: Path = Paths.get(fixHome(s.noDrive)) // .toAbsolutePath - def toFile: File = new File(s) + def toPath: Path = Paths.get(fixHome(s)) // .toAbsolutePath + def toFile: File = File(s) def absPath: String = s.toFile.absPath + def relpath: String = s.norm.replaceFirst(s"${cwd.norm}/","") def isFile: Boolean = s.toFile.isFile def isDirectory: Boolean = s.toFile.isDirectory def exists: Boolean = s.toFile.exists @@ -192,7 +196,6 @@ object ScriptTestEnv { extension(p: Path) { def norm: String = p.normalize.toString.replace('\\', '/') - def noDrive = p.norm match { case str if str.drop(1).take(1) == ":" => str.drop(2) case str => str @@ -244,7 +247,8 @@ object ScriptTestEnv { if scalacPath.isFile then scalacPath.replaceAll("/bin/scalac", "") else envOrElse("SCALA_HOME", "not-found").norm - lazy val envJavaHome: String = envOrElse("JAVA_HOME", whichJava.parent(2)).norm + lazy val javaParent: String = whichJava.norm.replace("/bin/[^/]*$","") + lazy val envJavaHome: String = envOrElse("JAVA_HOME", javaParent) lazy val cyghome = envOrElse("CYGWIN", "") lazy val msyshome = envOrElse("MSYS", "") From 2e45931db973514fffd44530a4251c3f03193941 Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 15 Dec 2021 19:15:06 -0700 Subject: [PATCH 11/15] remove trailing java from JAVA_HOME value; shorten comand lines with relpath --- compiler/test/dotty/tools/scripting/BashScriptsTests.scala | 3 ++- compiler/test/dotty/tools/scripting/ScriptTestEnv.scala | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 929019b66464..d97af1ff7624 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -104,7 +104,8 @@ class BashScriptsTests: /* verify that `dist/bin/scala` can set system properties via -D when executing compiled script via -jar envtest.jar */ @Test def saveAndRunWithDProperty = - val commandline = Seq("SCALA_OPTS= ", scalaPath, "-save", envtestScala).mkString(" ") + val commandline2 = Seq("SCALA_OPTS= ", scalaPath.relpath, s"-Dkey=$tag", testJar.relpath) + val commandline = Seq("SCALA_OPTS= ", scalaPath.relpath, "-save", envtestScala.relpath).mkString(" ") val (_, _, _, _) = bashCommand(commandline) // compile jar, discard output val testJar = testFile("envtest.jar") // jar is created by the previous bashCommand() if (testJar.isFile){ diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index 5d58091f2433..a37918f93ed2 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -247,7 +247,7 @@ object ScriptTestEnv { if scalacPath.isFile then scalacPath.replaceAll("/bin/scalac", "") else envOrElse("SCALA_HOME", "not-found").norm - lazy val javaParent: String = whichJava.norm.replace("/bin/[^/]*$","") + lazy val javaParent: String = whichJava.parent(2) lazy val envJavaHome: String = envOrElse("JAVA_HOME", javaParent) lazy val cyghome = envOrElse("CYGWIN", "") lazy val msyshome = envOrElse("MSYS", "") From 2dcd72252f453b400a6522c4eafe351644d118b7 Mon Sep 17 00:00:00 2001 From: philwalk Date: Wed, 15 Dec 2021 21:24:49 -0700 Subject: [PATCH 12/15] Update BashScriptsTests.scala remove duplicate --- compiler/test/dotty/tools/scripting/BashScriptsTests.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index d97af1ff7624..9c3db63ae337 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -104,7 +104,6 @@ class BashScriptsTests: /* verify that `dist/bin/scala` can set system properties via -D when executing compiled script via -jar envtest.jar */ @Test def saveAndRunWithDProperty = - val commandline2 = Seq("SCALA_OPTS= ", scalaPath.relpath, s"-Dkey=$tag", testJar.relpath) val commandline = Seq("SCALA_OPTS= ", scalaPath.relpath, "-save", envtestScala.relpath).mkString(" ") val (_, _, _, _) = bashCommand(commandline) // compile jar, discard output val testJar = testFile("envtest.jar") // jar is created by the previous bashCommand() From 7a6a6ac709324dffac2097e82abd8e8628523c53 Mon Sep 17 00:00:00 2001 From: Andrzej Ratajczak <32793002+BarkingBad@users.noreply.github.com> Date: Thu, 16 Dec 2021 11:17:06 +0100 Subject: [PATCH 13/15] Update BashScriptsTests.scala Fix not-updating property --- compiler/test/dotty/tools/scripting/BashScriptsTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 9c3db63ae337..402e618733b9 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -17,7 +17,7 @@ import ScriptTestEnv.* */ object BashScriptsTests: lazy val argsfile = createArgsFile() // avoid problems caused by drive letter - lazy val testFiles = scripts("/scripting") + def testFiles = scripts("/scripting") @AfterClass def cleanup: Unit = { val af = argsfile.toFile From 64b2ddc0ac08e58728b7d6db07a1b8e84b4e2f41 Mon Sep 17 00:00:00 2001 From: Phil Date: Mon, 10 Jan 2022 09:03:16 -0700 Subject: [PATCH 14/15] fix plus workaround for #13760 --- .../src/dotty/tools/MainGenericRunner.scala | 13 +++++++--- .../test-resources/scripting/sqlDateError.sc | 6 +++++ .../tools/scripting/BashScriptsTests.scala | 24 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) create mode 100755 compiler/test-resources/scripting/sqlDateError.sc diff --git a/compiler/src/dotty/tools/MainGenericRunner.scala b/compiler/src/dotty/tools/MainGenericRunner.scala index fbc947ce175a..8fa177eb66f6 100644 --- a/compiler/src/dotty/tools/MainGenericRunner.scala +++ b/compiler/src/dotty/tools/MainGenericRunner.scala @@ -81,6 +81,9 @@ case class Settings( def withSave: Settings = this.copy(save = true) + def noSave: Settings = + this.copy(save = false) + def withModeShouldBePossibleRun: Settings = this.copy(modeShouldBePossibleRun = true) @@ -135,6 +138,8 @@ object MainGenericRunner { ) case "-save" :: tail => process(tail, settings.withSave) + case "-nosave" :: tail => + process(tail, settings.noSave) case "-with-compiler" :: tail => process(tail, settings.withCompiler) case (o @ javaOption(striped)) :: tail => @@ -207,16 +212,18 @@ object MainGenericRunner { case ExecuteMode.Script => val targetScript = Paths.get(settings.targetScript).toFile val targetJar = settings.targetScript.replaceAll("[.][^\\/]*$", "")+".jar" - val precompiledJar = Paths.get(targetJar).toFile + val precompiledJar = File(targetJar) val mainClass = if !precompiledJar.isFile then "" else Jar(targetJar).mainClass.getOrElse("") - val jarIsValid = mainClass.nonEmpty && precompiledJar.lastModified >= targetScript.lastModified + val jarIsValid = mainClass.nonEmpty && precompiledJar.lastModified >= targetScript.lastModified && settings.save if jarIsValid then // precompiledJar exists, is newer than targetScript, and manifest defines a mainClass sys.props("script.path") = targetScript.toPath.toAbsolutePath.normalize.toString val scalaClasspath = ClasspathFromClassloader(Thread.currentThread().getContextClassLoader).split(classpathSeparator) val newClasspath = (settings.classPath.flatMap(_.split(classpathSeparator).filter(_.nonEmpty)) ++ removeCompiler(scalaClasspath) :+ ".").map(File(_).toURI.toURL) if mainClass.nonEmpty then - ObjectRunner.runAndCatch(newClasspath :+ File(targetJar).toURI.toURL, mainClass, settings.scriptArgs) + ObjectRunner.runAndCatch(newClasspath :+ File(targetJar).toURI.toURL, mainClass, settings.scriptArgs).foreach { (t: Throwable) => + t.printStackTrace() + } else Some(IllegalArgumentException(s"No main class defined in manifest in jar: $precompiledJar")) else diff --git a/compiler/test-resources/scripting/sqlDateError.sc b/compiler/test-resources/scripting/sqlDateError.sc new file mode 100755 index 000000000000..b9a47d245d1a --- /dev/null +++ b/compiler/test-resources/scripting/sqlDateError.sc @@ -0,0 +1,6 @@ +#!bin/scala -nosave + +def main(args: Array[String]): Unit = { + println(new java.sql.Date(100L)) + System.err.println("SCALA_OPTS="+Option(System.getenv("SCALA_OPTS")).getOrElse("")) +} diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 402e618733b9..13c40ce72ff9 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -188,3 +188,27 @@ class BashScriptsTests: if valid then printf(s"\n===> success: classpath begins with %s, as reported by [%s]\n", workingDirectory, scriptFile.getName) assert(valid, s"script ${scriptFile.absPath} did not report valid java.class.path first entry") + /* + * verify that individual scripts can override -save with -nosave (needed to address #13760). + */ + @Test def sqlDateTest = + val scriptBase = "sqlDateError" + val scriptFile = testFiles.find(_.getName == s"$scriptBase.sc").get + val testJar = testFile(s"$scriptBase.jar") // jar should not be created when scriptFile runs + printf("===> verify '-save' is cancelled by '-nosave' in script hashbang.`\n") + val envPairs = List(("SCALA_OPTS", "-save")) + val (validTest, exitCode, stdout, stderr) = bashCommand(scriptFile.absPath, envPairs) + printf("stdout: %s\n", stdout.mkString("\n","\n","")) + if verifyValid(validTest) then + // the expectation is that the script prints '1969-12-31' + val expected = "1969-12-31" + // stdout might be polluted with an ANSI color prefix, so be careful + printf("expected[%s]\n", expected) + val valid = stdout.contains(expected) + if (!valid) then + stdout.foreach { printf("stdout[%s]\n", _) } + stderr.foreach { printf("stderr[%s]\n", _) } + if valid then printf(s"\n===> success: scripts can override -save via -nosave\n") + assert(valid, s"script ${scriptFile.absPath} did not report valid java.class.path first entry") + assert(!testJar.exists,s"unexpected, jar file [$testJar] was created") + From 459329ee96431204a7d371890cf58581eaee585f Mon Sep 17 00:00:00 2001 From: philwalk Date: Mon, 10 Jan 2022 17:39:57 -0700 Subject: [PATCH 15/15] Update BashScriptsTests.scala removed time zone dependency of `BashScriptsTests` `sqlDateTest`, corrected comments --- .../dotty/tools/scripting/BashScriptsTests.scala | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 13c40ce72ff9..6bb41b7266df 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -196,19 +196,15 @@ class BashScriptsTests: val scriptFile = testFiles.find(_.getName == s"$scriptBase.sc").get val testJar = testFile(s"$scriptBase.jar") // jar should not be created when scriptFile runs printf("===> verify '-save' is cancelled by '-nosave' in script hashbang.`\n") - val envPairs = List(("SCALA_OPTS", "-save")) - val (validTest, exitCode, stdout, stderr) = bashCommand(scriptFile.absPath, envPairs) + val (validTest, exitCode, stdout, stderr) = bashCommand(s"SCALA_OPTS=-save ${scriptFile.absPath}") printf("stdout: %s\n", stdout.mkString("\n","\n","")) if verifyValid(validTest) then - // the expectation is that the script prints '1969-12-31' - val expected = "1969-12-31" - // stdout might be polluted with an ANSI color prefix, so be careful - printf("expected[%s]\n", expected) - val valid = stdout.contains(expected) + // the script should print '1969-12-31' or '1970-01-01', depending on time zone + // stdout can be polluted with an ANSI color prefix, in some test environments + val valid = stdout.mkString("").matches(""".*\d{4}-\d{2}-\d{2}.*""") if (!valid) then stdout.foreach { printf("stdout[%s]\n", _) } stderr.foreach { printf("stderr[%s]\n", _) } if valid then printf(s"\n===> success: scripts can override -save via -nosave\n") - assert(valid, s"script ${scriptFile.absPath} did not report valid java.class.path first entry") + assert(valid, s"script ${scriptFile.absPath} reported unexpected value for java.sql.Date ${stdout.mkString("\n")}") assert(!testJar.exists,s"unexpected, jar file [$testJar] was created") -