From 121d746932a31e6acff98ef2c66ce549110c1f2b Mon Sep 17 00:00:00 2001 From: Phil Date: Sat, 6 Mar 2021 15:34:11 -0700 Subject: [PATCH 01/15] expand wildcard classpath entries; not os-dependent, required by jar manifest; fixes #10761 --- compiler/src/dotty/tools/scripting/Main.scala | 13 +++++---- .../tools/scripting/ScriptingDriver.scala | 29 ++++++++++++++----- .../tools/scripting/ScriptingTests.scala | 7 +++-- 3 files changed, 32 insertions(+), 17 deletions(-) mode change 100644 => 100755 compiler/src/dotty/tools/scripting/Main.scala mode change 100644 => 100755 compiler/src/dotty/tools/scripting/ScriptingDriver.scala diff --git a/compiler/src/dotty/tools/scripting/Main.scala b/compiler/src/dotty/tools/scripting/Main.scala old mode 100644 new mode 100755 index ded809b9f675..fd69fc0ff5ec --- a/compiler/src/dotty/tools/scripting/Main.scala +++ b/compiler/src/dotty/tools/scripting/Main.scala @@ -32,10 +32,10 @@ object Main: def main(args: Array[String]): Unit = val (compilerArgs, scriptFile, scriptArgs, saveJar, invokeFlag) = distinguishArgs(args) val driver = ScriptingDriver(compilerArgs, scriptFile, scriptArgs) - try driver.compileAndRun { (outDir:Path, classpath:String, mainClass: String) => + try driver.compileAndRun { (outDir:Path, classpathEntries:Seq[Path], mainClass: String) => if saveJar then // write a standalone jar to the script parent directory - writeJarfile(outDir, scriptFile, scriptArgs, classpath, mainClass) + writeJarfile(outDir, scriptFile, scriptArgs, classpathEntries, mainClass) invokeFlag } catch @@ -47,10 +47,10 @@ object Main: throw e.getCause private def writeJarfile(outDir: Path, scriptFile: File, scriptArgs:Array[String], - classpath:String, mainClassName: String): Unit = + classpathEntries:Seq[Path], mainClassName: String): Unit = - val javaClasspath = sys.props("java.class.path") - val runtimeClasspath = s"${classpath}$pathsep$javaClasspath" + //val javaClasspath = sys.props("java.class.path") + //val runtimeClasspath = s"${classpath}$pathsep$javaClasspath" val jarTargetDir: Path = Option(scriptFile.toPath.toAbsolutePath.getParent) match { case None => sys.error(s"no parent directory for script file [$scriptFile]") @@ -60,7 +60,8 @@ object Main: def scriptBasename = scriptFile.getName.takeWhile(_!='.') val jarPath = s"$jarTargetDir/$scriptBasename.jar" - val cpPaths = runtimeClasspath.split(pathsep).map(_.toUrl) + //val cpPaths = runtimeClasspath.split(pathsep).map(_.toUrl) + val cpPaths = classpathEntries.map { _.toString.toUrl } import java.util.jar.Attributes.Name val cpString:String = cpPaths.distinct.mkString(" ") diff --git a/compiler/src/dotty/tools/scripting/ScriptingDriver.scala b/compiler/src/dotty/tools/scripting/ScriptingDriver.scala old mode 100644 new mode 100755 index af0858acf658..7c1109754e0f --- a/compiler/src/dotty/tools/scripting/ScriptingDriver.scala +++ b/compiler/src/dotty/tools/scripting/ScriptingDriver.scala @@ -1,6 +1,6 @@ package dotty.tools.scripting -import java.nio.file.{ Files, Path } +import java.nio.file.{ Files, Paths, Path } import java.io.File import java.net.{ URL, URLClassLoader } import java.lang.reflect.{ Modifier, Method } @@ -17,7 +17,7 @@ import dotty.tools.dotc.config.Settings.Setting._ import sys.process._ class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs: Array[String]) extends Driver: - def compileAndRun(pack:(Path, String, String) => Boolean = null): Unit = + def compileAndRun(pack:(Path, Seq[Path], String) => Boolean = null): Unit = val outDir = Files.createTempDirectory("scala3-scripting") setup(compilerArgs :+ scriptFile.getAbsolutePath, initCtx.fresh) match case Some((toCompile, rootCtx)) => @@ -28,11 +28,23 @@ class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs: throw ScriptingException("Errors encountered during compilation") try - val (mainClass, mainMethod) = detectMainClassAndMethod(outDir, ctx.settings.classpath.value, scriptFile) + val classpath = s"${ctx.settings.classpath.value}${pathsep}${sys.props("java.class.path")}" + val classpathEntries: Seq[Path] = + classpath.split(pathsep).toIndexedSeq.flatMap { entry => + val f = Paths.get(entry).toAbsolutePath.normalize.toFile + // expand wildcard classpath entries + if (f.getName == "*" && f.getParentFile.isDirectory){ + f.getParentFile.listFiles.filter { _.getName.toLowerCase.endsWith(".jar") }.map { _.toPath }.toSeq + } else { + Seq(f.toPath) + } + }.toIndexedSeq + + val (mainClass, mainMethod) = detectMainClassAndMethod(outDir, classpathEntries, scriptFile) val invokeMain: Boolean = Option(pack) match case Some(func) => - func(outDir, ctx.settings.classpath.value, mainClass) + func(outDir, classpathEntries, mainClass) case None => true end match @@ -52,11 +64,12 @@ class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs: target.delete() end deleteFile - private def detectMainClassAndMethod(outDir: Path, classpath: String, + private def detectMainClassAndMethod(outDir: Path, classpathEntries: Seq[Path], scriptFile: File): (String, Method) = - val outDirURL = outDir.toUri.toURL - val classpathUrls = classpath.split(pathsep).map(File(_).toURI.toURL) - val cl = URLClassLoader(classpathUrls :+ outDirURL) + + val classpathUrls = (classpathEntries :+ outDir).map { _.toUri.toURL } + + val cl = URLClassLoader(classpathUrls.toArray) def collectMainMethods(target: File, path: String): List[(String, Method)] = val nameWithoutExtension = target.getName.takeWhile(_ != '.') diff --git a/compiler/test/dotty/tools/scripting/ScriptingTests.scala b/compiler/test/dotty/tools/scripting/ScriptingTests.scala index e7399c68f09a..c0198507ed4d 100644 --- a/compiler/test/dotty/tools/scripting/ScriptingTests.scala +++ b/compiler/test/dotty/tools/scripting/ScriptingTests.scala @@ -3,6 +3,7 @@ package tools package scripting import java.io.File +import java.nio.file.Path import org.junit.Test @@ -65,7 +66,7 @@ class ScriptingTests: ), scriptFile = scriptFile, scriptArgs = scriptArgs - ).compileAndRun { (path:java.nio.file.Path,classpath:String, mainClass:String) => + ).compileAndRun { (path:java.nio.file.Path,classpathEntries:Seq[Path], mainClass:String) => printf("mainClass from ScriptingDriver: %s\n",mainClass) true // call compiled script main method } @@ -128,7 +129,7 @@ class ScriptingTests: compilerArgs = Array("-classpath", TestConfiguration.basicClasspath), scriptFile = scriptFile, scriptArgs = Array.empty[String] - ).compileAndRun { (path:java.nio.file.Path,classpath:String, mainClass:String) => + ).compileAndRun { (path:java.nio.file.Path,classpathEntries:Seq[Path], mainClass:String) => printf("success: no call to main method in mainClass: %s\n",mainClass) false // no call to compiled script main method } @@ -141,7 +142,7 @@ class ScriptingTests: compilerArgs = Array("-classpath", TestConfiguration.basicClasspath), scriptFile = scriptFile, scriptArgs = Array.empty[String] - ).compileAndRun { (path:java.nio.file.Path,classpath:String, mainClass:String) => + ).compileAndRun { (path:java.nio.file.Path,classpathEntries:Seq[Path], mainClass:String) => printf("call main method in mainClass: %s\n",mainClass) true // call compiled script main method, create touchedFile } From 54a2dc93301af8c192217a5c9de9f1a645f2470a Mon Sep 17 00:00:00 2001 From: Phil Date: Sat, 6 Mar 2021 15:54:56 -0700 Subject: [PATCH 02/15] removed obsolete comments --- compiler/src/dotty/tools/scripting/Main.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/src/dotty/tools/scripting/Main.scala b/compiler/src/dotty/tools/scripting/Main.scala index fd69fc0ff5ec..19c44c7ca083 100755 --- a/compiler/src/dotty/tools/scripting/Main.scala +++ b/compiler/src/dotty/tools/scripting/Main.scala @@ -49,9 +49,6 @@ object Main: private def writeJarfile(outDir: Path, scriptFile: File, scriptArgs:Array[String], classpathEntries:Seq[Path], mainClassName: String): Unit = - //val javaClasspath = sys.props("java.class.path") - //val runtimeClasspath = s"${classpath}$pathsep$javaClasspath" - val jarTargetDir: Path = Option(scriptFile.toPath.toAbsolutePath.getParent) match { case None => sys.error(s"no parent directory for script file [$scriptFile]") case Some(parent) => parent @@ -60,7 +57,6 @@ object Main: def scriptBasename = scriptFile.getName.takeWhile(_!='.') val jarPath = s"$jarTargetDir/$scriptBasename.jar" - //val cpPaths = runtimeClasspath.split(pathsep).map(_.toUrl) val cpPaths = classpathEntries.map { _.toString.toUrl } import java.util.jar.Attributes.Name From f5daaa2cdbde14d5752be9485d2e72db1720049b Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 9 Mar 2021 11:47:13 -0700 Subject: [PATCH 03/15] leverage dotty.tools.io.ClassPath.expandPath ; allow windows */ --- compiler/src/dotty/tools/io/ClassPath.scala | 2 +- .../tools/scripting/ScriptingDriver.scala | 14 +----- .../test/dotty/tools/io/ClasspathTest.scala | 48 +++++++++++++++++++ 3 files changed, 51 insertions(+), 13 deletions(-) create mode 100755 compiler/test/dotty/tools/io/ClasspathTest.scala diff --git a/compiler/src/dotty/tools/io/ClassPath.scala b/compiler/src/dotty/tools/io/ClassPath.scala index d7218108e944..e49430cc3226 100644 --- a/compiler/src/dotty/tools/io/ClassPath.scala +++ b/compiler/src/dotty/tools/io/ClassPath.scala @@ -132,7 +132,7 @@ object ClassPath { dir.list.filter(x => filt(x.name) && (x.isDirectory || isJarOrZip(x))).map(_.path).toList if (pattern == "*") lsDir(Directory(".")) - else if (pattern.endsWith(wildSuffix)) lsDir(Directory(pattern dropRight 2)) + else if (pattern.endsWith(wildSuffix) || pattern.endsWith("/*")) lsDir(Directory(pattern dropRight 2)) else if (pattern.contains('*')) { try { val regexp = ("^" + pattern.replace("""\*""", """.*""") + "$").r diff --git a/compiler/src/dotty/tools/scripting/ScriptingDriver.scala b/compiler/src/dotty/tools/scripting/ScriptingDriver.scala index 7c1109754e0f..ae428d069a29 100755 --- a/compiler/src/dotty/tools/scripting/ScriptingDriver.scala +++ b/compiler/src/dotty/tools/scripting/ScriptingDriver.scala @@ -10,7 +10,7 @@ import scala.jdk.CollectionConverters._ import dotty.tools.dotc.{ Driver, Compiler } import dotty.tools.dotc.core.Contexts, Contexts.{ Context, ContextBase, ctx } import dotty.tools.dotc.config.CompilerCommand -import dotty.tools.io.{ PlainDirectory, Directory } +import dotty.tools.io.{ PlainDirectory, Directory, ClassPath } import dotty.tools.dotc.reporting.Reporter import dotty.tools.dotc.config.Settings.Setting._ @@ -29,17 +29,7 @@ class ScriptingDriver(compilerArgs: Array[String], scriptFile: File, scriptArgs: try val classpath = s"${ctx.settings.classpath.value}${pathsep}${sys.props("java.class.path")}" - val classpathEntries: Seq[Path] = - classpath.split(pathsep).toIndexedSeq.flatMap { entry => - val f = Paths.get(entry).toAbsolutePath.normalize.toFile - // expand wildcard classpath entries - if (f.getName == "*" && f.getParentFile.isDirectory){ - f.getParentFile.listFiles.filter { _.getName.toLowerCase.endsWith(".jar") }.map { _.toPath }.toSeq - } else { - Seq(f.toPath) - } - }.toIndexedSeq - + val classpathEntries: Seq[Path] = ClassPath.expandPath(classpath, expandStar=true).map { Paths.get(_) } val (mainClass, mainMethod) = detectMainClassAndMethod(outDir, classpathEntries, scriptFile) val invokeMain: Boolean = Option(pack) match diff --git a/compiler/test/dotty/tools/io/ClasspathTest.scala b/compiler/test/dotty/tools/io/ClasspathTest.scala new file mode 100755 index 000000000000..086051640fcb --- /dev/null +++ b/compiler/test/dotty/tools/io/ClasspathTest.scala @@ -0,0 +1,48 @@ +package dotty.tools.io + +import org.junit.Test + +import java.io.File +import dotty.tools.io.AbstractFile +import java.nio.file.{Files, Paths} +import java.nio.file.StandardCopyOption._ +import java.nio.file.attribute.PosixFilePermissions +import dotty.tools.io.{ PlainDirectory, Directory, ClassPath } + +class ClasspathTest { + + def pathsep = sys.props("path.separator") + + // + // Cope with wildcard classpath entries, exercised with -classpath + // + // Verify that Windows users not forced to use backslash in classpath. + // + @Test def testWildcards(): Unit = + import dotty.tools.io.ClassPath + val outDir = Files.createTempDirectory("classpath-test") + try + val compilerLib = "dist/target/pack/lib" + for src <- Paths.get(compilerLib).toFile.listFiles.toList.take(5) do + val dest = Paths.get(s"$outDir/${src.getName}") + printf("copy: %s\n",Files.copy(src.toPath,dest)) // ,REPLACE_EXISTING,COPY_ATTRIBUTES)) + + //outDir.toFile.listFiles.toList.foreach { printf("%s\n",_) } + val cp = Seq(s"$compilerLib/*",s"$outDir/*","not-a-real-directory/*").mkString(pathsep).replace('\\','/') + + // need to expand wildcard classpath entries + val entries = ClassPath.expandPath(cp) + for entry <- entries.take(10) do + println(entry) + + finally + deleteFile(outDir.toFile) + + + private def deleteFile(target: File): Unit = + if target.isDirectory then + for member <- target.listFiles.toList + do deleteFile(member) + target.delete() + end deleteFile +} From 5e90176ab84308162e35e9fb0a70d84bfba427eb Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 9 Mar 2021 12:44:26 -0700 Subject: [PATCH 04/15] errors populating temp directory with jars not treated as a test failure --- compiler/test/dotty/tools/io/ClasspathTest.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compiler/test/dotty/tools/io/ClasspathTest.scala b/compiler/test/dotty/tools/io/ClasspathTest.scala index 086051640fcb..5bf1a0951c22 100755 --- a/compiler/test/dotty/tools/io/ClasspathTest.scala +++ b/compiler/test/dotty/tools/io/ClasspathTest.scala @@ -23,9 +23,13 @@ class ClasspathTest { val outDir = Files.createTempDirectory("classpath-test") try val compilerLib = "dist/target/pack/lib" - for src <- Paths.get(compilerLib).toFile.listFiles.toList.take(5) do - val dest = Paths.get(s"$outDir/${src.getName}") - printf("copy: %s\n",Files.copy(src.toPath,dest)) // ,REPLACE_EXISTING,COPY_ATTRIBUTES)) + val libdir = Paths.get(compilerLib).toFile + if libdir.exists then + try for src <- libdir.listFiles.toList.take(5) do + val dest = Paths.get(s"$outDir/${src.getName}") + printf("copy: %s\n",Files.copy(src.toPath,dest)) // ,REPLACE_EXISTING,COPY_ATTRIBUTES)) + catch + case _:NullPointerException => // ignore errors adding jars to outDir //outDir.toFile.listFiles.toList.foreach { printf("%s\n",_) } val cp = Seq(s"$compilerLib/*",s"$outDir/*","not-a-real-directory/*").mkString(pathsep).replace('\\','/') From 15f9965316c466df08929b237b9fb0aee9fca52d Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 9 Mar 2021 14:18:48 -0700 Subject: [PATCH 05/15] add comment to modification in io/ClassPath.scala --- compiler/src/dotty/tools/io/ClassPath.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/src/dotty/tools/io/ClassPath.scala b/compiler/src/dotty/tools/io/ClassPath.scala index e49430cc3226..eeefabddd19f 100644 --- a/compiler/src/dotty/tools/io/ClassPath.scala +++ b/compiler/src/dotty/tools/io/ClassPath.scala @@ -132,6 +132,7 @@ object ClassPath { dir.list.filter(x => filt(x.name) && (x.isDirectory || isJarOrZip(x))).map(_.path).toList if (pattern == "*") lsDir(Directory(".")) + // On Windows the JDK supports forward slash or backslash in classpath entries else if (pattern.endsWith(wildSuffix) || pattern.endsWith("/*")) lsDir(Directory(pattern dropRight 2)) else if (pattern.contains('*')) { try { From c58106c8b43ca322879a231df504d1751affd625 Mon Sep 17 00:00:00 2001 From: Phil Date: Thu, 11 Mar 2021 06:46:35 -0700 Subject: [PATCH 06/15] expand wildcards in MacroClassLoader --- compiler/src/dotty/tools/dotc/core/MacroClassLoader.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/core/MacroClassLoader.scala b/compiler/src/dotty/tools/dotc/core/MacroClassLoader.scala index 261b1f26a64d..896da4044b42 100644 --- a/compiler/src/dotty/tools/dotc/core/MacroClassLoader.scala +++ b/compiler/src/dotty/tools/dotc/core/MacroClassLoader.scala @@ -3,6 +3,7 @@ package dotty.tools.dotc.core import dotty.tools.dotc.core.Contexts._ import dotty.tools.dotc.util.Property import dotty.tools.dotc.reporting.trace +import dotty.tools.io.ClassPath import scala.collection.mutable @@ -20,7 +21,8 @@ object MacroClassLoader { ctx.setProperty(MacroClassLoaderKey, makeMacroClassLoader(using ctx)) private def makeMacroClassLoader(using Context): ClassLoader = trace("new macro class loader") { - val urls = ctx.settings.classpath.value.split(java.io.File.pathSeparatorChar).map(cp => java.nio.file.Paths.get(cp).toUri.toURL) + val entries = ClassPath.expandPath(ctx.settings.classpath.value, expandStar=true) + val urls = entries.map(cp => java.nio.file.Paths.get(cp).toUri.toURL).toArray val out = ctx.settings.outputDir.value.jpath.toUri.toURL // to find classes in case of suspended compilation new java.net.URLClassLoader(urls :+ out, getClass.getClassLoader) } From bbd0cca545201260299466f5c8e32273a9fd44e9 Mon Sep 17 00:00:00 2001 From: Phil Date: Thu, 11 Mar 2021 14:02:20 -0700 Subject: [PATCH 07/15] remove redundant import; extend io/ClasspathTest.scala ; add classpathTest.sc --- .../test-resources/scripting/classpathTest.sc | 33 ++++++++++++++ .../test/dotty/tools/io/ClasspathTest.scala | 44 +++++++++++++------ dist/bin/scala | 9 +++- 3 files changed, 72 insertions(+), 14 deletions(-) create mode 100755 compiler/test-resources/scripting/classpathTest.sc diff --git a/compiler/test-resources/scripting/classpathTest.sc b/compiler/test-resources/scripting/classpathTest.sc new file mode 100755 index 000000000000..8113ef2034ce --- /dev/null +++ b/compiler/test-resources/scripting/classpathTest.sc @@ -0,0 +1,33 @@ +#!dist/target/pack/bin/scala -classpath "dist/target/pack/lib/*" +!# + +/* + * a single wildcard -classpath entry is vulnerable to being converted by Windows jdk + * to a list of jar files. To prevent this Windows-only behavior, a semicolon is + * added to the classpath entry by dist/bin/scala to prevent globbing. + * + * The symptom of failure is that args contains lots of jar file paths. + */ +import java.nio.file.Paths + +// expecting classpath to contain scala compiler lib jars +def main(args: Array[String]): Unit = + assert(args.isEmpty,s"args contains ${args.length} arguments, but was expecting none") + + val expected = Paths.get("dist/target/pack/lib").toFile.listFiles.toList.map { _.getName }.filter { _.endsWith(".jar") } + def psep = java.io.File.pathSeparator + val classpath = sys.props("java.class.path") + val entries = classpath.split(psep).map { Paths.get(_).toFile.getName } + + val expectSet = expected.toSet + val entrySet = entries.toSet + + if ((expectSet intersect entrySet).isEmpty) then + for entry <- expected do + printf("expect:%s\n",entry) + for entry <- entries do + printf("found:%s\n",entry) + + assert ((expectSet intersect entrySet).nonEmpty, s"expected[${expected.size}] but found [${entries.size}]") + + printf("success!\n") diff --git a/compiler/test/dotty/tools/io/ClasspathTest.scala b/compiler/test/dotty/tools/io/ClasspathTest.scala index 5bf1a0951c22..7505352e5602 100755 --- a/compiler/test/dotty/tools/io/ClasspathTest.scala +++ b/compiler/test/dotty/tools/io/ClasspathTest.scala @@ -19,25 +19,43 @@ class ClasspathTest { // Verify that Windows users not forced to use backslash in classpath. // @Test def testWildcards(): Unit = - import dotty.tools.io.ClassPath val outDir = Files.createTempDirectory("classpath-test") try val compilerLib = "dist/target/pack/lib" val libdir = Paths.get(compilerLib).toFile if libdir.exists then - try for src <- libdir.listFiles.toList.take(5) do - val dest = Paths.get(s"$outDir/${src.getName}") - printf("copy: %s\n",Files.copy(src.toPath,dest)) // ,REPLACE_EXISTING,COPY_ATTRIBUTES)) + val libjarFiles = libdir.listFiles.toList.take(5) + try + for src <- libjarFiles do + val dest = Paths.get(s"$outDir/${src.getName}") + printf("copy: %s\n",Files.copy(src.toPath,dest)) // ,REPLACE_EXISTING,COPY_ATTRIBUTES)) + + val cp = Seq(s"$outDir/*","not-a-real-directory/*").mkString(pathsep).replace('\\','/') + + val libjars = libjarFiles.map { _.getName }.toSet + + // expand wildcard classpath entries, ignoring invalid entries + val entries = ClassPath.expandPath(cp).map { Paths.get(_).toFile.getName } + + // require one-to-one matches + assert(libjars == entries.toSet) + + printf("%d entries\n",entries.size) + printf("%d libjars\n",libjars.size) + + for entry <- libjars do + printf("libdir[%s]\n",entry) + + for entry <- entries do + printf("expand[%s]\n",entry) + + // verify that expanded classpath has expected jar names + for jar <- libjars do + assert(entries.contains(jar)) + catch - case _:NullPointerException => // ignore errors adding jars to outDir - - //outDir.toFile.listFiles.toList.foreach { printf("%s\n",_) } - val cp = Seq(s"$compilerLib/*",s"$outDir/*","not-a-real-directory/*").mkString(pathsep).replace('\\','/') - - // need to expand wildcard classpath entries - val entries = ClassPath.expandPath(cp) - for entry <- entries.take(10) do - println(entry) + case _:NullPointerException => // no test if unable to copy jars to outDir + finally deleteFile(outDir.toFile) diff --git a/dist/bin/scala b/dist/bin/scala index bfcd30c14cd0..fbca003d1613 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -53,7 +53,7 @@ truncated_params="${*#-}" # options_indicator != 0 if at least one parameter is not an option options_indicator=$(( ${#all_params} - ${#truncated_params} - $# )) -[ -n "$SCALA_OPTS" ] && set -- $SCALA_OPTS "$@" +[ -n "$SCALA_OPTS" ] && set -- $SCALA_OPTS $@ while [[ $# -gt 0 ]]; do case "$1" in @@ -67,6 +67,13 @@ while [[ $# -gt 0 ]]; do ;; -cp | -classpath) CLASS_PATH="$2" + if [[ $cygwin || $mingw || $msys ]]; then + # in Windows, need to protect a single wildcard entry from jdk globbing. + # (jdk globbing replaces a single wildcard arg to multiple jar file args!) + if [[ $CLASS_PATH =~ \* && ! $CLASS_PATH =~ \; ]]; then + CLASS_PATH=";$CLASS_PATH" + fi + fi class_path_count+=1 shift shift From 4bfa8024b712ad029c368661d2064084e5a49a04 Mon Sep 17 00:00:00 2001 From: Phil Date: Thu, 11 Mar 2021 15:30:00 -0700 Subject: [PATCH 08/15] remove wildcard classpath test script --- .../test-resources/scripting/classpathTest.sc | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100755 compiler/test-resources/scripting/classpathTest.sc diff --git a/compiler/test-resources/scripting/classpathTest.sc b/compiler/test-resources/scripting/classpathTest.sc deleted file mode 100755 index 8113ef2034ce..000000000000 --- a/compiler/test-resources/scripting/classpathTest.sc +++ /dev/null @@ -1,33 +0,0 @@ -#!dist/target/pack/bin/scala -classpath "dist/target/pack/lib/*" -!# - -/* - * a single wildcard -classpath entry is vulnerable to being converted by Windows jdk - * to a list of jar files. To prevent this Windows-only behavior, a semicolon is - * added to the classpath entry by dist/bin/scala to prevent globbing. - * - * The symptom of failure is that args contains lots of jar file paths. - */ -import java.nio.file.Paths - -// expecting classpath to contain scala compiler lib jars -def main(args: Array[String]): Unit = - assert(args.isEmpty,s"args contains ${args.length} arguments, but was expecting none") - - val expected = Paths.get("dist/target/pack/lib").toFile.listFiles.toList.map { _.getName }.filter { _.endsWith(".jar") } - def psep = java.io.File.pathSeparator - val classpath = sys.props("java.class.path") - val entries = classpath.split(psep).map { Paths.get(_).toFile.getName } - - val expectSet = expected.toSet - val entrySet = entries.toSet - - if ((expectSet intersect entrySet).isEmpty) then - for entry <- expected do - printf("expect:%s\n",entry) - for entry <- entries do - printf("found:%s\n",entry) - - assert ((expectSet intersect entrySet).nonEmpty, s"expected[${expected.size}] but found [${entries.size}]") - - printf("success!\n") From 8c72495dc7e5008e31e0fdeb8f1a64654cdf4035 Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 17 Mar 2021 15:00:12 -0600 Subject: [PATCH 09/15] cat mods.txt --- .../tools/scripting/ClasspathTests.scala | 134 ++++++++++++++++++ .../tools/scripting/ScriptingTests.scala | 3 +- dist/bin/scalac | 8 +- 3 files changed, 142 insertions(+), 3 deletions(-) create mode 100755 compiler/test/dotty/tools/scripting/ClasspathTests.scala diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala new file mode 100755 index 000000000000..9244f1f2897a --- /dev/null +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -0,0 +1,134 @@ +package dotty +package tools +package scripting + +import java.io.File +import java.nio.file.{Files, Paths, Path} + +import org.junit.Test + +import vulpix.TestConfiguration + +import scala.sys.process._ +import dotty.tools.dotc.config.Properties._ + +/** Runs all tests contained in `compiler/test-resources/scripting/` */ +class ClasspathTests: + extension (str: String) def dropExtension = + str.reverse.dropWhile(_ != '.').drop(1).reverse + + extension(f: File) def absPath = + f.getAbsolutePath.replace('\\','/') + + // only interested in classpath test scripts + def testFiles = scripts("/scripting").filter { _.getName.matches("classpath.*[.]sc") } + + /* + * Call test scripts + */ + @Test def scriptingJarTest = + for scriptFile <- testFiles do + val relpath = scriptFile.toPath.relpath.norm + printf("===> test script name [%s]\n",relpath) + printf("%s\n",relpath) + val bashExe = which("bash") + printf("bash is [%s]\n",bashExe) + + // ask [dist/bin/scalac] to echo generated command line so we can verify some things + val cmd = Array(bashExe,"-c",s"SCALAC_ECHO_TEST=1 dist/target/pack/bin/scala -classpath '*/lib' $relpath") + cmd.foreach { printf("cmd[%s]\n",_) } + val javaCommandLine = exec(cmd:_*).mkString(" ").split(" ").filter { _.trim.nonEmpty } + val args = scala.collection.mutable.Queue(javaCommandLine:_*) + args.dequeueWhile( _ != "dotty.tools.scripting.Main") + + def consumeNext = args.dequeue() + + // assert that we found "dotty.tools.scripting.Main" + assert(consumeNext == "dotty.tools.scripting.Main") + val mainArgs = args.copyToArray(Array.ofDim[String](args.length)) + + // display command line starting with "dotty.tools.scripting.Main" + args.foreach { line => + printf("%s\n",line) + } + + // expecting -classpath next + assert(consumeNext.replaceAll("'","") == "-classpath") + + // 2nd arg to scripting.Main is 'lib/*', with semicolon added if Windows jdk + + // PR #10761: verify that [dist/bin/scala] -classpath processing adds $psep to wildcard if Windows + val classpathValue = consumeNext + assert( !isWin || classpathValue.contains(psep) ) + + // expecting -script next + assert(consumeNext.replaceAll("'","") == "-script") + + // PR #10761: verify that Windows jdk did not expand single wildcard classpath to multiple file paths + if javaCommandLine.last != relpath then + printf("last: %s\nrelp: %s\n",javaCommandLine.last,relpath) + assert(javaCommandLine.last == relpath,s"unexpected args passed to scripting.Main") + + +lazy val cwd = Paths.get(".").toAbsolutePath + +import scala.jdk.CollectionConverters._ +lazy val env:Map[String,String] = System.getenv.asScala.toMap + +def exec(cmd: String *): Seq[String] = Process(cmd).lazyLines_!.toList + +def which(str:String) = + var out = "" + path.find { entry => + val it = Paths.get(entry).toAbsolutePath + it.toFile.isDirectory && { + var testpath = s"$it/$str".norm + val test = Paths.get(testpath) + if test.toFile.exists then + out = testpath + true + else + val test = Paths.get(s"$it/$str.exe".norm) + if test.toFile.exists then + out = testpath + true + else + false + } + } + + out + +def bashExe = which("bash") +def path = envOrElse("PATH","").split(psep).toList +def psep = sys.props("path.separator") + +extension(p:Path) + def relpath: Path = cwd.relativize(p) + def norm: String = p.toString.replace('\\','/') + +extension(path: String) + // Normalize path separator, convert relative path to absolute + def norm: String = + path.replace('\\', '/') match { + case s if s.secondChar == ":" => s // .drop(2) // path without drive letter + case s if s.startsWith("./") => s.drop(2) + case s => s + } + + def parent: String = norm.replaceAll("/[^/]*$","") + + // convert to absolute path relative to cwd. + def absPath: String = norm match + case str if str.isAbsolute => norm + case _ => Paths.get(userDir,norm).toString.norm + + def isDir: Boolean = Files.isDirectory(Paths.get(path)) + + def toUrl: String = Paths.get(absPath).toUri.toURL.toString + + // Treat norm paths with a leading '/' as absolute. + // Windows java.io.File#isAbsolute treats them as relative. + def isAbsolute = path.norm.startsWith("/") || (isWin && path.secondChar == ":") + def secondChar: String = path.take(2).drop(1).mkString("") + diff --git a/compiler/test/dotty/tools/scripting/ScriptingTests.scala b/compiler/test/dotty/tools/scripting/ScriptingTests.scala index c0198507ed4d..b043b6c4a7bd 100644 --- a/compiler/test/dotty/tools/scripting/ScriptingTests.scala +++ b/compiler/test/dotty/tools/scripting/ScriptingTests.scala @@ -18,7 +18,8 @@ class ScriptingTests: extension(f: File) def absPath = f.getAbsolutePath.replace('\\','/') - def testFiles = scripts("/scripting") + // classpath tests managed by scripting.ClasspathTests.scala + def testFiles = scripts("/scripting").filter { ! _.getName.startsWith("classpath") } def script2jar(scriptFile: File) = val jarName = s"${scriptFile.getName.dropExtension}.jar" diff --git a/dist/bin/scalac b/dist/bin/scalac index 74bfc4a603fe..692fd9d2fbfa 100755 --- a/dist/bin/scalac +++ b/dist/bin/scalac @@ -119,8 +119,12 @@ classpathArgs if [ "$PROG_NAME" == "$ScriptingMain" ]; then scripting_string="-script $target_script ${scripting_args[@]}" fi - -eval "\"$JAVACMD\"" \ +if [ -n "$SCALAC_ECHO_TEST" ]; then + EVAL=echo # to test with EVAL=echo +else + EVAL=eval +fi +$EVAL "\"$JAVACMD\"" \ ${JAVA_OPTS:-$default_java_opts} \ "${DEBUG-}" \ "${java_args[@]}" \ From 8efca1d54aad13a0e5bef272daff04b982021ef3 Mon Sep 17 00:00:00 2001 From: Phil Walker Date: Wed, 17 Mar 2021 22:08:50 -0600 Subject: [PATCH 10/15] updated scripting/ClasspathTests.scala for sbt.bat environment --- .../scripting/classpathReport.sc | 3 ++ .../tools/scripting/ClasspathTests.scala | 39 +++++++++++++------ 2 files changed, 31 insertions(+), 11 deletions(-) create mode 100755 compiler/test-resources/scripting/classpathReport.sc diff --git a/compiler/test-resources/scripting/classpathReport.sc b/compiler/test-resources/scripting/classpathReport.sc new file mode 100755 index 000000000000..de61df4c911d --- /dev/null +++ b/compiler/test-resources/scripting/classpathReport.sc @@ -0,0 +1,3 @@ +#!dist/target/pack/bin/scala -classpath 'dist/target/pack/lib/*' +def main(args: Array[String]): Unit = + println(sys.props("java.class.path")) diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index 9244f1f2897a..d8e174c7d9c0 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -26,29 +26,35 @@ class ClasspathTests: /* * Call test scripts */ - @Test def scriptingJarTest = + @Test def scalacEchoTest = for scriptFile <- testFiles do val relpath = scriptFile.toPath.relpath.norm printf("===> test script name [%s]\n",relpath) printf("%s\n",relpath) - val bashExe = which("bash") printf("bash is [%s]\n",bashExe) + val echoTest = "SCALAC_ECHO_TEST=1" + val bashCmdline = s"SCALA_OPTS= $echoTest dist/target/pack/bin/scala -classpath 'lib/*' $relpath" + // ask [dist/bin/scalac] to echo generated command line so we can verify some things - val cmd = Array(bashExe,"-c",s"SCALAC_ECHO_TEST=1 dist/target/pack/bin/scala -classpath '*/lib' $relpath") - cmd.foreach { printf("cmd[%s]\n",_) } + val cmd = Array(bashExe,"-c",bashCmdline) + + cmd.foreach { printf("[%s]\n",_) } + val javaCommandLine = exec(cmd:_*).mkString(" ").split(" ").filter { _.trim.nonEmpty } - val args = scala.collection.mutable.Queue(javaCommandLine:_*) - args.dequeueWhile( _ != "dotty.tools.scripting.Main") + javaCommandLine.foreach { printf("scalac-java-command[%s]\n",_) } + + val output = scala.collection.mutable.Queue(javaCommandLine:_*) + output.dequeueWhile( _ != "dotty.tools.scripting.Main") - def consumeNext = args.dequeue() + def consumeNext = if output.isEmpty then "" else output.dequeue() // assert that we found "dotty.tools.scripting.Main" assert(consumeNext == "dotty.tools.scripting.Main") - val mainArgs = args.copyToArray(Array.ofDim[String](args.length)) + val mainArgs = output.copyToArray(Array.ofDim[String](output.length)) // display command line starting with "dotty.tools.scripting.Main" - args.foreach { line => + output.foreach { line => printf("%s\n",line) } @@ -59,7 +65,9 @@ class ClasspathTests: // PR #10761: verify that [dist/bin/scala] -classpath processing adds $psep to wildcard if Windows val classpathValue = consumeNext - assert( !isWin || classpathValue.contains(psep) ) + printf("classpath value [%s]\n",classpathValue) + if isWin then printf("cygwin[%s], mingw[%s], msys[%s]\n",cygwin,mingw,msys) + assert( !winshell || classpathValue.contains(psep) ) // expecting -script next assert(consumeNext.replaceAll("'","") == "-script") @@ -67,7 +75,7 @@ class ClasspathTests: // PR #10761: verify that Windows jdk did not expand single wildcard classpath to multiple file paths if javaCommandLine.last != relpath then printf("last: %s\nrelp: %s\n",javaCommandLine.last,relpath) - assert(javaCommandLine.last == relpath,s"unexpected args passed to scripting.Main") + assert(javaCommandLine.last == relpath,s"unexpected output passed to scripting.Main") lazy val cwd = Paths.get(".").toAbsolutePath @@ -100,9 +108,18 @@ def which(str:String) = out def bashExe = which("bash") +def unameExe = which("uname") def path = envOrElse("PATH","").split(psep).toList def psep = sys.props("path.separator") +def cygwin = ostype == "cygwin" +def mingw = ostype == "mingw" +def msys = ostype == "msys" +def winshell = cygwin || mingw || msys + +def ostypeFull = if unameExe.nonEmpty then exec(unameExe).mkString else "" +def ostype = ostypeFull.toLowerCase.takeWhile{ cc => cc >= 'a' && cc <='z' || cc >= 'A' && cc <= 'Z' } + extension(p:Path) def relpath: Path = cwd.relativize(p) def norm: String = p.toString.replace('\\','/') From bd06fad459d55b84350f0c8a01760bab31f72b1f Mon Sep 17 00:00:00 2001 From: Phil Date: Wed, 17 Mar 2021 23:08:59 -0600 Subject: [PATCH 11/15] update java.class.path with wildcard-expanded classpath, for called script --- compiler/src/dotty/tools/scripting/Main.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/src/dotty/tools/scripting/Main.scala b/compiler/src/dotty/tools/scripting/Main.scala index 19c44c7ca083..6ffcc908cd3a 100755 --- a/compiler/src/dotty/tools/scripting/Main.scala +++ b/compiler/src/dotty/tools/scripting/Main.scala @@ -33,6 +33,8 @@ object Main: val (compilerArgs, scriptFile, scriptArgs, saveJar, invokeFlag) = distinguishArgs(args) val driver = ScriptingDriver(compilerArgs, scriptFile, scriptArgs) try driver.compileAndRun { (outDir:Path, classpathEntries:Seq[Path], mainClass: String) => + // write expanded classpath to java.class.path property, so called script can see it + sys.props("java.class.path") = classpathEntries.map(_.toString).mkString(pathsep) if saveJar then // write a standalone jar to the script parent directory writeJarfile(outDir, scriptFile, scriptArgs, classpathEntries, mainClass) From 721390d6a3662858b1a6ee9a057f1140f0f3a77a Mon Sep 17 00:00:00 2001 From: Phil Date: Thu, 18 Mar 2021 01:43:07 -0600 Subject: [PATCH 12/15] tests for script internal and external wildcard classpath use cases --- .../scripting/classpathReport.sc | 8 +- .../tools/scripting/ClasspathTests.scala | 76 ++++++++++++++++--- dist/bin/scala | 19 +++-- dist/bin/scalac | 7 +- 4 files changed, 83 insertions(+), 27 deletions(-) diff --git a/compiler/test-resources/scripting/classpathReport.sc b/compiler/test-resources/scripting/classpathReport.sc index de61df4c911d..e61e334c79a1 100755 --- a/compiler/test-resources/scripting/classpathReport.sc +++ b/compiler/test-resources/scripting/classpathReport.sc @@ -1,3 +1,9 @@ #!dist/target/pack/bin/scala -classpath 'dist/target/pack/lib/*' + +import java.nio.file.Paths + def main(args: Array[String]): Unit = - println(sys.props("java.class.path")) + val cwd = Paths.get(".").toAbsolutePath.toString.replace('\\','/').replaceAll("/$","") + printf("cwd: %s\n",cwd) + printf("classpath: %s\n",sys.props("java.class.path")) + diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index d8e174c7d9c0..c4dafcf64a2c 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -22,27 +22,35 @@ class ClasspathTests: // only interested in classpath test scripts def testFiles = scripts("/scripting").filter { _.getName.matches("classpath.*[.]sc") } + val testScriptName = "classpathReport.sc" + def testScript = testFiles.find { _.getName == testScriptName } match + case None => sys.error(s"test script not found: ${testScriptName}") + case Some(file) => file + + def packBinScalaExists:Boolean = + val scalaScriptPath = Paths.get("dist/target/pack/bin/scala") + scalaScriptPath.toFile.exists /* - * Call test scripts + * verify java command line generated by scalac. */ @Test def scalacEchoTest = - for scriptFile <- testFiles do - val relpath = scriptFile.toPath.relpath.norm - printf("===> test script name [%s]\n",relpath) - printf("%s\n",relpath) - printf("bash is [%s]\n",bashExe) - + val relpath = testScript.toPath.relpath.norm + printf("===> scalacEchoTest for script [%s]\n",relpath) + printf("bash is [%s]\n",bashExe) + + if packBinScalaExists then val echoTest = "SCALAC_ECHO_TEST=1" - val bashCmdline = s"SCALA_OPTS= $echoTest dist/target/pack/bin/scala -classpath 'lib/*' $relpath" + val bashCmdline = s"SCALA_OPTS= $echoTest dist/target/pack/bin/scala -classpath '$wildcardEntry' $relpath" // ask [dist/bin/scalac] to echo generated command line so we can verify some things val cmd = Array(bashExe,"-c",bashCmdline) - cmd.foreach { printf("[%s]\n",_) } + //cmd.foreach { printf("[%s]\n",_) } val javaCommandLine = exec(cmd:_*).mkString(" ").split(" ").filter { _.trim.nonEmpty } - javaCommandLine.foreach { printf("scalac-java-command[%s]\n",_) } + printf("\n==================== isWin[%s], cygwin[%s], mingw[%s], msys[%s]\n",isWin,cygwin,mingw,msys) + javaCommandLine.foreach { printf("java-command[%s]\n",_) } val output = scala.collection.mutable.Queue(javaCommandLine:_*) output.dequeueWhile( _ != "dotty.tools.scripting.Main") @@ -66,7 +74,6 @@ class ClasspathTests: // PR #10761: verify that [dist/bin/scala] -classpath processing adds $psep to wildcard if Windows val classpathValue = consumeNext printf("classpath value [%s]\n",classpathValue) - if isWin then printf("cygwin[%s], mingw[%s], msys[%s]\n",cygwin,mingw,msys) assert( !winshell || classpathValue.contains(psep) ) // expecting -script next @@ -77,12 +84,57 @@ class ClasspathTests: printf("last: %s\nrelp: %s\n",javaCommandLine.last,relpath) assert(javaCommandLine.last == relpath,s"unexpected output passed to scripting.Main") + /* + * verify classpath reported by called script. + */ + @Test def hashbangClasspathVerifyTest = + val relpath = testScript.toPath.relpath.norm + printf("===> hashbangClasspathVerifyTest for script [%s]\n",relpath) + printf("bash is [%s]\n",bashExe) + + if false && packBinScalaExists then + val bashCmdline = s"SCALA_OPTS= $relpath" + val cmd = Array(bashExe,"-c",bashCmdline) + + cmd.foreach { printf("[%s]\n",_) } + + // test script reports the classpath it sees + val scriptOutput = exec(cmd:_*) + val scriptCwd = findTaggedLine("cwd",scriptOutput) + printf("script ran in directory [%s]\n",scriptCwd) + val scriptCp = findTaggedLine("classpath",scriptOutput) + + val hashbangClasspathJars = scriptCp.split(psep).map { _.getName }.sorted.distinct + val packlibJars = listJars(s"$scriptCwd/dist/target/pack/lib").sorted.distinct + + // verify that the classpath set in the hashbang line is effective + if hashbangClasspathJars.size != packlibJars.size then + printf("%d test script jars in classpath\n",hashbangClasspathJars.size) + printf("%d jar files in dist/target/pack/lib\n",packlibJars.size) + + assert(hashbangClasspathJars.size == packlibJars.size) + +//////////////// end of tests //////////////// lazy val cwd = Paths.get(".").toAbsolutePath +lazy val wildcardEntry = "dist/target/pack/lib/*" + +def listJars(dir: String) = + val packlibDir = Paths.get(dir).toFile + if packlibDir.isDirectory then + packlibDir.listFiles.toList.map { _.getName }.filter { _.endsWith(".jar") } + else + Nil import scala.jdk.CollectionConverters._ lazy val env:Map[String,String] = System.getenv.asScala.toMap +// script output expected as ": " +def findTaggedLine(tag: String, lines: Seq[String]): String = + lines.find { _.startsWith(tag) } match + case None => sys.error(s"no $tag: found in script output") + case Some(cwd) => cwd.dropWhile( _ != ' ').trim // discard tag + def exec(cmd: String *): Seq[String] = Process(cmd).lazyLines_!.toList def which(str:String) = @@ -125,6 +177,8 @@ extension(p:Path) def norm: String = p.toString.replace('\\','/') extension(path: String) + def getName: String = norm.replaceAll(".*/","") + // Normalize path separator, convert relative path to absolute def norm: String = path.replace('\\', '/') match { diff --git a/dist/bin/scala b/dist/bin/scala index fbca003d1613..f6c177036059 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -53,7 +53,7 @@ truncated_params="${*#-}" # options_indicator != 0 if at least one parameter is not an option options_indicator=$(( ${#all_params} - ${#truncated_params} - $# )) -[ -n "$SCALA_OPTS" ] && set -- $SCALA_OPTS $@ +[ -n "$SCALA_OPTS" ] && set -- $SCALA_OPTS "$@" while [[ $# -gt 0 ]]; do case "$1" in @@ -65,19 +65,18 @@ while [[ $# -gt 0 ]]; do execute_run=true shift ;; - -cp | -classpath) - CLASS_PATH="$2" - if [[ $cygwin || $mingw || $msys ]]; then - # in Windows, need to protect a single wildcard entry from jdk globbing. - # (jdk globbing replaces a single wildcard arg to multiple jar file args!) - if [[ $CLASS_PATH =~ \* && ! $CLASS_PATH =~ \; ]]; then - CLASS_PATH=";$CLASS_PATH" - fi - fi + -cp | -classpath ) + CLASS_PATH="$2${PSEP}" class_path_count+=1 shift shift ;; + -cp*|-classpath*) + # hashbang can combine args, e.g. "-classpath 'lib/*'" + CLASS_PATH="${1#* *}${PSEP}" + echo "CLASS_PATH[$CLASS_PATH]" 1>&2 + class_path_count+=1 + ;; -with-compiler) with_compiler=true shift diff --git a/dist/bin/scalac b/dist/bin/scalac index 692fd9d2fbfa..b2ade34d42c8 100755 --- a/dist/bin/scalac +++ b/dist/bin/scalac @@ -119,11 +119,8 @@ classpathArgs if [ "$PROG_NAME" == "$ScriptingMain" ]; then scripting_string="-script $target_script ${scripting_args[@]}" fi -if [ -n "$SCALAC_ECHO_TEST" ]; then - EVAL=echo # to test with EVAL=echo -else - EVAL=eval -fi + +EVAL=eval ; [ -n "$SCALAC_ECHO_TEST" ] && EVAL=echo $EVAL "\"$JAVACMD\"" \ ${JAVA_OPTS:-$default_java_opts} \ "${DEBUG-}" \ From d02fc0513ea00ef6c418c1afd50031d400c7957c Mon Sep 17 00:00:00 2001 From: Phil Date: Thu, 18 Mar 2021 02:09:02 -0600 Subject: [PATCH 13/15] added missing shift in new -classpath* case in dist/bin/scala; fixed various style errors --- compiler/src/dotty/tools/scripting/Main.scala | 4 +- .../scripting/classpathReport.sc | 6 +- .../tools/scripting/ClasspathTests.scala | 56 +++++++++---------- .../tools/scripting/ScriptingTests.scala | 46 +++++++-------- dist/bin/scala | 2 +- 5 files changed, 57 insertions(+), 57 deletions(-) diff --git a/compiler/src/dotty/tools/scripting/Main.scala b/compiler/src/dotty/tools/scripting/Main.scala index 6ffcc908cd3a..97d0b91d4440 100755 --- a/compiler/src/dotty/tools/scripting/Main.scala +++ b/compiler/src/dotty/tools/scripting/Main.scala @@ -10,7 +10,7 @@ object Main: All arguments afterwards are script arguments.*/ private def distinguishArgs(args: Array[String]): (Array[String], File, Array[String], Boolean, Boolean) = val (leftArgs, rest) = args.splitAt(args.indexOf("-script")) - assert(rest.size >= 2,s"internal error: rest == Array(${rest.mkString(",")})") + assert(rest.size >= 2, s"internal error: rest == Array(${rest.mkString(",")})") val file = File(rest(1)) val scriptArgs = rest.drop(2) @@ -91,7 +91,7 @@ object Main: // convert to absolute path relative to cwd. def absPath: String = norm match case str if str.isAbsolute => norm - case _ => Paths.get(userDir,norm).toString.norm + case _ => Paths.get(userDir, norm).toString.norm def toUrl: String = Paths.get(absPath).toUri.toURL.toString diff --git a/compiler/test-resources/scripting/classpathReport.sc b/compiler/test-resources/scripting/classpathReport.sc index e61e334c79a1..5ccfd6faca76 100755 --- a/compiler/test-resources/scripting/classpathReport.sc +++ b/compiler/test-resources/scripting/classpathReport.sc @@ -3,7 +3,7 @@ import java.nio.file.Paths def main(args: Array[String]): Unit = - val cwd = Paths.get(".").toAbsolutePath.toString.replace('\\','/').replaceAll("/$","") - printf("cwd: %s\n",cwd) - printf("classpath: %s\n",sys.props("java.class.path")) + val cwd = Paths.get(".").toAbsolutePath.toString.replace('\\', '/').replaceAll("/$", "") + printf("cwd: %s\n", cwd) + printf("classpath: %s\n", sys.props("java.class.path")) diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index c4dafcf64a2c..5f25c95ef479 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -18,7 +18,7 @@ class ClasspathTests: str.reverse.dropWhile(_ != '.').drop(1).reverse extension(f: File) def absPath = - f.getAbsolutePath.replace('\\','/') + f.getAbsolutePath.replace('\\', '/') // only interested in classpath test scripts def testFiles = scripts("/scripting").filter { _.getName.matches("classpath.*[.]sc") } @@ -36,21 +36,21 @@ class ClasspathTests: */ @Test def scalacEchoTest = val relpath = testScript.toPath.relpath.norm - printf("===> scalacEchoTest for script [%s]\n",relpath) - printf("bash is [%s]\n",bashExe) + printf("===> scalacEchoTest for script [%s]\n", relpath) + printf("bash is [%s]\n", bashExe) if packBinScalaExists then val echoTest = "SCALAC_ECHO_TEST=1" val bashCmdline = s"SCALA_OPTS= $echoTest dist/target/pack/bin/scala -classpath '$wildcardEntry' $relpath" // ask [dist/bin/scalac] to echo generated command line so we can verify some things - val cmd = Array(bashExe,"-c",bashCmdline) + val cmd = Array(bashExe, "-c", bashCmdline) - //cmd.foreach { printf("[%s]\n",_) } + //cmd.foreach { printf("[%s]\n", _) } val javaCommandLine = exec(cmd:_*).mkString(" ").split(" ").filter { _.trim.nonEmpty } - printf("\n==================== isWin[%s], cygwin[%s], mingw[%s], msys[%s]\n",isWin,cygwin,mingw,msys) - javaCommandLine.foreach { printf("java-command[%s]\n",_) } + printf("\n==================== isWin[%s], cygwin[%s], mingw[%s], msys[%s]\n", isWin, cygwin, mingw, msys) + javaCommandLine.foreach { printf("java-command[%s]\n", _) } val output = scala.collection.mutable.Queue(javaCommandLine:_*) output.dequeueWhile( _ != "dotty.tools.scripting.Main") @@ -63,54 +63,54 @@ class ClasspathTests: // display command line starting with "dotty.tools.scripting.Main" output.foreach { line => - printf("%s\n",line) + printf("%s\n", line) } // expecting -classpath next - assert(consumeNext.replaceAll("'","") == "-classpath") + assert(consumeNext.replaceAll("'", "") == "-classpath") // 2nd arg to scripting.Main is 'lib/*', with semicolon added if Windows jdk // PR #10761: verify that [dist/bin/scala] -classpath processing adds $psep to wildcard if Windows val classpathValue = consumeNext - printf("classpath value [%s]\n",classpathValue) + printf("classpath value [%s]\n", classpathValue) assert( !winshell || classpathValue.contains(psep) ) // expecting -script next - assert(consumeNext.replaceAll("'","") == "-script") + assert(consumeNext.replaceAll("'", "") == "-script") // PR #10761: verify that Windows jdk did not expand single wildcard classpath to multiple file paths if javaCommandLine.last != relpath then - printf("last: %s\nrelp: %s\n",javaCommandLine.last,relpath) - assert(javaCommandLine.last == relpath,s"unexpected output passed to scripting.Main") + printf("last: %s\nrelp: %s\n", javaCommandLine.last, relpath) + assert(javaCommandLine.last == relpath, s"unexpected output passed to scripting.Main") /* * verify classpath reported by called script. */ @Test def hashbangClasspathVerifyTest = val relpath = testScript.toPath.relpath.norm - printf("===> hashbangClasspathVerifyTest for script [%s]\n",relpath) - printf("bash is [%s]\n",bashExe) + printf("===> hashbangClasspathVerifyTest for script [%s]\n", relpath) + printf("bash is [%s]\n", bashExe) if false && packBinScalaExists then val bashCmdline = s"SCALA_OPTS= $relpath" - val cmd = Array(bashExe,"-c",bashCmdline) + val cmd = Array(bashExe, "-c", bashCmdline) - cmd.foreach { printf("[%s]\n",_) } + cmd.foreach { printf("[%s]\n", _) } // test script reports the classpath it sees val scriptOutput = exec(cmd:_*) - val scriptCwd = findTaggedLine("cwd",scriptOutput) - printf("script ran in directory [%s]\n",scriptCwd) - val scriptCp = findTaggedLine("classpath",scriptOutput) + val scriptCwd = findTaggedLine("cwd", scriptOutput) + printf("script ran in directory [%s]\n", scriptCwd) + val scriptCp = findTaggedLine("classpath", scriptOutput) val hashbangClasspathJars = scriptCp.split(psep).map { _.getName }.sorted.distinct val packlibJars = listJars(s"$scriptCwd/dist/target/pack/lib").sorted.distinct // verify that the classpath set in the hashbang line is effective if hashbangClasspathJars.size != packlibJars.size then - printf("%d test script jars in classpath\n",hashbangClasspathJars.size) - printf("%d jar files in dist/target/pack/lib\n",packlibJars.size) + printf("%d test script jars in classpath\n", hashbangClasspathJars.size) + printf("%d jar files in dist/target/pack/lib\n", packlibJars.size) assert(hashbangClasspathJars.size == packlibJars.size) @@ -127,7 +127,7 @@ def listJars(dir: String) = Nil import scala.jdk.CollectionConverters._ -lazy val env:Map[String,String] = System.getenv.asScala.toMap +lazy val env:Map[String, String] = System.getenv.asScala.toMap // script output expected as ": " def findTaggedLine(tag: String, lines: Seq[String]): String = @@ -161,7 +161,7 @@ def which(str:String) = def bashExe = which("bash") def unameExe = which("uname") -def path = envOrElse("PATH","").split(psep).toList +def path = envOrElse("PATH", "").split(psep).toList def psep = sys.props("path.separator") def cygwin = ostype == "cygwin" @@ -174,10 +174,10 @@ def ostype = ostypeFull.toLowerCase.takeWhile{ cc => cc >= 'a' && cc <='z' || cc extension(p:Path) def relpath: Path = cwd.relativize(p) - def norm: String = p.toString.replace('\\','/') + def norm: String = p.toString.replace('\\', '/') extension(path: String) - def getName: String = norm.replaceAll(".*/","") + def getName: String = norm.replaceAll(".*/", "") // Normalize path separator, convert relative path to absolute def norm: String = @@ -187,12 +187,12 @@ extension(path: String) case s => s } - def parent: String = norm.replaceAll("/[^/]*$","") + def parent: String = norm.replaceAll("/[^/]*$", "") // convert to absolute path relative to cwd. def absPath: String = norm match case str if str.isAbsolute => norm - case _ => Paths.get(userDir,norm).toString.norm + case _ => Paths.get(userDir, norm).toString.norm def isDir: Boolean = Files.isDirectory(Paths.get(path)) diff --git a/compiler/test/dotty/tools/scripting/ScriptingTests.scala b/compiler/test/dotty/tools/scripting/ScriptingTests.scala index b043b6c4a7bd..25fdfe14f5c7 100644 --- a/compiler/test/dotty/tools/scripting/ScriptingTests.scala +++ b/compiler/test/dotty/tools/scripting/ScriptingTests.scala @@ -16,17 +16,17 @@ class ScriptingTests: str.reverse.dropWhile(_ != '.').drop(1).reverse extension(f: File) def absPath = - f.getAbsolutePath.replace('\\','/') + f.getAbsolutePath.replace('\\', '/') // classpath tests managed by scripting.ClasspathTests.scala def testFiles = scripts("/scripting").filter { ! _.getName.startsWith("classpath") } def script2jar(scriptFile: File) = val jarName = s"${scriptFile.getName.dropExtension}.jar" - File(scriptFile.getParent,jarName) + File(scriptFile.getParent, jarName) def showScriptUnderTest(scriptFile: File): Unit = - printf("===> test script name [%s]\n",scriptFile.getName) + printf("===> test script name [%s]\n", scriptFile.getName) val argss: Map[String, Array[String]] = ( for @@ -42,11 +42,11 @@ class ScriptingTests: if scriptFile.getName.endsWith(extension) name = scriptFile.getName.dropExtension scriptArgs = argss.getOrElse(name, Array.empty[String]) - yield scriptFile -> scriptArgs).toList.sortBy { (file,args) => file.getName } + yield scriptFile -> scriptArgs).toList.sortBy { (file, args) => file.getName } - def callExecutableJar(script: File,jar: File, scriptArgs: Array[String] = Array.empty[String]) = { + def callExecutableJar(script: File, jar: File, scriptArgs: Array[String] = Array.empty[String]) = { import scala.sys.process._ - val cmd = Array("java",s"-Dscript.path=${script.getName}","-jar",jar.absPath) + val cmd = Array("java", s"-Dscript.path=${script.getName}", "-jar", jar.absPath) ++ scriptArgs Process(cmd).lazyLines_!.foreach { println } } @@ -55,7 +55,7 @@ class ScriptingTests: * Call .scala scripts without -save option, verify no jar created */ @Test def scriptingDriverTests = - for (scriptFile,scriptArgs) <- scalaFilesWithArgs(".scala") do + for (scriptFile, scriptArgs) <- scalaFilesWithArgs(".scala") do showScriptUnderTest(scriptFile) val unexpectedJar = script2jar(scriptFile) unexpectedJar.delete @@ -67,8 +67,8 @@ class ScriptingTests: ), scriptFile = scriptFile, scriptArgs = scriptArgs - ).compileAndRun { (path:java.nio.file.Path,classpathEntries:Seq[Path], mainClass:String) => - printf("mainClass from ScriptingDriver: %s\n",mainClass) + ).compileAndRun { (path:java.nio.file.Path, classpathEntries:Seq[Path], mainClass:String) => + printf("mainClass from ScriptingDriver: %s\n", mainClass) true // call compiled script main method } assert(! unexpectedJar.exists, s"not expecting jar file: ${unexpectedJar.absPath}") @@ -77,7 +77,7 @@ class ScriptingTests: * Call .sc scripts without -save option, verify no jar created */ @Test def scriptingMainTests = - for (scriptFile,scriptArgs) <- scalaFilesWithArgs(".sc") do + for (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") do showScriptUnderTest(scriptFile) val unexpectedJar = script2jar(scriptFile) unexpectedJar.delete @@ -95,7 +95,7 @@ class ScriptingTests: * Call .sc scripts with -save option, verify jar is created. */ @Test def scriptingJarTest = - for (scriptFile,scriptArgs) <- scalaFilesWithArgs(".sc") do + for (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") do showScriptUnderTest(scriptFile) val expectedJar = script2jar(scriptFile) expectedJar.delete @@ -109,7 +109,7 @@ class ScriptingTests: Main.main(mainArgs) - printf("===> test script jar name [%s]\n",expectedJar.getName) + printf("===> test script jar name [%s]\n", expectedJar.getName) assert(expectedJar.exists) callExecutableJar(scriptFile, expectedJar, scriptArgs) @@ -130,8 +130,8 @@ class ScriptingTests: compilerArgs = Array("-classpath", TestConfiguration.basicClasspath), scriptFile = scriptFile, scriptArgs = Array.empty[String] - ).compileAndRun { (path:java.nio.file.Path,classpathEntries:Seq[Path], mainClass:String) => - printf("success: no call to main method in mainClass: %s\n",mainClass) + ).compileAndRun { (path:java.nio.file.Path, classpathEntries:Seq[Path], mainClass:String) => + printf("success: no call to main method in mainClass: %s\n", mainClass) false // no call to compiled script main method } touchedFile.delete @@ -143,14 +143,14 @@ class ScriptingTests: compilerArgs = Array("-classpath", TestConfiguration.basicClasspath), scriptFile = scriptFile, scriptArgs = Array.empty[String] - ).compileAndRun { (path:java.nio.file.Path,classpathEntries:Seq[Path], mainClass:String) => - printf("call main method in mainClass: %s\n",mainClass) + ).compileAndRun { (path:java.nio.file.Path, classpathEntries:Seq[Path], mainClass:String) => + printf("call main method in mainClass: %s\n", mainClass) true // call compiled script main method, create touchedFile } if touchedFile.exists then - printf("success: script created file %s\n",touchedFile) - if touchedFile.exists then printf("success: created file %s\n",touchedFile) + printf("success: script created file %s\n", touchedFile) + if touchedFile.exists then printf("success: created file %s\n", touchedFile) assert( touchedFile.exists, s"expected to find file ${touchedFile}" ) /* @@ -170,15 +170,15 @@ class ScriptingTests: expectedJar.delete Main.main(mainArgs) // create executable jar - printf("===> test script jar name [%s]\n",expectedJar.getName) - assert(expectedJar.exists,s"unable to create executable jar [$expectedJar]") + printf("===> test script jar name [%s]\n", expectedJar.getName) + assert(expectedJar.exists, s"unable to create executable jar [$expectedJar]") touchedFile.delete - assert(!touchedFile.exists,s"unable to delete ${touchedFile}") - printf("calling executable jar %s\n",expectedJar) + assert(!touchedFile.exists, s"unable to delete ${touchedFile}") + printf("calling executable jar %s\n", expectedJar) callExecutableJar(scriptFile, expectedJar) if touchedFile.exists then - printf("success: executable jar created file %s\n",touchedFile) + printf("success: executable jar created file %s\n", touchedFile) assert( touchedFile.exists, s"expected to find file ${touchedFile}" ) def touchFileScript = testFiles.find(_.getName == "touchFile.sc").get diff --git a/dist/bin/scala b/dist/bin/scala index f6c177036059..562e1c9be2e6 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -74,8 +74,8 @@ while [[ $# -gt 0 ]]; do -cp*|-classpath*) # hashbang can combine args, e.g. "-classpath 'lib/*'" CLASS_PATH="${1#* *}${PSEP}" - echo "CLASS_PATH[$CLASS_PATH]" 1>&2 class_path_count+=1 + shift ;; -with-compiler) with_compiler=true From 0dc35d84c0fcc398f8e20e19d1ed9b410ac9d344 Mon Sep 17 00:00:00 2001 From: Phil Date: Thu, 18 Mar 2021 02:48:55 -0600 Subject: [PATCH 14/15] code style fixes --- compiler/test/dotty/tools/io/ClasspathTest.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/test/dotty/tools/io/ClasspathTest.scala b/compiler/test/dotty/tools/io/ClasspathTest.scala index 7505352e5602..35437ef17390 100755 --- a/compiler/test/dotty/tools/io/ClasspathTest.scala +++ b/compiler/test/dotty/tools/io/ClasspathTest.scala @@ -28,9 +28,9 @@ class ClasspathTest { try for src <- libjarFiles do val dest = Paths.get(s"$outDir/${src.getName}") - printf("copy: %s\n",Files.copy(src.toPath,dest)) // ,REPLACE_EXISTING,COPY_ATTRIBUTES)) + printf("copy: %s\n", Files.copy(src.toPath, dest)) - val cp = Seq(s"$outDir/*","not-a-real-directory/*").mkString(pathsep).replace('\\','/') + val cp = Seq(s"$outDir/*", "not-a-real-directory/*").mkString(pathsep).replace('\\', '/') val libjars = libjarFiles.map { _.getName }.toSet @@ -40,14 +40,14 @@ class ClasspathTest { // require one-to-one matches assert(libjars == entries.toSet) - printf("%d entries\n",entries.size) - printf("%d libjars\n",libjars.size) + printf("%d entries\n", entries.size) + printf("%d libjars\n", libjars.size) for entry <- libjars do - printf("libdir[%s]\n",entry) + printf("libdir[%s]\n", entry) for entry <- entries do - printf("expand[%s]\n",entry) + printf("expand[%s]\n", entry) // verify that expanded classpath has expected jar names for jar <- libjars do From cb64423cde2999acd46302f0b199a8a2b12fdb21 Mon Sep 17 00:00:00 2001 From: Phil Date: Sun, 28 Mar 2021 11:24:38 -0600 Subject: [PATCH 15/15] replace SCALAC_ECHO_TEST scalac modification; test modified copy of scalac --- .../tools/scripting/ClasspathTests.scala | 45 ++++++++++++++----- dist/bin/scalac | 3 +- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index 5f25c95ef479..97db62d41240 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -10,15 +10,14 @@ import org.junit.Test import vulpix.TestConfiguration import scala.sys.process._ +import scala.jdk.CollectionConverters._ import dotty.tools.dotc.config.Properties._ /** Runs all tests contained in `compiler/test-resources/scripting/` */ class ClasspathTests: - extension (str: String) def dropExtension = - str.reverse.dropWhile(_ != '.').drop(1).reverse - - extension(f: File) def absPath = - f.getAbsolutePath.replace('\\', '/') + val packBinDir = "dist/target/pack/bin" + val scalaCopy = makeTestableScriptCopy("scala") + val scalacCopy = makeTestableScriptCopy("scalac") // only interested in classpath test scripts def testFiles = scripts("/scripting").filter { _.getName.matches("classpath.*[.]sc") } @@ -27,9 +26,24 @@ class ClasspathTests: case None => sys.error(s"test script not found: ${testScriptName}") case Some(file) => file - def packBinScalaExists:Boolean = - val scalaScriptPath = Paths.get("dist/target/pack/bin/scala") - scalaScriptPath.toFile.exists + def getScriptPath(scriptName: String): Path = Paths.get(s"$packBinDir/$scriptName") + + def exists(scriptPath: Path): Boolean = Files.exists(scriptPath) + def packBinScalaExists:Boolean = exists(Paths.get(s"$packBinDir/scala")) + + // create edited copy of [dist/bin/scala] and [dist/bin/scalac] for scalacEchoTest + def makeTestableScriptCopy(scriptName: String): Path = + val scriptPath: Path = getScriptPath(scriptName) + val scriptCopy: Path = getScriptPath(s"$scriptName-copy") + if Files.exists(scriptPath) then + val lines = Files.readAllLines(scriptPath).asScala.map { + _.replaceAll("/scalac", "/scalac-copy"). + replaceFirst("^eval(.*JAVACMD.*)", "echo $1") + } + val bytes = (lines.mkString("\n")+"\n").getBytes + Files.write(scriptCopy, bytes) + + scriptCopy /* * verify java command line generated by scalac. @@ -40,8 +54,8 @@ class ClasspathTests: printf("bash is [%s]\n", bashExe) if packBinScalaExists then - val echoTest = "SCALAC_ECHO_TEST=1" - val bashCmdline = s"SCALA_OPTS= $echoTest dist/target/pack/bin/scala -classpath '$wildcardEntry' $relpath" + val echoTest = "" // "SCALAC_ECHO_TEST=1" + val bashCmdline = s"SCALA_OPTS= $echoTest ${scalaCopy.norm} -classpath '$wildcardEntry' $relpath" // ask [dist/bin/scalac] to echo generated command line so we can verify some things val cmd = Array(bashExe, "-c", bashCmdline) @@ -58,7 +72,10 @@ class ClasspathTests: def consumeNext = if output.isEmpty then "" else output.dequeue() // assert that we found "dotty.tools.scripting.Main" - assert(consumeNext == "dotty.tools.scripting.Main") + val str = consumeNext + if str != "dotty.tools.scripting.Main" then + + assert(str == "dotty.tools.scripting.Main", s"found [$str]") val mainArgs = output.copyToArray(Array.ofDim[String](output.length)) // display command line starting with "dotty.tools.scripting.Main" @@ -203,3 +220,9 @@ extension(path: String) def isAbsolute = path.norm.startsWith("/") || (isWin && path.secondChar == ":") def secondChar: String = path.take(2).drop(1).mkString("") +extension (str: String) def dropExtension = + str.reverse.dropWhile(_ != '.').drop(1).reverse + +//extension(f: File) def absPath = +//f.getAbsolutePath.replace('\\', '/') + diff --git a/dist/bin/scalac b/dist/bin/scalac index b2ade34d42c8..74bfc4a603fe 100755 --- a/dist/bin/scalac +++ b/dist/bin/scalac @@ -120,8 +120,7 @@ if [ "$PROG_NAME" == "$ScriptingMain" ]; then scripting_string="-script $target_script ${scripting_args[@]}" fi -EVAL=eval ; [ -n "$SCALAC_ECHO_TEST" ] && EVAL=echo -$EVAL "\"$JAVACMD\"" \ +eval "\"$JAVACMD\"" \ ${JAVA_OPTS:-$default_java_opts} \ "${DEBUG-}" \ "${java_args[@]}" \