From 4426f99d3c1167b5883b1d0f964c13c4b0fe9c94 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Mon, 22 Apr 2024 10:23:54 +0200 Subject: [PATCH 1/9] Everything needed for the 3.5.0-RC1 release (except skipping windows CI) --- bin/scala | 2 +- bin/test/TestScripts.scala | 2 +- .../src/dotty/tools/MainGenericRunner.scala | 16 ++ .../tools/coursier/CoursierScalaTests.scala | 2 +- .../scripting/classpathReport.sc | 4 +- .../test-resources/scripting/envtestNu.sc | 2 + .../test-resources/scripting/scriptPathNu.sc | 13 ++ compiler/test-resources/scripting/showArgs.sc | 2 +- .../test-resources/scripting/showArgsNu.sc | 6 + .../test-resources/scripting/sqlDateError.sc | 2 +- .../scripting/sqlDateErrorNu.sc | 6 + .../scripting/unglobClasspath.sc | 8 +- .../tools/scripting/BashExitCodeTests.scala | 16 +- .../tools/scripting/BashScriptsTests.scala | 42 ++-- .../tools/scripting/ClasspathTests.scala | 16 +- .../tools/scripting/ExpressionTest.scala | 2 +- .../dotty/tools/scripting/ScriptTestEnv.scala | 15 +- .../tools/scripting/ScriptingTests.scala | 10 +- dist/bin/cli-common | 160 +++++++++++++++ dist/bin/scala | 63 +++--- dist/bin/scala.bat | 95 ++++----- dist/bin/scala_legacy | 72 +++++++ dist/bin/scalac.bat | 3 + dist/bin/scaladoc.bat | 4 + project/Build.scala | 32 ++- project/Modes.scala | 6 +- project/RepublishPlugin.scala | 193 ++++++++++++++++++ project/scripts/bootstrappedOnlyCmdTests | 26 ++- project/scripts/winCmdTests | 4 +- project/scripts/winCmdTests.bat | 4 +- tests/run-with-compiler/i14541.scala | 1 + 31 files changed, 675 insertions(+), 154 deletions(-) create mode 100755 compiler/test-resources/scripting/envtestNu.sc create mode 100755 compiler/test-resources/scripting/scriptPathNu.sc create mode 100755 compiler/test-resources/scripting/showArgsNu.sc create mode 100755 compiler/test-resources/scripting/sqlDateErrorNu.sc create mode 100644 dist/bin/cli-common create mode 100755 dist/bin/scala_legacy create mode 100644 project/RepublishPlugin.scala diff --git a/bin/scala b/bin/scala index 66ec9a5774c7..85c1ac91d08f 100755 --- a/bin/scala +++ b/bin/scala @@ -2,4 +2,4 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.." -"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scala" "$@" +"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scala" "--power" "$@" "--offline" "--server=false" diff --git a/bin/test/TestScripts.scala b/bin/test/TestScripts.scala index bada140580fc..4a2fd9a05c83 100644 --- a/bin/test/TestScripts.scala +++ b/bin/test/TestScripts.scala @@ -57,7 +57,7 @@ class TestScripts { s"bin/scalac script did not run properly. Output:$lineSep$dotcOutput" ) - val (retDotr, dotrOutput) = executeScript("./bin/scala HelloWorld") + val (retDotr, dotrOutput) = executeScript("./bin/scala -M HelloWorld") assert( retDotr == 0 && dotrOutput == "hello world\n", s"Running hello world exited with status: $retDotr and output: $dotrOutput" diff --git a/compiler/src/dotty/tools/MainGenericRunner.scala b/compiler/src/dotty/tools/MainGenericRunner.scala index 1540cc86d7a6..5b238693a135 100644 --- a/compiler/src/dotty/tools/MainGenericRunner.scala +++ b/compiler/src/dotty/tools/MainGenericRunner.scala @@ -266,6 +266,22 @@ object MainGenericRunner { run(settings.withExecuteMode(ExecuteMode.Run)) else run(settings.withExecuteMode(ExecuteMode.Repl)) + end run + + val ranByCoursierBootstrap = + sys.props.isDefinedAt("coursier.mainJar") + || sys.props.get("bootstrap.mainClass").filter(_ == "dotty.tools.MainGenericRunner").isDefined + + val silenced = sys.props.get("scala.use_legacy_launcher") == Some("true") + + if !silenced then + Console.err.println(s"[warning] MainGenericRunner class is deprecated since Scala 3.5.0, and Scala CLI features will not work.") + Console.err.println(s"[warning] Please be sure to update to the Scala CLI launcher to use the new features.") + if ranByCoursierBootstrap then + Console.err.println(s"[warning] It appears that your Coursier-based Scala installation is misconfigured.") + Console.err.println(s"[warning] To update to the new Scala CLI runner, please update (coursier, cs) commands first before re-installing scala.") + Console.err.println(s"[warning] Check the Scala 3.5.0 release notes to troubleshoot your installation.") + run(settings) match case Some(ex: (StringDriverException | ScriptingException)) => errorFn(ex.getMessage) diff --git a/compiler/test-coursier/dotty/tools/coursier/CoursierScalaTests.scala b/compiler/test-coursier/dotty/tools/coursier/CoursierScalaTests.scala index b8dfa833c437..115803d79dc1 100644 --- a/compiler/test-coursier/dotty/tools/coursier/CoursierScalaTests.scala +++ b/compiler/test-coursier/dotty/tools/coursier/CoursierScalaTests.scala @@ -166,7 +166,7 @@ object CoursierScalaTests: case Nil => args case _ => "--" +: args val newJOpts = jOpts.map(s => s"--java-opt ${s.stripPrefix("-J")}").mkString(" ") - execCmd("./cs", (s"""launch "org.scala-lang:scala3-compiler_3:${sys.env("DOTTY_BOOTSTRAPPED_VERSION")}" $newJOpts --main-class "$entry" --property "scala.usejavacp=true"""" +: newOptions)*)._2 + execCmd("./cs", (s"""launch "org.scala-lang:scala3-compiler_3:${sys.env("DOTTY_BOOTSTRAPPED_VERSION")}" $newJOpts --main-class "$entry" --property "scala.usejavacp=true" --property "scala.use_legacy_launcher=true"""" +: newOptions)*)._2 /** Get coursier script */ @BeforeClass def setup(): Unit = diff --git a/compiler/test-resources/scripting/classpathReport.sc b/compiler/test-resources/scripting/classpathReport.sc index a9eacbbba1f7..cc68c4b1d52e 100755 --- a/compiler/test-resources/scripting/classpathReport.sc +++ b/compiler/test-resources/scripting/classpathReport.sc @@ -1,8 +1,8 @@ -#!bin/scala -classpath 'dist/target/pack/lib/*' +#!/usr/bin/env bin/scala import java.nio.file.Paths -def main(args: Array[String]): Unit = +// def main(args: Array[String]): Unit = // MIGRATION: Scala CLI expects `*.sc` files to be straight-line code val cwd = Paths.get(".").toAbsolutePath.normalize.toString.norm printf("cwd: %s\n", cwd) printf("classpath: %s\n", sys.props("java.class.path").norm) diff --git a/compiler/test-resources/scripting/envtestNu.sc b/compiler/test-resources/scripting/envtestNu.sc new file mode 100755 index 000000000000..fe4cd7851b0a --- /dev/null +++ b/compiler/test-resources/scripting/envtestNu.sc @@ -0,0 +1,2 @@ +// MIGRATION: Scala CLI expects `*.sc` files to be straight-line code + println("Hello " + util.Properties.propOrNull("key")) diff --git a/compiler/test-resources/scripting/scriptPathNu.sc b/compiler/test-resources/scripting/scriptPathNu.sc new file mode 100755 index 000000000000..bb3e459654b9 --- /dev/null +++ b/compiler/test-resources/scripting/scriptPathNu.sc @@ -0,0 +1,13 @@ +#!/usr/bin/env bin/scala + +// THIS FILE IS RAN WITH SCALA CLI, which wraps scripts exposing scriptPath and args variables + +args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) } + +if !scriptPath.endsWith("scriptPathNu.sc") then + printf( s"incorrect script.path defined as [$scriptPath]") +else + printf("scriptPath: %s\n", scriptPath) // report the value + +extension(s: String) + def norm: String = s.replace('\\', '/') diff --git a/compiler/test-resources/scripting/showArgs.sc b/compiler/test-resources/scripting/showArgs.sc index 28f16a9022b3..8ef08f8962b0 100755 --- a/compiler/test-resources/scripting/showArgs.sc +++ b/compiler/test-resources/scripting/showArgs.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env scala +#!/usr/bin/env bin/scala // precise output format expected by BashScriptsTests.scala def main(args: Array[String]): Unit = diff --git a/compiler/test-resources/scripting/showArgsNu.sc b/compiler/test-resources/scripting/showArgsNu.sc new file mode 100755 index 000000000000..f4c1aa6af257 --- /dev/null +++ b/compiler/test-resources/scripting/showArgsNu.sc @@ -0,0 +1,6 @@ +#!/usr/bin/env bin/scala + +// precise output format expected by BashScriptsTests.scala +// MIGRATION: Scala CLI expects `*.sc` files to be straight-line code +for (a,i) <- args.zipWithIndex do + printf(s"arg %2d:[%s]\n",i,a) diff --git a/compiler/test-resources/scripting/sqlDateError.sc b/compiler/test-resources/scripting/sqlDateError.sc index ceff98f40cad..35160fd6fcd5 100755 --- a/compiler/test-resources/scripting/sqlDateError.sc +++ b/compiler/test-resources/scripting/sqlDateError.sc @@ -1,4 +1,4 @@ -#!bin/scala +#!/usr/bin/env bin/scala def main(args: Array[String]): Unit = { println(new java.sql.Date(100L)) diff --git a/compiler/test-resources/scripting/sqlDateErrorNu.sc b/compiler/test-resources/scripting/sqlDateErrorNu.sc new file mode 100755 index 000000000000..a6f1bd50297d --- /dev/null +++ b/compiler/test-resources/scripting/sqlDateErrorNu.sc @@ -0,0 +1,6 @@ +#!/usr/bin/env bin/scala + +// def main(args: Array[String]): Unit = { MIGRATION: Scala CLI expects `*.sc` files to be straight-line code + println(new java.sql.Date(100L)) + System.err.println("SCALA_OPTS="+Option(System.getenv("SCALA_OPTS")).getOrElse("")) +// } diff --git a/compiler/test-resources/scripting/unglobClasspath.sc b/compiler/test-resources/scripting/unglobClasspath.sc index 796697cdedf2..deab2b8982ac 100755 --- a/compiler/test-resources/scripting/unglobClasspath.sc +++ b/compiler/test-resources/scripting/unglobClasspath.sc @@ -1,8 +1,6 @@ -#!bin/scala -classpath 'dist/target/pack/lib/*' +// won't compile unless classpath is set correctly +import dotty.tools.tasty.TastyFormat -// won't compile unless the hashbang line sets classpath -import org.jline.terminal.Terminal - -def main(args: Array[String]) = +// def main(args: Array[String]) = // MIGRATION: Scala CLI expects `*.sc` files to be straight-line code val cp = sys.props("java.class.path") printf("unglobbed classpath: %s\n", cp) diff --git a/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala b/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala index 9b65522fc549..a06103137f62 100644 --- a/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala +++ b/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala @@ -29,7 +29,7 @@ class BashExitCodeTests: }, expectedExitCode, exitCode) // Helpers for running scala, scalac, and scalac without the output directory ("raw") - def scala(args: String*) = verifyExit(scalaPath, args*) + def scala(args: String*) = verifyExit(scalaPath, ("--power" +: "--offline" +: "--server=false" +: args)*) def scalacRaw(args: String*) = verifyExit(scalacPath, args*) def scalac(args: String*) = scalacRaw(("-d" +: tmpDir +: args)*) @@ -38,12 +38,16 @@ class BashExitCodeTests: Files.write(Files.createTempFile(tmpDir.toPath, getClass.getSimpleName, suffix), body.getBytes(UTF_8)).absPath @Test def neg = scalac(f("@main def Test = prin"))(1) - @Test def run = scalac(f("@main def Test = ???"))(0) & scala("-classpath", tmpDir, "Test")(1) - @Test def pos = scalac(f("@main def Test = ()"))(0) & scala("-classpath", tmpDir, "Test")(0) + @Test def run = scalac(f("@main def Test = ???"))(0) & scala("-classpath", tmpDir, "-M", "Test")(1) + @Test def pos = scalac(f("@main def Test = ()"))(0) & scala("-classpath", tmpDir, "-M", "Test")(0) - @Test def runNeg = scala(f("@main def Test = prin", ".sc"))(1) - @Test def runRun = scala(f("@main def Test = ???", ".sc"))(1) - @Test def runPos = scala(f("@main def Test = ()", ".sc"))(0) + @Test def runNeg_script = scala(f("prin", ".sc"))(1) + @Test def runRun_script = scala(f("???", ".sc"))(1) + @Test def runPos_script = scala(f("()", ".sc"))(0) + + @Test def runNeg = scala(f("@main def Test = prin", ".scala"))(1) + @Test def runRun = scala(f("@main def Test = ???", ".scala"))(1) + @Test def runPos = scala(f("@main def Test = ()", ".scala"))(0) @Test def scNeg = scalac("-script", f("@main def Test = prin", ".sc"))(1) @Test def scRun = scalac("-script", f("@main def Test = ???", ".sc"))(1) diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index f3f364754e20..25bc54e2dcbe 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -5,7 +5,7 @@ package scripting import scala.language.unsafeNulls import java.nio.file.Paths -import org.junit.{Test, AfterClass} +import org.junit.{Test, Ignore, AfterClass} import org.junit.Assert.assertEquals import org.junit.Assume.assumeFalse import org.junit.experimental.categories.Category @@ -50,7 +50,9 @@ object BashScriptsTests: val testScriptArgs = Seq( "a", "b", "c", "-repl", "-run", "-script", "-debug" ) - val showArgsScript = testFiles.find(_.getName == "showArgs.sc").get.absPath + val Seq(showArgsScript, showArgsScalaCli) = Seq("showArgs.sc", "showArgsNu.sc").map { name => + testFiles.find(_.getName == name).get.absPath + } def testFile(name: String): String = val file = testFiles.find(_.getName == name) match { @@ -64,13 +66,13 @@ object BashScriptsTests: } file - val Seq(envtestSc, envtestScala) = Seq("envtest.sc", "envtest.scala").map { testFile(_) } + val Seq(envtestNuSc, envtestScala) = Seq("envtestNu.sc", "envtest.scala").map { testFile(_) } // create command line with given options, execute specified script, return stdout def callScript(tag: String, script: String, keyPre: String): String = val keyArg = s"$keyPre=$tag" printf("pass tag [%s] via [%s] to script [%s]\n", tag, keyArg, script) - val cmd: String = Seq("SCALA_OPTS= ", scalaPath, keyArg, script).mkString(" ") + val cmd: String = Seq("SCALA_OPTS= ", scalaPath, "run", keyArg, "--power", "--offline", "--server=false", script).mkString(" ") printf("cmd: [%s]\n", cmd) val (validTest, exitCode, stdout, stderr) = bashCommand(cmd) stderr.filter { !_.contains("Inappropriate ioctl") }.foreach { System.err.printf("stderr [%s]\n", _) } @@ -84,13 +86,15 @@ class BashScriptsTests: ////////////////////////// begin tests ////////////////////// /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.sc */ + @Ignore // SCALA CLI does not support `-J` to pass java properties, only things like -Xmx5g @Test def verifyScJProperty = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) val tag = "World1" - val stdout = callScript(tag, envtestSc, s"-J-Dkey") + val stdout = callScript(tag, envtestNuSc, s"-J-Dkey") assertEquals( s"Hello $tag", stdout) /* verify that `dist/bin/scala` correctly passes args to the jvm via -J-D for script envtest.scala */ + @Ignore // SCALA CLI does not support `-J` to pass java properties, only things like -Xmx5g @Test def verifyScalaJProperty = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) val tag = "World2" @@ -101,7 +105,7 @@ class BashScriptsTests: @Test def verifyScDProperty = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) val tag = "World3" - val stdout = callScript(tag, envtestSc, s"-Dkey") + val stdout = callScript(tag, envtestNuSc, s"-Dkey") assertEquals(s"Hello $tag", stdout) /* verify that `dist/bin/scala` can set system properties via -D for envtest.scala */ @@ -114,7 +118,9 @@ class BashScriptsTests: /* verify that `dist/bin/scala` can set system properties via -D when executing compiled script via -jar envtest.jar */ @Test def saveAndRunWithDProperty = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - val commandline = Seq("SCALA_OPTS= ", scalaPath.relpath, "-save", envtestScala.relpath).mkString(" ") + val libOut = envtestScala.relpath.stripSuffix(".scala") + ".jar" + val commandline = Seq( + "SCALA_OPTS= ", scalaPath.relpath, "--power", "package", envtestScala.relpath, "-o", libOut, "--library", "--offline", "--server=false").mkString(" ") val (_, _, _, _) = bashCommand(commandline) // compile jar, discard output val testJar = testFile("envtest.jar") // jar is created by the previous bashCommand() if (testJar.isFile){ @@ -124,7 +130,8 @@ class BashScriptsTests: } val tag = "World5" - val commandline2 = Seq("SCALA_OPTS= ", scalaPath.relpath, s"-Dkey=$tag", testJar.relpath) + val commandline2 = Seq( + "SCALA_OPTS= ", scalaPath.relpath, "run", s"-Dkey=$tag", "-classpath", testJar.relpath, "--power", "--offline", "--server=false") printf("cmd[%s]\n", commandline2.mkString(" ")) val (validTest, exitCode, stdout, stderr) = bashCommand(commandline2.mkString(" ")) assertEquals(s"Hello $tag", stdout.mkString("/n")) @@ -148,7 +155,11 @@ class BashScriptsTests: /* verify `dist/bin/scala` non-interference with command line args following script name */ @Test def verifyScalaArgs = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - val commandline = (Seq("SCALA_OPTS= ", scalaPath, showArgsScript) ++ testScriptArgs).mkString(" ") + val commandline = ( + Seq("SCALA_OPTS= ", scalaPath, showArgsScalaCli) + ++ Seq("--power", "--offline", "--server=false") + ++ ("--" +: testScriptArgs) + ).mkString(" ") val (validTest, exitCode, stdout, stderr) = bashCommand(commandline) if verifyValid(validTest) then var fail = false @@ -162,13 +173,13 @@ class BashScriptsTests: assert(stdout == expectedOutput) /* - * verify that scriptPath.sc sees a valid script.path property, - * and that it's value is the path to "scriptPath.sc". + * verify that scriptPathNu.sc sees a valid script.path property, + * and that it's value is the path to "scriptPathNu.sc". */ @Category(Array(classOf[BootstrappedOnlyTests])) @Test def verifyScriptPathProperty = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - val scriptFile = testFiles.find(_.getName == "scriptPath.sc").get + val scriptFile = testFiles.find(_.getName == "scriptPathNu.sc").get val expected = s"${scriptFile.getName}" printf("===> verify valid system property script.path is reported by script [%s]\n", scriptFile.getName) printf("calling scriptFile: %s\n", scriptFile) @@ -177,8 +188,8 @@ class BashScriptsTests: stdout.foreach { printf("stdout: [%s]\n", _) } stderr.foreach { printf("stderr: [%s]\n", _) } val valid = stdout.exists { _.endsWith(expected) } - if valid then printf("# valid script.path reported by [%s]\n", scriptFile.getName) - assert(valid, s"script ${scriptFile.absPath} did not report valid script.path value") + if valid then printf("# valid scriptPath reported by [%s]\n", scriptFile.getName) + assert(valid, s"script ${scriptFile.absPath} did not report valid scriptPath value") /* * verify SCALA_OPTS can specify an @argsfile when launching a scala script in `dist/bin/scala`. @@ -208,7 +219,7 @@ class BashScriptsTests: */ @Test def sqlDateTest = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - val scriptBase = "sqlDateError" + val scriptBase = "sqlDateErrorNu" val scriptFile = testFiles.find(_.getName == s"$scriptBase.sc").get val testJar = testFile(s"$scriptBase.jar") // jar should not be created when scriptFile runs val tj = Paths.get(testJar).toFile @@ -236,7 +247,6 @@ class BashScriptsTests: printf("===> verify -e is properly handled by `dist/bin/scala`\n") val expected = "9" val expression = s"println(3*3)" - val cmd = s"bin/scala -e $expression" val (validTest, exitCode, stdout, stderr) = bashCommand(s"""bin/scala -e '$expression'""") val result = stdout.filter(_.nonEmpty).mkString("") printf("stdout: %s\n", result) diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index 4fd1211698f6..5107af5eee43 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -51,7 +51,7 @@ class ClasspathTests: // convert scriptCp to a list of files val hashbangJars: List[File] = scriptCp.split(psep).map { _.toFile }.toList val hashbangClasspathJars = hashbangJars.map { _.name }.sorted.distinct // get jar basenames, remove duplicates - val packlibDir = s"$scriptCwd/$packLibDir" // classpathReport.sc specifies a wildcard classpath in this directory + val packlibDir: String = ??? /* ??? was s"$scriptCwd/$packLibDir" */ // classpathReport.sc specifies a wildcard classpath in this directory val packlibJars: List[File] = listJars(packlibDir) // classpath entries expected to have been reported by the script printf("%d jar files in dist/target/pack/lib\n", packlibJars.size) @@ -84,11 +84,23 @@ class ClasspathTests: case Some(file) => file val relpath = testScript.toPath.relpath.norm + val scalaCommand = scalaPath.relpath.norm printf("===> unglobClasspathVerifyTest for script [%s]\n", relpath) printf("bash is [%s]\n", bashExe) if packBinScalaExists then - val bashCmdline = s"set +x ; SCALA_OPTS= $relpath" + val sv = packScalaVersion + val tastyDirGlob = s"$packMavenDir/org/scala-lang/tasty-core_3/$sv/*" + // ^^^^^^^^^^^^^ + // the classpath is a glob pattern that should be unglobbed by scala command, + // otherwise the script could not compile because it references a class + // from tasty-core + + val bashCmdline = Seq( + "set +x ;", + "SCALA_OPTS=", + scalaCommand, "run", "--classpath", s"'$tastyDirGlob'", "--power", "--offline", "--server=false", relpath + ).mkString(" ") val cmd = Array(bashExe, "-c", bashCmdline) cmd.foreach { printf("[%s]\n", _) } diff --git a/compiler/test/dotty/tools/scripting/ExpressionTest.scala b/compiler/test/dotty/tools/scripting/ExpressionTest.scala index 6b5248e67f08..02963f50ee52 100755 --- a/compiler/test/dotty/tools/scripting/ExpressionTest.scala +++ b/compiler/test/dotty/tools/scripting/ExpressionTest.scala @@ -44,7 +44,7 @@ class ExpressionTest: assert(success) def getResult(expression: String): String = - val (_, _, stdout, stderr) = bashCommand(s"$scalaPath -e '$expression'") + val (_, _, stdout, stderr) = bashCommand(s"$scalaPath -e '$expression' --power --offline --server=false") printf("stdout: %s\n", stdout.mkString("|")) printf("stderr: %s\n", stderr.mkString("\n", "\n", "")) stdout.filter(_.nonEmpty).mkString("") diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index 1db92d5415b4..a52014f14704 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -125,9 +125,22 @@ object ScriptTestEnv { def packBinDir = "dist/target/pack/bin" - def packLibDir = "dist/target/pack/lib" + // def packLibDir = "dist/target/pack/lib" // replaced by packMavenDir + def packMavenDir = "dist/target/pack/maven2" + def packVersionFile = "dist/target/pack/VERSION" def packBinScalaExists: Boolean = Files.exists(Paths.get(s"$packBinDir/scala")) + def packScalaVersion: String = { + val versionFile = Paths.get(packVersionFile) + if Files.exists(versionFile) then + val lines = Files.readAllLines(versionFile).asScala + lines.find { _.startsWith("version:=") } match + case Some(line) => line.drop(9) + case None => sys.error(s"no version:= found in $packVersionFile") + else + sys.error(s"no $packVersionFile found") + } + def listJars(dir: String): List[File] = val packlibDir = Paths.get(dir).toFile if packlibDir.isDirectory then diff --git a/compiler/test/dotty/tools/scripting/ScriptingTests.scala b/compiler/test/dotty/tools/scripting/ScriptingTests.scala index 5ec417090504..713695b62f4a 100644 --- a/compiler/test/dotty/tools/scripting/ScriptingTests.scala +++ b/compiler/test/dotty/tools/scripting/ScriptingTests.scala @@ -47,7 +47,10 @@ class ScriptingTests: */ @Test def scriptingMainTests = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - for (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") do + for + (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") + if !scriptFile.getName().endsWith("Nu.sc") + do showScriptUnderTest(scriptFile) val unexpectedJar = script2jar(scriptFile) unexpectedJar.delete @@ -66,7 +69,10 @@ class ScriptingTests: */ @Test def scriptingJarTest = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - for (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") do + for + (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") + if !scriptFile.getName().endsWith("Nu.sc") + do showScriptUnderTest(scriptFile) val expectedJar = script2jar(scriptFile) expectedJar.delete diff --git a/dist/bin/cli-common b/dist/bin/cli-common new file mode 100644 index 000000000000..d295d58916da --- /dev/null +++ b/dist/bin/cli-common @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +#/*-------------------------------------------------------------------------- +# * Credits: This script is based on the script generated by sbt-pack. +# *--------------------------------------------------------------------------*/ + +# save terminal settings +saved_stty=$(stty -g 2>/dev/null) +# clear on error so we don't later try to restore them +if [[ ! $? ]]; then + saved_stty="" +fi + +# restore stty settings (echo in particular) +function restoreSttySettings() { + stty $saved_stty + saved_stty="" +} + +scala_exit_status=127 +function onExit() { + [[ "$saved_stty" != "" ]] && restoreSttySettings + exit $scala_exit_status +} + +#/*-------------------------------------------------------------------------- +# * SECTION FOR JAVA COMMAND +# *--------------------------------------------------------------------------*/ + +# to reenable echo if we are interrupted before completing. +trap onExit INT TERM EXIT + +unset cygwin mingw msys darwin conemu + +# COLUMNS is used together with command line option '-pageWidth'. +if command -v tput >/dev/null 2>&1; then + export COLUMNS="$(tput -Tdumb cols)" +fi + +case "`uname`" in + CYGWIN*) cygwin=true + ;; + MINGW*) mingw=true + ;; + MSYS*) msys=true + ;; + Darwin*) darwin=true + if [ -z "$JAVA_VERSION" ] ; then + JAVA_VERSION="CurrentJDK" + else + echo "Using Java version: $JAVA_VERSION" 1>&2 + fi + if [ -z "$JAVA_HOME" ] ; then + JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home + fi + JAVACMD="`which java`" + ;; +esac + +unset CYGPATHCMD +if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then + # ConEmu terminal is incompatible with jna-5.*.jar + [[ (${CONEMUANSI-} || ${ConEmuANSI-}) ]] && conemu=true + # cygpath is used by various windows shells: cygwin, git-sdk, gitbash, msys, etc. + CYGPATHCMD=`which cygpath 2>/dev/null` + case "$TERM" in + rxvt* | xterm* | cygwin*) + stty -icanon min 1 -echo + JAVA_OPTS="$JAVA_OPTS -Djline.terminal=unix" + ;; + esac +fi + +# Resolve JAVA_HOME from javac command path +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + javaExecutable="`readlink -f \"$javaExecutable\"`" + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "${JAVACMD-}" ] ; then + if [ -n "${JAVA_HOME-}" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." + echo " We cannot execute $JAVACMD" + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSPATH_SUFFIX="" +# Path separator used in EXTRA_CLASSPATH +PSEP=":" + +# translate paths to Windows-mixed format before running java +if [ -n "${CYGPATHCMD-}" ]; then + [ -n "${PROG_HOME-}" ] && + PROG_HOME=`"$CYGPATHCMD" -am "$PROG_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`"$CYGPATHCMD" -am "$JAVA_HOME"` + CLASSPATH_SUFFIX=";" + PSEP=";" +elif [[ ${mingw-} || ${msys-} ]]; then + # For Mingw / Msys, convert paths from UNIX format before anything is touched + [ -n "$PROG_HOME" ] && + PROG_HOME="`(cd "$PROG_HOME"; pwd -W | sed 's|/|\\\\|g')`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd -W | sed 's|/|\\\\|g')`" + CLASSPATH_SUFFIX=";" + PSEP=";" +fi + +#/*-------------------------------------------------- +# * The code below is for Dotty +# *-------------------------------------------------*/ + +find_lib () { + for lib in "$PROG_HOME"/lib/$1 ; do + if [[ -f "$lib" ]]; then + if [ -n "$CYGPATHCMD" ]; then + "$CYGPATHCMD" -am "$lib" + elif [[ $mingw || $msys ]]; then + echo "$lib" | sed 's|/|\\\\|g' + else + echo "$lib" + fi + return + fi + done +} + +SCALA_CLI_JAR="$PROG_HOME/etc/scala-cli.jar" + +declare -a scala_args + +addScala () { + scala_args+=("'$1'") +} diff --git a/dist/bin/scala b/dist/bin/scala index bd69d40c2b97..3040c5a9a0f3 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -26,47 +26,40 @@ if [ -z "${PROG_HOME-}" ] ; then cd "$saveddir" fi -source "$PROG_HOME/bin/common" +source "$PROG_HOME/bin/cli-common" +SCALA_VERSION="" +# iterate through lines in VERSION_SRC +while IFS= read -r line; do + # if line starts with "version:=" then extract the version + if [[ "$line" == version:=* ]]; then + SCALA_VERSION="${line#version:=}" + break + fi +done < "$PROG_HOME/VERSION" + +# assert that SCALA_VERSION is not empty +if [ -z "$SCALA_VERSION" ]; then + echo "Failed to extract Scala version from $PROG_HOME/VERSION" + exit 1 +fi + +MVN_REPOSITORY="file://$PROG_HOME/maven2" + +# escape all script arguments while [[ $# -gt 0 ]]; do - case "$1" in - -D*) - # pass to scala as well: otherwise we lose it sometimes when we - # need it, e.g. communicating with a server compiler. - # respect user-supplied -Dscala.usejavacp - addJava "$1" - addScala "$1" - shift - ;; - -J*) - # as with -D, pass to scala even though it will almost - # never be used. - addJava "${1:2}" - addScala "$1" - shift - ;; - -classpath*) - if [ "$1" != "${1##* }" ]; then - # -classpath and its value have been supplied in a single string e.g. "-classpath 'lib/*'" - A=$1 ; shift # consume $1 before adding its substrings back - set -- $A "$@" # split $1 on whitespace and put it back - else - addScala "$1" - shift - fi - ;; - *) - addScala "$1" - shift - ;; - esac + addScala "$1" + shift done # exec here would prevent onExit from being called, leaving terminal in unusable state -compilerJavaClasspathArgs [ -z "${ConEmuPID-}" -o -n "${cygwin-}" ] && export MSYSTEM= PWD= # workaround for #12405 -eval "\"$JAVACMD\"" "${java_args[@]}" "-Dscala.home=\"$PROG_HOME\"" "-classpath \"$jvm_cp_args\"" "dotty.tools.MainGenericRunner" "-classpath \"$jvm_cp_args\"" "${scala_args[@]}" +eval "\"$JAVACMD\"" \ + "-jar \"$SCALA_CLI_JAR\"" \ + "--prog-name scala" \ + "--cli-default-scala-version \"$SCALA_VERSION\"" \ + "-r \"$MVN_REPOSITORY\"" \ + "${scala_args[@]}" scala_exit_status=$? - onExit diff --git a/dist/bin/scala.bat b/dist/bin/scala.bat index ca908fd340be..78336272055b 100644 --- a/dist/bin/scala.bat +++ b/dist/bin/scala.bat @@ -14,14 +14,15 @@ for %%f in ("%~dp0.") do ( call "%_PROG_HOME%\bin\common.bat" if not %_EXITCODE%==0 goto end -call :args %* - @rem ######################################################################### @rem ## Main -call :compilerJavaClasspathArgs +call :setScalaOpts + +@rem we need to escape % in the java command path, for some reason this doesnt work in common.bat +set "_JAVACMD=!_JAVACMD:%%=%%%%!" -call "%_JAVACMD%" %_JAVA_ARGS% "-Dscala.home=%_PROG_HOME%" -classpath "%_JVM_CP_ARGS%" dotty.tools.MainGenericRunner -classpath "%_JVM_CP_ARGS%" %_SCALA_ARGS% +call "%_JAVACMD%" "-jar" "%SCALA_CLI_JAR%" "--prog-name" "scala" "--cli-default-scala-version" "%_SCALA_VERSION%" "-r" "%MVN_REPOSITORY%" %* if not %ERRORLEVEL%==0 ( set _EXITCODE=1& goto end ) goto end @@ -29,62 +30,42 @@ goto end @rem ######################################################################### @rem ## Subroutines -:args -set _JAVA_ARGS= -set _SCALA_ARGS= -set _SCALA_CPATH= - -:args_loop -if "%~1"=="" goto args_done -set "__ARG=%~1" -if "%__ARG:~0,2%"=="-D" ( - @rem pass to scala as well: otherwise we lose it sometimes when we - @rem need it, e.g. communicating with a server compiler. - set _JAVA_ARGS=!_JAVA_ARGS! "%__ARG%" - set _SCALA_ARGS=!_SCALA_ARGS! "%__ARG%" -) else if "%__ARG:~0,2%"=="-J" ( - @rem as with -D, pass to scala even though it will almost - @rem never be used. - set _JAVA_ARGS=!_JAVA_ARGS! %__ARG:~2% - set _SCALA_ARGS=!_SCALA_ARGS! "%__ARG%" -) else if "%__ARG%"=="-classpath" ( - set "_SCALA_CPATH=%~2" - shift -) else if "%__ARG%"=="-cp" ( - set "_SCALA_CPATH=%~2" - shift -) else ( - set _SCALA_ARGS=!_SCALA_ARGS! "%__ARG%" +:setScalaOpts + +@REM sfind the index of the first colon in _PROG_HOME +set "index=0" +set "char=!_PROG_HOME:~%index%,1!" +:findColon +if not "%char%"==":" ( + set /a "index+=1" + set "char=!_PROG_HOME:~%index%,1!" + goto :findColon ) -shift -goto args_loop -:args_done -goto :eof -@rem output parameter: _JVM_CP_ARGS -:compilerJavaClasspathArgs -set __TOOLCHAIN= -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA_LIB%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_LIB%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA_ASM%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SBT_INTF%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_INTF%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_COMP%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_TASTY_CORE%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_STAGING%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_TASTY_INSPECTOR%%_PSEP%" - -@rem # jline -set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_READER%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_TERMINAL%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_TERMINAL_JNA%%_PSEP%" -set "__TOOLCHAIN=%__TOOLCHAIN%%_JNA%%_PSEP%" - -if defined _SCALA_CPATH ( - set "_JVM_CP_ARGS=%__TOOLCHAIN%%_SCALA_CPATH%" -) else ( - set "_JVM_CP_ARGS=%__TOOLCHAIN%" +@REM set _PROG_HOME to the substring from the first colon to the end +set "_PROG_HOME_SUB=!_PROG_HOME:~%index%!" +@REM strip initial character +set "_PROG_HOME_SUB=!_PROG_HOME_SUB:~1!" + +@REM set drive to substring from 0 to the first colon +set "_PROG_HOME_DRIVE=!_PROG_HOME:~0,%index%!" + + + +set "_SCALA_VERSION=" +set "MVN_REPOSITORY=file://%_PROG_HOME_DRIVE%\%_PROG_HOME_SUB:\=/%/maven2" +set "SCALA_CLI_JAR=%_PROG_HOME%\etc\scala-cli.jar" + +@rem read for version:=_SCALA_VERSION in VERSION_FILE +FOR /F "usebackq delims=" %%G IN ("%_PROG_HOME%\VERSION") DO ( + SET "line=%%G" + IF "!line:~0,9!"=="version:=" ( + SET "_SCALA_VERSION=!line:~9!" + GOTO :foundVersion + ) ) + +:foundVersion goto :eof @rem ######################################################################### diff --git a/dist/bin/scala_legacy b/dist/bin/scala_legacy new file mode 100755 index 000000000000..bd69d40c2b97 --- /dev/null +++ b/dist/bin/scala_legacy @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Try to autodetect real location of the script +if [ -z "${PROG_HOME-}" ] ; then + ## resolve links - $0 may be a link to PROG_HOME + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + PROG_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + PROG_HOME=`cd "$PROG_HOME" && pwd` + + cd "$saveddir" +fi + +source "$PROG_HOME/bin/common" + +while [[ $# -gt 0 ]]; do + case "$1" in + -D*) + # pass to scala as well: otherwise we lose it sometimes when we + # need it, e.g. communicating with a server compiler. + # respect user-supplied -Dscala.usejavacp + addJava "$1" + addScala "$1" + shift + ;; + -J*) + # as with -D, pass to scala even though it will almost + # never be used. + addJava "${1:2}" + addScala "$1" + shift + ;; + -classpath*) + if [ "$1" != "${1##* }" ]; then + # -classpath and its value have been supplied in a single string e.g. "-classpath 'lib/*'" + A=$1 ; shift # consume $1 before adding its substrings back + set -- $A "$@" # split $1 on whitespace and put it back + else + addScala "$1" + shift + fi + ;; + *) + addScala "$1" + shift + ;; + esac +done + +# exec here would prevent onExit from being called, leaving terminal in unusable state +compilerJavaClasspathArgs +[ -z "${ConEmuPID-}" -o -n "${cygwin-}" ] && export MSYSTEM= PWD= # workaround for #12405 +eval "\"$JAVACMD\"" "${java_args[@]}" "-Dscala.home=\"$PROG_HOME\"" "-classpath \"$jvm_cp_args\"" "dotty.tools.MainGenericRunner" "-classpath \"$jvm_cp_args\"" "${scala_args[@]}" +scala_exit_status=$? + + +onExit diff --git a/dist/bin/scalac.bat b/dist/bin/scalac.bat index cb1a76471f70..c8cd0babe60b 100644 --- a/dist/bin/scalac.bat +++ b/dist/bin/scalac.bat @@ -21,6 +21,9 @@ call :args %* call :compilerJavaClasspathArgs +@rem we need to escape % in the java command path, for some reason this doesnt work in common.bat +set "_JAVACMD=!_JAVACMD:%%=%%%%!" + call "%_JAVACMD%" %_JAVA_ARGS% -classpath "%_JVM_CP_ARGS%" "-Dscala.usejavacp=true" "-Dscala.home=%_PROG_HOME%" dotty.tools.MainGenericCompiler %_SCALA_ARGS% if not %ERRORLEVEL%==0 ( set _EXITCODE=1 diff --git a/dist/bin/scaladoc.bat b/dist/bin/scaladoc.bat index bcc0d71788a3..c30a4689244c 100644 --- a/dist/bin/scaladoc.bat +++ b/dist/bin/scaladoc.bat @@ -26,6 +26,10 @@ call :classpathArgs if defined JAVA_OPTS ( set _JAVA_OPTS=%JAVA_OPTS% ) else ( set _JAVA_OPTS=%_DEFAULT_JAVA_OPTS% ) + +@rem we need to escape % in the java command path, for some reason this doesnt work in common.bat +set "_JAVACMD=!_JAVACMD:%%=%%%%!" + call "%_JAVACMD%" %_JAVA_OPTS% %_JAVA_DEBUG% %_JAVA_ARGS% ^ -classpath "%_CLASS_PATH%" ^ -Dscala.usejavacp=true ^ diff --git a/project/Build.scala b/project/Build.scala index 921fbcd80b90..232c5f24d13d 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -12,6 +12,8 @@ import pl.project13.scala.sbt.JmhPlugin import pl.project13.scala.sbt.JmhPlugin.JmhKeys.Jmh import sbt.Package.ManifestAttributes import sbt.PublishBinPlugin.autoImport._ +import dotty.tools.sbtplugin.RepublishPlugin +import dotty.tools.sbtplugin.RepublishPlugin.autoImport._ import sbt.plugins.SbtPlugin import sbt.ScriptedPlugin.autoImport._ import xerial.sbt.pack.PackPlugin @@ -26,6 +28,8 @@ import sbttastymima.TastyMiMaPlugin import sbttastymima.TastyMiMaPlugin.autoImport._ import scala.util.Properties.isJavaAtLeast +import scala.collection.mutable + import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._ import org.scalajs.linker.interface.{ModuleInitializer, StandardConfig} @@ -114,6 +118,9 @@ object Build { */ val mimaPreviousLTSDottyVersion = "3.3.0" + /** Version of Scala CLI to download */ + val scalaCliLauncherVersion = "1.3.1" + object CompatMode { final val BinaryCompatible = 0 final val SourceAndBinaryCompatible = 1 @@ -2114,13 +2121,22 @@ object Build { packMain := Map(), publishArtifact := false, packGenerateMakefile := false, - packExpandedClasspath := true, - packArchiveName := "scala3-" + dottyVersion + packArchiveName := "scala3-" + dottyVersion, + republishRepo := target.value / "republish", + republishLaunchers := { + val cliV = scalaCliLauncherVersion + Seq( + ("scala-cli.jar", cliV, url(s"https://github.com/VirtusLab/scala-cli/releases/download/v$cliV/scala-cli.jar")) + ) + }, + Compile / pack := (Compile / pack).dependsOn(republish).value, ) lazy val dist = project.asDist(Bootstrapped) .settings( packResourceDir += (baseDirectory.value / "bin" -> "bin"), + packResourceDir += (republishRepo.value / "maven2" -> "maven2"), + packResourceDir += (republishRepo.value / "etc" -> "etc"), ) private def customMimaReportBinaryIssues(issueFilterLocation: String) = mimaReportBinaryIssues := { @@ -2255,9 +2271,19 @@ object Build { def asDist(implicit mode: Mode): Project = project. enablePlugins(PackPlugin). + enablePlugins(RepublishPlugin). withCommonSettings. - dependsOn(`scala3-interfaces`, dottyCompiler, dottyLibrary, tastyCore, `scala3-staging`, `scala3-tasty-inspector`, scaladoc). settings(commonDistSettings). + dependsOn( + `scala3-interfaces`, + dottyCompiler, + dottyLibrary, + tastyCore, + `scala3-staging`, + `scala3-tasty-inspector`, + scaladoc, + `scala3-sbt-bridge`, // for scala-cli + ). bootstrappedSettings( target := baseDirectory.value / "target" // override setting in commonBootstrappedSettings ) diff --git a/project/Modes.scala b/project/Modes.scala index eddb5a3f1a7b..fcc13dea8a89 100644 --- a/project/Modes.scala +++ b/project/Modes.scala @@ -1,4 +1,4 @@ -import sbt.{Project, ProjectReference, SettingsDefinition} +import sbt.{Project, ProjectReference, SettingsDefinition, Plugins} object Modes { @@ -25,5 +25,9 @@ object Modes { def bootstrappedDependsOn(s: sbt.ClasspathDep[ProjectReference]*)(implicit mode: Mode): Project = if (mode == NonBootstrapped) project else project.dependsOn(s: _*) + /** Plugins only if the mode is bootstrapped */ + def bootstrappedEnablePlugins(ns: Plugins*)(implicit mode: Mode): Project = + if (mode == NonBootstrapped) project else project.enablePlugins(ns: _*) + } } diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala new file mode 100644 index 000000000000..0b71c9ecb6df --- /dev/null +++ b/project/RepublishPlugin.scala @@ -0,0 +1,193 @@ +package dotty.tools.sbtplugin + +import sbt._ +import xerial.sbt.pack.PackPlugin +import sbt.Keys._ +import sbt.AutoPlugin +import sbt.PublishBinPlugin +import sbt.PublishBinPlugin.autoImport._ +import sbt.io.Using +import sbt.util.CacheImplicits._ + +import scala.collection.mutable +import java.nio.file.Files + +/** This local plugin provides ways of publishing a project classpath and library dependencies to + * .a local repository */ +object RepublishPlugin extends AutoPlugin { + override def trigger = allRequirements + override def requires = super.requires && PublishBinPlugin && PackPlugin + + object autoImport { + val republishProjectRefs = taskKey[Seq[ProjectRef]]("fetch the classpath deps from the project.") + val republishLocalResolved = taskKey[Seq[ResolvedArtifacts]]("resolve local artifacts for distribution.") + val republishAllResolved = taskKey[Seq[ResolvedArtifacts]]("Resolve the dependencies for the distribution") + val republishClasspath = taskKey[Set[File]]("cache the dependencies for the distribution") + val republishFetchLaunchers = taskKey[Set[File]]("cache the launcher deps for the distribution") + val republish = taskKey[File]("cache the dependencies and download launchers for the distribution") + val republishRepo = settingKey[File]("the location to store the republished artifacts.") + val republishLaunchers = settingKey[Seq[(String, String, URL)]]("launchers to download. Sequence of (name, version, URL).") + } + + import autoImport._ + + case class SimpleModuleId(org: String, name: String, revision: String) { + override def toString = s"$org:$name:$revision" + } + case class ResolvedArtifacts(id: SimpleModuleId, jar: File, pom: File) + + override val projectSettings: Seq[Def.Setting[_]] = Def.settings( + republishLocalResolved / republishProjectRefs := { + val proj = thisProjectRef.value + val deps = buildDependencies.value + + deps.classpathRefs(proj) + }, + republishLocalResolved := Def.taskDyn { + val deps = (republishLocalResolved / republishProjectRefs).value + val publishAllLocalBin = deps.map({ d => ((d / publishLocalBin / packagedArtifacts)) }).join + val resolveId = deps.map({ d => ((d / projectID)) }).join + Def.task { + val published = publishAllLocalBin.value + val ids = resolveId.value + + ids.zip(published).map({ case (id, as) => + val simpleId = { + val disabled = CrossVersion.disabled + val name0 = id.crossVersion match { + case cv: CrossVersion.Binary => + // projectID does not add binary suffix + (s"${id.name}_${cv.prefix}${cv.suffix}3") + .ensuring(!id.name.endsWith("_3") && id.revision.startsWith("3.")) + case _ => id.name + } + SimpleModuleId(id.organization, name0, id.revision) + } + var jarOrNull: File = null + var pomOrNull: File = null + as.foreach({ case (a, f) => + if (a.`type` == "jar") { + jarOrNull = f + } else if (a.`type` == "pom") { + pomOrNull = f + } + }) + assert(jarOrNull != null, s"Could not find jar for ${id}") + assert(pomOrNull != null, s"Could not find pom for ${id}") + ResolvedArtifacts(simpleId, jarOrNull, pomOrNull) + }) + } + }.value, + republishAllResolved := { + val localResolved = republishLocalResolved.value + val report = (thisProjectRef / updateFull).value + + val found = mutable.Map.empty[SimpleModuleId, ResolvedArtifacts] + val evicted = mutable.Set.empty[SimpleModuleId] + + localResolved.foreach({ resolved => + val simpleId = resolved.id + evicted += simpleId.copy(revision = simpleId.revision + "-nonbootstrapped") + found(simpleId) = resolved + }) + + report.allModuleReports.foreach { mr => + val simpleId = { + val id = mr.module + SimpleModuleId(id.organization, id.name, id.revision) + } + + if (!found.contains(simpleId) && !evicted(simpleId)) { + var jarOrNull: File = null + var pomOrNull: File = null + mr.artifacts.foreach({ case (a, f) => + if (a.`type` == "jar" || a.`type` == "bundle") { + jarOrNull = f + } else if (a.`type` == "pom") { + pomOrNull = f + } + }) + assert(jarOrNull != null, s"Could not find jar for ${simpleId}") + if (pomOrNull == null) { + val jarPath = jarOrNull.toPath + // we found the jar, so assume we can resolve a sibling pom file + val pomPath = jarPath.resolveSibling(jarPath.getFileName.toString.stripSuffix(".jar") + ".pom") + assert(Files.exists(pomPath), s"Could not find pom for ${simpleId}") + pomOrNull = pomPath.toFile + } + found(simpleId) = ResolvedArtifacts(simpleId, jarOrNull, pomOrNull) + } + + } + found.values.toSeq + }, + republishClasspath := { + val s = streams.value + val resolved = republishAllResolved.value + val cacheDir = republishRepo.value + + val log = s.log + val mavenRepo = cacheDir / "maven2" + IO.createDirectory(mavenRepo) + resolved.map { ra => + log.info(s"[republish] publishing ${ra.id} to $mavenRepo...") + val jar = ra.jar + val pom = ra.pom + + val pathElems = ra.id.org.split('.').toVector :+ ra.id.name :+ ra.id.revision + val artifactDir = pathElems.foldLeft(mavenRepo)(_ / _) + IO.createDirectory(artifactDir) + IO.copyFile(jar, artifactDir / jar.getName) + IO.copyFile(pom, artifactDir / pom.getName) + artifactDir + }.toSet + }, + republishFetchLaunchers := { + val s = streams.value + val log = s.log + val repoDir = republishRepo.value + val launcherVersions = republishLaunchers.value + + val etc = repoDir / "etc" + + val store = s.cacheStoreFactory / "versions" + + def work(dest: File, launcher: URL) = { + IO.delete(dest) + Using.urlInputStream(launcher) { in => + IO.createDirectory(etc) + log.info(s"[republish] Downloading $launcher to $dest...") + IO.transfer(in, dest) + log.info(s"[republish] Downloaded $launcher to $dest...") + } + dest + } + + val allLaunchers = { + for ((name, version, launcher) <- launcherVersions) yield { + val dest = etc / name + + val id = name.replaceAll("[^a-zA-Z0-9]", "_") + + val fetchAction = Tracked.inputChanged[String, File](store.make(id)) { (inChanged, version) => + if (inChanged || !Files.exists(dest.toPath)) { + work(dest, launcher) + } else { + log.info(s"[republish] Using cached $launcher at $dest...") + dest + } + } + + fetchAction(version) + } + } + allLaunchers.toSet + }, + republish := { + val cacheDir = republishRepo.value + val artifacts = republishClasspath.value + val launchers = republishFetchLaunchers.value + cacheDir + } + ) +} diff --git a/project/scripts/bootstrappedOnlyCmdTests b/project/scripts/bootstrappedOnlyCmdTests index 4e18e3a1d4a4..f3d730f8f494 100755 --- a/project/scripts/bootstrappedOnlyCmdTests +++ b/project/scripts/bootstrappedOnlyCmdTests @@ -14,32 +14,38 @@ echo "testing scala.quoted.Expr.run from sbt scala" "$SBT" ";scala3-compiler-bootstrapped/scalac -with-compiler tests/run-staging/quote-run.scala; scala3-compiler-bootstrapped/scala -with-compiler Test" > "$tmp" grep -qe "val a: scala.Int = 3" "$tmp" - # setup for `scalac`/`scala` script tests "$SBT" dist/pack +echo "capturing scala version from dist/target/pack/VERSION" +IFS=':=' read -ra versionProps < "$ROOT/dist/target/pack/VERSION" # temporarily set IFS to ':=' to split versionProps +[ ${#versionProps[@]} -eq 3 ] && \ + [ ${versionProps[0]} = "version" ] && \ + [ -n ${versionProps[2]} ] || die "Expected non-empty 'version' property in $ROOT/dist/target/pack/VERSION" +scala_version=${versionProps[2]} + # check that `scalac` compiles and `scala` runs it echo "testing ./bin/scalac and ./bin/scala" clear_out "$OUT" ./bin/scalac "$SOURCE" -d "$OUT" -./bin/scala -classpath "$OUT" "$MAIN" > "$tmp" +./bin/scala -classpath "$OUT" -M "$MAIN" > "$tmp" test "$EXPECTED_OUTPUT" = "$(cat "$tmp")" # Test scaladoc based on compiled classes ./bin/scaladoc -project Staging -d "$OUT1" "$OUT" clear_out "$OUT1" -# check that `scalac` and `scala` works for staging +# check that `scalac` and `scala` works for staging. clear_out "$OUT" ./bin/scalac tests/run-staging/i4044f.scala -d "$OUT" -./bin/scala -with-compiler -classpath "$OUT" Test > "$tmp" +./bin/scala -with-compiler -classpath "$OUT" -M Test > "$tmp" # check that `scalac -from-tasty` compiles and `scala` runs it echo "testing ./bin/scalac -from-tasty and scala -classpath" clear_out "$OUT1" ./bin/scalac "$SOURCE" -d "$OUT" ./bin/scalac -from-tasty -d "$OUT1" "$OUT/$TASTY" -./bin/scala -classpath "$OUT1" "$MAIN" > "$tmp" +./bin/scala -classpath "$OUT1" -M "$MAIN" > "$tmp" test "$EXPECTED_OUTPUT" = "$(cat "$tmp")" # check that `sbt scalac -decompile` runs @@ -91,7 +97,7 @@ clear_out "$OUT" grep -qe "Usage: scalac " "$tmp" ./bin/scala -help > "$tmp" 2>&1 -grep -qe "Usage: scala " "$tmp" +grep -qe "See 'scala --help' to read about a specific subcommand." "$tmp" ./bin/scala -d hello.jar tests/run/hello.scala ls hello.jar @@ -102,14 +108,6 @@ clear_out "$OUT" echo "Bug12973().check" | TERM=dumb ./bin/scala -cp "$OUT/out.jar" > "$tmp" 2>&1 grep -qe "Bug12973 is fixed" "$tmp" -echo "capturing scala version from dist/target/pack/VERSION" -cwd=$(pwd) -IFS=':=' read -ra versionProps < "$cwd/dist/target/pack/VERSION" # temporarily set IFS to ':=' to split versionProps -[ ${#versionProps[@]} -eq 3 ] && \ - [ ${versionProps[0]} = "version" ] && \ - [ -n ${versionProps[2]} ] || die "Expected non-empty 'version' property in $cwd/dist/target/pack/VERSION" -scala_version=${versionProps[2]} - echo "testing -sourcepath with incremental compile: inlining changed inline def into a def" # Here we will test that a changed inline method symbol loaded from the sourcepath (-sourcepath compiler option) # will have its `defTree` correctly set when its method body is required for inlining. diff --git a/project/scripts/winCmdTests b/project/scripts/winCmdTests index d287b60992b2..2dffff5b196a 100644 --- a/project/scripts/winCmdTests +++ b/project/scripts/winCmdTests @@ -5,6 +5,6 @@ PREFIX="dist/target/pack" SOURCE="tests/pos/HelloWorld.scala" $PREFIX/bin/scalac @project/scripts/options "$SOURCE" $PREFIX/bin/scalac -d out "$SOURCE" -$PREFIX/bin/scala -classpath out HelloWorld -$PREFIX/bin/scala -classpath out -J-Xmx512m HelloWorld +$PREFIX/bin/scala --power -classpath out -M HelloWorld --offline '--server=false' +$PREFIX/bin/scala --power -classpath out -J -Xmx512m -M HelloWorld --offline '--server=false' mkdir -p _site && $PREFIX/bin/scaladoc -d _site -project Hello "$SOURCE" diff --git a/project/scripts/winCmdTests.bat b/project/scripts/winCmdTests.bat index ee9b8237c694..d9b594d560ab 100644 --- a/project/scripts/winCmdTests.bat +++ b/project/scripts/winCmdTests.bat @@ -14,10 +14,10 @@ if not %ERRORLEVEL%==0 endlocal& exit /b 1 call "%_PREFIX%\bin\scalac.bat" -d "%_OUT_DIR%" "%_SOURCE%" if not %ERRORLEVEL%==0 endlocal& exit /b 1 -call "%_PREFIX%\bin\scala.bat" -classpath "%_OUT_DIR%" HelloWorld +call "%_PREFIX%\bin\scala.bat" --power -classpath "%_OUT_DIR%" -M HelloWorld --offline --server=false if not %ERRORLEVEL%==0 endlocal& exit /b 1 -call "%_PREFIX%\bin\scala.bat" -classpath "%_OUT_DIR%" -J-Xmx512m HelloWorld +call "%_PREFIX%\bin\scala.bat" --power -classpath "%_OUT_DIR%" -J -Xmx512m -M HelloWorld --offline --server=false if not %ERRORLEVEL%==0 endlocal& exit /b 1 if not exist "%_SITE_DIR%" mkdir "%_SITE_DIR%" diff --git a/tests/run-with-compiler/i14541.scala b/tests/run-with-compiler/i14541.scala index 0fdfb89674d5..2b942007c5b6 100644 --- a/tests/run-with-compiler/i14541.scala +++ b/tests/run-with-compiler/i14541.scala @@ -6,6 +6,7 @@ object Test: def main(args: Array[String]): Unit = getClass.getClassLoader.run("echo", List("hello", "raw", "world")) // caution: uses "SCALA_OPTS" + sys.props("scala.use_legacy_launcher") = "true" dotty.tools.MainGenericRunner.main(Array("--class-path", classpath, "echo", "hello", "run", "world")) @main def echo(args: String*): Unit = println { From 8fda9cf4556434076932a7f305e60a5d9a7a9a96 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Wed, 8 May 2024 13:42:05 +0200 Subject: [PATCH 2/9] Add native launchers, Restore CI tests on Windows --- .github/workflows/ci.yaml | 4 +- bin/scala | 35 ++++- build.sbt | 5 + .../scripting/argfileClasspath.sc | 9 -- ...hReport.sc => classpathReport_scalacli.sc} | 2 +- .../scripting/cpArgumentsFile.txt | 1 - compiler/test-resources/scripting/envtest.sc | 2 + .../test-resources/scripting/envtestNu.sc | 2 - .../scripting/envtest_scalacli.sc | 3 + compiler/test-resources/scripting/hashBang.sc | 2 +- .../test-resources/scripting/hashBang.scala | 4 +- .../test-resources/scripting/scriptName.scala | 2 +- .../test-resources/scripting/scriptPath.sc | 2 +- ...scriptPathNu.sc => scriptPath_scalacli.sc} | 2 +- compiler/test-resources/scripting/showArgs.sc | 2 +- .../{showArgsNu.sc => showArgs_scalacli.sc} | 3 +- .../test-resources/scripting/sqlDateError.sc | 2 +- .../scripting/sqlDateErrorNu.sc | 6 - .../scripting/sqlDateError_scalacli.sc | 6 + .../test-resources/scripting/touchFile.sc | 2 +- .../scripting/unglobClasspath.sc | 6 - .../scripting/unglobClasspath_scalacli.sc | 9 ++ .../test/dotty/tools/io/ClasspathTest.scala | 4 +- .../tools/scripting/BashExitCodeTests.scala | 6 +- .../tools/scripting/BashScriptsTests.scala | 20 +-- .../tools/scripting/ClasspathTests.scala | 18 ++- .../tools/scripting/ExpressionTest.scala | 4 + .../dotty/tools/scripting/ScriptTestEnv.scala | 45 +++++-- .../tools/scripting/ScriptingTests.scala | 16 +-- compiler/test/dotty/tools/utils.scala | 13 +- dist/bin-native-overrides/cli-common-platform | 3 + .../cli-common-platform.bat | 3 + dist/bin/cli-common | 56 +------- dist/bin/cli-common-platform | 5 + dist/bin/cli-common-platform.bat | 5 + dist/bin/common | 39 +----- dist/bin/common-java | 40 ++++++ dist/bin/scala | 5 +- dist/bin/scala.bat | 8 +- project/Build.scala | 91 ++++++++----- project/DottyVersion.scala | 24 ++++ project/RepublishPlugin.scala | 125 +++++++++++++++--- project/VersionUtil.scala | 2 + project/scripts/bootstrappedOnlyCmdTests | 7 + project/scripts/cmdTestsCommon.inc.sh | 14 ++ project/scripts/echoArgs.sc | 6 + project/scripts/winCmdTests | 2 +- project/scripts/winCmdTests.bat | 2 +- 48 files changed, 444 insertions(+), 230 deletions(-) delete mode 100755 compiler/test-resources/scripting/argfileClasspath.sc rename compiler/test-resources/scripting/{classpathReport.sc => classpathReport_scalacli.sc} (91%) delete mode 100755 compiler/test-resources/scripting/cpArgumentsFile.txt delete mode 100755 compiler/test-resources/scripting/envtestNu.sc create mode 100755 compiler/test-resources/scripting/envtest_scalacli.sc rename compiler/test-resources/scripting/{scriptPathNu.sc => scriptPath_scalacli.sc} (87%) rename compiler/test-resources/scripting/{showArgsNu.sc => showArgs_scalacli.sc} (68%) delete mode 100755 compiler/test-resources/scripting/sqlDateErrorNu.sc create mode 100755 compiler/test-resources/scripting/sqlDateError_scalacli.sc delete mode 100755 compiler/test-resources/scripting/unglobClasspath.sc create mode 100755 compiler/test-resources/scripting/unglobClasspath_scalacli.sc create mode 100644 dist/bin-native-overrides/cli-common-platform create mode 100644 dist/bin-native-overrides/cli-common-platform.bat create mode 100644 dist/bin/cli-common-platform create mode 100644 dist/bin/cli-common-platform.bat create mode 100644 dist/bin/common-java create mode 100644 project/DottyVersion.scala create mode 100644 project/scripts/echoArgs.sc diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 12e90eb9d653..8ed18be1a703 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -230,7 +230,7 @@ jobs: shell: cmd - name: build binary - run: sbt "dist/pack" & bash -version + run: sbt "dist-win-x64/pack" & bash -version shell: cmd - name: cygwin tests @@ -270,7 +270,7 @@ jobs: uses: actions/checkout@v4 - name: Test - run: sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test" + run: sbt ";dist-win-x64/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test" shell: cmd - name: Scala.js Test diff --git a/bin/scala b/bin/scala index 85c1ac91d08f..2df274fe95ba 100755 --- a/bin/scala +++ b/bin/scala @@ -2,4 +2,37 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.." -"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scala" "--power" "$@" "--offline" "--server=false" +scala_args() { + + declare -a CLI_ARGS + declare -a SCRIPT_ARGS + declare DISABLE_BLOOP=1 + + while (( "$#" )); do + case "$1" in + "--") + shift + SCRIPT_ARGS+=("--") + SCRIPT_ARGS+=("$@") + break + ;; + "clean") + CLI_ARGS+=("$1") + DISABLE_BLOOP=0 # clean command should not add --offline --server=false + shift + ;; + *) + CLI_ARGS+=("$1") + shift + ;; + esac + done + + if [ $DISABLE_BLOOP -eq 1 ]; then + CLI_ARGS+=("--offline" "--server=false") + fi + + echo "--power ${CLI_ARGS[@]} ${SCRIPT_ARGS[@]}" +} + +"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scala" $(scala_args "$@") diff --git a/build.sbt b/build.sbt index 1bc74e5e23fb..705ef0ba0eb7 100644 --- a/build.sbt +++ b/build.sbt @@ -28,6 +28,11 @@ val `scaladoc-js-main` = Build.`scaladoc-js-main` val `scaladoc-js-contributors` = Build.`scaladoc-js-contributors` val `scala3-bench-run` = Build.`scala3-bench-run` val dist = Build.dist +val `dist-mac-x64` = Build.`dist-mac-x64` +val `dist-mac-aarch64` = Build.`dist-mac-aarch64` +val `dist-win-x64` = Build.`dist-win-x64` +val `dist-linux-x64` = Build.`dist-linux-x64` +val `dist-linux-aarch64` = Build.`dist-linux-aarch64` val `community-build` = Build.`community-build` val `sbt-community-build` = Build.`sbt-community-build` val `scala3-presentation-compiler` = Build.`scala3-presentation-compiler` diff --git a/compiler/test-resources/scripting/argfileClasspath.sc b/compiler/test-resources/scripting/argfileClasspath.sc deleted file mode 100755 index c31371ba8934..000000000000 --- a/compiler/test-resources/scripting/argfileClasspath.sc +++ /dev/null @@ -1,9 +0,0 @@ -#!dist/target/pack/bin/scala @compiler/test-resources/scripting/cpArgumentsFile.txt - -import java.nio.file.Paths - -def main(args: Array[String]): Unit = - val cwd = Paths.get(".").toAbsolutePath.toString.replace('\\', '/').replaceAll("/$", "") - printf("cwd: %s\n", cwd) - printf("classpath: %s\n", sys.props("java.class.path")) - diff --git a/compiler/test-resources/scripting/classpathReport.sc b/compiler/test-resources/scripting/classpathReport_scalacli.sc similarity index 91% rename from compiler/test-resources/scripting/classpathReport.sc rename to compiler/test-resources/scripting/classpathReport_scalacli.sc index cc68c4b1d52e..0b2552b3ac84 100755 --- a/compiler/test-resources/scripting/classpathReport.sc +++ b/compiler/test-resources/scripting/classpathReport_scalacli.sc @@ -1,5 +1,5 @@ #!/usr/bin/env bin/scala - +// This file is a Scala CLI script. import java.nio.file.Paths // def main(args: Array[String]): Unit = // MIGRATION: Scala CLI expects `*.sc` files to be straight-line code diff --git a/compiler/test-resources/scripting/cpArgumentsFile.txt b/compiler/test-resources/scripting/cpArgumentsFile.txt deleted file mode 100755 index 73037eb7d9bc..000000000000 --- a/compiler/test-resources/scripting/cpArgumentsFile.txt +++ /dev/null @@ -1 +0,0 @@ --classpath dist/target/pack/lib/* diff --git a/compiler/test-resources/scripting/envtest.sc b/compiler/test-resources/scripting/envtest.sc index b2fde1b32339..724580449229 100755 --- a/compiler/test-resources/scripting/envtest.sc +++ b/compiler/test-resources/scripting/envtest.sc @@ -1,2 +1,4 @@ +// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class + def main(args: Array[String]): Unit = println("Hello " + util.Properties.propOrNull("key")) diff --git a/compiler/test-resources/scripting/envtestNu.sc b/compiler/test-resources/scripting/envtestNu.sc deleted file mode 100755 index fe4cd7851b0a..000000000000 --- a/compiler/test-resources/scripting/envtestNu.sc +++ /dev/null @@ -1,2 +0,0 @@ -// MIGRATION: Scala CLI expects `*.sc` files to be straight-line code - println("Hello " + util.Properties.propOrNull("key")) diff --git a/compiler/test-resources/scripting/envtest_scalacli.sc b/compiler/test-resources/scripting/envtest_scalacli.sc new file mode 100755 index 000000000000..993ea1691640 --- /dev/null +++ b/compiler/test-resources/scripting/envtest_scalacli.sc @@ -0,0 +1,3 @@ +// This file is a Scala CLI script. + +println("Hello " + util.Properties.propOrNull("key")) diff --git a/compiler/test-resources/scripting/hashBang.sc b/compiler/test-resources/scripting/hashBang.sc index d767bd1a1592..98884bc050c0 100755 --- a/compiler/test-resources/scripting/hashBang.sc +++ b/compiler/test-resources/scripting/hashBang.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env scala +#!/usr/bin/env fake-program-to-test-hashbang-removal # comment STUFF=nada !# diff --git a/compiler/test-resources/scripting/hashBang.scala b/compiler/test-resources/scripting/hashBang.scala index 1aab26269f86..b7bf6b541854 100755 --- a/compiler/test-resources/scripting/hashBang.scala +++ b/compiler/test-resources/scripting/hashBang.scala @@ -1,8 +1,8 @@ -#!/usr/bin/env scala +#!/usr/bin/env fake-program-to-test-hashbang-removal # comment STUFF=nada !# - +// everything above this point should be ignored by the compiler def main(args: Array[String]): Unit = System.err.printf("mainClassFromStack: %s\n",mainFromStack) assert(mainFromStack.contains("hashBang"),s"fromStack[$mainFromStack]") diff --git a/compiler/test-resources/scripting/scriptName.scala b/compiler/test-resources/scripting/scriptName.scala index 21aec32fe0bb..7e479197d567 100755 --- a/compiler/test-resources/scripting/scriptName.scala +++ b/compiler/test-resources/scripting/scriptName.scala @@ -1,4 +1,4 @@ -#!/usr/bin/env scala +// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class def main(args: Array[String]): Unit = val name = Option(sys.props("script.name")) match { diff --git a/compiler/test-resources/scripting/scriptPath.sc b/compiler/test-resources/scripting/scriptPath.sc index 46cd5e8a7385..e29e659d09d4 100755 --- a/compiler/test-resources/scripting/scriptPath.sc +++ b/compiler/test-resources/scripting/scriptPath.sc @@ -1,4 +1,4 @@ -#!dist/target/pack/bin/scala +// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class def main(args: Array[String]): Unit = args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) } diff --git a/compiler/test-resources/scripting/scriptPathNu.sc b/compiler/test-resources/scripting/scriptPath_scalacli.sc similarity index 87% rename from compiler/test-resources/scripting/scriptPathNu.sc rename to compiler/test-resources/scripting/scriptPath_scalacli.sc index bb3e459654b9..c13888d0e4b1 100755 --- a/compiler/test-resources/scripting/scriptPathNu.sc +++ b/compiler/test-resources/scripting/scriptPath_scalacli.sc @@ -4,7 +4,7 @@ args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) } -if !scriptPath.endsWith("scriptPathNu.sc") then +if !scriptPath.endsWith("scriptPath_scalacli.sc") then printf( s"incorrect script.path defined as [$scriptPath]") else printf("scriptPath: %s\n", scriptPath) // report the value diff --git a/compiler/test-resources/scripting/showArgs.sc b/compiler/test-resources/scripting/showArgs.sc index 8ef08f8962b0..69d552b9cf5f 100755 --- a/compiler/test-resources/scripting/showArgs.sc +++ b/compiler/test-resources/scripting/showArgs.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env bin/scala +// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class // precise output format expected by BashScriptsTests.scala def main(args: Array[String]): Unit = diff --git a/compiler/test-resources/scripting/showArgsNu.sc b/compiler/test-resources/scripting/showArgs_scalacli.sc similarity index 68% rename from compiler/test-resources/scripting/showArgsNu.sc rename to compiler/test-resources/scripting/showArgs_scalacli.sc index f4c1aa6af257..4591ac159345 100755 --- a/compiler/test-resources/scripting/showArgsNu.sc +++ b/compiler/test-resources/scripting/showArgs_scalacli.sc @@ -1,6 +1,7 @@ #!/usr/bin/env bin/scala +// This file is a Scala CLI script. + // precise output format expected by BashScriptsTests.scala -// MIGRATION: Scala CLI expects `*.sc` files to be straight-line code for (a,i) <- args.zipWithIndex do printf(s"arg %2d:[%s]\n",i,a) diff --git a/compiler/test-resources/scripting/sqlDateError.sc b/compiler/test-resources/scripting/sqlDateError.sc index 35160fd6fcd5..e7c3a623c6c1 100755 --- a/compiler/test-resources/scripting/sqlDateError.sc +++ b/compiler/test-resources/scripting/sqlDateError.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env bin/scala +// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class def main(args: Array[String]): Unit = { println(new java.sql.Date(100L)) diff --git a/compiler/test-resources/scripting/sqlDateErrorNu.sc b/compiler/test-resources/scripting/sqlDateErrorNu.sc deleted file mode 100755 index a6f1bd50297d..000000000000 --- a/compiler/test-resources/scripting/sqlDateErrorNu.sc +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bin/scala - -// def main(args: Array[String]): Unit = { MIGRATION: Scala CLI expects `*.sc` files to be straight-line code - println(new java.sql.Date(100L)) - System.err.println("SCALA_OPTS="+Option(System.getenv("SCALA_OPTS")).getOrElse("")) -// } diff --git a/compiler/test-resources/scripting/sqlDateError_scalacli.sc b/compiler/test-resources/scripting/sqlDateError_scalacli.sc new file mode 100755 index 000000000000..10b58821a6e4 --- /dev/null +++ b/compiler/test-resources/scripting/sqlDateError_scalacli.sc @@ -0,0 +1,6 @@ +#!/usr/bin/env bin/scala + +// This file is a Scala CLI script. + +println(new java.sql.Date(100L)) +System.err.println("SCALA_OPTS="+Option(System.getenv("SCALA_OPTS")).getOrElse("")) diff --git a/compiler/test-resources/scripting/touchFile.sc b/compiler/test-resources/scripting/touchFile.sc index 974f8a64d192..b46b3c99d786 100755 --- a/compiler/test-resources/scripting/touchFile.sc +++ b/compiler/test-resources/scripting/touchFile.sc @@ -1,4 +1,4 @@ -#!/usr/bin/env scala +// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class import java.io.File diff --git a/compiler/test-resources/scripting/unglobClasspath.sc b/compiler/test-resources/scripting/unglobClasspath.sc deleted file mode 100755 index deab2b8982ac..000000000000 --- a/compiler/test-resources/scripting/unglobClasspath.sc +++ /dev/null @@ -1,6 +0,0 @@ -// won't compile unless classpath is set correctly -import dotty.tools.tasty.TastyFormat - -// def main(args: Array[String]) = // MIGRATION: Scala CLI expects `*.sc` files to be straight-line code - val cp = sys.props("java.class.path") - printf("unglobbed classpath: %s\n", cp) diff --git a/compiler/test-resources/scripting/unglobClasspath_scalacli.sc b/compiler/test-resources/scripting/unglobClasspath_scalacli.sc new file mode 100755 index 000000000000..ccc4cf667085 --- /dev/null +++ b/compiler/test-resources/scripting/unglobClasspath_scalacli.sc @@ -0,0 +1,9 @@ +// This file is a Scala CLI script. + +import dotty.tools.tasty.TastyFormat +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// not visible on default classpath, "compiler/test/dotty/tools/scripting/ClasspathTests.scala" +// adds it to classpath via a compiler argument `-classpath 'org/scala-lang/tasty-core_3/$VERSION/*'` + +val cp = sys.props("java.class.path") +printf("unglobbed classpath: %s\n", cp) diff --git a/compiler/test/dotty/tools/io/ClasspathTest.scala b/compiler/test/dotty/tools/io/ClasspathTest.scala index a0fef65afdec..1b5e338b67e4 100755 --- a/compiler/test/dotty/tools/io/ClasspathTest.scala +++ b/compiler/test/dotty/tools/io/ClasspathTest.scala @@ -15,6 +15,8 @@ class ClasspathTest { def pathsep = sys.props("path.separator") + def isWindows: Boolean = scala.util.Properties.isWin + // // Cope with wildcard classpath entries, exercised with -classpath // @@ -23,7 +25,7 @@ class ClasspathTest { @Test def testWildcards(): Unit = val outDir = Files.createTempDirectory("classpath-test") try - val compilerLib = "dist/target/pack/lib" + val compilerLib = s"${if isWindows then "dist-win-x64" else "dist"}/target/pack/lib" val libdir = Paths.get(compilerLib).toFile if libdir.exists then val libjarFiles = libdir.listFiles.toList.take(5) diff --git a/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala b/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala index a06103137f62..a1cfb129ba41 100644 --- a/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala +++ b/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala @@ -16,7 +16,11 @@ import ScriptTestEnv.* class BashExitCodeTests: private var myTmpDir: String | Null = null private lazy val tmpDir = { myTmpDir = Files.createTempDirectory("exit-code-tests").toFile.absPath; myTmpDir } - @After def cleanup(): Unit = if myTmpDir != null then io.Directory(myTmpDir).deleteRecursively() + @After def cleanup(): Unit = { + if myTmpDir != null then io.Directory(myTmpDir).deleteRecursively() + + cleanupScalaCLIDirs() + } /** Verify the exit code of running `cmd args*`. */ def verifyExit(cmd: String, args: String*)(expectedExitCode: Int): Unit = diff --git a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala index 25bc54e2dcbe..6af863f0fccd 100644 --- a/compiler/test/dotty/tools/scripting/BashScriptsTests.scala +++ b/compiler/test/dotty/tools/scripting/BashScriptsTests.scala @@ -25,11 +25,13 @@ object BashScriptsTests: def testFiles = scripts("/scripting") @AfterClass def cleanup: Unit = { + cleanupScalaCLIDirs() + val af = argsfile.toFile - if (af.exists) { + if af.exists then af.delete() - } } + printf("osname[%s]\n", osname) printf("uname[%s]\n", ostypeFull) printf("using JAVA_HOME=%s\n", envJavaHome) @@ -50,7 +52,7 @@ object BashScriptsTests: val testScriptArgs = Seq( "a", "b", "c", "-repl", "-run", "-script", "-debug" ) - val Seq(showArgsScript, showArgsScalaCli) = Seq("showArgs.sc", "showArgsNu.sc").map { name => + val Seq(showArgsScript, showArgsScalaCli) = Seq("showArgs.sc", "showArgs_scalacli.sc").map { name => testFiles.find(_.getName == name).get.absPath } @@ -66,7 +68,7 @@ object BashScriptsTests: } file - val Seq(envtestNuSc, envtestScala) = Seq("envtestNu.sc", "envtest.scala").map { testFile(_) } + val Seq(envtestNuSc, envtestScala) = Seq("envtest_scalacli.sc", "envtest.scala").map { testFile(_) } // create command line with given options, execute specified script, return stdout def callScript(tag: String, script: String, keyPre: String): String = @@ -173,13 +175,13 @@ class BashScriptsTests: assert(stdout == expectedOutput) /* - * verify that scriptPathNu.sc sees a valid script.path property, - * and that it's value is the path to "scriptPathNu.sc". + * verify that scriptPath_scalacli.sc sees a valid script.path property, + * and that it's value is the path to "scriptPath_scalacli.sc". */ @Category(Array(classOf[BootstrappedOnlyTests])) @Test def verifyScriptPathProperty = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - val scriptFile = testFiles.find(_.getName == "scriptPathNu.sc").get + val scriptFile = testFiles.find(_.getName == "scriptPath_scalacli.sc").get val expected = s"${scriptFile.getName}" printf("===> verify valid system property script.path is reported by script [%s]\n", scriptFile.getName) printf("calling scriptFile: %s\n", scriptFile) @@ -196,7 +198,7 @@ class BashScriptsTests: */ @Test def verifyScalaOpts = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - val scriptFile = testFiles.find(_.getName == "classpathReport.sc").get + val scriptFile = testFiles.find(_.getName == "classpathReport_scalacli.sc").get printf("===> verify SCALA_OPTS='@argsfile' is properly handled by `dist/bin/scala`\n") val envPairs = List(("SCALA_OPTS", s"@$argsfile")) val (validTest, exitCode, stdout, stderr) = bashCommand(scriptFile.absPath, envPairs) @@ -219,7 +221,7 @@ class BashScriptsTests: */ @Test def sqlDateTest = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - val scriptBase = "sqlDateErrorNu" + val scriptBase = "sqlDateError_scalacli" val scriptFile = testFiles.find(_.getName == s"$scriptBase.sc").get val testJar = testFile(s"$scriptBase.jar") // jar should not be created when scriptFile runs val tj = Paths.get(testJar).toFile diff --git a/compiler/test/dotty/tools/scripting/ClasspathTests.scala b/compiler/test/dotty/tools/scripting/ClasspathTests.scala index 5107af5eee43..d5f13065ccb3 100755 --- a/compiler/test/dotty/tools/scripting/ClasspathTests.scala +++ b/compiler/test/dotty/tools/scripting/ClasspathTests.scala @@ -11,8 +11,12 @@ import org.junit.{Test, Ignore, AfterClass} import vulpix.TestConfiguration import ScriptTestEnv.* -/** Test java command line generated by bin/scala and bin/scalac */ +object ClasspathTests: + @AfterClass def cleanup: Unit = { + cleanupScalaCLIDirs() + } +/** Test java command line generated by bin/scala and bin/scalac */ class ClasspathTests: /* * Test disabled (temporarily). @@ -24,7 +28,7 @@ class ClasspathTests: @Ignore @Test def hashbangClasspathVerifyTest = { // only interested in classpath test scripts - val testScriptName = "classpathReport.sc" + val testScriptName = "classpathReport_scalacli.sc" val testScript = scripts("/scripting").find { _.getName.matches(testScriptName) } match case None => sys.error(s"test script not found: ${testScriptName}") case Some(file) => file @@ -39,7 +43,7 @@ class ClasspathTests: cmd.foreach { printf("[%s]\n", _) } - // classpathReport.sc is expected to produce two lines: + // classpathReport_scalacli.sc is expected to produce two lines: // cwd: // classpath: @@ -51,10 +55,10 @@ class ClasspathTests: // convert scriptCp to a list of files val hashbangJars: List[File] = scriptCp.split(psep).map { _.toFile }.toList val hashbangClasspathJars = hashbangJars.map { _.name }.sorted.distinct // get jar basenames, remove duplicates - val packlibDir: String = ??? /* ??? was s"$scriptCwd/$packLibDir" */ // classpathReport.sc specifies a wildcard classpath in this directory + val packlibDir: String = ??? /* ??? was s"$scriptCwd/$packLibDir" */ // classpathReport_scalacli.sc specifies a wildcard classpath in this directory val packlibJars: List[File] = listJars(packlibDir) // classpath entries expected to have been reported by the script - printf("%d jar files in dist/target/pack/lib\n", packlibJars.size) + printf(s"%d jar files in $packDir/lib\n", packlibJars.size) printf("%d test script jars in classpath\n", hashbangClasspathJars.size) val (diff: Set[File], msg: String) = if (packlibJars.size > hashbangClasspathJars.size) { @@ -63,7 +67,7 @@ class ClasspathTests: (hashbangJars.toSet -- packlibJars.toSet , "only in hashbang classpath") } // verify that the script hasbang classpath setting was effective at supplementing the classpath - // (a minimal subset of jars below dist/target/pack/lib are always be in the classpath) + // (a minimal subset of jars below dist*/target/pack/lib are always be in the classpath) val missingClasspathEntries = if hashbangClasspathJars.size != packlibJars.size then printf("packlib dir [%s]\n", packlibDir) printf("hashbangClasspathJars: %s\n", hashbangJars.map { _.relpath.norm }.mkString("\n ", "\n ", "")) @@ -78,7 +82,7 @@ class ClasspathTests: * verify classpath is unglobbed by MainGenericRunner. */ @Test def unglobClasspathVerifyTest = { - val testScriptName = "unglobClasspath.sc" + val testScriptName = "unglobClasspath_scalacli.sc" val testScript = scripts("/scripting").find { _.name.matches(testScriptName) } match case None => sys.error(s"test script not found: ${testScriptName}") case Some(file) => file diff --git a/compiler/test/dotty/tools/scripting/ExpressionTest.scala b/compiler/test/dotty/tools/scripting/ExpressionTest.scala index 02963f50ee52..bc42860253b0 100755 --- a/compiler/test/dotty/tools/scripting/ExpressionTest.scala +++ b/compiler/test/dotty/tools/scripting/ExpressionTest.scala @@ -55,6 +55,10 @@ class ExpressionTest: object ExpressionTest: + @AfterClass def cleanup(): Unit = { + cleanupScalaCLIDirs() + } + def main(args: Array[String]): Unit = val tests = new ExpressionTest println("\n=== verifyCommandLineExpression ===") diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index a52014f14704..de205acfc7ae 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -15,7 +15,7 @@ import scala.jdk.CollectionConverters.* /** * Common Code for supporting scripting tests. * To override the path to the bash executable, set TEST_BASH= - * To specify where `dist/target/pack/bin` resides, set TEST_CWD= + * To specify where `dist[*]/target/pack/bin` resides, set TEST_CWD= * Test scripts run in a bash env, so paths are converted to forward slash via .norm. */ object ScriptTestEnv { @@ -28,6 +28,32 @@ object ScriptTestEnv { def whichJava: String = whichExe("java") def whichBash: String = whichExe("bash") + def cleanupScalaCLIDirs(): Unit = { + val scriptingDir = io.Directory(scriptsDir("/scripting").getPath) + val dottyDir = io.Directory(workingDirectory) + + val residueDirs = Seq( + (scriptingDir / ".bsp"), + (scriptingDir / ".scala-build"), + (dottyDir / ".scala-build") + ) + + for f <- residueDirs do + f.deleteRecursively() + + val bspDir = dottyDir / ".bsp" + (bspDir / "scala.json").delete() + if bspDir.isEmpty then bspDir.delete() + } + + lazy val packDir: String = + if winshell then + "dist-win-x64/target/pack" + else + "dist/target/pack" + + def packBinDir: String = s"$packDir/bin" + lazy val workingDirectory: String = { val dirstr = if testCwd.nonEmpty then if verbose then printf("TEST_CWD set to [%s]\n", testCwd) @@ -36,7 +62,7 @@ object ScriptTestEnv { userDir // userDir, if TEST_CWD not set // issue warning if things don't look right - val test = Paths.get(s"$dirstr/dist/target/pack/bin").normalize + val test = Paths.get(s"$dirstr/$packBinDir").normalize if !test.isDirectory then printf("warning: not found below working directory: %s\n", test.norm) @@ -46,7 +72,7 @@ object ScriptTestEnv { def envPath: String = envOrElse("PATH", "") // remove duplicate entries in path - def supplementedPath: String = s"dist/target/pack/bin$psep$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath".norm + def supplementedPath: String = s"$packBinDir$psep$envJavaHome/bin$psep$envScalaHome/bin$psep$envPath".norm def adjustedPathEntries: List[String] = supplementedPath.norm.split(psep).toList.distinct def adjustedPath: String = adjustedPathEntries.mkString(psep) def envPathEntries: List[String] = envPath.split(psep).toList.distinct @@ -124,10 +150,9 @@ object ScriptTestEnv { } yield line - def packBinDir = "dist/target/pack/bin" - // def packLibDir = "dist/target/pack/lib" // replaced by packMavenDir - def packMavenDir = "dist/target/pack/maven2" - def packVersionFile = "dist/target/pack/VERSION" + // def packLibDir = s"$packDir/lib" // replaced by packMavenDir + def packMavenDir = s"$packDir/maven2" + def packVersionFile = s"$packDir/VERSION" def packBinScalaExists: Boolean = Files.exists(Paths.get(s"$packBinDir/scala")) def packScalaVersion: String = { @@ -248,8 +273,8 @@ object ScriptTestEnv { lazy val cwd: Path = Paths.get(".").toAbsolutePath.normalize lazy val (scalacPath: String, scalaPath: String) = { - val scalac = s"$workingDirectory/dist/target/pack/bin/scalac".toPath.normalize - val scala = s"$workingDirectory/dist/target/pack/bin/scala".toPath.normalize + val scalac = s"$workingDirectory/$packBinDir/scalac".toPath.normalize + val scala = s"$workingDirectory/$packBinDir/scala".toPath.normalize (scalac.norm, scala.norm) } @@ -257,7 +282,7 @@ object ScriptTestEnv { // use optional TEST_BASH if defined, otherwise, bash must be in PATH // envScalaHome is: - // dist/target/pack, if present + // dist[*]/target/pack, if present // else, SCALA_HOME if defined // else, not defined lazy val envScalaHome = diff --git a/compiler/test/dotty/tools/scripting/ScriptingTests.scala b/compiler/test/dotty/tools/scripting/ScriptingTests.scala index 713695b62f4a..8d07cb137917 100644 --- a/compiler/test/dotty/tools/scripting/ScriptingTests.scala +++ b/compiler/test/dotty/tools/scripting/ScriptingTests.scala @@ -17,7 +17,11 @@ import org.junit.Assume.assumeFalse /** Runs all tests contained in `compiler/test-resources/scripting/` */ class ScriptingTests: // classpath tests managed by scripting.ClasspathTests.scala - def testFiles = scripts("/scripting").filter { ! _.getName.toLowerCase.contains("classpath") } + def testFiles = scripts("/scripting").filter { sc => + val name = sc.getName.toLowerCase + !name.contains("classpath") + && !name.contains("_scalacli") + } /* * Call .scala scripts without -save option, verify no jar created @@ -47,10 +51,7 @@ class ScriptingTests: */ @Test def scriptingMainTests = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - for - (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") - if !scriptFile.getName().endsWith("Nu.sc") - do + for (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") do showScriptUnderTest(scriptFile) val unexpectedJar = script2jar(scriptFile) unexpectedJar.delete @@ -69,10 +70,7 @@ class ScriptingTests: */ @Test def scriptingJarTest = assumeFalse("Scripts do not yet support Scala 2 library TASTy", Properties.usingScalaLibraryTasty) - for - (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") - if !scriptFile.getName().endsWith("Nu.sc") - do + for (scriptFile, scriptArgs) <- scalaFilesWithArgs(".sc") do showScriptUnderTest(scriptFile) val expectedJar = script2jar(scriptFile) expectedJar.delete diff --git a/compiler/test/dotty/tools/utils.scala b/compiler/test/dotty/tools/utils.scala index a8c480088e08..d17edbaa855e 100644 --- a/compiler/test/dotty/tools/utils.scala +++ b/compiler/test/dotty/tools/utils.scala @@ -20,14 +20,19 @@ import dotc.config.CommandLineParser object Dummy def scripts(path: String): Array[File] = { - val dir = new File(Dummy.getClass.getResource(path).getPath) - assert(dir.exists && dir.isDirectory, "Couldn't load scripts dir") + val dir = scriptsDir(path) dir.listFiles.filter { f => val path = if f.isDirectory then f.getPath + "/" else f.getPath Properties.testsFilter.isEmpty || Properties.testsFilter.exists(path.contains) } } +def scriptsDir(path: String): File = { + val dir = new File(Dummy.getClass.getResource(path).getPath) + assert(dir.exists && dir.isDirectory, "Couldn't load scripts dir") + dir +} + extension (f: File) def absPath = f.getAbsolutePath.replace('\\', '/') @@ -101,10 +106,10 @@ def toolArgsParse(lines: List[String], filename: Option[String]): List[(String,S case toolArg(name, args) => List((name, args)) case _ => Nil } ++ - lines.flatMap { + lines.flatMap { case directiveOptionsArg(args) => List(("scalac", args)) case directiveJavacOptions(args) => List(("javac", args)) - case _ => Nil + case _ => Nil } import org.junit.Test diff --git a/dist/bin-native-overrides/cli-common-platform b/dist/bin-native-overrides/cli-common-platform new file mode 100644 index 000000000000..49803d6282c5 --- /dev/null +++ b/dist/bin-native-overrides/cli-common-platform @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +SCALA_CLI_CMD_BASH=("\"$PROG_HOME/bin/scala-cli\"") diff --git a/dist/bin-native-overrides/cli-common-platform.bat b/dist/bin-native-overrides/cli-common-platform.bat new file mode 100644 index 000000000000..62b6a1029c13 --- /dev/null +++ b/dist/bin-native-overrides/cli-common-platform.bat @@ -0,0 +1,3 @@ +@echo off + +set SCALA_CLI_CMD_WIN="%_PROG_HOME%\bin\scala-cli.exe" \ No newline at end of file diff --git a/dist/bin/cli-common b/dist/bin/cli-common index d295d58916da..eb77c9924d06 100644 --- a/dist/bin/cli-common +++ b/dist/bin/cli-common @@ -71,44 +71,7 @@ if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then esac fi -# Resolve JAVA_HOME from javac command path -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - javaExecutable="`readlink -f \"$javaExecutable\"`" - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "${JAVACMD-}" ] ; then - if [ -n "${JAVA_HOME-}" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." - echo " We cannot execute $JAVACMD" - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi +source "$PROG_HOME/bin/cli-common-platform" CLASSPATH_SUFFIX="" # Path separator used in EXTRA_CLASSPATH @@ -136,23 +99,6 @@ fi # * The code below is for Dotty # *-------------------------------------------------*/ -find_lib () { - for lib in "$PROG_HOME"/lib/$1 ; do - if [[ -f "$lib" ]]; then - if [ -n "$CYGPATHCMD" ]; then - "$CYGPATHCMD" -am "$lib" - elif [[ $mingw || $msys ]]; then - echo "$lib" | sed 's|/|\\\\|g' - else - echo "$lib" - fi - return - fi - done -} - -SCALA_CLI_JAR="$PROG_HOME/etc/scala-cli.jar" - declare -a scala_args addScala () { diff --git a/dist/bin/cli-common-platform b/dist/bin/cli-common-platform new file mode 100644 index 000000000000..ee1b4768dd51 --- /dev/null +++ b/dist/bin/cli-common-platform @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source "$PROG_HOME/bin/common-java" + +SCALA_CLI_CMD_BASH=("\"$JAVACMD\"" "-jar \"$PROG_HOME/bin/scala-cli.jar\"") diff --git a/dist/bin/cli-common-platform.bat b/dist/bin/cli-common-platform.bat new file mode 100644 index 000000000000..99103266c1d9 --- /dev/null +++ b/dist/bin/cli-common-platform.bat @@ -0,0 +1,5 @@ +@echo off + +@rem we need to escape % in the java command path, for some reason this doesnt work in common.bat +set "_JAVACMD=!_JAVACMD:%%=%%%%!" +set SCALA_CLI_CMD_WIN="%_JAVACMD%" "-jar" "%_PROG_HOME%\bin\scala-cli.jar" \ No newline at end of file diff --git a/dist/bin/common b/dist/bin/common index e3e4253938fb..f9dd594c566b 100755 --- a/dist/bin/common +++ b/dist/bin/common @@ -67,44 +67,7 @@ if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then esac fi -# Resolve JAVA_HOME from javac command path -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - javaExecutable="`readlink -f \"$javaExecutable\"`" - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "${JAVACMD-}" ] ; then - if [ -n "${JAVA_HOME-}" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." - echo " We cannot execute $JAVACMD" - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi +source "$PROG_HOME/bin/common-java" CLASSPATH_SUFFIX="" # Path separator used in EXTRA_CLASSPATH diff --git a/dist/bin/common-java b/dist/bin/common-java new file mode 100644 index 000000000000..fbc8f309543c --- /dev/null +++ b/dist/bin/common-java @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Resolve JAVA_HOME from javac command path +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + javaExecutable="`readlink -f \"$javaExecutable\"`" + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "${JAVACMD-}" ] ; then + if [ -n "${JAVA_HOME-}" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." + echo " We cannot execute $JAVACMD" + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi diff --git a/dist/bin/scala b/dist/bin/scala index 3040c5a9a0f3..6b5930e3c215 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -54,8 +54,9 @@ done # exec here would prevent onExit from being called, leaving terminal in unusable state [ -z "${ConEmuPID-}" -o -n "${cygwin-}" ] && export MSYSTEM= PWD= # workaround for #12405 -eval "\"$JAVACMD\"" \ - "-jar \"$SCALA_CLI_JAR\"" \ + +# SCALA_CLI_CMD_BASH is an array, set by cli-common-platform +eval "${SCALA_CLI_CMD_BASH[@]}" \ "--prog-name scala" \ "--cli-default-scala-version \"$SCALA_VERSION\"" \ "-r \"$MVN_REPOSITORY\"" \ diff --git a/dist/bin/scala.bat b/dist/bin/scala.bat index 78336272055b..8b4c08da553f 100644 --- a/dist/bin/scala.bat +++ b/dist/bin/scala.bat @@ -19,10 +19,9 @@ if not %_EXITCODE%==0 goto end call :setScalaOpts -@rem we need to escape % in the java command path, for some reason this doesnt work in common.bat -set "_JAVACMD=!_JAVACMD:%%=%%%%!" +@rem SCALA_CLI_CMD_WIN is an array, set in cli-common-platform.bat +call %SCALA_CLI_CMD_WIN% "--prog-name" "scala" "--cli-default-scala-version" "%_SCALA_VERSION%" "-r" "%MVN_REPOSITORY%" %* -call "%_JAVACMD%" "-jar" "%SCALA_CLI_JAR%" "--prog-name" "scala" "--cli-default-scala-version" "%_SCALA_VERSION%" "-r" "%MVN_REPOSITORY%" %* if not %ERRORLEVEL%==0 ( set _EXITCODE=1& goto end ) goto end @@ -54,7 +53,8 @@ set "_PROG_HOME_DRIVE=!_PROG_HOME:~0,%index%!" set "_SCALA_VERSION=" set "MVN_REPOSITORY=file://%_PROG_HOME_DRIVE%\%_PROG_HOME_SUB:\=/%/maven2" -set "SCALA_CLI_JAR=%_PROG_HOME%\etc\scala-cli.jar" + +call "%_PROG_HOME%\bin\cli-common-platform.bat" @rem read for version:=_SCALA_VERSION in VERSION_FILE FOR /F "usebackq delims=" %%G IN ("%_PROG_HOME%\VERSION") DO ( diff --git a/project/Build.scala b/project/Build.scala index 232c5f24d13d..a6cd0c302ecf 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -86,10 +86,7 @@ object DottyJSPlugin extends AutoPlugin { object Build { import ScaladocConfigs._ - - val referenceVersion = "3.4.2-RC1" - - val baseVersion = "3.5.1-RC1" + import versionhelpers.DottyVersion._ // LTS or Next val versionLine = "Next" @@ -158,24 +155,6 @@ object Build { val dottyGithubUrl = "https://github.com/scala/scala3" val dottyGithubRawUserContentUrl = "https://raw.githubusercontent.com/scala/scala3" - - val isRelease = sys.env.get("RELEASEBUILD") == Some("yes") - - val dottyVersion = { - def isNightly = sys.env.get("NIGHTLYBUILD") == Some("yes") - if (isRelease) - baseVersion - else if (isNightly) - baseVersion + "-bin-" + VersionUtil.commitDate + "-" + VersionUtil.gitHash + "-NIGHTLY" - else - baseVersion + "-bin-SNAPSHOT" - } - val dottyNonBootstrappedVersion = { - // Make sure sbt always computes the scalaBinaryVersion correctly - val bin = if (!dottyVersion.contains("-bin")) "-bin" else "" - dottyVersion + bin + "-nonbootstrapped" - } - val sbtCommunityBuildVersion = "0.1.0-SNAPSHOT" val agentOptions = List( @@ -627,7 +606,7 @@ object Build { val contents = //2.11.11.v20170413-090219-8a413ba7cc s"""version.number=${version.value} |maven.version.number=${version.value} - |git.hash=${VersionUtil.gitHash} + |git.hash=${versionhelpers.VersionUtil.gitHash} |copyright.string=Copyright 2002-$currentYear, LAMP/EPFL """.stripMargin @@ -667,7 +646,7 @@ object Build { // Spawn new JVM in run and test // Add git-hash used to package the distribution to the manifest to know it in runtime and report it in REPL - packageOptions += ManifestAttributes(("Git-Hash", VersionUtil.gitHash)), + packageOptions += ManifestAttributes(("Git-Hash", versionhelpers.VersionUtil.gitHash)), javaOptions ++= { val log = streams.value.log @@ -2121,22 +2100,63 @@ object Build { packMain := Map(), publishArtifact := false, packGenerateMakefile := false, - packArchiveName := "scala3-" + dottyVersion, republishRepo := target.value / "republish", - republishLaunchers := { - val cliV = scalaCliLauncherVersion - Seq( - ("scala-cli.jar", cliV, url(s"https://github.com/VirtusLab/scala-cli/releases/download/v$cliV/scala-cli.jar")) - ) - }, + packResourceDir += (republishRepo.value / "bin" -> "bin"), + packResourceDir += (republishRepo.value / "maven2" -> "maven2"), Compile / pack := (Compile / pack).dependsOn(republish).value, ) lazy val dist = project.asDist(Bootstrapped) .settings( - packResourceDir += (baseDirectory.value / "bin" -> "bin"), - packResourceDir += (republishRepo.value / "maven2" -> "maven2"), - packResourceDir += (republishRepo.value / "etc" -> "etc"), + packArchiveName := "scala3-" + dottyVersion, + republishBinDir := baseDirectory.value / "bin", + republishLaunchers += + ("scala-cli.jar" -> s"https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli.jar") + ) + + lazy val `dist-mac-x64` = project.in(file("dist/mac-x64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-x86_64-apple-darwin", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-apple-darwin.gz") + ) + + lazy val `dist-mac-aarch64` = project.in(file("dist/mac-aarch64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-aarch64-apple-darwin", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-aarch64-apple-darwin.gz") + ) + + lazy val `dist-win-x64` = project.in(file("dist/win-x64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-win32", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli.exe" -> s"zip+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-pc-win32.zip!/scala-cli.exe") + ) + + lazy val `dist-linux-x64` = project.in(file("dist/linux-x64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-linux", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-pc-linux.gz") + ) + + lazy val `dist-linux-aarch64` = project.in(file("dist/linux-aarch64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-aarch64-pc-linux", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-aarch64-pc-linux.gz") ) private def customMimaReportBinaryIssues(issueFilterLocation: String) = mimaReportBinaryIssues := { @@ -2297,6 +2317,7 @@ object Build { object ScaladocConfigs { import Build._ + import versionhelpers.DottyVersion._ private lazy val currentYear: String = java.util.Calendar.getInstance().get(java.util.Calendar.YEAR).toString def dottyExternalMapping = ".*scala/.*::scaladoc3::https://dotty.epfl.ch/api/" @@ -2376,7 +2397,7 @@ object ScaladocConfigs { .add(UseJavacp(true)) .add(ProjectName("scaladoc")) .add(OutputDir("scaladoc/output/self")) - .add(Revision(VersionUtil.gitHash)) + .add(Revision(versionhelpers.VersionUtil.gitHash)) .add(ExternalMappings(List(dottyExternalMapping, javaExternalMapping))) .withTargets((Compile / classDirectory).value.getAbsolutePath :: Nil) } diff --git a/project/DottyVersion.scala b/project/DottyVersion.scala new file mode 100644 index 000000000000..393cca1a251b --- /dev/null +++ b/project/DottyVersion.scala @@ -0,0 +1,24 @@ +package versionhelpers + +object DottyVersion { + val isRelease = sys.env.get("RELEASEBUILD") == Some("yes") + + val referenceVersion = "3.4.2-RC1" + + val baseVersion = "3.5.1-RC1" + + val dottyVersion = { + def isNightly = sys.env.get("NIGHTLYBUILD") == Some("yes") + if (isRelease) + baseVersion + else if (isNightly) + baseVersion + "-bin-" + VersionUtil.commitDate + "-" + VersionUtil.gitHash + "-NIGHTLY" + else + baseVersion + "-bin-SNAPSHOT" + } + val dottyNonBootstrappedVersion = { + // Make sure sbt always computes the scalaBinaryVersion correctly + val bin = if (!dottyVersion.contains("-bin")) "-bin" else "" + dottyVersion + bin + "-nonbootstrapped" + } +} diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala index 0b71c9ecb6df..a74fd0f9a6fb 100644 --- a/project/RepublishPlugin.scala +++ b/project/RepublishPlugin.scala @@ -11,10 +11,46 @@ import sbt.util.CacheImplicits._ import scala.collection.mutable import java.nio.file.Files +import versionhelpers.DottyVersion._ + +import java.nio.file.attribute.PosixFilePermission +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ /** This local plugin provides ways of publishing a project classpath and library dependencies to * .a local repository */ object RepublishPlugin extends AutoPlugin { + + private object FileUtil { + + def tryMakeExecutable(path: Path): Boolean = + try { + val perms = Files.getPosixFilePermissions(path).asScala.toSet + + var newPerms = perms + if (perms(PosixFilePermission.OWNER_READ)) + newPerms += PosixFilePermission.OWNER_EXECUTE + if (perms(PosixFilePermission.GROUP_READ)) + newPerms += PosixFilePermission.GROUP_EXECUTE + if (perms(PosixFilePermission.OTHERS_READ)) + newPerms += PosixFilePermission.OTHERS_EXECUTE + + if (newPerms != perms) + Files.setPosixFilePermissions( + path, + newPerms.asJava + ) + + true + } + catch { + case _: UnsupportedOperationException => + false + } + + } + override def trigger = allRequirements override def requires = super.requires && PublishBinPlugin && PackPlugin @@ -24,9 +60,12 @@ object RepublishPlugin extends AutoPlugin { val republishAllResolved = taskKey[Seq[ResolvedArtifacts]]("Resolve the dependencies for the distribution") val republishClasspath = taskKey[Set[File]]("cache the dependencies for the distribution") val republishFetchLaunchers = taskKey[Set[File]]("cache the launcher deps for the distribution") + val republishPrepareBin = taskKey[File]("prepare the bin directory, including launchers and scripts.") + val republishBinDir = settingKey[File]("where to find static files for the bin dir.") + val republishBinOverrides = settingKey[Seq[File]]("files to override those in bin-dir.") val republish = taskKey[File]("cache the dependencies and download launchers for the distribution") val republishRepo = settingKey[File]("the location to store the republished artifacts.") - val republishLaunchers = settingKey[Seq[(String, String, URL)]]("launchers to download. Sequence of (name, version, URL).") + val republishLaunchers = settingKey[Seq[(String, String)]]("launchers to download. Sequence of (name, version, URL).") } import autoImport._ @@ -37,6 +76,8 @@ object RepublishPlugin extends AutoPlugin { case class ResolvedArtifacts(id: SimpleModuleId, jar: File, pom: File) override val projectSettings: Seq[Def.Setting[_]] = Def.settings( + republishLaunchers := Seq.empty, + republishBinOverrides := Seq.empty, republishLocalResolved / republishProjectRefs := { val proj = thisProjectRef.value val deps = buildDependencies.value @@ -87,7 +128,9 @@ object RepublishPlugin extends AutoPlugin { localResolved.foreach({ resolved => val simpleId = resolved.id - evicted += simpleId.copy(revision = simpleId.revision + "-nonbootstrapped") + if (simpleId.revision == dottyVersion) { + evicted += simpleId.copy(revision = dottyNonBootstrappedVersion) + } found(simpleId) = resolved }) @@ -147,42 +190,94 @@ object RepublishPlugin extends AutoPlugin { val log = s.log val repoDir = republishRepo.value val launcherVersions = republishLaunchers.value + val libexec = republishPrepareBin.value - val etc = repoDir / "etc" + val dlCache = repoDir / "cache" val store = s.cacheStoreFactory / "versions" - def work(dest: File, launcher: URL) = { - IO.delete(dest) - Using.urlInputStream(launcher) { in => - IO.createDirectory(etc) - log.info(s"[republish] Downloading $launcher to $dest...") - IO.transfer(in, dest) - log.info(s"[republish] Downloaded $launcher to $dest...") + def work(name: String, dest: File, launcher: String): File = { + val (launcherURL, workFile, prefix, subPart) = { + if (launcher.startsWith("gz+")) { + IO.createDirectory(dlCache) + val launcherURL = url(launcher.stripPrefix("gz+")) + (launcherURL, dlCache / s"$name.gz", "gz", "") + } else if (launcher.startsWith("zip+")) { + IO.createDirectory(dlCache) + val (urlPart, subPath) = launcher.split("!/") match { + case Array(urlPart, subPath) => (urlPart, subPath) + case _ => + throw new MessageOnlyException(s"[republish] Invalid zip+ URL, expected ! to mark subpath: $launcher") + } + val launcherURL = url(urlPart.stripPrefix("zip+")) + (launcherURL, dlCache / s"$name.zip", "zip", subPath) + } else { + IO.createDirectory(libexec) + (url(launcher), dest, "", "") + } + } + IO.delete(workFile) + Using.urlInputStream(launcherURL) { in => + log.info(s"[republish] Downloading $launcherURL to $workFile...") + IO.transfer(in, workFile) + log.info(s"[republish] Downloaded $launcherURL to $workFile...") + } + if (prefix == "gz") { + IO.delete(dest) + Using.fileInputStream(workFile) { in => + Using.gzipInputStream(in) { gzIn => + IO.transfer(gzIn, dest) + } + } + log.info(s"[republish] uncompressed gz file $workFile to $dest...") + FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable + IO.delete(workFile) + } else if (prefix == "zip") { + IO.delete(dest) + val files = IO.unzip(workFile, dlCache, new ExactFilter(subPart)) + val extracted = files.headOption.getOrElse(throw new MessageOnlyException(s"[republish] No files extracted from $workFile matching $subPart")) + log.info(s"[republish] unzipped $workFile to $extracted...") + IO.move(extracted, dest) + log.info(s"[republish] moved $extracted to $dest...") + FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable + IO.delete(workFile) } dest } val allLaunchers = { - for ((name, version, launcher) <- launcherVersions) yield { - val dest = etc / name + for ((name, launcher) <- launcherVersions) yield { + val dest = libexec / name val id = name.replaceAll("[^a-zA-Z0-9]", "_") - val fetchAction = Tracked.inputChanged[String, File](store.make(id)) { (inChanged, version) => + val fetchAction = Tracked.inputChanged[String, File](store.make(id)) { (inChanged, launcher) => if (inChanged || !Files.exists(dest.toPath)) { - work(dest, launcher) + work(name, dest, launcher) } else { log.info(s"[republish] Using cached $launcher at $dest...") dest } } - fetchAction(version) + fetchAction(launcher) } } allLaunchers.toSet }, + republishPrepareBin := { + val baseDir = baseDirectory.value + val srcBin = republishBinDir.value + val overrides = republishBinOverrides.value + val repoDir = republishRepo.value + + val targetBin = repoDir / "bin" + IO.copyDirectory(srcBin, targetBin) + overrides.foreach { dir => + IO.copyDirectory(dir, targetBin, overwrite = true) + } + targetBin + }, republish := { val cacheDir = republishRepo.value val artifacts = republishClasspath.value diff --git a/project/VersionUtil.scala b/project/VersionUtil.scala index 1d385d75631d..cef73a66ddc6 100644 --- a/project/VersionUtil.scala +++ b/project/VersionUtil.scala @@ -1,3 +1,5 @@ +package versionhelpers + import org.eclipse.jgit.storage.file.FileRepositoryBuilder import org.eclipse.jgit.lib.{Constants, ObjectId, Ref, Repository} import org.eclipse.jgit.revwalk.{RevCommit, RevWalk} diff --git a/project/scripts/bootstrappedOnlyCmdTests b/project/scripts/bootstrappedOnlyCmdTests index f3d730f8f494..fc1ff89f560d 100755 --- a/project/scripts/bootstrappedOnlyCmdTests +++ b/project/scripts/bootstrappedOnlyCmdTests @@ -101,6 +101,13 @@ grep -qe "See 'scala --help' to read about a specific subcommand." "$t ./bin/scala -d hello.jar tests/run/hello.scala ls hello.jar +clear_cli_dotfiles tests/run + +# check that `scala` runs scripts with args +echo "testing ./bin/scala with arguments" +./bin/scala run project/scripts/echoArgs.sc -- abc true 123 > "$tmp" +test "$EXPECTED_OUTPUT_ARGS" = "$(cat "$tmp")" +clear_cli_dotfiles project/scripts echo "testing i12973" clear_out "$OUT" diff --git a/project/scripts/cmdTestsCommon.inc.sh b/project/scripts/cmdTestsCommon.inc.sh index a37ab757c057..d041613e46a8 100644 --- a/project/scripts/cmdTestsCommon.inc.sh +++ b/project/scripts/cmdTestsCommon.inc.sh @@ -9,6 +9,7 @@ SOURCE="tests/pos/HelloWorld.scala" MAIN="HelloWorld" TASTY="HelloWorld.tasty" EXPECTED_OUTPUT="hello world" +EXPECTED_OUTPUT_ARGS="[0:abc],[1:true],[2:123]" OUT=$(mktemp -d) OUT1=$(mktemp -d) @@ -24,3 +25,16 @@ clear_out() local out="$1" rm -rf "$out"/* } + +clear_cli_dotfiles() +{ + local out="$1" + rm -rf "$out"/.bsp + rm -rf "$out"/.scala-build + + rm -f "$ROOT"/.bsp/scala.json + if [ -z "$(ls -A "$ROOT"/.bsp)" ]; then + rm -rf "$ROOT"/.bsp + fi + rm -rf "$ROOT"/.scala-build +} diff --git a/project/scripts/echoArgs.sc b/project/scripts/echoArgs.sc new file mode 100644 index 000000000000..cb9acbb6ad2e --- /dev/null +++ b/project/scripts/echoArgs.sc @@ -0,0 +1,6 @@ +// This is a Scala CLI script + +val formatted = + (for (arg, i) <- args.zipWithIndex yield + s"[$i:$arg]").mkString(",") +println(formatted) diff --git a/project/scripts/winCmdTests b/project/scripts/winCmdTests index 2dffff5b196a..54519f9a318d 100644 --- a/project/scripts/winCmdTests +++ b/project/scripts/winCmdTests @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -PREFIX="dist/target/pack" +PREFIX="dist/win-x64/target/pack" SOURCE="tests/pos/HelloWorld.scala" $PREFIX/bin/scalac @project/scripts/options "$SOURCE" $PREFIX/bin/scalac -d out "$SOURCE" diff --git a/project/scripts/winCmdTests.bat b/project/scripts/winCmdTests.bat index d9b594d560ab..8ee1cbe4be68 100644 --- a/project/scripts/winCmdTests.bat +++ b/project/scripts/winCmdTests.bat @@ -2,7 +2,7 @@ setlocal @rem paths are relative to the root project directory -set "_PREFIX=dist\target\pack" +set "_PREFIX=dist\win-x64\target\pack" set "_SOURCE=tests\pos\HelloWorld.scala" set "_OUT_DIR=out" set "_SITE_DIR=_site" From f6c27b684895e4d3d06922be434cdff32130beb6 Mon Sep 17 00:00:00 2001 From: Hamza REMMAL Date: Thu, 23 May 2024 10:07:51 +0200 Subject: [PATCH 3/9] Add GitHub Workflow to test the launchers --- .github/workflows/launchers.yml | 89 +++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/launchers.yml diff --git a/.github/workflows/launchers.yml b/.github/workflows/launchers.yml new file mode 100644 index 000000000000..1e9ffdc7a38f --- /dev/null +++ b/.github/workflows/launchers.yml @@ -0,0 +1,89 @@ +name: Test CLI Launchers on all the platforms +on: + push: + workflow_dispatch: + +jobs: + linux-x64: + name: Deploy and Test on Linux x64 architecture + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'sbt' + - name: Build the launcher command + run: sbt "dist-linux-x64/pack" + - run: ./dist/linux-x64/target/pack/bin/scala --version + + linux-aarch64: + name: Deploy and Test on Linux ARM64 architecture + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'sbt' + # https://github.com/actions/runner-images/issues/9369 + - name: Install sbt + run: brew install sbt + - name: Build the launcher command + run: sbt "dist-linux-aarch64/pack" + - run: ./dist/linux-aarch64/target/pack/bin/scala --version + + mac-x64: + name: Deploy and Test on Mac x64 architecture + runs-on: macos-13 + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'sbt' + # https://github.com/actions/runner-images/issues/9369 + - name: Install sbt + run: brew install sbt + - name: Build the launcher command + run: sbt "dist-mac-x64/pack" + - run: ./dist/mac-x64/target/pack/bin/scala --version + + mac-aarch64: + name: Deploy and Test on Mac ARM64 architecture + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'sbt' + # https://github.com/actions/runner-images/issues/9369 + - name: Install sbt + run: brew install sbt + - name: Build the launcher command + run: sbt "dist-mac-aarch64/pack" + - run: ./dist/mac-aarch64/target/pack/bin/scala --version + + win-x64: + name: Deploy and Test on Windows x64 architecture + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'sbt' + - name: Build the launcher command + run: sbt "dist-win-x64/pack" + - run: ./dist/win-x64/target/pack/bin/scala --version From 0bf7e752ddac228f31b9de9d7341ae6376a9083b Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Fri, 24 May 2024 18:24:44 +0200 Subject: [PATCH 4/9] use coursier.jar launcher subprocess to fetch instead. --- project/Build.scala | 14 +- project/RepublishPlugin.scala | 371 +++++++++++++++++++++------------- 2 files changed, 245 insertions(+), 140 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index a6cd0c302ecf..464ed3ab1266 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -116,7 +116,8 @@ object Build { val mimaPreviousLTSDottyVersion = "3.3.0" /** Version of Scala CLI to download */ - val scalaCliLauncherVersion = "1.3.1" + val scalaCliLauncherVersion = "1.3.2" + val coursierJarVersion = "2.1.10" object CompatMode { final val BinaryCompatible = 0 @@ -2110,8 +2111,10 @@ object Build { .settings( packArchiveName := "scala3-" + dottyVersion, republishBinDir := baseDirectory.value / "bin", - republishLaunchers += - ("scala-cli.jar" -> s"https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli.jar") + republishLaunchers ++= Seq( + ("scala-cli.jar" -> s"https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli.jar"), + ("coursier.jar" -> s"https://github.com/coursier/coursier/releases/download/v$coursierJarVersion/coursier.jar") + ) ) lazy val `dist-mac-x64` = project.in(file("dist/mac-x64")).asDist(Bootstrapped) @@ -2119,6 +2122,7 @@ object Build { republishBinDir := (dist / republishBinDir).value, packArchiveName := (dist / packArchiveName).value + "-x86_64-apple-darwin", republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishFetchCoursier := (dist / republishFetchCoursier).value, republishLaunchers += ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-apple-darwin.gz") ) @@ -2128,6 +2132,7 @@ object Build { republishBinDir := (dist / republishBinDir).value, packArchiveName := (dist / packArchiveName).value + "-aarch64-apple-darwin", republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishFetchCoursier := (dist / republishFetchCoursier).value, republishLaunchers += ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-aarch64-apple-darwin.gz") ) @@ -2137,6 +2142,7 @@ object Build { republishBinDir := (dist / republishBinDir).value, packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-win32", republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishFetchCoursier := (dist / republishFetchCoursier).value, republishLaunchers += ("scala-cli.exe" -> s"zip+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-pc-win32.zip!/scala-cli.exe") ) @@ -2146,6 +2152,7 @@ object Build { republishBinDir := (dist / republishBinDir).value, packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-linux", republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishFetchCoursier := (dist / republishFetchCoursier).value, republishLaunchers += ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-pc-linux.gz") ) @@ -2155,6 +2162,7 @@ object Build { republishBinDir := (dist / republishBinDir).value, packArchiveName := (dist / packArchiveName).value + "-aarch64-pc-linux", republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishFetchCoursier := (dist / republishFetchCoursier).value, republishLaunchers += ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-aarch64-pc-linux.gz") ) diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala index a74fd0f9a6fb..a5a8115b4b51 100644 --- a/project/RepublishPlugin.scala +++ b/project/RepublishPlugin.scala @@ -22,6 +22,7 @@ import scala.jdk.CollectionConverters._ * .a local repository */ object RepublishPlugin extends AutoPlugin { + /** copied from github.com/coursier/coursier */ private object FileUtil { def tryMakeExecutable(path: Path): Boolean = @@ -60,8 +61,10 @@ object RepublishPlugin extends AutoPlugin { val republishAllResolved = taskKey[Seq[ResolvedArtifacts]]("Resolve the dependencies for the distribution") val republishClasspath = taskKey[Set[File]]("cache the dependencies for the distribution") val republishFetchLaunchers = taskKey[Set[File]]("cache the launcher deps for the distribution") + val republishFetchCoursier = taskKey[File]("cache the coursier.jar for resolving the local maven repo.") val republishPrepareBin = taskKey[File]("prepare the bin directory, including launchers and scripts.") val republishBinDir = settingKey[File]("where to find static files for the bin dir.") + val republishCoursierDir = settingKey[File]("where to download the coursier launcher jar.") val republishBinOverrides = settingKey[Seq[File]]("files to override those in bin-dir.") val republish = taskKey[File]("cache the dependencies and download launchers for the distribution") val republishRepo = settingKey[File]("the location to store the republished artifacts.") @@ -73,9 +76,213 @@ object RepublishPlugin extends AutoPlugin { case class SimpleModuleId(org: String, name: String, revision: String) { override def toString = s"$org:$name:$revision" } - case class ResolvedArtifacts(id: SimpleModuleId, jar: File, pom: File) + case class ResolvedArtifacts(id: SimpleModuleId, jar: Option[File], pom: Option[File]) + + class NameFilter(kind: String, filter: String => Boolean) { + def apply(name: String): Boolean = filter(name) + override def toString = kind + } + + object NameFilter { + final val Coursier = new NameFilter("coursier", _ == "coursier.jar") + final val Generic = new NameFilter("generic", _ != "coursier.jar") + } + + private def republishResolvedArtifacts(resolved: Seq[ResolvedArtifacts], mavenRepo: File, logOpt: Option[Logger]): Set[File] = { + IO.createDirectory(mavenRepo) + resolved.map { ra => + for (log <- logOpt) + log.info(s"[republish] publishing ${ra.id} to $mavenRepo...") + val jarOpt = ra.jar + val pomOpt = ra.pom + + assert(jarOpt.nonEmpty || pomOpt.nonEmpty, s"Neither jar nor pom found for ${ra.id}") + + val pathElems = ra.id.org.split('.').toVector :+ ra.id.name :+ ra.id.revision + val artifactDir = pathElems.foldLeft(mavenRepo)(_ / _) + IO.createDirectory(artifactDir) + for (pom <- pomOpt) IO.copyFile(pom, artifactDir / pom.getName) + for (jar <- jarOpt) IO.copyFile(jar, artifactDir / jar.getName) + artifactDir + }.toSet + } + + private def coursierCmd(jar: File, cache: File, log: Logger, args: Seq[String]): Unit = { + val jar0 = jar.getAbsolutePath.toString + val javaHome = sys.props.get("java.home").getOrElse { + throw new MessageOnlyException("java.home property not set") + } + val javaCmd = { + val cmd = if (scala.util.Properties.isWin) "java.exe" else "java" + (file(javaHome) / "bin" / cmd).getAbsolutePath + } + val env = Map("COURSIER_CACHE" -> cache.getAbsolutePath.toString) + val cmdLine = Seq(javaCmd, "-jar", jar0) ++ args + // invoke cmdLine with env + val p = new ProcessBuilder(cmdLine: _*).inheritIO() + p.environment().putAll(env.asJava) + val proc = p.start() + log.info(s"[republish] Running with env ${env}: coursier.jar with args ${args.mkString(" ")}") + proc.waitFor() + if (proc.exitValue() != 0) + throw new MessageOnlyException(s"Error running coursier.jar with args ${args.mkString(" ")}") + } + + private def coursierFetch(coursierJar: File, log: Logger, cacheDir: File, localRepo: File, libs: Seq[String]): Unit = { + val localRepoArg = { + val path = localRepo.getAbsolutePath + if (scala.util.Properties.isWin) { + val path0 = path.replace('\\', '/') + s"file:///$path0" // extra root slash for Windows paths + } + else + s"file://$path" + } + + IO.createDirectory(cacheDir) + for (lib <- libs) { + log.info(s"[republish] Fetching $lib with coursier.jar...") + coursierCmd(coursierJar, cacheDir, log, + Seq( + "fetch", + "--repository", localRepoArg, + lib + ) + ) + } + } + + /**Resolve the transitive library dependencies of `libs` to `csrCacheDir`. + */ + private def resolveLibraryDeps( + coursierJar: File, + log: Logger, + csrCacheDir: File, + localRepo: File, + resolvedLocal: Seq[ResolvedArtifacts]): Seq[ResolvedArtifacts] = { + + // publish the local artifacts to the local repo, so coursier can resolve them + republishResolvedArtifacts(resolvedLocal, localRepo, logOpt = None) + + coursierFetch(coursierJar, log, csrCacheDir, localRepo, resolvedLocal.map(_.id.toString)) + + val maven2Root = java.nio.file.Files.walk(csrCacheDir.toPath) + .filter(_.getFileName.toString == "maven2") + .findFirst() + .orElseThrow(() => new MessageOnlyException(s"Could not find maven2 directory in $csrCacheDir")) + + def pathToArtifact(p: Path): ResolvedArtifacts = { + // relative path from maven2Root + val lastAsString = p.getFileName.toString + val relP = maven2Root.relativize(p) + val parts = relP.iterator().asScala.map(_.toString).toVector + val (orgParts :+ name :+ rev :+ _) = parts + val id = SimpleModuleId(orgParts.mkString("."), name, rev) + if (lastAsString.endsWith(".jar")) { + ResolvedArtifacts(id, Some(p.toFile), None) + } else { + ResolvedArtifacts(id, None, Some(p.toFile)) + } + } + + java.nio.file.Files.walk(maven2Root) + .filter(p => { + val lastAsString = p.getFileName.toString + lastAsString.endsWith(".pom") || lastAsString.endsWith(".jar") + }) + .map[ResolvedArtifacts](pathToArtifact(_)) + .iterator() + .asScala + .toSeq + } + + private def fetchFilesTask(libexecT: Def.Initialize[Task[File]], nameFilter: NameFilter) = Def.task[Set[File]] { + val s = streams.value + val log = s.log + val repoDir = republishRepo.value + val launcherVersions = republishLaunchers.value + val libexec = libexecT.value + + val dlCache = s.cacheDirectory / "republish-launchers" + + val store = s.cacheStoreFactory / "versions" + + def work(name: String, dest: File, launcher: String): File = { + val (launcherURL, workFile, prefix, subPart) = { + if (launcher.startsWith("gz+")) { + IO.createDirectory(dlCache) + val launcherURL = url(launcher.stripPrefix("gz+")) + (launcherURL, dlCache / s"$name.gz", "gz", "") + } else if (launcher.startsWith("zip+")) { + IO.createDirectory(dlCache) + val (urlPart, subPath) = launcher.split("!/") match { + case Array(urlPart, subPath) => (urlPart, subPath) + case _ => + throw new MessageOnlyException(s"[republish] Invalid zip+ URL, expected ! to mark subpath: $launcher") + } + val launcherURL = url(urlPart.stripPrefix("zip+")) + (launcherURL, dlCache / s"$name.zip", "zip", subPath) + } else { + IO.createDirectory(libexec) + (url(launcher), dest, "", "") + } + } + IO.delete(workFile) + Using.urlInputStream(launcherURL) { in => + log.info(s"[republish] Downloading $launcherURL to $workFile...") + IO.transfer(in, workFile) + log.info(s"[republish] Downloaded $launcherURL to $workFile...") + } + if (prefix == "gz") { + IO.delete(dest) + Using.fileInputStream(workFile) { in => + Using.gzipInputStream(in) { gzIn => + IO.transfer(gzIn, dest) + } + } + log.info(s"[republish] uncompressed gz file $workFile to $dest...") + FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable + IO.delete(workFile) + } else if (prefix == "zip") { + IO.delete(dest) + val files = IO.unzip(workFile, dlCache, new ExactFilter(subPart)) + val extracted = files.headOption.getOrElse(throw new MessageOnlyException(s"[republish] No files extracted from $workFile matching $subPart")) + log.info(s"[republish] unzipped $workFile to $extracted...") + IO.move(extracted, dest) + log.info(s"[republish] moved $extracted to $dest...") + FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable + IO.delete(workFile) + } + dest + } + + val allLaunchers = { + val filtered = launcherVersions.filter { case (name, _) => nameFilter(name) } + if (filtered.isEmpty) + throw new MessageOnlyException(s"[republish] No $nameFilter launchers to fetch, check the build configuration for ${republishLaunchers.key.label}.") + + for ((name, launcher) <- filtered) yield { + val dest = libexec / name + + val id = name.replaceAll("[^a-zA-Z0-9]", "_") + + val fetchAction = Tracked.inputChanged[String, File](store.make(id)) { (inChanged, launcher) => + if (inChanged || !Files.exists(dest.toPath)) { + work(name, dest, launcher) + } else { + log.info(s"[republish] Using cached $name launcher ($launcher).") + dest + } + } + + fetchAction(launcher) + } + } + allLaunchers.toSet + } override val projectSettings: Seq[Def.Setting[_]] = Def.settings( + republishCoursierDir := republishRepo.value / "coursier", republishLaunchers := Seq.empty, republishBinOverrides := Seq.empty, republishLocalResolved / republishProjectRefs := { @@ -94,7 +301,6 @@ object RepublishPlugin extends AutoPlugin { ids.zip(published).map({ case (id, as) => val simpleId = { - val disabled = CrossVersion.disabled val name0 = id.crossVersion match { case cv: CrossVersion.Binary => // projectID does not add binary suffix @@ -115,155 +321,46 @@ object RepublishPlugin extends AutoPlugin { }) assert(jarOrNull != null, s"Could not find jar for ${id}") assert(pomOrNull != null, s"Could not find pom for ${id}") - ResolvedArtifacts(simpleId, jarOrNull, pomOrNull) + ResolvedArtifacts(simpleId, Some(jarOrNull), Some(pomOrNull)) }) } }.value, republishAllResolved := { - val localResolved = republishLocalResolved.value + val resolvedLocal = republishLocalResolved.value + val coursierJar = republishFetchCoursier.value val report = (thisProjectRef / updateFull).value + val s = streams.value + val lm = (republishAllResolved / dependencyResolution).value + val cacheDir = republishRepo.value - val found = mutable.Map.empty[SimpleModuleId, ResolvedArtifacts] - val evicted = mutable.Set.empty[SimpleModuleId] - - localResolved.foreach({ resolved => - val simpleId = resolved.id - if (simpleId.revision == dottyVersion) { - evicted += simpleId.copy(revision = dottyNonBootstrappedVersion) - } - found(simpleId) = resolved - }) - - report.allModuleReports.foreach { mr => - val simpleId = { - val id = mr.module - SimpleModuleId(id.organization, id.name, id.revision) - } - - if (!found.contains(simpleId) && !evicted(simpleId)) { - var jarOrNull: File = null - var pomOrNull: File = null - mr.artifacts.foreach({ case (a, f) => - if (a.`type` == "jar" || a.`type` == "bundle") { - jarOrNull = f - } else if (a.`type` == "pom") { - pomOrNull = f - } - }) - assert(jarOrNull != null, s"Could not find jar for ${simpleId}") - if (pomOrNull == null) { - val jarPath = jarOrNull.toPath - // we found the jar, so assume we can resolve a sibling pom file - val pomPath = jarPath.resolveSibling(jarPath.getFileName.toString.stripSuffix(".jar") + ".pom") - assert(Files.exists(pomPath), s"Could not find pom for ${simpleId}") - pomOrNull = pomPath.toFile - } - found(simpleId) = ResolvedArtifacts(simpleId, jarOrNull, pomOrNull) - } + val log = s.log + val csrCacheDir = s.cacheDirectory / "csr-cache" + val localRepo = s.cacheDirectory / "localRepo" / "maven2" + + // resolve the transitive dependencies of the local artifacts + val resolvedLibs = resolveLibraryDeps(coursierJar, log, csrCacheDir, localRepo, resolvedLocal) + + // the combination of local artifacts and resolved transitive dependencies + val merged = + (resolvedLocal ++ resolvedLibs).groupBy(_.id).values.map(_.reduce { (ra1, ra2) => + val jar = ra1.jar.orElse(ra2.jar) + val pom = ra1.pom.orElse(ra2.pom) + ResolvedArtifacts(ra1.id, jar, pom) + }) - } - found.values.toSeq + merged.toSeq }, republishClasspath := { val s = streams.value val resolved = republishAllResolved.value val cacheDir = republishRepo.value - - val log = s.log - val mavenRepo = cacheDir / "maven2" - IO.createDirectory(mavenRepo) - resolved.map { ra => - log.info(s"[republish] publishing ${ra.id} to $mavenRepo...") - val jar = ra.jar - val pom = ra.pom - - val pathElems = ra.id.org.split('.').toVector :+ ra.id.name :+ ra.id.revision - val artifactDir = pathElems.foldLeft(mavenRepo)(_ / _) - IO.createDirectory(artifactDir) - IO.copyFile(jar, artifactDir / jar.getName) - IO.copyFile(pom, artifactDir / pom.getName) - artifactDir - }.toSet + republishResolvedArtifacts(resolved, cacheDir / "maven2", logOpt = Some(s.log)) }, republishFetchLaunchers := { - val s = streams.value - val log = s.log - val repoDir = republishRepo.value - val launcherVersions = republishLaunchers.value - val libexec = republishPrepareBin.value - - val dlCache = repoDir / "cache" - - val store = s.cacheStoreFactory / "versions" - - def work(name: String, dest: File, launcher: String): File = { - val (launcherURL, workFile, prefix, subPart) = { - if (launcher.startsWith("gz+")) { - IO.createDirectory(dlCache) - val launcherURL = url(launcher.stripPrefix("gz+")) - (launcherURL, dlCache / s"$name.gz", "gz", "") - } else if (launcher.startsWith("zip+")) { - IO.createDirectory(dlCache) - val (urlPart, subPath) = launcher.split("!/") match { - case Array(urlPart, subPath) => (urlPart, subPath) - case _ => - throw new MessageOnlyException(s"[republish] Invalid zip+ URL, expected ! to mark subpath: $launcher") - } - val launcherURL = url(urlPart.stripPrefix("zip+")) - (launcherURL, dlCache / s"$name.zip", "zip", subPath) - } else { - IO.createDirectory(libexec) - (url(launcher), dest, "", "") - } - } - IO.delete(workFile) - Using.urlInputStream(launcherURL) { in => - log.info(s"[republish] Downloading $launcherURL to $workFile...") - IO.transfer(in, workFile) - log.info(s"[republish] Downloaded $launcherURL to $workFile...") - } - if (prefix == "gz") { - IO.delete(dest) - Using.fileInputStream(workFile) { in => - Using.gzipInputStream(in) { gzIn => - IO.transfer(gzIn, dest) - } - } - log.info(s"[republish] uncompressed gz file $workFile to $dest...") - FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable - IO.delete(workFile) - } else if (prefix == "zip") { - IO.delete(dest) - val files = IO.unzip(workFile, dlCache, new ExactFilter(subPart)) - val extracted = files.headOption.getOrElse(throw new MessageOnlyException(s"[republish] No files extracted from $workFile matching $subPart")) - log.info(s"[republish] unzipped $workFile to $extracted...") - IO.move(extracted, dest) - log.info(s"[republish] moved $extracted to $dest...") - FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable - IO.delete(workFile) - } - dest - } - - val allLaunchers = { - for ((name, launcher) <- launcherVersions) yield { - val dest = libexec / name - - val id = name.replaceAll("[^a-zA-Z0-9]", "_") - - val fetchAction = Tracked.inputChanged[String, File](store.make(id)) { (inChanged, launcher) => - if (inChanged || !Files.exists(dest.toPath)) { - work(name, dest, launcher) - } else { - log.info(s"[republish] Using cached $launcher at $dest...") - dest - } - } - - fetchAction(launcher) - } - } - allLaunchers.toSet + fetchFilesTask(republishPrepareBin, NameFilter.Generic).value + }, + republishFetchCoursier := { + fetchFilesTask(republishCoursierDir.toTask, NameFilter.Coursier).value.head }, republishPrepareBin := { val baseDir = baseDirectory.value From db7c2548711d4ca4b0edbe9220e36903327e6bcb Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Fri, 17 May 2024 20:38:57 +0200 Subject: [PATCH 5/9] use native launchers in bash tests cleanup Build.scala --- .github/workflows/ci.yaml | 14 ++- .github/workflows/launchers.yml | 18 ++-- bin/common | 9 +- bin/common-platform | 63 ++++++++++++ bin/scala | 4 +- bin/scalac | 2 +- bin/scaladoc | 2 +- build.sbt | 6 +- .../test/dotty/tools/io/ClasspathTest.scala | 2 +- .../tools/scripting/BashExitCodeTests.scala | 2 +- .../dotty/tools/scripting/ScriptTestEnv.scala | 32 +++++-- dist/bin/cli-common-platform | 2 - dist/bin/common | 95 +------------------ dist/bin/common-java | 40 -------- dist/bin/{cli-common => common-shared} | 53 +++++++++-- dist/bin/scala | 6 +- dist/bin/scala.bat | 16 +--- project/Build.scala | 44 ++++++--- project/DottyVersion.scala | 24 ----- project/Modes.scala | 4 - project/RepublishPlugin.scala | 41 ++++---- project/VersionUtil.scala | 2 - project/scripts/bootstrappedOnlyCmdTests | 10 +- project/scripts/buildScalaBinary | 8 ++ project/scripts/cmdTestsCommon.inc.sh | 3 + project/scripts/winCmdTests | 2 +- project/scripts/winCmdTests.bat | 2 +- .../src/main/scala/a/zz.scala | 6 ++ 28 files changed, 246 insertions(+), 266 deletions(-) create mode 100755 bin/common-platform delete mode 100644 dist/bin/common-java rename dist/bin/{cli-common => common-shared} (65%) delete mode 100644 project/DottyVersion.scala create mode 100755 project/scripts/buildScalaBinary create mode 100644 tests/cmdTest-sbt-tests/sourcepath-with-inline/src/main/scala/a/zz.scala diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ed18be1a703..de1f74c641db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -141,7 +141,8 @@ jobs: - name: Cmd Tests run: | - ./project/scripts/sbt ";dist/pack; scala3-bootstrapped/compile; scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*; scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test" + ./project/scripts/buildScalaBinary + ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test" ./project/scripts/cmdTests ./project/scripts/bootstrappedOnlyCmdTests @@ -230,7 +231,7 @@ jobs: shell: cmd - name: build binary - run: sbt "dist-win-x64/pack" & bash -version + run: sbt "dist-win-x86_64/pack" & bash -version shell: cmd - name: cygwin tests @@ -269,8 +270,12 @@ jobs: - name: Git Checkout uses: actions/checkout@v4 + - name: build binary + run: sbt "dist-win-x86_64/pack" + shell: cmd + - name: Test - run: sbt ";dist-win-x64/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test" + run: sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test" shell: cmd - name: Scala.js Test @@ -596,7 +601,8 @@ jobs: - name: Test run: | - ./project/scripts/sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*" + ./project/scripts/buildScalaBinary + ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*" ./project/scripts/cmdTests ./project/scripts/bootstrappedOnlyCmdTests diff --git a/.github/workflows/launchers.yml b/.github/workflows/launchers.yml index 1e9ffdc7a38f..0692a9f4377d 100644 --- a/.github/workflows/launchers.yml +++ b/.github/workflows/launchers.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: jobs: - linux-x64: + linux-x86_64: name: Deploy and Test on Linux x64 architecture runs-on: ubuntu-latest steps: @@ -16,8 +16,8 @@ jobs: distribution: 'temurin' cache: 'sbt' - name: Build the launcher command - run: sbt "dist-linux-x64/pack" - - run: ./dist/linux-x64/target/pack/bin/scala --version + run: sbt "dist-linux-x86_64/pack" + - run: ./dist/linux-x86_64/target/pack/bin/scala --version linux-aarch64: name: Deploy and Test on Linux ARM64 architecture @@ -37,7 +37,7 @@ jobs: run: sbt "dist-linux-aarch64/pack" - run: ./dist/linux-aarch64/target/pack/bin/scala --version - mac-x64: + mac-x86_64: name: Deploy and Test on Mac x64 architecture runs-on: macos-13 steps: @@ -52,8 +52,8 @@ jobs: - name: Install sbt run: brew install sbt - name: Build the launcher command - run: sbt "dist-mac-x64/pack" - - run: ./dist/mac-x64/target/pack/bin/scala --version + run: sbt "dist-mac-x86_64/pack" + - run: ./dist/mac-x86_64/target/pack/bin/scala --version mac-aarch64: name: Deploy and Test on Mac ARM64 architecture @@ -73,7 +73,7 @@ jobs: run: sbt "dist-mac-aarch64/pack" - run: ./dist/mac-aarch64/target/pack/bin/scala --version - win-x64: + win-x86_64: name: Deploy and Test on Windows x64 architecture runs-on: windows-latest steps: @@ -85,5 +85,5 @@ jobs: distribution: 'temurin' cache: 'sbt' - name: Build the launcher command - run: sbt "dist-win-x64/pack" - - run: ./dist/win-x64/target/pack/bin/scala --version + run: sbt "dist-win-x86_64/pack" + - run: ./dist/win-x86_64/target/pack/bin/scala --version diff --git a/bin/common b/bin/common index 7d3aa7148265..37b2ebd1ff93 100755 --- a/bin/common +++ b/bin/common @@ -9,15 +9,18 @@ target="$1" shift # Mutates $@ by deleting the first element ($1) +# set the $DIST_PROJECT and $DIST_DIR variables +source "$ROOT/bin/common-platform" + # Marker file used to obtain the date of latest call to sbt-back -version="$ROOT/dist/target/pack/VERSION" +version="$ROOT/$DIST_DIR/target/pack/VERSION" # Create the target if absent or if file changed in ROOT/compiler new_files="$(find "$ROOT/compiler" \( -iname "*.scala" -o -iname "*.java" \) -newer "$version" 2> /dev/null)" if [ ! -f "$version" ] || [ ! -z "$new_files" ]; then echo "Building Dotty..." - (cd $ROOT && sbt "dist/pack") + (cd $ROOT && sbt "$DIST_PROJECT/pack") fi -"$target" "$@" +"$ROOT/$DIST_DIR/target/pack/bin/$target" "$@" diff --git a/bin/common-platform b/bin/common-platform new file mode 100755 index 000000000000..648e0195e7e6 --- /dev/null +++ b/bin/common-platform @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +unset cygwin mingw msys darwin + +# COLUMNS is used together with command line option '-pageWidth'. +if command -v tput >/dev/null 2>&1; then + export COLUMNS="$(tput -Tdumb cols)" +fi + +case "`uname`" in + CYGWIN*) cygwin=true + ;; + MINGW*) mingw=true + ;; + MSYS*) msys=true + ;; + Darwin*) darwin=true + ;; +esac + +unset DIST_PROJECT DIST_DIR + +if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then + DIST_PROJECT="dist-win-x86_64" + DIST_DIR="dist/win-x86_64" +else + # OS and arch logic taken from https://github.com/VirtusLab/scala-cli/blob/main/scala-cli.sh + unset arch ARCH_NORM + arch=$(uname -m) + if [[ "$arch" == "aarch64" ]] || [[ "$arch" == "x86_64" ]]; then + ARCH_NORM="$arch" + elif [[ "$arch" == "amd64" ]]; then + ARCH_NORM="x86_64" + elif [[ "$arch" == "arm64" ]]; then + ARCH_NORM="aarch64" + else + ARCH_NORM="unknown" + fi + + if [ "$(expr substr $(uname -s) 1 5 2>/dev/null)" == "Linux" ]; then + if [[ "$ARCH_NORM" == "unknown" ]]; then + echo >&2 "unknown Linux CPU architecture, defaulting to JVM launcher" + DIST_PROJECT="dist" + DIST_DIR="dist" + else + DIST_PROJECT="dist-linux-$ARCH_NORM" + DIST_DIR="dist/linux-$ARCH_NORM" + fi + elif [ "$(uname)" == "Darwin" ]; then + if [[ "$ARCH_NORM" == "unknown" ]]; then + echo >&2 "unknown Darwin CPU architecture, defaulting to JVM launcher" + DIST_PROJECT="dist" + DIST_DIR="dist" + else + DIST_PROJECT="dist-mac-$ARCH_NORM" + DIST_DIR="dist/mac-$ARCH_NORM" + fi + else + echo >&2 "unknown OS, defaulting to JVM launcher" + DIST_PROJECT="dist" + DIST_DIR="dist" + fi +fi diff --git a/bin/scala b/bin/scala index 2df274fe95ba..e87c4391806b 100755 --- a/bin/scala +++ b/bin/scala @@ -16,7 +16,7 @@ scala_args() { SCRIPT_ARGS+=("$@") break ;; - "clean") + "clean" | "version" | "--version" | "-version" | "help" | "--help" | "-help") CLI_ARGS+=("$1") DISABLE_BLOOP=0 # clean command should not add --offline --server=false shift @@ -35,4 +35,4 @@ scala_args() { echo "--power ${CLI_ARGS[@]} ${SCRIPT_ARGS[@]}" } -"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scala" $(scala_args "$@") +"$ROOT/bin/common" "scala" $(scala_args "$@") diff --git a/bin/scalac b/bin/scalac index faeb48d92d87..d141b9a6c6bb 100755 --- a/bin/scalac +++ b/bin/scalac @@ -2,4 +2,4 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.." -"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scalac" "$@" +"$ROOT/bin/common" "scalac" "$@" diff --git a/bin/scaladoc b/bin/scaladoc index 11a754c6579f..02decabb9ae3 100755 --- a/bin/scaladoc +++ b/bin/scaladoc @@ -2,4 +2,4 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.." -"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scaladoc" "$@" +"$ROOT/bin/common" "scaladoc" "$@" diff --git a/build.sbt b/build.sbt index 705ef0ba0eb7..f357044c91ca 100644 --- a/build.sbt +++ b/build.sbt @@ -28,10 +28,10 @@ val `scaladoc-js-main` = Build.`scaladoc-js-main` val `scaladoc-js-contributors` = Build.`scaladoc-js-contributors` val `scala3-bench-run` = Build.`scala3-bench-run` val dist = Build.dist -val `dist-mac-x64` = Build.`dist-mac-x64` +val `dist-mac-x86_64` = Build.`dist-mac-x86_64` val `dist-mac-aarch64` = Build.`dist-mac-aarch64` -val `dist-win-x64` = Build.`dist-win-x64` -val `dist-linux-x64` = Build.`dist-linux-x64` +val `dist-win-x86_64` = Build.`dist-win-x86_64` +val `dist-linux-x86_64` = Build.`dist-linux-x86_64` val `dist-linux-aarch64` = Build.`dist-linux-aarch64` val `community-build` = Build.`community-build` val `sbt-community-build` = Build.`sbt-community-build` diff --git a/compiler/test/dotty/tools/io/ClasspathTest.scala b/compiler/test/dotty/tools/io/ClasspathTest.scala index 1b5e338b67e4..333f2b8062b0 100755 --- a/compiler/test/dotty/tools/io/ClasspathTest.scala +++ b/compiler/test/dotty/tools/io/ClasspathTest.scala @@ -25,7 +25,7 @@ class ClasspathTest { @Test def testWildcards(): Unit = val outDir = Files.createTempDirectory("classpath-test") try - val compilerLib = s"${if isWindows then "dist-win-x64" else "dist"}/target/pack/lib" + val compilerLib = s"${if isWindows then "dist-win-x86_64" else "dist"}/target/pack/lib" val libdir = Paths.get(compilerLib).toFile if libdir.exists then val libjarFiles = libdir.listFiles.toList.take(5) diff --git a/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala b/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala index a1cfb129ba41..857f5ef378e7 100644 --- a/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala +++ b/compiler/test/dotty/tools/scripting/BashExitCodeTests.scala @@ -33,7 +33,7 @@ class BashExitCodeTests: }, expectedExitCode, exitCode) // Helpers for running scala, scalac, and scalac without the output directory ("raw") - def scala(args: String*) = verifyExit(scalaPath, ("--power" +: "--offline" +: "--server=false" +: args)*) + def scala(args: String*) = verifyExit(scalaPath, ("--power" +: args :+ "--offline" :+ "--server=false")*) def scalacRaw(args: String*) = verifyExit(scalacPath, args*) def scalac(args: String*) = scalacRaw(("-d" +: tmpDir +: args)*) diff --git a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala index de205acfc7ae..dd1cc04bb58a 100644 --- a/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala +++ b/compiler/test/dotty/tools/scripting/ScriptTestEnv.scala @@ -5,6 +5,7 @@ package scripting import scala.language.unsafeNulls import java.io.File +import java.util.Locale import java.nio.file.{Path, Paths, Files} import dotty.tools.dotc.config.Properties.* @@ -46,11 +47,23 @@ object ScriptTestEnv { if bspDir.isEmpty then bspDir.delete() } - lazy val packDir: String = - if winshell then - "dist-win-x64/target/pack" - else - "dist/target/pack" + lazy val nativePackDir: Option[String] = { + def nativeDir(os: String, arch: String) = Some(s"dist/$os-$arch/target/pack") + def nativeOs(os: String) = archNorm match + case arch @ ("aarch64" | "x86_64") => nativeDir(os, arch) + case _ => None + + if winshell then nativeDir("win", "x86_64") // assume x86_64 for now + else if linux then nativeOs("linux") + else if mac then nativeOs("mac") + else None + } + + def jvmPackDir() = + println("warning: unknown OS architecture combination, defaulting to JVM launcher.") + "dist/target/pack" + + def packDir: String = nativePackDir.getOrElse(jvmPackDir()) def packBinDir: String = s"$packDir/bin" @@ -81,11 +94,18 @@ object ScriptTestEnv { def unameExe = which("uname") def ostypeFull = if unameExe.nonEmpty then exec(unameExe).mkString else "" - def ostype = ostypeFull.toLowerCase.takeWhile{ cc => cc >= 'a' && cc <='z' || cc >= 'A' && cc <= 'Z' } + def ostype = ostypeFull.toLowerCase(Locale.ROOT).takeWhile{ cc => cc >= 'a' && cc <='z' || cc >= 'A' && cc <= 'Z' } + def archFull = if unameExe.nonEmpty then exec(unameExe, "-m").mkString else "" + def archNorm = archFull match + case "arm64" => "aarch64" + case "amd64" => "x86_64" + case id => id def cygwin = ostype == "cygwin" def mingw = ostype == "mingw" def msys = ostype == "msys" + def linux = ostype == "linux" + def mac = ostype == "darwin" def winshell: Boolean = cygwin || mingw || msys def which(str: String) = diff --git a/dist/bin/cli-common-platform b/dist/bin/cli-common-platform index ee1b4768dd51..a5906e882bb4 100644 --- a/dist/bin/cli-common-platform +++ b/dist/bin/cli-common-platform @@ -1,5 +1,3 @@ #!/usr/bin/env bash -source "$PROG_HOME/bin/common-java" - SCALA_CLI_CMD_BASH=("\"$JAVACMD\"" "-jar \"$PROG_HOME/bin/scala-cli.jar\"") diff --git a/dist/bin/common b/dist/bin/common index f9dd594c566b..4a0152fbc4cb 100755 --- a/dist/bin/common +++ b/dist/bin/common @@ -1,95 +1,6 @@ #!/usr/bin/env bash -#/*-------------------------------------------------------------------------- -# * Credits: This script is based on the script generated by sbt-pack. -# *--------------------------------------------------------------------------*/ - -# save terminal settings -saved_stty=$(stty -g 2>/dev/null) -# clear on error so we don't later try to restore them -if [[ ! $? ]]; then - saved_stty="" -fi - -# restore stty settings (echo in particular) -function restoreSttySettings() { - stty $saved_stty - saved_stty="" -} - -scala_exit_status=127 -function onExit() { - [[ "$saved_stty" != "" ]] && restoreSttySettings - exit $scala_exit_status -} - -# to reenable echo if we are interrupted before completing. -trap onExit INT TERM EXIT - -unset cygwin mingw msys darwin conemu - -# COLUMNS is used together with command line option '-pageWidth'. -if command -v tput >/dev/null 2>&1; then - export COLUMNS="$(tput -Tdumb cols)" -fi - -case "`uname`" in - CYGWIN*) cygwin=true - ;; - MINGW*) mingw=true - ;; - MSYS*) msys=true - ;; - Darwin*) darwin=true - if [ -z "$JAVA_VERSION" ] ; then - JAVA_VERSION="CurrentJDK" - else - echo "Using Java version: $JAVA_VERSION" 1>&2 - fi - if [ -z "$JAVA_HOME" ] ; then - JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/${JAVA_VERSION}/Home - fi - JAVACMD="`which java`" - ;; -esac - -unset CYGPATHCMD -if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then - # ConEmu terminal is incompatible with jna-5.*.jar - [[ (${CONEMUANSI-} || ${ConEmuANSI-}) ]] && conemu=true - # cygpath is used by various windows shells: cygwin, git-sdk, gitbash, msys, etc. - CYGPATHCMD=`which cygpath 2>/dev/null` - case "$TERM" in - rxvt* | xterm* | cygwin*) - stty -icanon min 1 -echo - JAVA_OPTS="$JAVA_OPTS -Djline.terminal=unix" - ;; - esac -fi - -source "$PROG_HOME/bin/common-java" - -CLASSPATH_SUFFIX="" -# Path separator used in EXTRA_CLASSPATH -PSEP=":" - -# translate paths to Windows-mixed format before running java -if [ -n "${CYGPATHCMD-}" ]; then - [ -n "${PROG_HOME-}" ] && - PROG_HOME=`"$CYGPATHCMD" -am "$PROG_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`"$CYGPATHCMD" -am "$JAVA_HOME"` - CLASSPATH_SUFFIX=";" - PSEP=";" -elif [[ ${mingw-} || ${msys-} ]]; then - # For Mingw / Msys, convert paths from UNIX format before anything is touched - [ -n "$PROG_HOME" ] && - PROG_HOME="`(cd "$PROG_HOME"; pwd -W | sed 's|/|\\\\|g')`" - [ -n "$JAVA_HOME" ] && - JAVA_HOME="`(cd "$JAVA_HOME"; pwd -W | sed 's|/|\\\\|g')`" - CLASSPATH_SUFFIX=";" - PSEP=";" -fi +source "$PROG_HOME/bin/common-shared" #/*-------------------------------------------------- # * The code below is for Dotty @@ -168,16 +79,12 @@ ReplMain=dotty.tools.repl.Main ScriptingMain=dotty.tools.scripting.Main declare -a java_args -declare -a scala_args declare -a residual_args declare -a script_args addJava () { java_args+=("'$1'") } -addScala () { - scala_args+=("'$1'") -} addResidual () { residual_args+=("'$1'") } diff --git a/dist/bin/common-java b/dist/bin/common-java deleted file mode 100644 index fbc8f309543c..000000000000 --- a/dist/bin/common-java +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# Resolve JAVA_HOME from javac command path -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - javaExecutable="`readlink -f \"$javaExecutable\"`" - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "${JAVACMD-}" ] ; then - if [ -n "${JAVA_HOME-}" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." - echo " We cannot execute $JAVACMD" - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi diff --git a/dist/bin/cli-common b/dist/bin/common-shared similarity index 65% rename from dist/bin/cli-common rename to dist/bin/common-shared index eb77c9924d06..8c85993a5283 100644 --- a/dist/bin/cli-common +++ b/dist/bin/common-shared @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# Common options for both scala-cli and java based launchers + #/*-------------------------------------------------------------------------- # * Credits: This script is based on the script generated by sbt-pack. # *--------------------------------------------------------------------------*/ @@ -23,10 +25,6 @@ function onExit() { exit $scala_exit_status } -#/*-------------------------------------------------------------------------- -# * SECTION FOR JAVA COMMAND -# *--------------------------------------------------------------------------*/ - # to reenable echo if we are interrupted before completing. trap onExit INT TERM EXIT @@ -71,16 +69,55 @@ if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then esac fi -source "$PROG_HOME/bin/cli-common-platform" +# Resolve JAVA_HOME from javac command path +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + javaExecutable="`readlink -f \"$javaExecutable\"`" + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "${JAVACMD-}" ] ; then + if [ -n "${JAVA_HOME-}" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." + echo " We cannot execute $JAVACMD" + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi CLASSPATH_SUFFIX="" # Path separator used in EXTRA_CLASSPATH PSEP=":" +PROG_HOME_URI="file://$PROG_HOME" # translate paths to Windows-mixed format before running java if [ -n "${CYGPATHCMD-}" ]; then [ -n "${PROG_HOME-}" ] && PROG_HOME=`"$CYGPATHCMD" -am "$PROG_HOME"` + PROG_HOME_URI="file:///$PROG_HOME" # Add extra root dir prefix [ -n "$JAVA_HOME" ] && JAVA_HOME=`"$CYGPATHCMD" -am "$JAVA_HOME"` CLASSPATH_SUFFIX=";" @@ -89,18 +126,14 @@ elif [[ ${mingw-} || ${msys-} ]]; then # For Mingw / Msys, convert paths from UNIX format before anything is touched [ -n "$PROG_HOME" ] && PROG_HOME="`(cd "$PROG_HOME"; pwd -W | sed 's|/|\\\\|g')`" + PROG_HOME_URI="file:///$PROG_HOME" # Add extra root dir prefix [ -n "$JAVA_HOME" ] && JAVA_HOME="`(cd "$JAVA_HOME"; pwd -W | sed 's|/|\\\\|g')`" CLASSPATH_SUFFIX=";" PSEP=";" fi -#/*-------------------------------------------------- -# * The code below is for Dotty -# *-------------------------------------------------*/ - declare -a scala_args - addScala () { scala_args+=("'$1'") } diff --git a/dist/bin/scala b/dist/bin/scala index 6b5930e3c215..c6c6f8807a64 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -26,7 +26,8 @@ if [ -z "${PROG_HOME-}" ] ; then cd "$saveddir" fi -source "$PROG_HOME/bin/cli-common" +source "$PROG_HOME/bin/common-shared" +source "$PROG_HOME/bin/cli-common-platform" SCALA_VERSION="" # iterate through lines in VERSION_SRC @@ -44,7 +45,7 @@ if [ -z "$SCALA_VERSION" ]; then exit 1 fi -MVN_REPOSITORY="file://$PROG_HOME/maven2" +MVN_REPOSITORY="$PROG_HOME_URI/maven2" # escape all script arguments while [[ $# -gt 0 ]]; do @@ -61,6 +62,7 @@ eval "${SCALA_CLI_CMD_BASH[@]}" \ "--cli-default-scala-version \"$SCALA_VERSION\"" \ "-r \"$MVN_REPOSITORY\"" \ "${scala_args[@]}" + scala_exit_status=$? onExit diff --git a/dist/bin/scala.bat b/dist/bin/scala.bat index 8b4c08da553f..d473facbbb1c 100644 --- a/dist/bin/scala.bat +++ b/dist/bin/scala.bat @@ -19,6 +19,8 @@ if not %_EXITCODE%==0 goto end call :setScalaOpts +call "%_PROG_HOME%\bin\cli-common-platform.bat" + @rem SCALA_CLI_CMD_WIN is an array, set in cli-common-platform.bat call %SCALA_CLI_CMD_WIN% "--prog-name" "scala" "--cli-default-scala-version" "%_SCALA_VERSION%" "-r" "%MVN_REPOSITORY%" %* @@ -41,20 +43,8 @@ if not "%char%"==":" ( goto :findColon ) -@REM set _PROG_HOME to the substring from the first colon to the end -set "_PROG_HOME_SUB=!_PROG_HOME:~%index%!" -@REM strip initial character -set "_PROG_HOME_SUB=!_PROG_HOME_SUB:~1!" - -@REM set drive to substring from 0 to the first colon -set "_PROG_HOME_DRIVE=!_PROG_HOME:~0,%index%!" - - - set "_SCALA_VERSION=" -set "MVN_REPOSITORY=file://%_PROG_HOME_DRIVE%\%_PROG_HOME_SUB:\=/%/maven2" - -call "%_PROG_HOME%\bin\cli-common-platform.bat" +set "MVN_REPOSITORY=file:///%_PROG_HOME:\=/%/maven2" @rem read for version:=_SCALA_VERSION in VERSION_FILE FOR /F "usebackq delims=" %%G IN ("%_PROG_HOME%\VERSION") DO ( diff --git a/project/Build.scala b/project/Build.scala index 464ed3ab1266..f6c905800f40 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -28,7 +28,6 @@ import sbttastymima.TastyMiMaPlugin import sbttastymima.TastyMiMaPlugin.autoImport._ import scala.util.Properties.isJavaAtLeast -import scala.collection.mutable import org.portablescala.sbtplatformdeps.PlatformDepsPlugin.autoImport._ import org.scalajs.linker.interface.{ModuleInitializer, StandardConfig} @@ -86,7 +85,10 @@ object DottyJSPlugin extends AutoPlugin { object Build { import ScaladocConfigs._ - import versionhelpers.DottyVersion._ + + val referenceVersion = "3.4.2-RC1" + + val baseVersion = "3.5.1-RC1" // LTS or Next val versionLine = "Next" @@ -117,6 +119,7 @@ object Build { /** Version of Scala CLI to download */ val scalaCliLauncherVersion = "1.3.2" + /** Version of Coursier to download for initializing the local maven repo of Scala command */ val coursierJarVersion = "2.1.10" object CompatMode { @@ -156,6 +159,24 @@ object Build { val dottyGithubUrl = "https://github.com/scala/scala3" val dottyGithubRawUserContentUrl = "https://raw.githubusercontent.com/scala/scala3" + + val isRelease = sys.env.get("RELEASEBUILD") == Some("yes") + + val dottyVersion = { + def isNightly = sys.env.get("NIGHTLYBUILD") == Some("yes") + if (isRelease) + baseVersion + else if (isNightly) + baseVersion + "-bin-" + VersionUtil.commitDate + "-" + VersionUtil.gitHash + "-NIGHTLY" + else + baseVersion + "-bin-SNAPSHOT" + } + val dottyNonBootstrappedVersion = { + // Make sure sbt always computes the scalaBinaryVersion correctly + val bin = if (!dottyVersion.contains("-bin")) "-bin" else "" + dottyVersion + bin + "-nonbootstrapped" + } + val sbtCommunityBuildVersion = "0.1.0-SNAPSHOT" val agentOptions = List( @@ -607,7 +628,7 @@ object Build { val contents = //2.11.11.v20170413-090219-8a413ba7cc s"""version.number=${version.value} |maven.version.number=${version.value} - |git.hash=${versionhelpers.VersionUtil.gitHash} + |git.hash=${VersionUtil.gitHash} |copyright.string=Copyright 2002-$currentYear, LAMP/EPFL """.stripMargin @@ -647,7 +668,7 @@ object Build { // Spawn new JVM in run and test // Add git-hash used to package the distribution to the manifest to know it in runtime and report it in REPL - packageOptions += ManifestAttributes(("Git-Hash", versionhelpers.VersionUtil.gitHash)), + packageOptions += ManifestAttributes(("Git-Hash", VersionUtil.gitHash)), javaOptions ++= { val log = streams.value.log @@ -2111,13 +2132,13 @@ object Build { .settings( packArchiveName := "scala3-" + dottyVersion, republishBinDir := baseDirectory.value / "bin", - republishLaunchers ++= Seq( + republishCoursier += + ("coursier.jar" -> s"https://github.com/coursier/coursier/releases/download/v$coursierJarVersion/coursier.jar"), + republishLaunchers += ("scala-cli.jar" -> s"https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli.jar"), - ("coursier.jar" -> s"https://github.com/coursier/coursier/releases/download/v$coursierJarVersion/coursier.jar") - ) ) - lazy val `dist-mac-x64` = project.in(file("dist/mac-x64")).asDist(Bootstrapped) + lazy val `dist-mac-x86_64` = project.in(file("dist/mac-x86_64")).asDist(Bootstrapped) .settings( republishBinDir := (dist / republishBinDir).value, packArchiveName := (dist / packArchiveName).value + "-x86_64-apple-darwin", @@ -2137,7 +2158,7 @@ object Build { ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-aarch64-apple-darwin.gz") ) - lazy val `dist-win-x64` = project.in(file("dist/win-x64")).asDist(Bootstrapped) + lazy val `dist-win-x86_64` = project.in(file("dist/win-x86_64")).asDist(Bootstrapped) .settings( republishBinDir := (dist / republishBinDir).value, packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-win32", @@ -2147,7 +2168,7 @@ object Build { ("scala-cli.exe" -> s"zip+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-pc-win32.zip!/scala-cli.exe") ) - lazy val `dist-linux-x64` = project.in(file("dist/linux-x64")).asDist(Bootstrapped) + lazy val `dist-linux-x86_64` = project.in(file("dist/linux-x86_64")).asDist(Bootstrapped) .settings( republishBinDir := (dist / republishBinDir).value, packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-linux", @@ -2325,7 +2346,6 @@ object Build { object ScaladocConfigs { import Build._ - import versionhelpers.DottyVersion._ private lazy val currentYear: String = java.util.Calendar.getInstance().get(java.util.Calendar.YEAR).toString def dottyExternalMapping = ".*scala/.*::scaladoc3::https://dotty.epfl.ch/api/" @@ -2405,7 +2425,7 @@ object ScaladocConfigs { .add(UseJavacp(true)) .add(ProjectName("scaladoc")) .add(OutputDir("scaladoc/output/self")) - .add(Revision(versionhelpers.VersionUtil.gitHash)) + .add(Revision(VersionUtil.gitHash)) .add(ExternalMappings(List(dottyExternalMapping, javaExternalMapping))) .withTargets((Compile / classDirectory).value.getAbsolutePath :: Nil) } diff --git a/project/DottyVersion.scala b/project/DottyVersion.scala deleted file mode 100644 index 393cca1a251b..000000000000 --- a/project/DottyVersion.scala +++ /dev/null @@ -1,24 +0,0 @@ -package versionhelpers - -object DottyVersion { - val isRelease = sys.env.get("RELEASEBUILD") == Some("yes") - - val referenceVersion = "3.4.2-RC1" - - val baseVersion = "3.5.1-RC1" - - val dottyVersion = { - def isNightly = sys.env.get("NIGHTLYBUILD") == Some("yes") - if (isRelease) - baseVersion - else if (isNightly) - baseVersion + "-bin-" + VersionUtil.commitDate + "-" + VersionUtil.gitHash + "-NIGHTLY" - else - baseVersion + "-bin-SNAPSHOT" - } - val dottyNonBootstrappedVersion = { - // Make sure sbt always computes the scalaBinaryVersion correctly - val bin = if (!dottyVersion.contains("-bin")) "-bin" else "" - dottyVersion + bin + "-nonbootstrapped" - } -} diff --git a/project/Modes.scala b/project/Modes.scala index fcc13dea8a89..8f34aa5870e4 100644 --- a/project/Modes.scala +++ b/project/Modes.scala @@ -25,9 +25,5 @@ object Modes { def bootstrappedDependsOn(s: sbt.ClasspathDep[ProjectReference]*)(implicit mode: Mode): Project = if (mode == NonBootstrapped) project else project.dependsOn(s: _*) - /** Plugins only if the mode is bootstrapped */ - def bootstrappedEnablePlugins(ns: Plugins*)(implicit mode: Mode): Project = - if (mode == NonBootstrapped) project else project.enablePlugins(ns: _*) - } } diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala index a5a8115b4b51..6efb6208a152 100644 --- a/project/RepublishPlugin.scala +++ b/project/RepublishPlugin.scala @@ -11,7 +11,6 @@ import sbt.util.CacheImplicits._ import scala.collection.mutable import java.nio.file.Files -import versionhelpers.DottyVersion._ import java.nio.file.attribute.PosixFilePermission import java.nio.file.{Files, Path} @@ -68,7 +67,8 @@ object RepublishPlugin extends AutoPlugin { val republishBinOverrides = settingKey[Seq[File]]("files to override those in bin-dir.") val republish = taskKey[File]("cache the dependencies and download launchers for the distribution") val republishRepo = settingKey[File]("the location to store the republished artifacts.") - val republishLaunchers = settingKey[Seq[(String, String)]]("launchers to download. Sequence of (name, version, URL).") + val republishLaunchers = settingKey[Seq[(String, String)]]("launchers to download. Sequence of (name, URL).") + val republishCoursier = settingKey[Seq[(String, String)]]("coursier launcher to download. Sequence of (name, URL).") } import autoImport._ @@ -78,16 +78,6 @@ object RepublishPlugin extends AutoPlugin { } case class ResolvedArtifacts(id: SimpleModuleId, jar: Option[File], pom: Option[File]) - class NameFilter(kind: String, filter: String => Boolean) { - def apply(name: String): Boolean = filter(name) - override def toString = kind - } - - object NameFilter { - final val Coursier = new NameFilter("coursier", _ == "coursier.jar") - final val Generic = new NameFilter("generic", _ != "coursier.jar") - } - private def republishResolvedArtifacts(resolved: Seq[ResolvedArtifacts], mavenRepo: File, logOpt: Option[Logger]): Set[File] = { IO.createDirectory(mavenRepo) resolved.map { ra => @@ -107,7 +97,7 @@ object RepublishPlugin extends AutoPlugin { }.toSet } - private def coursierCmd(jar: File, cache: File, log: Logger, args: Seq[String]): Unit = { + private def coursierCmd(jar: File, cache: File, args: Seq[String]): Unit = { val jar0 = jar.getAbsolutePath.toString val javaHome = sys.props.get("java.home").getOrElse { throw new MessageOnlyException("java.home property not set") @@ -122,7 +112,6 @@ object RepublishPlugin extends AutoPlugin { val p = new ProcessBuilder(cmdLine: _*).inheritIO() p.environment().putAll(env.asJava) val proc = p.start() - log.info(s"[republish] Running with env ${env}: coursier.jar with args ${args.mkString(" ")}") proc.waitFor() if (proc.exitValue() != 0) throw new MessageOnlyException(s"Error running coursier.jar with args ${args.mkString(" ")}") @@ -142,7 +131,7 @@ object RepublishPlugin extends AutoPlugin { IO.createDirectory(cacheDir) for (lib <- libs) { log.info(s"[republish] Fetching $lib with coursier.jar...") - coursierCmd(coursierJar, cacheDir, log, + coursierCmd(coursierJar, cacheDir, Seq( "fetch", "--repository", localRepoArg, @@ -196,11 +185,14 @@ object RepublishPlugin extends AutoPlugin { .toSeq } - private def fetchFilesTask(libexecT: Def.Initialize[Task[File]], nameFilter: NameFilter) = Def.task[Set[File]] { + private def fetchFilesTask( + libexecT: Def.Initialize[Task[File]], + srcs: SettingKey[Seq[(String, String)]], + strict: Boolean) = Def.task[Set[File]] { val s = streams.value val log = s.log val repoDir = republishRepo.value - val launcherVersions = republishLaunchers.value + val launcherVersions = srcs.value val libexec = libexecT.value val dlCache = s.cacheDirectory / "republish-launchers" @@ -241,7 +233,6 @@ object RepublishPlugin extends AutoPlugin { } } log.info(s"[republish] uncompressed gz file $workFile to $dest...") - FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable IO.delete(workFile) } else if (prefix == "zip") { IO.delete(dest) @@ -250,18 +241,17 @@ object RepublishPlugin extends AutoPlugin { log.info(s"[republish] unzipped $workFile to $extracted...") IO.move(extracted, dest) log.info(s"[republish] moved $extracted to $dest...") - FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable IO.delete(workFile) } + FileUtil.tryMakeExecutable(dest.toPath) dest } val allLaunchers = { - val filtered = launcherVersions.filter { case (name, _) => nameFilter(name) } - if (filtered.isEmpty) - throw new MessageOnlyException(s"[republish] No $nameFilter launchers to fetch, check the build configuration for ${republishLaunchers.key.label}.") + if (strict && launcherVersions.isEmpty) + throw new MessageOnlyException(s"[republish] No launchers to fetch, check the build configuration for ${srcs.key.label}.") - for ((name, launcher) <- filtered) yield { + for ((name, launcher) <- launcherVersions) yield { val dest = libexec / name val id = name.replaceAll("[^a-zA-Z0-9]", "_") @@ -284,6 +274,7 @@ object RepublishPlugin extends AutoPlugin { override val projectSettings: Seq[Def.Setting[_]] = Def.settings( republishCoursierDir := republishRepo.value / "coursier", republishLaunchers := Seq.empty, + republishCoursier := Seq.empty, republishBinOverrides := Seq.empty, republishLocalResolved / republishProjectRefs := { val proj = thisProjectRef.value @@ -357,10 +348,10 @@ object RepublishPlugin extends AutoPlugin { republishResolvedArtifacts(resolved, cacheDir / "maven2", logOpt = Some(s.log)) }, republishFetchLaunchers := { - fetchFilesTask(republishPrepareBin, NameFilter.Generic).value + fetchFilesTask(republishPrepareBin, republishLaunchers, strict = true).value }, republishFetchCoursier := { - fetchFilesTask(republishCoursierDir.toTask, NameFilter.Coursier).value.head + fetchFilesTask(republishCoursierDir.toTask, republishCoursier, strict = true).value.head }, republishPrepareBin := { val baseDir = baseDirectory.value diff --git a/project/VersionUtil.scala b/project/VersionUtil.scala index cef73a66ddc6..1d385d75631d 100644 --- a/project/VersionUtil.scala +++ b/project/VersionUtil.scala @@ -1,5 +1,3 @@ -package versionhelpers - import org.eclipse.jgit.storage.file.FileRepositoryBuilder import org.eclipse.jgit.lib.{Constants, ObjectId, Ref, Repository} import org.eclipse.jgit.revwalk.{RevCommit, RevWalk} diff --git a/project/scripts/bootstrappedOnlyCmdTests b/project/scripts/bootstrappedOnlyCmdTests index fc1ff89f560d..11c35a7028cc 100755 --- a/project/scripts/bootstrappedOnlyCmdTests +++ b/project/scripts/bootstrappedOnlyCmdTests @@ -15,13 +15,13 @@ echo "testing scala.quoted.Expr.run from sbt scala" grep -qe "val a: scala.Int = 3" "$tmp" # setup for `scalac`/`scala` script tests -"$SBT" dist/pack +"$SBT" "$DIST_PROJECT/pack" -echo "capturing scala version from dist/target/pack/VERSION" -IFS=':=' read -ra versionProps < "$ROOT/dist/target/pack/VERSION" # temporarily set IFS to ':=' to split versionProps +echo "capturing scala version from $DIST_DIR/target/pack/VERSION" +IFS=':=' read -ra versionProps < "$ROOT/$DIST_DIR/target/pack/VERSION" # temporarily set IFS to ':=' to split versionProps [ ${#versionProps[@]} -eq 3 ] && \ [ ${versionProps[0]} = "version" ] && \ - [ -n ${versionProps[2]} ] || die "Expected non-empty 'version' property in $ROOT/dist/target/pack/VERSION" + [ -n ${versionProps[2]} ] || die "Expected non-empty 'version' property in $ROOT/$DIST_DIR/target/pack/VERSION" scala_version=${versionProps[2]} # check that `scalac` compiles and `scala` runs it @@ -77,7 +77,7 @@ echo "testing sbt scalac with suspension" clear_out "$OUT" "$SBT" "scala3-compiler-bootstrapped/scalac -d $OUT tests/pos-macros/macros-in-same-project-1/Bar.scala tests/pos-macros/macros-in-same-project-1/Foo.scala" > "$tmp" -# echo ":quit" | ./dist/target/pack/bin/scala # not supported by CI +# echo ":quit" | ./$DIST_DIR/target/pack/bin/scala # not supported by CI echo "testing ./bin/scaladoc" clear_out "$OUT1" diff --git a/project/scripts/buildScalaBinary b/project/scripts/buildScalaBinary new file mode 100755 index 000000000000..552831c07b41 --- /dev/null +++ b/project/scripts/buildScalaBinary @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +source $(dirname $0)/cmdTestsCommon.inc.sh + +# build the scala/scalac/scaladoc binary, where scala is native for the current platform. +"$SBT" "$DIST_PROJECT/pack" diff --git a/project/scripts/cmdTestsCommon.inc.sh b/project/scripts/cmdTestsCommon.inc.sh index d041613e46a8..bccb4aa56ac1 100644 --- a/project/scripts/cmdTestsCommon.inc.sh +++ b/project/scripts/cmdTestsCommon.inc.sh @@ -15,6 +15,9 @@ OUT=$(mktemp -d) OUT1=$(mktemp -d) tmp=$(mktemp) +# set the $DIST_PROJECT and $DIST_DIR variables +source "$ROOT/bin/common-platform" + die () { echo >&2 "$@" exit 1 diff --git a/project/scripts/winCmdTests b/project/scripts/winCmdTests index 54519f9a318d..fe6a43c7f68f 100644 --- a/project/scripts/winCmdTests +++ b/project/scripts/winCmdTests @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -PREFIX="dist/win-x64/target/pack" +PREFIX="dist/win-x86_64/target/pack" SOURCE="tests/pos/HelloWorld.scala" $PREFIX/bin/scalac @project/scripts/options "$SOURCE" $PREFIX/bin/scalac -d out "$SOURCE" diff --git a/project/scripts/winCmdTests.bat b/project/scripts/winCmdTests.bat index 8ee1cbe4be68..903f74d7ab98 100644 --- a/project/scripts/winCmdTests.bat +++ b/project/scripts/winCmdTests.bat @@ -2,7 +2,7 @@ setlocal @rem paths are relative to the root project directory -set "_PREFIX=dist\win-x64\target\pack" +set "_PREFIX=dist\win-x86_64\target\pack" set "_SOURCE=tests\pos\HelloWorld.scala" set "_OUT_DIR=out" set "_SITE_DIR=_site" diff --git a/tests/cmdTest-sbt-tests/sourcepath-with-inline/src/main/scala/a/zz.scala b/tests/cmdTest-sbt-tests/sourcepath-with-inline/src/main/scala/a/zz.scala new file mode 100644 index 000000000000..17a7488ccb1a --- /dev/null +++ b/tests/cmdTest-sbt-tests/sourcepath-with-inline/src/main/scala/a/zz.scala @@ -0,0 +1,6 @@ +package a + +object Foo: // note that `Foo` is defined in `zz.scala` + class Local + inline def foo(using Local): Nothing = + ??? From 709e91f3feb469d2d95227662a1c5edd42276f73 Mon Sep 17 00:00:00 2001 From: Hamza REMMAL Date: Wed, 29 May 2024 02:59:29 +0200 Subject: [PATCH 6/9] Do not run linux arm tests --- .github/workflows/launchers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/launchers.yml b/.github/workflows/launchers.yml index 0692a9f4377d..c5226dd44c80 100644 --- a/.github/workflows/launchers.yml +++ b/.github/workflows/launchers.yml @@ -1,6 +1,5 @@ name: Test CLI Launchers on all the platforms on: - push: workflow_dispatch: jobs: @@ -22,6 +21,7 @@ jobs: linux-aarch64: name: Deploy and Test on Linux ARM64 architecture runs-on: macos-latest + if: ${{ false }} steps: - uses: actions/checkout@v4 - name: Set up JDK 17 From b01a91fce4a91c806f63505d2cd9f3009786d530 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Wed, 29 May 2024 14:22:42 +0200 Subject: [PATCH 7/9] use --cli-version launcher option on windows --- dist/bin-native-overrides/cli-common-platform | 15 +++++++++++- .../cli-common-platform.bat | 17 ++++++++++++- project/Build.scala | 6 ++++- project/RepublishPlugin.scala | 24 +++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/dist/bin-native-overrides/cli-common-platform b/dist/bin-native-overrides/cli-common-platform index 49803d6282c5..1a11c770f91a 100644 --- a/dist/bin-native-overrides/cli-common-platform +++ b/dist/bin-native-overrides/cli-common-platform @@ -1,3 +1,16 @@ #!/usr/bin/env bash -SCALA_CLI_CMD_BASH=("\"$PROG_HOME/bin/scala-cli\"") +if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then + SCALA_CLI_VERSION="" + # iterate through lines in VERSION_SRC + while IFS= read -r line; do + # if line starts with "version:=" then extract the version + if [[ "$line" == cli_version:=* ]]; then + SCALA_CLI_VERSION="${line#cli_version:=}" + break + fi + done < "$PROG_HOME/EXTRA_PROPERTIES" + SCALA_CLI_CMD_BASH=("\"$PROG_HOME/bin/scala-cli\"" "--cli-version \"$SCALA_CLI_VERSION\"") +else + SCALA_CLI_CMD_BASH=("\"$PROG_HOME/bin/scala-cli\"") +fi diff --git a/dist/bin-native-overrides/cli-common-platform.bat b/dist/bin-native-overrides/cli-common-platform.bat index 62b6a1029c13..e0cfa40692b5 100644 --- a/dist/bin-native-overrides/cli-common-platform.bat +++ b/dist/bin-native-overrides/cli-common-platform.bat @@ -1,3 +1,18 @@ @echo off -set SCALA_CLI_CMD_WIN="%_PROG_HOME%\bin\scala-cli.exe" \ No newline at end of file +setlocal enabledelayedexpansion + +set "_SCALA_CLI_VERSION=" +@rem read for cli_version:=_SCALA_CLI_VERSION in EXTRA_PROPERTIES file +FOR /F "usebackq delims=" %%G IN ("%_PROG_HOME%\EXTRA_PROPERTIES") DO ( + SET "line=%%G" + IF "!line:~0,13!"=="cli_version:=" ( + SET "_SCALA_CLI_VERSION=!line:~13!" + GOTO :foundCliVersion + ) +) + +:foundCliVersion +endlocal & set "SCALA_CLI_VERSION=%_SCALA_CLI_VERSION%" + +set SCALA_CLI_CMD_WIN="%_PROG_HOME%\bin\scala-cli.exe" "--cli-version" "%SCALA_CLI_VERSION%" \ No newline at end of file diff --git a/project/Build.scala b/project/Build.scala index f6c905800f40..0786c47ad5fc 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -119,6 +119,8 @@ object Build { /** Version of Scala CLI to download */ val scalaCliLauncherVersion = "1.3.2" + /** Version of Scala CLI to download (on Windows - last known validated version) */ + val scalaCliLauncherVersionWindows = "1.3.2" /** Version of Coursier to download for initializing the local maven repo of Scala command */ val coursierJarVersion = "2.1.10" @@ -2164,8 +2166,10 @@ object Build { packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-win32", republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", republishFetchCoursier := (dist / republishFetchCoursier).value, + republishExtraProps += ("cli_version" -> scalaCliLauncherVersion), + mappings += (republishRepo.value / "etc" / "EXTRA_PROPERTIES" -> "EXTRA_PROPERTIES"), republishLaunchers += - ("scala-cli.exe" -> s"zip+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-pc-win32.zip!/scala-cli.exe") + ("scala-cli.exe" -> s"zip+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersionWindows/scala-cli-x86_64-pc-win32.zip!/scala-cli.exe") ) lazy val `dist-linux-x86_64` = project.in(file("dist/linux-x86_64")).asDist(Bootstrapped) diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala index 6efb6208a152..537c82d62cce 100644 --- a/project/RepublishPlugin.scala +++ b/project/RepublishPlugin.scala @@ -62,6 +62,7 @@ object RepublishPlugin extends AutoPlugin { val republishFetchLaunchers = taskKey[Set[File]]("cache the launcher deps for the distribution") val republishFetchCoursier = taskKey[File]("cache the coursier.jar for resolving the local maven repo.") val republishPrepareBin = taskKey[File]("prepare the bin directory, including launchers and scripts.") + val republishWriteExtraProps = taskKey[Option[File]]("write extra properties for the launchers.") val republishBinDir = settingKey[File]("where to find static files for the bin dir.") val republishCoursierDir = settingKey[File]("where to download the coursier launcher jar.") val republishBinOverrides = settingKey[Seq[File]]("files to override those in bin-dir.") @@ -69,6 +70,7 @@ object RepublishPlugin extends AutoPlugin { val republishRepo = settingKey[File]("the location to store the republished artifacts.") val republishLaunchers = settingKey[Seq[(String, String)]]("launchers to download. Sequence of (name, URL).") val republishCoursier = settingKey[Seq[(String, String)]]("coursier launcher to download. Sequence of (name, URL).") + val republishExtraProps = settingKey[Seq[(String, String)]]("extra properties for launchers.") } import autoImport._ @@ -276,6 +278,7 @@ object RepublishPlugin extends AutoPlugin { republishLaunchers := Seq.empty, republishCoursier := Seq.empty, republishBinOverrides := Seq.empty, + republishExtraProps := Seq.empty, republishLocalResolved / republishProjectRefs := { val proj = thisProjectRef.value val deps = buildDependencies.value @@ -366,10 +369,31 @@ object RepublishPlugin extends AutoPlugin { } targetBin }, + republishWriteExtraProps := { + val s = streams.value + val log = s.log + val extraProps = republishExtraProps.value + if (extraProps.isEmpty) { + log.info("[republish] No extra properties to write.") + None + } + else { + val repoDir = republishRepo.value + val propsFile = repoDir / "etc" / "EXTRA_PROPERTIES" + log.info(s"[republish] Writing extra properties to $propsFile...") + Using.fileWriter()(propsFile) { writer => + extraProps.foreach { case (k, v) => + writer.write(s"$k:=$v\n") + } + } + Some(propsFile) + } + }, republish := { val cacheDir = republishRepo.value val artifacts = republishClasspath.value val launchers = republishFetchLaunchers.value + val extraProps = republishWriteExtraProps.value cacheDir } ) From 84c6969bc5109701701026455968aa388053ee01 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Wed, 29 May 2024 17:08:30 +0200 Subject: [PATCH 8/9] add tests to launchers --- .github/workflows/launchers.yml | 32 ++++--- project/scripts/native-integration/bashTests | 84 +++++++++++++++++++ .../reportScalaVersion.scala | 4 + .../scripts/native-integration/winTests.bat | 19 +++++ 4 files changed, 126 insertions(+), 13 deletions(-) create mode 100755 project/scripts/native-integration/bashTests create mode 100644 project/scripts/native-integration/reportScalaVersion.scala create mode 100755 project/scripts/native-integration/winTests.bat diff --git a/.github/workflows/launchers.yml b/.github/workflows/launchers.yml index c5226dd44c80..719ac20b9828 100644 --- a/.github/workflows/launchers.yml +++ b/.github/workflows/launchers.yml @@ -14,9 +14,10 @@ jobs: java-version: '17' distribution: 'temurin' cache: 'sbt' - - name: Build the launcher command - run: sbt "dist-linux-x86_64/pack" - - run: ./dist/linux-x86_64/target/pack/bin/scala --version + - name: Build and test launcher command + run: ./project/scripts/native-integration/bashTests + env: + LAUNCHER_EXPECTED_PROJECT: "dist-linux-x86_64" linux-aarch64: name: Deploy and Test on Linux ARM64 architecture @@ -33,9 +34,10 @@ jobs: # https://github.com/actions/runner-images/issues/9369 - name: Install sbt run: brew install sbt - - name: Build the launcher command - run: sbt "dist-linux-aarch64/pack" - - run: ./dist/linux-aarch64/target/pack/bin/scala --version + - name: Build and test launcher command + run: ./project/scripts/native-integration/bashTests + env: + LAUNCHER_EXPECTED_PROJECT: "dist-linux-aarch64" mac-x86_64: name: Deploy and Test on Mac x64 architecture @@ -51,9 +53,10 @@ jobs: # https://github.com/actions/runner-images/issues/9369 - name: Install sbt run: brew install sbt - - name: Build the launcher command - run: sbt "dist-mac-x86_64/pack" - - run: ./dist/mac-x86_64/target/pack/bin/scala --version + - name: Build and test launcher command + run: ./project/scripts/native-integration/bashTests + env: + LAUNCHER_EXPECTED_PROJECT: "dist-mac-x86_64" mac-aarch64: name: Deploy and Test on Mac ARM64 architecture @@ -69,9 +72,10 @@ jobs: # https://github.com/actions/runner-images/issues/9369 - name: Install sbt run: brew install sbt - - name: Build the launcher command - run: sbt "dist-mac-aarch64/pack" - - run: ./dist/mac-aarch64/target/pack/bin/scala --version + - name: Build and test launcher command + run: ./project/scripts/native-integration/bashTests + env: + LAUNCHER_EXPECTED_PROJECT: "dist-mac-aarch64" win-x86_64: name: Deploy and Test on Windows x64 architecture @@ -86,4 +90,6 @@ jobs: cache: 'sbt' - name: Build the launcher command run: sbt "dist-win-x86_64/pack" - - run: ./dist/win-x86_64/target/pack/bin/scala --version + - name: Run the launcher command tests + run: './project/scripts/native-integration/winTests.bat' + shell: cmd diff --git a/project/scripts/native-integration/bashTests b/project/scripts/native-integration/bashTests new file mode 100755 index 000000000000..5fb77355238c --- /dev/null +++ b/project/scripts/native-integration/bashTests @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -eux + +#/*---------------*\ +# * SETUP VARS *# +# *---------------*/ + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/../../.." + +SBT="$ROOT/project/scripts/sbt" # if run on CI +# SBT="sbt" # if run locally + +# set the $DIST_PROJECT and $DIST_DIR variables +source "$ROOT/bin/common-platform" + +die () { + echo >&2 "$@" + exit 1 +} + +PROG_HOME="$DIST_DIR/target/pack" + +SOURCE="$ROOT/tests/pos/HelloWorld.scala" +SOURCE_VERSION="$ROOT/project/scripts/native-integration/reportScalaVersion.scala" + +clear_cli_dotfiles() +{ + local out="$1" + rm -rf "$out"/.bsp + rm -rf "$out"/.scala-build + + rm -f "$ROOT"/.bsp/scala.json + if [ -z "$(ls -A "$ROOT"/.bsp)" ]; then + rm -rf "$ROOT"/.bsp + fi + rm -rf "$ROOT"/.scala-build +} + +#/*---------------*\ +# * INITIALIZE *# +# *---------------*/ + +# build the distribution +"$SBT" "$DIST_PROJECT/pack" + +SCALA_VERSION="" +# iterate through lines in VERSION_SRC +while IFS= read -r line; do + # if line starts with "version:=" then extract the version + if [[ "$line" == version:=* ]]; then + SCALA_VERSION="${line#version:=}" + break + fi +done < "$PROG_HOME/VERSION" + +if [ -z "$SCALA_VERSION" ]; then + die "Could not find scala version in $PROG_HOME/VERSION" +fi + +#/*-------------------*\ +# * TESTING BEGINS *# +# *-------------------*/ + +echo "assert native launcher matches expected version" +if [ -z "$LAUNCHER_EXPECTED_PROJECT" ]; then + die "LAUNCHER_EXPECTED_PROJECT is not set in the environment" +fi +test "$LAUNCHER_EXPECTED_PROJECT" = "$DIST_PROJECT" + +echo "testing version output (default)" +std_output=$("$PROG_HOME/bin/scala" version --scala-version) +test "$SCALA_VERSION" = "$std_output" + +echo "testing run command" +std_output=$("$PROG_HOME/bin/scala" run "$SOURCE" --power --offline --server=false) +test "hello world" = "$std_output" +clear_cli_dotfiles "$ROOT/tests/pos" + +echo "testing run command (-with-compiler)" +std_output=$("$PROG_HOME/bin/scala" run "$SOURCE_VERSION" -with-compiler --power --offline --server=false) +test "$SCALA_VERSION" = "$std_output" +clear_cli_dotfiles "$ROOT/project/scripts/native-integration" + diff --git a/project/scripts/native-integration/reportScalaVersion.scala b/project/scripts/native-integration/reportScalaVersion.scala new file mode 100644 index 000000000000..dc6e93708a48 --- /dev/null +++ b/project/scripts/native-integration/reportScalaVersion.scala @@ -0,0 +1,4 @@ +// To be ran by Scala CLI (requires -with-compiler command line option) + +@main def reportScalaVersion: Unit = + println(dotty.tools.dotc.config.Properties.versionNumberString) diff --git a/project/scripts/native-integration/winTests.bat b/project/scripts/native-integration/winTests.bat new file mode 100755 index 000000000000..a85b2c8c2531 --- /dev/null +++ b/project/scripts/native-integration/winTests.bat @@ -0,0 +1,19 @@ +@echo off +setlocal + +@rem paths are relative to the root project directory +set "_PREFIX=dist\win-x86_64\target\pack" +set "_SOURCE=tests\pos\HelloWorld.scala" +set "_OUT_DIR=out" + +@rem if-tests mimic the non-existing bash instruction 'set -e'. +call "%_PREFIX%\bin\scalac.bat" "@project\scripts\options" "%_SOURCE%" +if not %ERRORLEVEL%==0 endlocal& exit /b 1 + +call "%_PREFIX%\bin\scalac.bat" -d "%_OUT_DIR%" "%_SOURCE%" +if not %ERRORLEVEL%==0 endlocal& exit /b 1 + +call "%_PREFIX%\bin\scala.bat" --power -classpath "%_OUT_DIR%" -M HelloWorld --offline --server=false +if not %ERRORLEVEL%==0 endlocal& exit /b 1 + +endlocal From aeafc081968cb171ef2ff38d9bcb519d6189860d Mon Sep 17 00:00:00 2001 From: Hamza REMMAL Date: Sat, 8 Jun 2024 12:07:23 +0100 Subject: [PATCH 9/9] Address the comments in the review --- .github/workflows/launchers.yml | 1 + compiler/src/dotty/tools/MainGenericRunner.scala | 2 +- project/Build.scala | 3 +-- project/Modes.scala | 2 +- project/scripts/buildScalaBinary | 6 +++++- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/launchers.yml b/.github/workflows/launchers.yml index 719ac20b9828..818e3b72b06b 100644 --- a/.github/workflows/launchers.yml +++ b/.github/workflows/launchers.yml @@ -1,5 +1,6 @@ name: Test CLI Launchers on all the platforms on: + pull_request: workflow_dispatch: jobs: diff --git a/compiler/src/dotty/tools/MainGenericRunner.scala b/compiler/src/dotty/tools/MainGenericRunner.scala index 5b238693a135..bf477f019cba 100644 --- a/compiler/src/dotty/tools/MainGenericRunner.scala +++ b/compiler/src/dotty/tools/MainGenericRunner.scala @@ -270,7 +270,7 @@ object MainGenericRunner { val ranByCoursierBootstrap = sys.props.isDefinedAt("coursier.mainJar") - || sys.props.get("bootstrap.mainClass").filter(_ == "dotty.tools.MainGenericRunner").isDefined + || sys.props.get("bootstrap.mainClass").contains("dotty.tools.MainGenericRunner") val silenced = sys.props.get("scala.use_legacy_launcher") == Some("true") diff --git a/project/Build.scala b/project/Build.scala index 0786c47ad5fc..c1a8800421a6 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -2323,8 +2323,7 @@ object Build { settings(scala3PresentationCompilerBuildInfo) def asDist(implicit mode: Mode): Project = project. - enablePlugins(PackPlugin). - enablePlugins(RepublishPlugin). + enablePlugins(PackPlugin, RepublishPlugin). withCommonSettings. settings(commonDistSettings). dependsOn( diff --git a/project/Modes.scala b/project/Modes.scala index 8f34aa5870e4..eddb5a3f1a7b 100644 --- a/project/Modes.scala +++ b/project/Modes.scala @@ -1,4 +1,4 @@ -import sbt.{Project, ProjectReference, SettingsDefinition, Plugins} +import sbt.{Project, ProjectReference, SettingsDefinition} object Modes { diff --git a/project/scripts/buildScalaBinary b/project/scripts/buildScalaBinary index 552831c07b41..7fc5275e5d8d 100755 --- a/project/scripts/buildScalaBinary +++ b/project/scripts/buildScalaBinary @@ -2,7 +2,11 @@ set -e -source $(dirname $0)/cmdTestsCommon.inc.sh +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/../.." +SBT="$ROOT/project/scripts/sbt" # if run on CI + +# set the $DIST_PROJECT and $DIST_DIR variables +source "$ROOT/bin/common-platform" # build the scala/scalac/scaladoc binary, where scala is native for the current platform. "$SBT" "$DIST_PROJECT/pack"