From ff48634676b17fe13692d4403b954e0a275fcf5f Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 25 Oct 2018 18:13:31 +0200 Subject: [PATCH 1/6] Allow tasty files to be the input of -from-tasty Also works for -decompile and any other compiler that has the -form-tatsy flag. --- compiler/src/dotty/tools/dotc/Driver.scala | 24 +++++++++++- .../tools/dotc/fromtasty/TastyFileUtil.scala | 39 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala diff --git a/compiler/src/dotty/tools/dotc/Driver.scala b/compiler/src/dotty/tools/dotc/Driver.scala index f319432199a6..b5105bb32871 100644 --- a/compiler/src/dotty/tools/dotc/Driver.scala +++ b/compiler/src/dotty/tools/dotc/Driver.scala @@ -1,13 +1,16 @@ package dotty.tools.dotc +import java.nio.file.{Files, Paths} + import dotty.tools.FatalError import config.CompilerCommand import core.Comments.{ContextDoc, ContextDocstrings} import core.Contexts.{Context, ContextBase} import core.Mode import reporting._ + import scala.util.control.NonFatal -import fromtasty.TASTYCompiler +import fromtasty.{TASTYCompiler, TastyFileUtil} /** Run the Dotty compiler. * @@ -132,7 +135,24 @@ class Driver { * if compilation succeeded. */ def process(args: Array[String], rootCtx: Context): Reporter = { - val (fileNames, ctx) = setup(args, rootCtx) + val (fileNames0, ctx0) = setup(args, rootCtx) + val (fileNames, ctx) = + if (ctx0.settings.fromTasty.value(ctx0)) { + // Resolve classpath and class names of tasty files + val (classPaths, classNames) = fileNames0.map { name => + val path = Paths.get(name) + if (!name.endsWith(".tasty")) ("", name) + else if (Files.exists(path)) TastyFileUtil.getClassName(path) + else { + ctx0.error(s"File $name does not exist.") + ("", name) + } + }.unzip + val ctx1 = ctx0.fresh + val classPaths1 = classPaths.distinct.filter(_ != "") + ctx1.setSetting(ctx1.settings.classpath, (ctx1.settings.classpath.value(ctx1) :: classPaths1).mkString(":")) + (classNames, ctx1) + } else (fileNames0, ctx0) doCompile(newCompiler(ctx), fileNames)(ctx) } diff --git a/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala b/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala new file mode 100644 index 000000000000..d0352553628c --- /dev/null +++ b/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala @@ -0,0 +1,39 @@ +package dotty.tools.dotc.fromtasty + +import java.nio.file.{Files, Path, Paths} + +import dotty.tools.dotc.core.Contexts.Context +import dotty.tools.dotc.core.NameKinds +import dotty.tools.dotc.core.Names.SimpleName +import dotty.tools.dotc.core.StdNames.nme +import dotty.tools.dotc.core.tasty.TastyUnpickler + +object TastyFileUtil { + + /** Get the class path and the class name including packages + * + * If + * ```scala + * package foo + * class Foo + * ``` + * then `getClassName("./out/foo/Foo.tasty") returns `("./out", "foo.Foo")` + */ + def getClassName(path: Path): (String, String) = { + assert(path.toString.endsWith(".tasty")) + assert(Files.exists(path)) + val bytes = Files.readAllBytes(path) + val unpickler: TastyUnpickler = new TastyUnpickler(bytes) + val className = + unpickler.nameAtRef.contents.iterator.takeWhile { + case name: SimpleName => name != nme.CONSTRUCTOR + case name => !name.is(NameKinds.ModuleClassName) + }.collect { + case name: SimpleName => name.toString + }.drop(1).toList + + val classpath = path.toString.replace(className.mkString("", "/", ".tasty"), "") + (classpath, className.mkString(".")) + } + +} From 15b912a5e75c419f6b4d62ed513626a8ccb03379 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 26 Oct 2018 18:53:46 +0200 Subject: [PATCH 2/6] Use File.pathSeparator --- compiler/src/dotty/tools/dotc/Driver.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/Driver.scala b/compiler/src/dotty/tools/dotc/Driver.scala index b5105bb32871..3b6607c14e78 100644 --- a/compiler/src/dotty/tools/dotc/Driver.scala +++ b/compiler/src/dotty/tools/dotc/Driver.scala @@ -150,7 +150,8 @@ class Driver { }.unzip val ctx1 = ctx0.fresh val classPaths1 = classPaths.distinct.filter(_ != "") - ctx1.setSetting(ctx1.settings.classpath, (ctx1.settings.classpath.value(ctx1) :: classPaths1).mkString(":")) + val fullClassPath = (ctx1.settings.classpath.value(ctx1) :: classPaths1).mkString(java.io.File.pathSeparator) + ctx1.setSetting(ctx1.settings.classpath, fullClassPath) (classNames, ctx1) } else (fileNames0, ctx0) doCompile(newCompiler(ctx), fileNames)(ctx) From 4e6adcfe02816cbe34c9efeff514ec169c0d4006 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 26 Oct 2018 19:22:06 +0200 Subject: [PATCH 3/6] Fix handling of empty package and ASTs name --- .../dotty/tools/dotc/core/tasty/DottyUnpickler.scala | 2 +- .../dotty/tools/dotc/core/tasty/TastyPrinter.scala | 2 +- .../dotty/tools/dotc/core/tasty/TreePickler.scala | 4 +++- .../dotty/tools/dotc/fromtasty/TastyFileUtil.scala | 12 +++++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala index fdb003a8a181..57125fed3e65 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala @@ -16,7 +16,7 @@ object DottyUnpickler { class BadSignature(msg: String) extends RuntimeException(msg) class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler]) - extends SectionUnpickler[TreeUnpickler]("ASTs") { + extends SectionUnpickler[TreeUnpickler](TreePickler.sectionName) { def unpickle(reader: TastyReader, nameAtRef: NameTable): TreeUnpickler = new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler, Seq.empty) } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index 4bd53cdb8d2f..afaecd8c0ef8 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -34,7 +34,7 @@ class TastyPrinter(bytes: Array[Byte])(implicit ctx: Context) { unpickle(new CommentSectionUnpickler) } - class TreeSectionUnpickler extends SectionUnpickler[Unit]("ASTs") { + class TreeSectionUnpickler extends SectionUnpickler[Unit](TreePickler.sectionName) { import TastyFormat._ def unpickle(reader: TastyReader, tastyName: NameTable): Unit = { import reader._ diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala index 2960c4ebc3f7..3c76acf4f2e1 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala @@ -17,6 +17,8 @@ import printing.Texts._ object TreePickler { + val sectionName = "ASTs" + case class Hole(idx: Int, args: List[tpd.Tree]) extends tpd.Tree { override def fallbackToText(printer: Printer): Text = s"[[$idx|" ~~ printer.toTextGlobal(args, ", ") ~~ "]]" @@ -25,7 +27,7 @@ object TreePickler { class TreePickler(pickler: TastyPickler) { val buf: TreeBuffer = new TreeBuffer - pickler.newSection("ASTs", buf) + pickler.newSection(TreePickler.sectionName, buf) import TreePickler._ import buf._ import pickler.nameBuffer.nameIndex diff --git a/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala b/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala index d0352553628c..cc28e2ced030 100644 --- a/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala +++ b/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala @@ -6,7 +6,7 @@ import dotty.tools.dotc.core.Contexts.Context import dotty.tools.dotc.core.NameKinds import dotty.tools.dotc.core.Names.SimpleName import dotty.tools.dotc.core.StdNames.nme -import dotty.tools.dotc.core.tasty.TastyUnpickler +import dotty.tools.dotc.core.tasty.{TastyUnpickler, TreePickler} object TastyFileUtil { @@ -25,14 +25,16 @@ object TastyFileUtil { val bytes = Files.readAllBytes(path) val unpickler: TastyUnpickler = new TastyUnpickler(bytes) val className = - unpickler.nameAtRef.contents.iterator.takeWhile { + unpickler.nameAtRef.contents.iterator.dropWhile { + name => name.toString == TreePickler.sectionName || name == nme.EMPTY_PACKAGE + }.takeWhile { case name: SimpleName => name != nme.CONSTRUCTOR case name => !name.is(NameKinds.ModuleClassName) }.collect { case name: SimpleName => name.toString - }.drop(1).toList - - val classpath = path.toString.replace(className.mkString("", "/", ".tasty"), "") + }.toList + val classInPath = className.mkString("", "/", ".tasty") + val classpath = path.toString.replace(classInPath, "") (classpath, className.mkString(".")) } From 830cdb1783c8c49a73d38d8e3cf2daf317a2f731 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Wed, 7 Nov 2018 22:39:36 +0000 Subject: [PATCH 4/6] Get name by reading the TASTy tree --- compiler/src/dotty/tools/dotc/Driver.scala | 10 +++- .../dotc/core/tasty/TastyClassName.scala | 58 +++++++++++++++++++ .../tools/dotc/fromtasty/TastyFileUtil.scala | 29 +++++----- 3 files changed, 79 insertions(+), 18 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala diff --git a/compiler/src/dotty/tools/dotc/Driver.scala b/compiler/src/dotty/tools/dotc/Driver.scala index 3b6607c14e78..8b337c3538c3 100644 --- a/compiler/src/dotty/tools/dotc/Driver.scala +++ b/compiler/src/dotty/tools/dotc/Driver.scala @@ -142,8 +142,14 @@ class Driver { val (classPaths, classNames) = fileNames0.map { name => val path = Paths.get(name) if (!name.endsWith(".tasty")) ("", name) - else if (Files.exists(path)) TastyFileUtil.getClassName(path) - else { + else if (Files.exists(path)) { + TastyFileUtil.getClassName(path) match { + case Some(res) => res + case _ => + ctx0.error(s"Could not load classname from $name.") + ("", name) + } + } else { ctx0.error(s"File $name does not exist.") ("", name) } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala new file mode 100644 index 000000000000..041a84da2912 --- /dev/null +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala @@ -0,0 +1,58 @@ +package dotty.tools.dotc +package core +package tasty + +import Contexts._, Decorators._ +import Names.{Name, TermName} +import StdNames.nme +import TastyUnpickler._ +import TastyBuffer.NameRef +import util.Positions.offsetToInt +import printing.Highlighting._ + +/** Reads the package and class name of the class contained in this TASTy */ +class TastyClassName(bytes: Array[Byte]) { + + val unpickler: TastyUnpickler = new TastyUnpickler(bytes) + import unpickler.{nameAtRef, unpickle} + + /** Returns a tuple with the package and class names */ + def readName(): Option[(TermName, TermName)] = unpickle(new TreeSectionUnpickler) + + class TreeSectionUnpickler extends SectionUnpickler[(TermName, TermName)](TreePickler.sectionName) { + import TastyFormat._ + def unpickle(reader: TastyReader, tastyName: NameTable): (TermName, TermName) = { + import reader._ + def readName() = { + val idx = readNat() + nameAtRef(NameRef(idx)) + } + def readNames(pack: TermName): (TermName, TermName) = { + val tag = readByte() + if (tag >= firstLengthTreeTag) { + val len = readNat() + val end = currentAddr + len + tag match { + case TYPEDEF => + val name = readName() + goto(end) + (pack, name) + case IMPORT | VALDEF => + goto(end) + readNames(pack) + case PACKAGE => + readNames(pack) + } + } + else tag match { + case TERMREFpkg | TYPEREFpkg => + val packName = readName() + readNames(packName) + case _ => + readNames(pack) + } + } + readNames(nme.EMPTY_PACKAGE) + } + } +} diff --git a/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala b/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala index cc28e2ced030..6f8be1e595c9 100644 --- a/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala +++ b/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala @@ -1,6 +1,8 @@ -package dotty.tools.dotc.fromtasty +package dotty.tools.dotc +package fromtasty import java.nio.file.{Files, Path, Paths} +import java.io import dotty.tools.dotc.core.Contexts.Context import dotty.tools.dotc.core.NameKinds @@ -17,25 +19,20 @@ object TastyFileUtil { * package foo * class Foo * ``` - * then `getClassName("./out/foo/Foo.tasty") returns `("./out", "foo.Foo")` + * then `getClassName("./out/foo/Foo.tasty") returns `Some(("./out", "foo.Foo"))` */ - def getClassName(path: Path): (String, String) = { + def getClassName(path: Path): Option[(String, String)] = { assert(path.toString.endsWith(".tasty")) assert(Files.exists(path)) val bytes = Files.readAllBytes(path) - val unpickler: TastyUnpickler = new TastyUnpickler(bytes) - val className = - unpickler.nameAtRef.contents.iterator.dropWhile { - name => name.toString == TreePickler.sectionName || name == nme.EMPTY_PACKAGE - }.takeWhile { - case name: SimpleName => name != nme.CONSTRUCTOR - case name => !name.is(NameKinds.ModuleClassName) - }.collect { - case name: SimpleName => name.toString - }.toList - val classInPath = className.mkString("", "/", ".tasty") - val classpath = path.toString.replace(classInPath, "") - (classpath, className.mkString(".")) + val names = new core.tasty.TastyClassName(bytes).readName() + names.map { case (packName, className) => + val fullName = s"$packName.${className.lastPart}" + val classInPath = fullName.replace(".", io.File.separator) + ".tasty" + val classpath = path.toString.replace(classInPath, "") + (classpath, fullName) + } + } } From c034eaf855364a3c3bdb2ebd86459e323a2cae3b Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 22 Nov 2018 15:53:37 +0100 Subject: [PATCH 5/6] Rename variables --- .../tools/dotc/core/tasty/TastyClassName.scala | 16 ++++++++-------- .../tools/dotc/fromtasty/TastyFileUtil.scala | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala index 041a84da2912..ab7ead709560 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala @@ -27,29 +27,29 @@ class TastyClassName(bytes: Array[Byte]) { val idx = readNat() nameAtRef(NameRef(idx)) } - def readNames(pack: TermName): (TermName, TermName) = { + def readNames(packageName: TermName): (TermName, TermName) = { val tag = readByte() if (tag >= firstLengthTreeTag) { val len = readNat() val end = currentAddr + len tag match { case TYPEDEF => - val name = readName() + val className = readName() goto(end) - (pack, name) + (packageName, className) case IMPORT | VALDEF => goto(end) - readNames(pack) + readNames(packageName) case PACKAGE => - readNames(pack) + readNames(packageName) } } else tag match { case TERMREFpkg | TYPEREFpkg => - val packName = readName() - readNames(packName) + val subPackageName = readName() + readNames(subPackageName) case _ => - readNames(pack) + readNames(packageName) } } readNames(nme.EMPTY_PACKAGE) diff --git a/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala b/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala index 6f8be1e595c9..89da2374276f 100644 --- a/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala +++ b/compiler/src/dotty/tools/dotc/fromtasty/TastyFileUtil.scala @@ -26,8 +26,8 @@ object TastyFileUtil { assert(Files.exists(path)) val bytes = Files.readAllBytes(path) val names = new core.tasty.TastyClassName(bytes).readName() - names.map { case (packName, className) => - val fullName = s"$packName.${className.lastPart}" + names.map { case (packageName, className) => + val fullName = s"$packageName.${className.lastPart}" val classInPath = fullName.replace(".", io.File.separator) + ".tasty" val classpath = path.toString.replace(classInPath, "") (classpath, fullName) From 0bf8b1723d101e1ec6bd3541b830a8c6b23744fe Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 22 Nov 2018 16:00:46 +0100 Subject: [PATCH 6/6] Move tasty setup logic and invert classpath order --- compiler/src/dotty/tools/dotc/Driver.scala | 55 ++++++++++++---------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/Driver.scala b/compiler/src/dotty/tools/dotc/Driver.scala index 8b337c3538c3..b1c9496e3a8f 100644 --- a/compiler/src/dotty/tools/dotc/Driver.scala +++ b/compiler/src/dotty/tools/dotc/Driver.scala @@ -57,7 +57,34 @@ class Driver { } val fileNames = CompilerCommand.checkUsage(summary, sourcesRequired)(ctx) - (fileNames, ctx) + fromTastySetup(fileNames, ctx) + } + + /** Setup extra classpath and figure out class names for tasty file inputs */ + private def fromTastySetup(fileNames0: List[String], ctx0: Context) = { + if (ctx0.settings.fromTasty.value(ctx0)) { + // Resolve classpath and class names of tasty files + val (classPaths, classNames) = fileNames0.map { name => + val path = Paths.get(name) + if (!name.endsWith(".tasty")) ("", name) + else if (Files.exists(path)) { + TastyFileUtil.getClassName(path) match { + case Some(res) => res + case _ => + ctx0.error(s"Could not load classname from $name.") + ("", name) + } + } else { + ctx0.error(s"File $name does not exist.") + ("", name) + } + }.unzip + val ctx1 = ctx0.fresh + val classPaths1 = classPaths.distinct.filter(_ != "") + val fullClassPath = (classPaths1 :+ ctx1.settings.classpath.value(ctx1)).mkString(java.io.File.pathSeparator) + ctx1.setSetting(ctx1.settings.classpath, fullClassPath) + (classNames, ctx1) + } else (fileNames0, ctx0) } /** Entry point to the compiler that can be conveniently used with Java reflection. @@ -135,31 +162,7 @@ class Driver { * if compilation succeeded. */ def process(args: Array[String], rootCtx: Context): Reporter = { - val (fileNames0, ctx0) = setup(args, rootCtx) - val (fileNames, ctx) = - if (ctx0.settings.fromTasty.value(ctx0)) { - // Resolve classpath and class names of tasty files - val (classPaths, classNames) = fileNames0.map { name => - val path = Paths.get(name) - if (!name.endsWith(".tasty")) ("", name) - else if (Files.exists(path)) { - TastyFileUtil.getClassName(path) match { - case Some(res) => res - case _ => - ctx0.error(s"Could not load classname from $name.") - ("", name) - } - } else { - ctx0.error(s"File $name does not exist.") - ("", name) - } - }.unzip - val ctx1 = ctx0.fresh - val classPaths1 = classPaths.distinct.filter(_ != "") - val fullClassPath = (ctx1.settings.classpath.value(ctx1) :: classPaths1).mkString(java.io.File.pathSeparator) - ctx1.setSetting(ctx1.settings.classpath, fullClassPath) - (classNames, ctx1) - } else (fileNames0, ctx0) + val (fileNames, ctx) = setup(args, rootCtx) doCompile(newCompiler(ctx), fileNames)(ctx) }