Skip to content

Commit d465c02

Browse files
nicolasstuckinicolasstucki
committed
Add TASTY quote pickling for trees
* Pickle/unpickle quotes * Add Splicer * Add tree interpreter for splicing staged expression * Add concrete implementations of Quoted
1 parent 94178df commit d465c02

36 files changed

+473
-43
lines changed

‎compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,18 @@ class DottyUnpickler(bytes: Array[Byte]) extends ClassfileParser.Embedded {
3838

3939
val unpickler = new TastyUnpickler(bytes)
4040
private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler)
41-
private val treeUnpickler = unpickler.unpickle(new TreeSectionUnpickler(posUnpicklerOpt)).get
41+
private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt)).get
4242

4343
/** Enter all toplevel classes and objects into their scopes
4444
* @param roots a set of SymDenotations that should be overwritten by unpickling
4545
*/
4646
def enter(roots: Set[SymDenotation])(implicit ctx: Context): Unit =
4747
treeUnpickler.enterTopLevel(roots)
4848

49+
protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler]): TreeSectionUnpickler = {
50+
new TreeSectionUnpickler(posUnpicklerOpt)
51+
}
52+
4953
/** Only used if `-Yretain-trees` is set. */
5054
private[this] var myBody: List[Tree] = _
5155
/** The unpickled trees, and the source file they come from. */

‎compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class TastyPrinter(bytes: Array[Byte])(implicit ctx: Context) {
6767
printName(); printTrees()
6868
case REFINEDtype =>
6969
printName(); printTree(); printTrees()
70-
case RETURN =>
70+
case RETURN | HOLE =>
7171
printNat(); printTrees()
7272
case METHODtype | POLYtype | TYPELAMBDAtype =>
7373
printTree()

‎compiler/src/dotty/tools/dotc/core/tasty/TastyString.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package dotty.tools.dotc.core.tasty
2+
3+
/** Utils for String representation of TASTY */
4+
object TastyString {
5+
6+
/** Decode the TASTY String into TASTY bytes */
7+
def stringToTasty(str: String): Array[Byte] = {
8+
val bytes = new Array[Byte](str.length)
9+
for (i <- str.indices) bytes(i) = str.charAt(i).toByte
10+
bytes
11+
}
12+
13+
/** Encode TASTY bytes into a TASTY String */
14+
def tastyToString(bytes: Array[Byte]): String = {
15+
val chars = new Array[Char](bytes.length)
16+
for (i <- bytes.indices) chars(i) = (bytes(i) & 0xff).toChar
17+
new String(chars)
18+
}
19+
20+
}

‎compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ class TreePickler(pickler: TastyPickler) {
300300
pickleName(sym.name)
301301
pickleParams
302302
tpt match {
303-
case templ: Template => pickleTree(tpt)
303+
case _: Template | _: Hole => pickleTree(tpt)
304304
case _ if tpt.isType => pickleTpt(tpt)
305305
}
306306
pickleTreeUnlessEmpty(rhs)

‎compiler/src/dotty/tools/dotc/interpreter/Interpreter.scala

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package dotty.tools.dotc
2+
package interpreter
3+
4+
import dotty.tools.dotc.ast.tpd
5+
import dotty.tools.dotc.ast.Trees._
6+
import dotty.tools.dotc.core.Constants._
7+
import dotty.tools.dotc.core.Contexts._
8+
import dotty.tools.dotc.core.Flags._
9+
import dotty.tools.dotc.core.Decorators._
10+
import dotty.tools.dotc.core.Symbols._
11+
12+
import scala.reflect.ClassTag
13+
import java.net.URLClassLoader
14+
15+
/** Tree interpreter that can interpret
16+
* * Literal constants
17+
* * Calls to static methods
18+
* * New objects with explicit `new` keyword
19+
* * Quoted code
20+
*
21+
* The interpreter assumes that all calls in the trees are to code that was
22+
* previously compiled and is present in the classpath of the current context.
23+
*/
24+
class Interpreter(implicit ctx: Context) {
25+
import tpd._
26+
27+
private[this] val classLoader = {
28+
val urls = ctx.settings.classpath.value.split(':').map(cp => java.nio.file.Paths.get(cp).toUri.toURL)
29+
new URLClassLoader(urls, getClass.getClassLoader)
30+
}
31+
32+
/** Returns the interpreted result of interpreting the code represented by the tree.
33+
* Return Some of the result or None if some error happen during the interpretation.
34+
*/
35+
def interpretTree[T](tree: Tree)(implicit ct: ClassTag[T]): Option[T] = {
36+
try {
37+
interpretTreeImpl(tree) match {
38+
case obj: T => Some(obj)
39+
case obj =>
40+
ctx.error(s"Interpreted tree returned a result of an unexpected type. Expected ${ct.runtimeClass} but was ${obj.getClass}", tree.pos)
41+
throw new StopInterpretation
42+
}
43+
} catch {
44+
case _: StopInterpretation => None
45+
}
46+
}
47+
48+
/** Returns the interpreted result of interpreting the code represented by the tree.
49+
* Returns the result of the interpreted tree.
50+
*
51+
* If some error is encountered while interpreting a ctx.error is emited and a StopInterpretation is thrown.
52+
*/
53+
private def interpretTreeImpl(tree: Tree): Object = {
54+
try {
55+
tree match {
56+
case Apply(_, quote :: Nil) if tree.symbol eq defn.quoteMethod =>
57+
new RawExpr(quote)
58+
case TypeApply(_, quote :: Nil) if tree.symbol eq defn.typeQuoteMethod =>
59+
new RawType(quote)
60+
61+
case Literal(Constant(c)) =>
62+
c.asInstanceOf[AnyRef]
63+
64+
case Apply(fn, args) if fn.symbol.isConstructor =>
65+
val cls = fn.symbol.owner
66+
val clazz = classLoader.loadClass(cls.symbol.fullName.toString)
67+
val paramClasses = paramsSig(fn.symbol)
68+
val args1: List[Object] = args.map(arg => interpretTreeImpl(arg))
69+
clazz.getConstructor(paramClasses: _*).newInstance(args1: _*).asInstanceOf[Object]
70+
71+
case Apply(fun, args) if fun.symbol.isStatic =>
72+
val clazz = classLoader.loadClass(fun.symbol.owner.companionModule.fullName.toString)
73+
val paramClasses = paramsSig(fun.symbol)
74+
val args1: List[Object] = args.map(arg => interpretTreeImpl(arg))
75+
val method = clazz.getMethod(fun.symbol.name.toString, paramClasses: _*)
76+
method.invoke(null, args1: _*)
77+
78+
case tree: RefTree if tree.symbol.isStatic =>
79+
val clazz = classLoader.loadClass(tree.symbol.owner.companionModule.fullName.toString)
80+
val method = clazz.getMethod(tree.name.toString)
81+
method.invoke(null)
82+
83+
case tree: RefTree if tree.symbol.is(Module) =>
84+
??? // TODO
85+
86+
case Inlined(_, bindings, expansion) =>
87+
if (bindings.nonEmpty) ??? // TODO evaluate bindings and add environment
88+
interpretTreeImpl(expansion)
89+
case _ =>
90+
val msg =
91+
if (tree.tpe.derivesFrom(defn.QuotedExprClass)) "Quote needs to be explicit or a call to a static method"
92+
else "Value needs to be a explicit or a call to a static method"
93+
ctx.error(msg, tree.pos)
94+
throw new StopInterpretation
95+
}
96+
} catch {
97+
case ex: NoSuchMethodException =>
98+
ctx.error("Could not find interpreted method in classpath: " + ex.getMessage, tree.pos)
99+
throw new StopInterpretation
100+
case ex: ClassNotFoundException =>
101+
ctx.error("Could not find interpreted class in classpath: " + ex.getMessage, tree.pos)
102+
throw new StopInterpretation
103+
case ex: RuntimeException =>
104+
ex.printStackTrace()
105+
ctx.error("A runtime exception occurred while interpreting: " + ex.getMessage, tree.pos)
106+
throw new StopInterpretation
107+
}
108+
}
109+
110+
/** List of classes of the parameters of the signature of `sym` */
111+
private def paramsSig(sym: Symbol): List[Class[_]] = {
112+
sym.signature.paramsSig.map { param =>
113+
val paramString = param.toString
114+
if (paramString == defn.BooleanClass.showFullName) classOf[Boolean]
115+
else if (paramString == defn.ByteClass.showFullName) classOf[Byte]
116+
else if (paramString == defn.CharClass.showFullName) classOf[Char]
117+
else if (paramString == defn.ShortClass.showFullName) classOf[Short]
118+
else if (paramString == defn.IntClass.showFullName) classOf[Int]
119+
else if (paramString == defn.LongClass.showFullName) classOf[Long]
120+
else if (paramString == defn.DoubleClass.showFullName) classOf[Float]
121+
else if (paramString == defn.DoubleClass.showFullName) classOf[Double]
122+
else classLoader.loadClass(paramString)
123+
}
124+
}
125+
126+
/** Exception that stops interpretation if some issue is found */
127+
private class StopInterpretation extends Exception
128+
129+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawExpr.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
class RawExpr(val tree: tpd.Tree) extends quoted.Expr[Any] with RawQuoted {
6+
override def toString: String = s"RawExpr(${tree.toString})"
7+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawQuoted.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
trait RawQuoted extends quoted.Quoted {
6+
def tree: tpd.Tree
7+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawType.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
class RawType(val tree: tpd.Tree) extends quoted.Type[Any] with RawQuoted {
6+
override def toString: String = s"RawType(${tree.toString})"
7+
}

‎compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package dotty.tools.dotc.quoted
2+
3+
import dotty.tools.dotc.ast.Trees._
4+
import dotty.tools.dotc.ast.tpd
5+
import dotty.tools.dotc.config.Printers._
6+
import dotty.tools.dotc.core.Constants.Constant
7+
import dotty.tools.dotc.core.Contexts._
8+
import dotty.tools.dotc.core.Decorators._
9+
import dotty.tools.dotc.core.Flags._
10+
import dotty.tools.dotc.core.Symbols._
11+
import dotty.tools.dotc.core.tasty._
12+
import dotty.tools.dotc.interpreter.RawQuoted
13+
14+
object PickledQuotes {
15+
import tpd._
16+
17+
/** Pickle the quote into a TASTY string */
18+
def pickleQuote(tree: Tree)(implicit ctx: Context): String = {
19+
if (ctx.reporter.hasErrors) "<error>"
20+
else {
21+
val encapsulated = encapsulateQuote(tree)
22+
val pickled = pickle(encapsulated)
23+
TastyString.tastyToString(pickled)
24+
}
25+
}
26+
27+
/** Transform the expression into it's fully spliced Tree */
28+
def quotedToTree(expr: quoted.Quoted)(implicit ctx: Context): Tree = expr match {
29+
case expr: quoted.TastyQuoted => unpickleQuote(expr)
30+
case expr: quoted.ValueExpr[_] => Literal(Constant(expr.value))
31+
case expr: RawQuoted => expr.tree
32+
}
33+
34+
/** Unpickle the tree contained in the TastyQuoted */
35+
private def unpickleQuote(expr: quoted.TastyQuoted)(implicit ctx: Context): Tree = {
36+
val tastyBytes = TastyString.stringToTasty(expr.tasty)
37+
val splices = expr.args.map {
38+
case arg: quoted.Quoted => quotedToTree(arg)
39+
case arg => arg
40+
}
41+
val unpickled = unpickle(tastyBytes, splices)
42+
unpickled match { // Expects `package _root_ { val ': Any = <tree> }`
43+
case PackageDef(_, (vdef: ValDef) :: Nil) => vdef.rhs
44+
}
45+
}
46+
47+
/** Encapsulate the tree in a top level `val`
48+
* `<tree>` ==> `package _root_ { val ': Any = <tree> }`
49+
*
50+
* Note: Trees for types are also encapsulated this way to preserve the holes in the tree.
51+
* Encapsulating the type of the tree in a `type ' = <tree.tpe>` can potentially
52+
* contain references to the outer environment.
53+
*/
54+
private def encapsulateQuote(tree: Tree)(implicit ctx: Context): Tree = {
55+
val sym = ctx.newSymbol(ctx.owner, "'".toTermName, Synthetic, defn.AnyType, coord = tree.pos)
56+
val quotedVal = ValDef(sym, tree).withPos(tree.pos)
57+
PackageDef(ref(defn.RootPackage).asInstanceOf[Ident], quotedVal :: Nil).withPos(tree.pos)
58+
}
59+
60+
// TASTY picklingtests/pos/quoteTest.scala
61+
62+
/** Pickle tree into it's TASTY bytes s*/
63+
private def pickle(tree: Tree)(implicit ctx: Context): Array[Byte] = {
64+
val pickler = new TastyPickler(defn.RootClass)
65+
val treePkl = pickler.treePkl
66+
treePkl.pickle(tree :: Nil)
67+
treePkl.compactify()
68+
pickler.addrOfTree = treePkl.buf.addrOfTree
69+
pickler.addrOfSym = treePkl.addrOfSym
70+
// if (tree.pos.exists)
71+
// new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
72+
73+
// other pickle sections go here.
74+
val pickled = pickler.assembleParts()
75+
76+
if (pickling ne noPrinter) {
77+
println(i"**** pickled quote of \n${tree.show}")
78+
new TastyPrinter(pickled).printContents()
79+
}
80+
81+
pickled
82+
}
83+
84+
/** Unpickle TASTY bytes into it's tree */
85+
private def unpickle(bytes: Array[Byte], splices: Seq[Any])(implicit ctx: Context): Tree = {
86+
val unpickler = new QuoteUnpickler(bytes, splices)
87+
unpickler.enter(roots = Set(defn.RootPackage))
88+
val tree = unpickler.body.head
89+
if (pickling ne noPrinter) {
90+
println(i"**** unpickled quote for \n${tree.show}")
91+
new TastyPrinter(bytes).printContents()
92+
}
93+
tree
94+
}
95+
}

‎compiler/src/dotty/tools/dotc/quoted/QuoteUnpickler.scala

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package dotty.tools.dotc.quoted
2+
3+
import dotty.tools.dotc.core.tasty._
4+
5+
object QuoteUnpickler {
6+
class QuotedTreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], splices: Seq[Any])
7+
extends DottyUnpickler.TreeSectionUnpickler(posUnpickler) {
8+
override def unpickle(reader: TastyReader, nameAtRef: TastyUnpickler.NameTable) =
9+
new TreeUnpickler(reader, nameAtRef, posUnpickler, splices)
10+
}
11+
}
12+
13+
/** A class for unpickling quoted Tasty trees and symbols.
14+
* @param bytes the bytearray containing the Tasty file from which we unpickle
15+
* @param splices splices that will fill the holes in the quote
16+
*/
17+
class QuoteUnpickler(bytes: Array[Byte], splices: Seq[Any]) extends DottyUnpickler(bytes) {
18+
import DottyUnpickler._
19+
import QuoteUnpickler._
20+
21+
protected override def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler]): TreeSectionUnpickler =
22+
new QuotedTreeSectionUnpickler(posUnpicklerOpt, splices)
23+
}

‎compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,11 @@ import tasty.TreePickler.Hole
1313
import MegaPhase.MiniPhase
1414
import SymUtils._
1515
import NameKinds.OuterSelectName
16+
1617
import scala.collection.mutable
18+
import dotty.tools.dotc.core.StdNames._
19+
import dotty.tools.dotc.quoted.PickledQuotes
20+
1721

1822
/** Translates quoted terms and types to `unpickle` method calls.
1923
* Checks that the phase consistency principle (PCP) holds.
@@ -29,17 +33,6 @@ class ReifyQuotes extends MacroTransform {
2933
protected def newTransformer(implicit ctx: Context): Transformer =
3034
new Reifier(inQuote = false, null, 0, new LevelInfo)
3135

32-
/** Serialize `tree`. Embedded splices are represented as nodes of the form
33-
*
34-
* Select(qual, sym)
35-
*
36-
* where `sym` is either `defn.QuotedExpr_~` or `defn.QuotedType_~`. For any splice,
37-
* the `qual` part should not be pickled, since it will be added separately later
38-
* as a splice.
39-
*/
40-
def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String =
41-
tree.show // TODO: replace with TASTY
42-
4336
private class LevelInfo {
4437
/** A map from locally defined symbols to the staging levels of their definitions */
4538
val levelOf = new mutable.HashMap[Symbol, Int]
@@ -276,7 +269,7 @@ class ReifyQuotes extends MacroTransform {
276269
ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr)
277270
.appliedToType(if (isType) body1.tpe else body1.tpe.widen)
278271
.appliedTo(
279-
Literal(Constant(pickleTree(body1, isType))),
272+
Literal(Constant(PickledQuotes.pickleQuote(body1))),
280273
SeqLiteral(splices, TypeTree(defn.AnyType)))
281274
}
282275
}.withPos(quote.pos)
@@ -336,7 +329,8 @@ class ReifyQuotes extends MacroTransform {
336329
case Inlined(call, bindings, expansion @ Select(body, name)) if expansion.symbol.isSplice =>
337330
// To maintain phase consistency, convert inlined expressions of the form
338331
// `{ bindings; ~expansion }` to `~{ bindings; expansion }`
339-
transform(cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name))
332+
if (level == 0) transform(Splicer.splice(cpy.Inlined(tree)(call, bindings, body)))
333+
else transform(cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name))
340334
case _: Import =>
341335
tree
342336
case tree: DefDef if tree.symbol.is(Macro) && level == 0 =>

0 commit comments

Comments
 (0)