Skip to content

Commit 0a90a2d

Browse files
nicolasstuckinicolasstucki
committed
Add TASTY quote pickling for trees
* Pickle/unpickle quotes * Add Splicer * Add tree interpreter for splicing staged expression * Add concrete implementations of Quoted
1 parent 2710717 commit 0a90a2d

30 files changed

+404
-33
lines changed

‎compiler/src/dotty/tools/dotc/CompilationUnit.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ class CompilationUnit(val source: SourceFile) {
2626
* is used in phase ReifyQuotes in order to avoid traversing a quote-less tree.
2727
*/
2828
var containsQuotes: Boolean = false
29+
30+
/** Will be reset to `true` if `untpdTree` contains `Quoted.unary_~` trees. The information
31+
* is used in phase ReifyQuotes in order to avoid traversing a splice-less tree.
32+
*/
33+
var containsSplices: Boolean = false
2934
}
3035

3136
object CompilationUnit {

‎compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class TastyPrinter(bytes: Array[Byte])(implicit ctx: Context) {
6767
printName(); printTrees()
6868
case REFINEDtype =>
6969
printName(); printTree(); printTrees()
70-
case RETURN =>
70+
case RETURN | HOLE =>
7171
printNat(); printTrees()
7272
case METHODtype | POLYtype | TYPELAMBDAtype =>
7373
printTree()

‎compiler/src/dotty/tools/dotc/core/tasty/TastyString.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package dotty.tools.dotc.core.tasty
2+
3+
/** Utils for String representation of TASTY */
4+
object TastyString {
5+
6+
/** Decode the TASTY String into TASTY bytes */
7+
def stringToTasty(str: String): Array[Byte] = {
8+
val bytes = new Array[Byte](str.length)
9+
for (i <- str.indices) bytes(i) = str.charAt(i).toByte
10+
bytes
11+
}
12+
13+
/** Encode TASTY bytes into a TASTY String */
14+
def tastyToString(bytes: Array[Byte]): String = {
15+
val chars = new Array[Char](bytes.length)
16+
for (i <- bytes.indices) chars(i) = (bytes(i) & 0xff).toChar
17+
new String(chars)
18+
}
19+
20+
}

‎compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ class TreePickler(pickler: TastyPickler) {
300300
pickleName(sym.name)
301301
pickleParams
302302
tpt match {
303-
case templ: Template => pickleTree(tpt)
303+
case _: Template | _: Hole => pickleTree(tpt)
304304
case _ if tpt.isType => pickleTpt(tpt)
305305
}
306306
pickleTreeUnlessEmpty(rhs)

‎compiler/src/dotty/tools/dotc/interpreter/Expressions.scala

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.core.Contexts._
4+
import dotty.tools.dotc.core.Constants._
5+
import dotty.tools.dotc.ast.tpd
6+
7+
object Expressions {
8+
import tpd._
9+
10+
/** Transform the expression into it's fully spliced Tree */
11+
def exprToTree(expr: quoted.Quoted)(implicit ctx: Context): Tree = expr match {
12+
case expr: quoted.TastyQuoted => PickledQuotes.unpickleQuote(expr)
13+
case expr: quoted.ValueExpr[_] => Literal(Constant(~expr))
14+
case expr: RawQuoted => expr.tree
15+
}
16+
17+
}

‎compiler/src/dotty/tools/dotc/interpreter/Interpreter.scala

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package dotty.tools.dotc
2+
package interpreter
3+
4+
import dotty.tools.dotc.ast.tpd
5+
import dotty.tools.dotc.ast.Trees._
6+
import dotty.tools.dotc.core.Constants._
7+
import dotty.tools.dotc.core.Contexts._
8+
import dotty.tools.dotc.core.Decorators._
9+
import dotty.tools.dotc.core.Symbols._
10+
11+
import scala.reflect.ClassTag
12+
import java.net.URLClassLoader
13+
14+
/** Tree interpreter that can interpret
15+
* * Literal constants
16+
* * Calls to static methods
17+
* * New objects with explicit `new` keyword
18+
* * Quoted code
19+
*
20+
* The interpreter assumes that all calls in the trees are to code that was
21+
* previously compiled and is present in the classpath of the current context.
22+
*/
23+
class Interpreter(implicit ctx: Context) {
24+
import tpd._
25+
26+
private[this] val classLoader = {
27+
val urls = ctx.settings.classpath.value.split(':').map(cp => java.nio.file.Paths.get(cp).toUri.toURL)
28+
new URLClassLoader(urls, getClass.getClassLoader)
29+
}
30+
31+
/** Returns the interpreted result of interpreting the code represented by the tree.
32+
* Return Some of the result or None if some error happen during the interpretation.
33+
*/
34+
def interpretTree[T](tree: Tree)(implicit ct: ClassTag[T]): Option[T] = {
35+
try {
36+
interpretTreeImpl(tree) match {
37+
case obj: T => Some(obj)
38+
case obj =>
39+
ctx.error(s"Interpreted tree returned a result of an unexpected type. Expected ${ct.runtimeClass} but was ${obj.getClass}", tree.pos)
40+
throw new StopInterpretation
41+
}
42+
} catch {
43+
case _: StopInterpretation => None
44+
}
45+
}
46+
47+
/** Returns the interpreted result of interpreting the code represented by the tree.
48+
* Returns the result of the interpreted tree.
49+
*
50+
* If some error is encountered while interpreting a ctx.error is emited and a StopInterpretation is thrown.
51+
*/
52+
private def interpretTreeImpl(tree: Tree): Object = {
53+
try {
54+
tree match {
55+
case Apply(_, quote :: Nil) if tree.symbol eq defn.quoteMethod =>
56+
new RawExpr(quote)
57+
case TypeApply(_, quote :: Nil) if tree.symbol eq defn.typeQuoteMethod =>
58+
new RawType(quote)
59+
60+
case Literal(Constant(c)) =>
61+
c.asInstanceOf[AnyRef]
62+
63+
case Apply(fn, args) if fn.symbol.isConstructor =>
64+
val cls = fn.symbol.owner
65+
val clazz = classLoader.loadClass(cls.symbol.fullName.toString)
66+
val paramClasses = paramsSig(fn.symbol)
67+
val args1: List[Object] = args.map(arg => interpretTreeImpl(arg))
68+
clazz.getConstructor(paramClasses: _*).newInstance(args1: _*).asInstanceOf[Object]
69+
70+
case Apply(fun, args) if fun.symbol.isStatic =>
71+
val clazz = classLoader.loadClass(fun.symbol.owner.companionModule.fullName.toString)
72+
val paramClasses = paramsSig(fun.symbol)
73+
val args1: List[Object] = args.map(arg => interpretTreeImpl(arg))
74+
val method = clazz.getMethod(fun.symbol.name.toString, paramClasses: _*)
75+
method.invoke(null, args1: _*)
76+
77+
case tree: RefTree if tree.symbol.isStatic =>
78+
val clazz = classLoader.loadClass(tree.symbol.owner.companionModule.fullName.toString)
79+
val method = clazz.getMethod(tree.name.toString)
80+
method.invoke(null)
81+
82+
case _ =>
83+
val msg =
84+
if (tree.tpe.derivesFrom(defn.QuotedExprClass)) "Quote needs to be explicit or a call to a static method"
85+
else "Value needs to be a explicit or a call to a static method"
86+
ctx.error(msg, tree.pos)
87+
throw new StopInterpretation
88+
}
89+
} catch {
90+
case ex: NoSuchMethodException =>
91+
ctx.error("Could not find interpreted method in classpath: " + ex.getMessage, tree.pos)
92+
throw new StopInterpretation
93+
case ex: ClassNotFoundException =>
94+
ctx.error("Could not find interpreted class in classpath: " + ex.getMessage, tree.pos)
95+
throw new StopInterpretation
96+
case ex: RuntimeException =>
97+
ex.printStackTrace()
98+
ctx.error("A runtime exception occurred while interpreting: " + ex.getMessage, tree.pos)
99+
throw new StopInterpretation
100+
}
101+
}
102+
103+
/** List of classes of the parameters of the signature of `sym` */
104+
private def paramsSig(sym: Symbol): List[Class[_]] = {
105+
sym.signature.paramsSig.map { param =>
106+
val paramString = param.toString
107+
if (paramString == defn.ByteClass.showFullName) classOf[Byte]
108+
else if (paramString == defn.CharClass.showFullName) classOf[Char]
109+
else if (paramString == defn.ShortClass.showFullName) classOf[Short]
110+
else if (paramString == defn.IntClass.showFullName) classOf[Int]
111+
else if (paramString == defn.LongClass.showFullName) classOf[Long]
112+
else if (paramString == defn.DoubleClass.showFullName) classOf[Double]
113+
else if (paramString == defn.DoubleClass.showFullName) classOf[Float]
114+
else classLoader.loadClass(paramString)
115+
}
116+
}
117+
118+
/** Exception that stops interpretation if some issue is found */
119+
private class StopInterpretation extends Exception
120+
121+
}

‎compiler/src/dotty/tools/dotc/interpreter/PickledQuotes.scala

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.Trees._
4+
import dotty.tools.dotc.ast.tpd
5+
import dotty.tools.dotc.config.Printers._
6+
import dotty.tools.dotc.core.Contexts._
7+
import dotty.tools.dotc.core.Decorators._
8+
import dotty.tools.dotc.core.Flags._
9+
import dotty.tools.dotc.core.NameKinds._
10+
import dotty.tools.dotc.core.StdNames._
11+
import dotty.tools.dotc.core.Symbols._
12+
import dotty.tools.dotc.core.Types._
13+
import dotty.tools.dotc.core.tasty._
14+
15+
object PickledQuotes {
16+
import tpd._
17+
18+
/** TODO */
19+
def pickleQuote(tree: Tree)(implicit ctx: Context): String = {
20+
if (ctx.reporter.hasErrors) "<error>"
21+
else {
22+
val encapsulated = encapsulateQuote(tree)
23+
val pickled = pickle(encapsulated)
24+
TastyString.tastyToString(pickled)
25+
}
26+
}
27+
28+
/** TODO */
29+
def unpickleQuote(expr: quoted.TastyQuoted)(implicit ctx: Context): Tree = {
30+
val tastyBytes = TastyString.stringToTasty(expr.tasty)
31+
val unpickled = unpickle(tastyBytes)
32+
unpickled match {
33+
case PackageDef(_, (vdef: ValDef) :: Nil) => vdef.rhs
34+
}
35+
}
36+
37+
/** Encapsulate the tree in a top level def.
38+
* Splices in the tree are replaced with hole represented as the `def`s parameters.
39+
*
40+
* `'{ "foo" + ~x + "bar" + ~y }`
41+
* ===>
42+
* `package _root_ { def '(hole$1: String, hole$2: String): String = "foo" + hole$1 + "bar" + hole$2 }`
43+
*/
44+
private def encapsulateQuote(tree: Tree)(implicit ctx: Context): Tree = {
45+
val quotedVal = SyntheticValDef("'".toTermName, tree)
46+
PackageDef(ref(defn.RootPackage).asInstanceOf[Ident], quotedVal :: Nil)
47+
}
48+
49+
// TASTY picklingtests/pos/quoteTest.scala
50+
51+
/** Pickle tree into it's TASTY bytes s*/
52+
private def pickle(tree: Tree)(implicit ctx: Context): Array[Byte] = {
53+
val pickler = new TastyPickler(defn.RootClass)
54+
val treePkl = pickler.treePkl
55+
treePkl.pickle(tree :: Nil)
56+
treePkl.compactify()
57+
pickler.addrOfTree = treePkl.buf.addrOfTree
58+
pickler.addrOfSym = treePkl.addrOfSym
59+
// if (tree.pos.exists)
60+
// new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
61+
62+
// other pickle sections go here.
63+
val pickled = pickler.assembleParts()
64+
65+
if (pickling ne noPrinter) {
66+
println(i"**** pickled quote of \n${tree.show}")
67+
new TastyPrinter(pickled).printContents()
68+
}
69+
70+
pickled
71+
}
72+
73+
/** Unpickle TASTY bytes into it's tree */
74+
private def unpickle(bytes: Array[Byte])(implicit ctx: Context): Tree = {
75+
val unpickler = new DottyUnpickler(bytes)
76+
unpickler.enter(roots = Set(defn.RootPackage))
77+
val tree = unpickler.body.head
78+
if (pickling ne noPrinter) {
79+
println(i"**** unpickled quote for \n${tree.show}")
80+
new TastyPrinter(bytes).printContents()
81+
}
82+
tree
83+
}
84+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawExpr.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
class RawExpr(val tree: tpd.Tree) extends quoted.Expr[Any] with RawQuoted {
6+
override def toString: String = s"RawExpr(${tree.toString})"
7+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawQuoted.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
trait RawQuoted extends quoted.Quoted {
6+
def tree: tpd.Tree
7+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawType.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
class RawType(val tree: tpd.Tree) extends quoted.Type[Any] with RawQuoted {
6+
override def toString: String = s"RawType(${tree.toString})"
7+
}

‎compiler/src/dotty/tools/dotc/transform/PostTyper.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,8 @@ class PostTyper extends MacroTransform with IdentityDenotTransformer { thisPhase
167167
case _ => tree
168168
}
169169
case tree @ Select(qual, name) =>
170+
if (tree.symbol == defn.QuotedExpr_~ || tree.symbol == defn.QuotedType_~)
171+
ctx.compilationUnit.containsSplices = true
170172
if (name.isTypeName) {
171173
Checking.checkRealizable(qual.tpe, qual.pos.focus)
172174
super.transform(tree)

‎compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ import MegaPhase.MiniPhase
1414
import SymUtils._
1515
import NameKinds.OuterSelectName
1616
import scala.collection.mutable
17+
import dotty.tools.dotc.core.StdNames._
18+
import dotty.tools.dotc.interpreter.PickledQuotes
19+
1720

1821
/** Translates quoted terms and types to `unpickle` method calls.
1922
* Checks that the phase consistency principle (PCP) holds.
@@ -24,22 +27,11 @@ class ReifyQuotes extends MacroTransform {
2427
override def phaseName: String = "reifyQuotes"
2528

2629
override def run(implicit ctx: Context): Unit =
27-
if (ctx.compilationUnit.containsQuotes) super.run
30+
if (ctx.compilationUnit.containsQuotes || ctx.compilationUnit.containsSplices) super.run
2831

2932
protected def newTransformer(implicit ctx: Context): Transformer =
3033
new Reifier(inQuote = false, null, 0, new LevelInfo)
3134

32-
/** Serialize `tree`. Embedded splices are represented as nodes of the form
33-
*
34-
* Select(qual, sym)
35-
*
36-
* where `sym` is either `defn.QuotedExpr_~` or `defn.QuotedType_~`. For any splice,
37-
* the `qual` part should not be pickled, since it will be added separately later
38-
* as a splice.
39-
*/
40-
def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String =
41-
tree.show // TODO: replace with TASTY
42-
4335
private class LevelInfo {
4436
/** A map from locally defined symbols to the staging levels of their definitions */
4537
val levelOf = new mutable.HashMap[Symbol, Int]
@@ -276,7 +268,7 @@ class ReifyQuotes extends MacroTransform {
276268
ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr)
277269
.appliedToType(if (isType) body1.tpe else body1.tpe.widen)
278270
.appliedTo(
279-
Literal(Constant(pickleTree(body1, isType))),
271+
Literal(Constant(PickledQuotes.pickleQuote(body1))),
280272
SeqLiteral(splices, TypeTree(defn.AnyType)))
281273
}
282274
}.withPos(quote.pos)

‎compiler/src/dotty/tools/dotc/transform/Splicer.scala

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package dotty.tools.dotc
2+
package transform
3+
4+
import dotty.tools.dotc.core.Contexts._
5+
import dotty.tools.dotc.core.Symbols._
6+
import dotty.tools.dotc.ast.Trees._
7+
import dotty.tools.dotc.ast.tpd
8+
import dotty.tools.dotc.interpreter._
9+
10+
/** Utility class to slice quoted expressions */
11+
object Splicer {
12+
import tpd._
13+
14+
/** Splice the Tree for a Quoted expression. `~'(xyz)` becomes `xyz`
15+
* and for `~xyz` the tree of `xyz` is interpreted for which the
16+
* resulting expression is return as a `Tree`
17+
*/
18+
def splice(tree: Tree)(implicit ctx: Context): Tree = tree match {
19+
case Apply(quote, quoted :: Nil) if quote.symbol == defn.quoteMethod => quoted
20+
case TypeApply(quote, quoted :: Nil) if quote.symbol == defn.typeQuoteMethod => quoted
21+
case tree: RefTree => reflectiveSplice(tree)
22+
case tree: Apply => reflectiveSplice(tree)
23+
case Inlined(_, _, expansion) => splice(expansion)
24+
}
25+
26+
/** Splice the Tree for a Quoted expression which is constructed via a reflective call to the given method */
27+
private def reflectiveSplice(tree: Tree)(implicit ctx: Context): Tree = {
28+
val interpreter = new Interpreter
29+
interpreter.interpretTree[quoted.Expr[_]](tree).map(Expressions.exprToTree(_)).getOrElse(tree)
30+
}
31+
32+
}

‎library/src/scala/quoted/Expr.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package scala.quoted
22

3-
class Expr[T] extends Quoted {
4-
def unary_~ : T = ???
3+
abstract class Expr[T] extends Quoted {
4+
def unary_~ : T = throw new Error("~ should have been compiled away")
55
def run: T = ???
66
}
77

0 commit comments

Comments
 (0)