From 85fa62701dc10682f82c3140638dbe10b8d2a289 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 2 Dec 2017 19:43:21 +0100 Subject: [PATCH 01/62] Fix #3597 - Don't check inline for params after pickling There was a discrepancy in that value parameters had a Deferred flag set before pickling but not after unpickling. This triggered a check that the (missing) rhs of an inline parameter was a constant. This commit changes the condition of the test to also exclude parameters. It also aligns frontend and unpickler in that the frontend will no longer mark term parameters as Deferred. --- compiler/src/dotty/tools/dotc/ast/TreeInfo.scala | 2 +- compiler/src/dotty/tools/dotc/core/Flags.scala | 2 +- compiler/src/dotty/tools/dotc/typer/Typer.scala | 2 +- tests/pos-from-tasty/i3597.scala | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 tests/pos-from-tasty/i3597.scala diff --git a/compiler/src/dotty/tools/dotc/ast/TreeInfo.scala b/compiler/src/dotty/tools/dotc/ast/TreeInfo.scala index da8759a5f5fc..2b2b167e2a85 100644 --- a/compiler/src/dotty/tools/dotc/ast/TreeInfo.scala +++ b/compiler/src/dotty/tools/dotc/ast/TreeInfo.scala @@ -264,7 +264,7 @@ trait UntypedTreeInfo extends TreeInfo[Untyped] { self: Trees.Instance[Untyped] */ def lacksDefinition(mdef: MemberDef)(implicit ctx: Context) = mdef match { case mdef: ValOrDefDef => - mdef.unforcedRhs == EmptyTree && !mdef.name.isConstructorName && !mdef.mods.is(ParamAccessor) + mdef.unforcedRhs == EmptyTree && !mdef.name.isConstructorName && !mdef.mods.is(TermParamOrAccessor) case mdef: TypeDef => def isBounds(rhs: Tree): Boolean = rhs match { case _: TypeBoundsTree => true diff --git a/compiler/src/dotty/tools/dotc/core/Flags.scala b/compiler/src/dotty/tools/dotc/core/Flags.scala index 66f17a3c7f85..60438a125334 100644 --- a/compiler/src/dotty/tools/dotc/core/Flags.scala +++ b/compiler/src/dotty/tools/dotc/core/Flags.scala @@ -575,7 +575,7 @@ object Flags { final val SyntheticOrPrivate = Synthetic | Private /** A deferred member or a parameter accessor (these don't have right hand sides) */ - final val DeferredOrParamAccessor = Deferred | ParamAccessor + final val DeferredOrParamOrAccessor = Deferred | Param | ParamAccessor /** value that's final or inline */ final val FinalOrInline = Final | Inline diff --git a/compiler/src/dotty/tools/dotc/typer/Typer.scala b/compiler/src/dotty/tools/dotc/typer/Typer.scala index 1519780ea8be..fcfdce97cc97 100644 --- a/compiler/src/dotty/tools/dotc/typer/Typer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Typer.scala @@ -1300,7 +1300,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit case rhs => typedExpr(rhs, tpt1.tpe) } val vdef1 = assignType(cpy.ValDef(vdef)(name, tpt1, rhs1), sym) - if (sym.is(Inline, butNot = DeferredOrParamAccessor)) + if (sym.is(Inline, butNot = DeferredOrParamOrAccessor)) checkInlineConformant(rhs1, em"right-hand side of inline $sym") patchIfLazy(vdef1) patchFinalVals(vdef1) diff --git a/tests/pos-from-tasty/i3597.scala b/tests/pos-from-tasty/i3597.scala new file mode 100644 index 000000000000..04e33975ea2c --- /dev/null +++ b/tests/pos-from-tasty/i3597.scala @@ -0,0 +1,3 @@ +object Test { + def bar(inline n: Int) = n +} From 63d02a65acf9173a89d75d08a8ace43a3f04f6a0 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 3 Dec 2017 15:06:31 +0100 Subject: [PATCH 02/62] Fix condition in expandPrivate Previous condition did not take into account private static members. Their "enclosing class" is the enclosing package, but we really want to count them as static members of their lexical enclosing class here. --- compiler/src/dotty/tools/dotc/transform/ExpandPrivate.scala | 2 +- tests/run/i3006b.check | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ExpandPrivate.scala b/compiler/src/dotty/tools/dotc/transform/ExpandPrivate.scala index 34bc6c6e2694..e5ee8556ec88 100644 --- a/compiler/src/dotty/tools/dotc/transform/ExpandPrivate.scala +++ b/compiler/src/dotty/tools/dotc/transform/ExpandPrivate.scala @@ -74,7 +74,7 @@ class ExpandPrivate extends MiniPhase with IdentityDenotTransformer { thisPhase private def ensurePrivateAccessible(d: SymDenotation)(implicit ctx: Context) = if (isVCPrivateParamAccessor(d)) d.ensureNotPrivate.installAfter(thisPhase) - else if (d.is(PrivateTerm) && !d.owner.is(Package) && d.owner != ctx.owner.enclosingClass) { + else if (d.is(PrivateTerm) && !d.owner.is(Package) && d.owner != ctx.owner.lexicallyEnclosingClass) { // Paths `p1` and `p2` are similar if they have a common suffix that follows // possibly different directory paths. That is, their common suffix extends // in both cases either to the start of the path or to a file separator character. diff --git a/tests/run/i3006b.check b/tests/run/i3006b.check index b62605500d51..3698829ef115 100644 --- a/tests/run/i3006b.check +++ b/tests/run/i3006b.check @@ -1,3 +1,3 @@ -Foo$$_$bar$1 -Foo$$_$bar$2 -Bar$$_$bar$1 +bar$1 +bar$2 +bar$1 From f93368d2a8c75f8dffec4b03a987c81af6875179 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 3 Dec 2017 15:07:09 +0100 Subject: [PATCH 03/62] Fix #3596: Handle supercall arguments in unpickler --- .../tools/dotc/core/tasty/TreeUnpickler.scala | 27 +++++++++++-------- tests/pos-from-tasty/i3596.scala | 2 ++ 2 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 tests/pos-from-tasty/i3596.scala diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index c1a06769a7c4..7b3c4edbfae2 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -55,6 +55,9 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi */ private[this] var seenRoots: Set[Symbol] = Set() + /** A map from unpickled class symbols to their local dummies */ + private[this] val localDummies = new mutable.HashMap[ClassSymbol, Symbol] + /** The root owner tree. See `OwnerTree` class definition. Set by `enterTopLevel`. */ private[this] var ownerTree: OwnerTree = _ @@ -424,7 +427,7 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi case VALDEF | DEFDEF | TYPEDEF | TYPEPARAM | PARAM => createMemberSymbol() case TEMPLATE => - val localDummy = ctx.newLocalDummy(ctx.owner) + val localDummy = localDummies(ctx.owner.asClass) registerSym(currentAddr, localDummy) localDummy case tag => @@ -478,6 +481,7 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi ctx.enter(sym) registerSym(start, sym) if (isClass) { + localDummies(sym.asClass) = ctx.newLocalDummy(sym) sym.completer.withDecls(newScope) forkAt(templateStart).indexTemplateParams()(localContext(sym)) } @@ -745,14 +749,15 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi else NoType cls.info = new TempClassInfo(cls.owner.thisType, cls, cls.unforcedDecls, assumedSelfType) val localDummy = symbolAtCurrent() + val parentCtx = ctx.withOwner(localDummy) assert(readByte() == TEMPLATE) val end = readEnd() val tparams = readIndexedParams[TypeDef](TYPEPARAM) val vparams = readIndexedParams[ValDef](PARAM) val parents = collectWhile(nextByte != SELFDEF && nextByte != DEFDEF) { nextByte match { - case APPLY | TYPEAPPLY => readTerm() - case _ => readTpt() + case APPLY | TYPEAPPLY => readTerm()(parentCtx) + case _ => readTpt()(parentCtx) } } val parentTypes = parents.map(_.tpe.dealias) @@ -766,13 +771,14 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi if (self.isEmpty) NoType else self.tpt.tpe) cls.setNoInitsFlags(fork.indexStats(end)) val constr = readIndexedDef().asInstanceOf[DefDef] + val mappedParents = parents.map(_.changeOwner(localDummy, constr.symbol)) val lazyStats = readLater(end, rdr => implicit ctx => { val stats = rdr.readIndexedStats(localDummy, end) tparams ++ vparams ++ stats }) setPos(start, - untpd.Template(constr, parents, self, lazyStats) + untpd.Template(constr, mappedParents, self, lazyStats) .withType(localDummy.termRef)) } @@ -922,15 +928,10 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi def readLengthTerm(): Tree = { val end = readEnd() - def localNonClassCtx = { - val ctx1 = ctx.fresh.setNewScope - if (ctx.owner.isClass) ctx1.setOwner(ctx1.newLocalDummy(ctx.owner)) else ctx1 - } - def readBlock(mkTree: (List[Tree], Tree) => Tree): Tree = { val exprReader = fork skipTree() - val localCtx = localNonClassCtx + val localCtx = ctx.fresh.setNewScope val stats = readStats(ctx.owner, end)(localCtx) val expr = exprReader.readTerm()(localCtx) mkTree(stats, expr) @@ -1030,7 +1031,11 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi case ANNOTATEDtpt => Annotated(readTpt(), readTerm()) case LAMBDAtpt => - val localCtx = localNonClassCtx + var localCtx = ctx.fresh.setNewScope + ctx.owner match { + case cls: ClassSymbol => localCtx = localCtx.setOwner(localDummies(cls)) + case _ => + } val tparams = readParams[TypeDef](TYPEPARAM)(localCtx) val body = readTpt()(localCtx) LambdaTypeTree(tparams, body) diff --git a/tests/pos-from-tasty/i3596.scala b/tests/pos-from-tasty/i3596.scala new file mode 100644 index 000000000000..6f122631779f --- /dev/null +++ b/tests/pos-from-tasty/i3596.scala @@ -0,0 +1,2 @@ +class Bar(ctor : Int => Int) +class Foo extends Bar(x => x) From 36ede6ffc5368415d1e6f0ffabc106119ca2801e Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 3 Dec 2017 15:50:53 +0100 Subject: [PATCH 04/62] When unpickling, enter only class members into enclosing scope. --- compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index 7b3c4edbfae2..6ae3d96f6eb7 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -478,7 +478,10 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi ctx.newSymbol(ctx.owner, name, flags, completer, privateWithin, coord) } sym.annotations = annots - ctx.enter(sym) + ctx.owner match { + case cls: ClassSymbol => cls.enter(sym) + case _ => + } registerSym(start, sym) if (isClass) { localDummies(sym.asClass) = ctx.newLocalDummy(sym) From bebefeccd3e3523f5c5709a681ab3c999d25774c Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 3 Dec 2017 17:28:18 +0100 Subject: [PATCH 05/62] Clean up treatment of local contexts in unpickler --- .../tools/dotc/core/tasty/TreeUnpickler.scala | 35 +++++++------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index 6ae3d96f6eb7..04c0a06bdba4 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -386,10 +386,8 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi private def noRhs(end: Addr): Boolean = currentAddr == end || isModifierTag(nextByte) - private def localContext(owner: Symbol)(implicit ctx: Context) = { - val lctx = ctx.fresh.setOwner(owner) - if (owner.isClass) lctx.setScope(owner.unforcedDecls) else lctx.setNewScope - } + private def localContext(owner: Symbol)(implicit ctx: Context) = + ctx.fresh.setOwner(owner) private def normalizeFlags(tag: Int, givenFlags: FlagSet, name: Name, isAbsType: Boolean, rhsIsEmpty: Boolean)(implicit ctx: Context): FlagSet = { val lacksDefinition = @@ -646,19 +644,18 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi } } + val localCtx = localContext(sym) + def readRhs(implicit ctx: Context) = if (noRhs(end)) EmptyTree else readLater(end, rdr => ctx => rdr.readTerm()(ctx)) - def localCtx = localContext(sym) - def ValDef(tpt: Tree) = ta.assignType(untpd.ValDef(sym.name.asTermName, tpt, readRhs(localCtx)), sym) def DefDef(tparams: List[TypeDef], vparamss: List[List[ValDef]], tpt: Tree) = ta.assignType( - untpd.DefDef( - sym.name.asTermName, tparams, vparamss, tpt, readRhs(localCtx)), + untpd.DefDef(sym.name.asTermName, tparams, vparamss, tpt, readRhs(localCtx)), sym) def TypeDef(rhs: Tree) = @@ -672,7 +669,7 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi case DEFDEF => val tparams = readParams[TypeDef](TYPEPARAM)(localCtx) val vparamss = readParamss(localCtx) - val tpt = readTpt() + val tpt = readTpt()(localCtx) val typeParams = tparams.map(_.symbol) val valueParamss = ctx.normalizeIfConstructor( vparamss.nestedMap(_.symbol), name == nme.CONSTRUCTOR) @@ -685,7 +682,7 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi } DefDef(tparams, vparamss, tpt) case VALDEF => - val tpt = readTpt() + val tpt = readTpt()(localCtx) sym.info = tpt.tpe ValDef(tpt) case TYPEDEF | TYPEPARAM => @@ -704,7 +701,7 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi } TypeDef(readTemplate(localCtx)) } else { - val rhs = readTpt() + val rhs = readTpt()(localCtx) sym.info = NoCompleter sym.info = rhs.tpe match { case _: TypeBounds | _: ClassInfo => checkNonCyclic(sym, rhs.tpe, reportErrors = false) @@ -713,7 +710,7 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi TypeDef(rhs) } case PARAM => - val tpt = readTpt() + val tpt = readTpt()(localCtx) if (noRhs(end)) { sym.info = tpt.tpe ValDef(tpt) @@ -934,9 +931,8 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi def readBlock(mkTree: (List[Tree], Tree) => Tree): Tree = { val exprReader = fork skipTree() - val localCtx = ctx.fresh.setNewScope - val stats = readStats(ctx.owner, end)(localCtx) - val expr = exprReader.readTerm()(localCtx) + val stats = readStats(ctx.owner, end) + val expr = exprReader.readTerm() mkTree(stats, expr) } @@ -1034,13 +1030,8 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi case ANNOTATEDtpt => Annotated(readTpt(), readTerm()) case LAMBDAtpt => - var localCtx = ctx.fresh.setNewScope - ctx.owner match { - case cls: ClassSymbol => localCtx = localCtx.setOwner(localDummies(cls)) - case _ => - } - val tparams = readParams[TypeDef](TYPEPARAM)(localCtx) - val body = readTpt()(localCtx) + val tparams = readParams[TypeDef](TYPEPARAM) + val body = readTpt() LambdaTypeTree(tparams, body) case TYPEBOUNDStpt => TypeBoundsTree(readTpt(), readTpt()) From 227f8be42e8b2b0842263835dc465b4dc6e9d957 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 3 Dec 2017 18:59:08 +0100 Subject: [PATCH 06/62] Drop localDummies again With the cleaned-up local scope handling we don't need them anymore. --- .../src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index 04c0a06bdba4..8052deda72e5 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -55,9 +55,6 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi */ private[this] var seenRoots: Set[Symbol] = Set() - /** A map from unpickled class symbols to their local dummies */ - private[this] val localDummies = new mutable.HashMap[ClassSymbol, Symbol] - /** The root owner tree. See `OwnerTree` class definition. Set by `enterTopLevel`. */ private[this] var ownerTree: OwnerTree = _ @@ -425,7 +422,7 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi case VALDEF | DEFDEF | TYPEDEF | TYPEPARAM | PARAM => createMemberSymbol() case TEMPLATE => - val localDummy = localDummies(ctx.owner.asClass) + val localDummy = ctx.newLocalDummy(ctx.owner) registerSym(currentAddr, localDummy) localDummy case tag => @@ -482,7 +479,6 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi } registerSym(start, sym) if (isClass) { - localDummies(sym.asClass) = ctx.newLocalDummy(sym) sym.completer.withDecls(newScope) forkAt(templateStart).indexTemplateParams()(localContext(sym)) } From 02eebc25137cc0c5d33c43bafc3eea6525aa7926 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 4 Dec 2017 11:43:48 +0100 Subject: [PATCH 07/62] Add prefix types Allow `~ stableId` as a type, where `stableId` is a stable term identifier. This is not the same as the second part of SIP 33, which proposed `~` as an operator over types. --- compiler/src/dotty/tools/dotc/ast/Desugar.scala | 3 ++- .../src/dotty/tools/dotc/parsing/Parsers.scala | 3 +++ docs/docs/internals/syntax.md | 1 + tests/pos/spliceTest.scala | 14 ++++++++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/pos/spliceTest.scala diff --git a/compiler/src/dotty/tools/dotc/ast/Desugar.scala b/compiler/src/dotty/tools/dotc/ast/Desugar.scala index e9d02525546e..4fea10f68c8a 100644 --- a/compiler/src/dotty/tools/dotc/ast/Desugar.scala +++ b/compiler/src/dotty/tools/dotc/ast/Desugar.scala @@ -1069,7 +1069,8 @@ object desugar { Select(t, op.name) } case PrefixOp(op, t) => - Select(t, nme.UNARY_PREFIX ++ op.name) + val nspace = if (ctx.mode.is(Mode.Type)) tpnme else nme + Select(t, nspace.UNARY_PREFIX ++ op.name) case Tuple(ts) => val arity = ts.length def tupleTypeRef = defn.TupleType(arity) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 171ccb641e14..fc4e176ada19 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -857,6 +857,7 @@ object Parsers { /** SimpleType ::= SimpleType TypeArgs * | SimpleType `#' id * | StableId + * | [‘-’ | ‘+’ | ‘~’ | ‘!’] StableId * | Path `.' type * | `(' ArgTypes `)' * | `_' TypeBounds @@ -875,6 +876,8 @@ object Parsers { val start = in.skipToken() typeBounds().withPos(Position(start, in.lastOffset, start)) } + else if (isIdent && nme.raw.isUnary(in.name)) + atPos(in.offset) { PrefixOp(termIdent(), path(thisOK = true)) } else path(thisOK = false, handleSingletonType) match { case r @ SingletonTypeTree(_) => r case r => convertToTypeId(r) diff --git a/docs/docs/internals/syntax.md b/docs/docs/internals/syntax.md index 0afe98abdc58..0f2789bd7b1b 100644 --- a/docs/docs/internals/syntax.md +++ b/docs/docs/internals/syntax.md @@ -131,6 +131,7 @@ AnnotType ::= SimpleType {Annotation} SimpleType ::= SimpleType TypeArgs AppliedTypeTree(t, args) | SimpleType ‘#’ id Select(t, name) | StableId + | [‘-’ | ‘+’ | ‘~’ | ‘!’] StableId PrefixOp(expr, op) | Path ‘.’ ‘type’ SingletonTypeTree(p) | ‘(’ ArgTypes ‘)’ Tuple(ts) | ‘_’ TypeBounds diff --git a/tests/pos/spliceTest.scala b/tests/pos/spliceTest.scala new file mode 100644 index 000000000000..c04afdcecb57 --- /dev/null +++ b/tests/pos/spliceTest.scala @@ -0,0 +1,14 @@ +class Expr[T] { + def unary_~ : T = ??? +} +class Type[T] { + type unary_~ = T +} +object Test { + + def f[T](t: Type[T], x: Expr[T]) = { + val y: t.unary_~ = x.unary_~ + val z: ~t = ~x + } + +} From 73038981e04d90ea569cc598baabb4189d57a909 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 4 Dec 2017 15:37:34 +0100 Subject: [PATCH 08/62] Cleanup of ElimJavaPackages Just came by this accidentally and noticed that it could be more idiomatic and terminology could be improved. --- .../dotc/transform/ElimJavaPackages.scala | 59 +++++++------------ 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ElimJavaPackages.scala b/compiler/src/dotty/tools/dotc/transform/ElimJavaPackages.scala index a823ce4d7ed7..9cd329fd96b9 100644 --- a/compiler/src/dotty/tools/dotc/transform/ElimJavaPackages.scala +++ b/compiler/src/dotty/tools/dotc/transform/ElimJavaPackages.scala @@ -1,46 +1,31 @@ -package dotty.tools.dotc.transform +package dotty.tools.dotc +package transform -import dotty.tools.dotc.ast.tpd._ -import dotty.tools.dotc.core.Contexts.Context -import dotty.tools.dotc.core.Flags._ -import dotty.tools.dotc.core.Types.{Type, TypeRef} -import dotty.tools.dotc.transform.MegaPhase.MiniPhase +import core._ +import Decorators._, Flags._, Types._, Contexts._, Symbols._ +import ast.tpd._ +import Flags._ +import MegaPhase.MiniPhase -/** - * Eliminates syntactic references to Java packages, so that there's no chance - * they accidentally end up in the backend. - */ -class ElimJavaPackages extends MiniPhase { +/** Eliminates syntactic references to package terms as prefixes of classes, so that there's no chance + * they accidentally end up in the backend. + */ +class ElimPackagePrefixes extends MiniPhase { - override def phaseName: String = "elimJavaPackages" + override def phaseName: String = "elimPackagePrefixes" - override def transformSelect(tree: Select)(implicit ctx: Context): Tree = { - if (isJavaPackage(tree)) { - assert(tree.tpe.isInstanceOf[TypeRef], s"Expected tree with type TypeRef, but got ${tree.tpe.show}") - Ident(tree.tpe.asInstanceOf[TypeRef]) - } else { - tree - } - } + override def transformSelect(tree: Select)(implicit ctx: Context): Tree = + if (isPackageClassRef(tree)) Ident(tree.tpe.asInstanceOf[TypeRef]) else tree - override def checkPostCondition(tree: Tree)(implicit ctx: Context): Unit = { - tree match { - case tree: Select => - assert(!isJavaPackage(tree), s"Unexpected reference to Java package in ${tree.show}") - case _ => () - } + override def checkPostCondition(tree: Tree)(implicit ctx: Context): Unit = tree match { + case tree: Select => + assert(!isPackageClassRef(tree), i"Unexpected reference to package in $tree") + case _ => } - /** - * Is the given tree a syntactic reference to a Java package? - */ - private def isJavaPackage(tree: Select)(implicit ctx: Context): Boolean = { - tree.tpe match { - case TypeRef(prefix, _) => - val flags = prefix.termSymbol.flags - // Testing for each flag separately is more efficient than using FlagConjunction. - flags.is(Package) && flags.is(JavaDefined) - case _ => false - } + /** Is the given tree a reference to a type in a package? */ + private def isPackageClassRef(tree: Select)(implicit ctx: Context): Boolean = tree.tpe match { + case TypeRef(prefix, _) => prefix.termSymbol.is(Package) + case _ => false } } From 141acaa876f66af9fdd8d288f08a0de00aec846b Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 4 Dec 2017 15:38:28 +0100 Subject: [PATCH 09/62] Rename ElimJavaPackages -> ElimPackagePrefixes That's a better description of what the miniphase does. --- .../{ElimJavaPackages.scala => ElimPackagePrefixes.scala} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename compiler/src/dotty/tools/dotc/transform/{ElimJavaPackages.scala => ElimPackagePrefixes.scala} (100%) diff --git a/compiler/src/dotty/tools/dotc/transform/ElimJavaPackages.scala b/compiler/src/dotty/tools/dotc/transform/ElimPackagePrefixes.scala similarity index 100% rename from compiler/src/dotty/tools/dotc/transform/ElimJavaPackages.scala rename to compiler/src/dotty/tools/dotc/transform/ElimPackagePrefixes.scala From 6a9c0277f2c80b1ee466a8b0fcb2a41c2f5a6f0b Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 4 Dec 2017 17:22:36 +0100 Subject: [PATCH 10/62] Initial meta framework classes and modules --- .../dotty/tools/dotc/core/Definitions.scala | 19 +++++++++++++++++-- library/src/scala/meta/Expr.scala | 5 +++++ library/src/scala/meta/Reifier.scala | 7 +++++++ library/src/scala/meta/Type.scala | 5 +++++ 4 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 library/src/scala/meta/Expr.scala create mode 100644 library/src/scala/meta/Reifier.scala create mode 100644 library/src/scala/meta/Type.scala diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index e3bcd1e1fda0..7963361de98e 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -290,14 +290,21 @@ class Definitions { /** Marker method to indicate an argument to a call-by-name parameter. * Created by byNameClosures and elimByName, eliminated by Erasure, */ - lazy val cbnArg = enterPolyMethod( - OpsPackageClass, nme.cbnArg, 1, + lazy val cbnArg = enterPolyMethod(OpsPackageClass, nme.cbnArg, 1, pt => MethodType(List(FunctionOf(Nil, pt.paramRefs(0))), pt.paramRefs(0))) /** Method representing a throw */ lazy val throwMethod = enterMethod(OpsPackageClass, nme.THROWkw, MethodType(List(ThrowableType), NothingType)) + /** Method representing a term quote */ + lazy val quoteMethod = enterPolyMethod(OpsPackageClass, nme.QUOTE, 1, + pt => MethodType(pt.paramRefs(0) :: Nil, MetaExprType.appliedTo(pt.paramRefs(0) :: Nil))) + + /** Method representing a type quote */ + lazy val typeQuoteMethod = enterPolyMethod(OpsPackageClass, nme.QUOTE, 1, + pt => MetaTypeType.appliedTo(pt.paramRefs(0) :: Nil)) + lazy val NothingClass: ClassSymbol = enterCompleteClassSymbol( ScalaPackageClass, tpnme.Nothing, AbstractFinal, List(AnyClass.typeRef)) def NothingType = NothingClass.typeRef @@ -585,6 +592,14 @@ class Definitions { def ClassTagClass(implicit ctx: Context) = ClassTagType.symbol.asClass def ClassTagModule(implicit ctx: Context) = ClassTagClass.companionModule + lazy val MetaExprType = ctx.requiredClassRef("scala.meta.Expr") + def MetaExprClass(implicit ctx: Context) = MetaExprType.symbol.asClass + + lazy val MetaTypeType = ctx.requiredClassRef("scala.meta.Type") + def MetaTypeClass(implicit ctx: Context) = MetaTypeType.symbol.asClass + + def Reifier_reify = ctx.requiredMethod("scala.meta.Reifier.reify") + lazy val EqType = ctx.requiredClassRef("scala.Eq") def EqClass(implicit ctx: Context) = EqType.symbol.asClass def EqModule(implicit ctx: Context) = EqClass.companionModule diff --git a/library/src/scala/meta/Expr.scala b/library/src/scala/meta/Expr.scala new file mode 100644 index 000000000000..a61c6145b0d6 --- /dev/null +++ b/library/src/scala/meta/Expr.scala @@ -0,0 +1,5 @@ +package scala.meta + +class Expr[T] { + def unary_~ : T = ??? +} diff --git a/library/src/scala/meta/Reifier.scala b/library/src/scala/meta/Reifier.scala new file mode 100644 index 000000000000..f7271d92cbb8 --- /dev/null +++ b/library/src/scala/meta/Reifier.scala @@ -0,0 +1,7 @@ +package scala.meta + +object Reifier { + + def reify(repr: String, args: Expr[_]*) = ??? + +} diff --git a/library/src/scala/meta/Type.scala b/library/src/scala/meta/Type.scala new file mode 100644 index 000000000000..dc19c0234b32 --- /dev/null +++ b/library/src/scala/meta/Type.scala @@ -0,0 +1,5 @@ +package scala.meta + +class Type[T] { + type unary_~ = T +} From e52fe742860e02471b6fddcda9f0536a3f53ff14 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 08:51:30 +0100 Subject: [PATCH 11/62] Utility method: reporting Can be used like this: .reporting(res => println(s"result is $res")) To achieve this previously, we'd need to store result in a val and then return the val. This is more effort, in particular if we want to just quickly add some debug output, to be removed later. --- compiler/src/dotty/tools/dotc/core/Decorators.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/core/Decorators.scala b/compiler/src/dotty/tools/dotc/core/Decorators.scala index 094d4092ac1a..ec8563bf3f8b 100644 --- a/compiler/src/dotty/tools/dotc/core/Decorators.scala +++ b/compiler/src/dotty/tools/dotc/core/Decorators.scala @@ -175,6 +175,10 @@ object Decorators { recur(enclosingInlineds, pos) } + implicit class reportingDeco[T](val x: T) extends AnyVal { + def reporting(op: T => String): T = { println(op(x)); x } + } + implicit class StringInterpolators(val sc: StringContext) extends AnyVal { /** General purpose string formatting */ From a5134cae6443ebe0a693d0c24ac0baa1bedbb7a3 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 08:52:36 +0100 Subject: [PATCH 12/62] Add quote syntax --- compiler/src/dotty/tools/dotc/ast/untpd.scala | 9 ++++++++ .../dotty/tools/dotc/parsing/Parsers.scala | 14 ++++++++++-- .../dotty/tools/dotc/parsing/Scanners.scala | 22 ++++++++++++++----- .../src/dotty/tools/dotc/parsing/Tokens.scala | 3 +++ docs/docs/internals/syntax.md | 3 +++ 5 files changed, 44 insertions(+), 7 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/ast/untpd.scala b/compiler/src/dotty/tools/dotc/ast/untpd.scala index e4b34d59c189..d1077a8c0488 100644 --- a/compiler/src/dotty/tools/dotc/ast/untpd.scala +++ b/compiler/src/dotty/tools/dotc/ast/untpd.scala @@ -78,6 +78,7 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo { override def isType = !isTerm } case class Throw(expr: Tree) extends TermTree + case class Quote(expr: Tree) extends TermTree case class WhileDo(cond: Tree, body: Tree) extends TermTree case class DoWhile(body: Tree, cond: Tree) extends TermTree case class ForYield(enums: List[Tree], expr: Tree) extends TermTree @@ -449,6 +450,10 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo { case tree: Throw if expr eq tree.expr => tree case _ => finalize(tree, untpd.Throw(expr)) } + def Quote(tree: Tree)(expr: Tree) = tree match { + case tree: Quote if expr eq tree.expr => tree + case _ => finalize(tree, untpd.Quote(expr)) + } def WhileDo(tree: Tree)(cond: Tree, body: Tree) = tree match { case tree: WhileDo if (cond eq tree.cond) && (body eq tree.body) => tree case _ => finalize(tree, untpd.WhileDo(cond, body)) @@ -507,6 +512,8 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo { cpy.Tuple(tree)(transform(trees)) case Throw(expr) => cpy.Throw(tree)(transform(expr)) + case Quote(expr) => + cpy.Quote(tree)(transform(expr)) case WhileDo(cond, body) => cpy.WhileDo(tree)(transform(cond), transform(body)) case DoWhile(body, cond) => @@ -554,6 +561,8 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo { this(x, trees) case Throw(expr) => this(x, expr) + case Quote(expr) => + this(x, expr) case WhileDo(cond, body) => this(this(x, cond), body) case DoWhile(body, cond) => diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index fc4e176ada19..da28195c1a5d 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1285,6 +1285,9 @@ object Parsers { /** SimpleExpr ::= new Template * | BlockExpr + * | ‘'’ BlockExpr + * | ‘'’ ‘(’ ExprsInParens ‘)’ + * | ‘'’ ‘[’ Type ‘]’ * | SimpleExpr1 [`_'] * SimpleExpr1 ::= literal * | xmlLiteral @@ -1313,6 +1316,15 @@ object Parsers { case LBRACE => canApply = false blockExpr() + case QPAREN => + in.token = LPAREN + atPos(in.offset)(Quote(simpleExpr())) + case QBRACE => + in.token = LBRACE + atPos(in.offset)(Quote(simpleExpr())) + case QBRACKET => + in.token = LBRACKET + atPos(in.offset)(Quote(inBrackets(typ()))) case NEW => canApply = false val start = in.skipToken() @@ -2077,8 +2089,6 @@ object Parsers { } } - - private def checkVarArgsRules(vparamss: List[List[untpd.ValDef]]): List[untpd.ValDef] = { def isVarArgs(tpt: Trees.Tree[Untyped]): Boolean = tpt match { case PostfixOp(_, op) if op.name == tpnme.raw.STAR => true diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 5677542c0605..15265e4ca5cb 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -540,9 +540,13 @@ object Scanners { def fetchSingleQuote() = { nextChar() if (isIdentifierStart(ch)) - charLitOr(() => getIdentRest()) + charLitOr { getIdentRest(); SYMBOLLIT } else if (isOperatorPart(ch) && (ch != '\\')) - charLitOr(() => getOperatorRest()) + charLitOr { getOperatorRest(); SYMBOLLIT } + else if (ch == '(' || ch == '{' || ch == '[') { + val tok = quote(ch) + charLitOr(tok) + } else { getLitChar() if (ch == '\'') { @@ -965,7 +969,7 @@ object Scanners { /** Parse character literal if current character is followed by \', * or follow with given op and return a symbol literal token */ - def charLitOr(op: () => Unit): Unit = { + def charLitOr(op: => Token): Unit = { putChar(ch) nextChar() if (ch == '\'') { @@ -973,11 +977,19 @@ object Scanners { token = CHARLIT setStrVal() } else { - op() - token = SYMBOLLIT + token = op strVal = name.toString + litBuf.clear() } } + + /** The opening quote bracket token corresponding to `c` */ + def quote(c: Char): Token = c match { + case '(' => QPAREN + case '{' => QBRACE + case '[' => QBRACKET + } + override def toString = showTokenDetailed(token) + { if ((identifierTokens contains token) || (literalTokens contains token)) " " + name diff --git a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala index 3a0655e72f75..507b7f250200 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala @@ -190,6 +190,9 @@ object Tokens extends TokensCommon { final val SUPERTYPE = 81; enter(SUPERTYPE, ">:") final val HASH = 82; enter(HASH, "#") final val VIEWBOUND = 84; enter(VIEWBOUND, "<%") // TODO: deprecate + final val QPAREN = 85; enter(QPAREN, "'(") + final val QBRACE = 86; enter(QBRACE, "'{") + final val QBRACKET = 87; enter(QBRACKET, "'[") /** XML mode */ final val XMLSTART = 96; enter(XMLSTART, "$XMLSTART$<") // TODO: deprecate diff --git a/docs/docs/internals/syntax.md b/docs/docs/internals/syntax.md index 0f2789bd7b1b..34bf8c390da2 100644 --- a/docs/docs/internals/syntax.md +++ b/docs/docs/internals/syntax.md @@ -183,6 +183,9 @@ InfixExpr ::= PrefixExpr PrefixExpr ::= [‘-’ | ‘+’ | ‘~’ | ‘!’] SimpleExpr PrefixOp(expr, op) SimpleExpr ::= ‘new’ Template New(templ) | BlockExpr + | ‘'’ BlockExpr + | ‘'’ ‘(’ ExprsInParens ‘)’ + | ‘'’ ‘[’ Type ‘]’ | SimpleExpr1 [‘_’] PostfixOp(expr, _) SimpleExpr1 ::= Literal | Path From f3ff4e98bc603816efa89422c8bb9bd9e50a7bc2 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 08:54:20 +0100 Subject: [PATCH 13/62] Add quote typechecking No phase consistency principle yet, we compute just the basic types. --- .../src/dotty/tools/dotc/core/Definitions.scala | 11 +++++++++-- .../src/dotty/tools/dotc/core/StdNames.scala | 1 + .../src/dotty/tools/dotc/core/Symbols.scala | 3 +++ compiler/src/dotty/tools/dotc/typer/Typer.scala | 17 +++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 7963361de98e..d380d3e0648c 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -595,10 +595,16 @@ class Definitions { lazy val MetaExprType = ctx.requiredClassRef("scala.meta.Expr") def MetaExprClass(implicit ctx: Context) = MetaExprType.symbol.asClass + def MetaExpr_~(implicit ctx: Context) = MetaExprClass.requiredMethod(nme.UNARY_~) + lazy val MetaTypeType = ctx.requiredClassRef("scala.meta.Type") def MetaTypeClass(implicit ctx: Context) = MetaTypeType.symbol.asClass - def Reifier_reify = ctx.requiredMethod("scala.meta.Reifier.reify") + def MetaType_~(implicit ctx: Context) = + MetaTypeClass.info.member(tpnme.UNARY_~).symbol.asType + + def Reifier_reifyExpr = ctx.requiredMethod("scala.meta.Reifier.reifyExpr") + def Reifier_reifyType = ctx.requiredMethod("scala.meta.Reifier.reifyType") lazy val EqType = ctx.requiredClassRef("scala.Eq") def EqClass(implicit ctx: Context) = EqType.symbol.asClass @@ -1086,7 +1092,8 @@ class Definitions { OpsPackageClass) /** Lists core methods that don't have underlying bytecode, but are synthesized on-the-fly in every reflection universe */ - lazy val syntheticCoreMethods = AnyMethods ++ ObjectMethods ++ List(String_+, throwMethod) + lazy val syntheticCoreMethods = + AnyMethods ++ ObjectMethods ++ List(String_+, throwMethod, quoteMethod, typeQuoteMethod) lazy val reservedScalaClassNames: Set[Name] = syntheticScalaClasses.map(_.name).toSet diff --git a/compiler/src/dotty/tools/dotc/core/StdNames.scala b/compiler/src/dotty/tools/dotc/core/StdNames.scala index be886b2b3bd7..10b0d0d896af 100644 --- a/compiler/src/dotty/tools/dotc/core/StdNames.scala +++ b/compiler/src/dotty/tools/dotc/core/StdNames.scala @@ -143,6 +143,7 @@ object StdNames { val INITIALIZER_PREFIX: N = "initial$" val COMPANION_MODULE_METHOD: N = "companion$module" val COMPANION_CLASS_METHOD: N = "companion$class" + val QUOTE: N = "'" val TRAIT_SETTER_SEPARATOR: N = str.TRAIT_SETTER_SEPARATOR // value types (and AnyRef) are all used as terms as well diff --git a/compiler/src/dotty/tools/dotc/core/Symbols.scala b/compiler/src/dotty/tools/dotc/core/Symbols.scala index 4b8730ead2f5..090dbaed0bf3 100644 --- a/compiler/src/dotty/tools/dotc/core/Symbols.scala +++ b/compiler/src/dotty/tools/dotc/core/Symbols.scala @@ -383,6 +383,9 @@ trait Symbols { this: Context => base.staticRef(path.toTermName).requiredSymbol(_ is Module).asTerm def requiredModuleRef(path: PreName): TermRef = requiredModule(path).termRef + + def requiredMethod(path: PreName): TermSymbol = + base.staticRef(path.toTermName).requiredSymbol(_ is Method).asTerm } object Symbols { diff --git a/compiler/src/dotty/tools/dotc/typer/Typer.scala b/compiler/src/dotty/tools/dotc/typer/Typer.scala index fcfdce97cc97..2fb9ccd3524e 100644 --- a/compiler/src/dotty/tools/dotc/typer/Typer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Typer.scala @@ -1072,6 +1072,22 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit Throw(expr1).withPos(tree.pos) } + def typedQuote(tree: untpd.Quote, pt: Type)(implicit ctx: Context): Tree = track("typedQuote") { + val untpd.Quote(body) = tree + val isType = body.isType + val resultClass = if (isType) defn.MetaTypeClass else defn.MetaExprClass + val proto1 = pt.baseType(resultClass) match { + case AppliedType(_, argType :: Nil) => argType + case _ => WildcardType + } + if (isType) + ref(defn.typeQuoteMethod).appliedToTypeTrees(typedType(body, proto1) :: Nil) + else { + val body1 = typed(body, proto1) + ref(defn.quoteMethod).appliedToType(body1.tpe.widen).appliedTo(body1) + } + } + def typedSeqLiteral(tree: untpd.SeqLiteral, pt: Type)(implicit ctx: Context): SeqLiteral = track("typedSeqLiteral") { val proto1 = pt.elemType match { case NoType => WildcardType @@ -1695,6 +1711,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit case tree: untpd.Super => typedSuper(tree, pt) case tree: untpd.SeqLiteral => typedSeqLiteral(tree, pt) case tree: untpd.Inlined => typedInlined(tree, pt) + case tree: untpd.Quote => typedQuote(tree, pt) case tree: untpd.TypeTree => typedTypeTree(tree, pt) case tree: untpd.SingletonTypeTree => typedSingletonTypeTree(tree) case tree: untpd.AndTypeTree => typedAndTypeTree(tree) From b21e638e64ec1f39bd248c55dce9f6ae617fa20b Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 08:59:38 +0100 Subject: [PATCH 14/62] Add quote reification framework The body of reify is still missing, for @nicolasstucki can fill in. For the moment, we just replace the tree to reify with its "_.show" output. --- compiler/src/dotty/tools/dotc/Compiler.scala | 9 +-- .../tools/dotc/transform/ReifyQuotes.scala | 67 +++++++++++++++++++ library/src/scala/meta/Reifier.scala | 4 +- 3 files changed, 75 insertions(+), 5 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index e318332826f6..f5fcc8c4c61e 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -49,7 +49,8 @@ class Compiler { List(new LinkAll), // Reload compilation units from TASTY for library code (if needed) List(new FirstTransform, // Some transformations to put trees into a canonical form new CheckReentrant, // Internal use only: Check that compiled program has no data races involving global vars - new ElimJavaPackages), // Eliminate syntactic references to Java packages + new ElimPackagePrefixes, // Eliminate syntactic references to Java packages + new ReifyQuotes), // Eliminate syntactic references to Java packages List(new CheckStatic, // Check restrictions that apply to @static members new ElimRepeated, // Rewrite vararg parameters and arguments new NormalizeFlags, // Rewrite some definition flags @@ -59,8 +60,8 @@ class Compiler { new ByNameClosures, // Expand arguments to by-name parameters to closures new LiftTry, // Put try expressions that might execute on non-empty stacks into their own methods new HoistSuperArgs, // Hoist complex arguments of supercalls to enclosing scope - new ClassOf, // Expand `Predef.classOf` calls. - new RefChecks), // Various checks mostly related to abstract members and overriding + new ClassOf, // Expand `Predef.classOf` calls. + new RefChecks), // Various checks mostly related to abstract members and overriding List(new TryCatchPatterns, // Compile cases in try/catch new PatternMatcher, // Compile pattern matches new ExplicitOuter, // Add accessors to outer classes from nested ones. @@ -68,7 +69,7 @@ class Compiler { new ShortcutImplicits, // Allow implicit functions without creating closures new CrossCastAnd, // Normalize selections involving intersection types. new Splitter), // Expand selections involving union types into conditionals - List(new PhantomArgLift, // Extracts the evaluation of phantom arguments placing them before the call. + List(new PhantomArgLift, // Extracts the evaluation of phantom arguments placing them before the call. new VCInlineMethods, // Inlines calls to value class methods new SeqLiterals, // Express vararg arguments as arrays new InterceptedMethods, // Special handling of `==`, `|=`, `getClass` methods diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala new file mode 100644 index 000000000000..b8b7e54b77b4 --- /dev/null +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -0,0 +1,67 @@ +package dotty.tools.dotc +package transform + +import core._ +import Decorators._, Flags._, Types._, Contexts._, Symbols._, Constants._ +import Flags._ +import ast.Trees._ +import MegaPhase.MiniPhase +import scala.collection.mutable + +/** Translates quoted terms and types to reify method calls. + */ +class ReifyQuotes extends MiniPhase { + import ast.tpd._ + + override def phaseName: String = "reifyQuotes" + + def reifyTree(tree: Tree)(implicit ctx: Context): String = i"($tree)" // TODO: replace with TASTY + def reifyType(tpe: Type)(implicit ctx: Context): String = i"[$tpe]" // TODO: replace with TASTY + + private def reifyCall(meth: Symbol, typeArg: Type, reified: String, splices: List[Tree])(implicit ctx: Context) = + ref(meth) + .appliedToType(typeArg) + .appliedTo( + Literal(Constant(reified)), + SeqLiteral(splices, TypeTree(defn.MetaExprType.appliedTo(TypeBounds.empty)))) + + override def transformApply(tree: Apply)(implicit ctx: Context): Tree = { + object liftSplices extends TreeMap { + val splices = new mutable.ListBuffer[Tree] + override def transform(tree: Tree)(implicit ctx: Context) = tree match { + case tree @ Select(qual, name) if tree.symbol == defn.MetaExpr_~ => + splices += qual + val placeHolder = Typed(EmptyTree, TypeTree(qual.tpe.widen)) + cpy.Select(tree)(placeHolder, name) + case _ => + super.transform(tree) + } + } + if (tree.fun.symbol == defn.quoteMethod) { + val body = tree.args.head + val reified = reifyTree(liftSplices.transform(body)) + reifyCall(defn.Reifier_reifyExpr, body.tpe.widen, reified, liftSplices.splices.toList) + } + else tree + } + + override def transformTypeApply(tree: TypeApply)(implicit ctx: Context): Tree = { + object liftSplices extends TypeMap { + val splices = new mutable.ListBuffer[Tree] + override def apply(t: Type) = t match { + case t @ TypeRef(pre: TermRef, _) if t.symbol == defn.MetaType_~ => + splices += ref(pre).withPos(tree.args.head.pos) + val placeHolder = SkolemType(pre.widen) + t.withPrefix(placeHolder) + case _ => + mapOver(t) + } + } + if (tree.fun.symbol == defn.typeQuoteMethod) { + val body = tree.args.head.tpe + val reified = reifyType(liftSplices(body)) + reifyCall(defn.Reifier_reifyType, body, reified, liftSplices.splices.toList) + } + else tree + } +} diff --git a/library/src/scala/meta/Reifier.scala b/library/src/scala/meta/Reifier.scala index f7271d92cbb8..e2fa4fe400f6 100644 --- a/library/src/scala/meta/Reifier.scala +++ b/library/src/scala/meta/Reifier.scala @@ -2,6 +2,8 @@ package scala.meta object Reifier { - def reify(repr: String, args: Expr[_]*) = ??? + def reifyExpr[T](repr: String, args: Seq[Expr[_]]): Expr[T] = ??? + + def reifyType[T](repr: String, args: Seq[Expr[_]]): Type[T] = ??? } From 7418cef7d0d55f52e40fe2ca5821e2924dd8be08 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 14:14:47 +0100 Subject: [PATCH 15/62] Support printing of Quote trees --- compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala index 279c3d7642b2..2251870900c2 100644 --- a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala +++ b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala @@ -598,6 +598,8 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { changePrec(GlobalPrec) { keywordStr("try ") ~ toText(expr) ~ " " ~ keywordStr("catch") ~ " {" ~ toText(handler) ~ "}" ~ optText(finalizer)(keywordStr(" finally ") ~ _) } + case Quote(tree) => + if (tree.isType) "'[" ~ toTextGlobal(tree) ~ "]" else "'(" ~ toTextGlobal(tree) ~ ")" case Thicket(trees) => "Thicket {" ~~ toTextGlobal(trees, "\n") ~~ "}" case _ => From 67a808a3dfec8b5b03220691f82fe31352a28f87 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 14:15:42 +0100 Subject: [PATCH 16/62] Fix isType/isTerm predicates for splices --- compiler/src/dotty/tools/dotc/ast/untpd.scala | 5 ++++- compiler/src/dotty/tools/dotc/parsing/Parsers.scala | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/ast/untpd.scala b/compiler/src/dotty/tools/dotc/ast/untpd.scala index d1077a8c0488..cd6952ed16ab 100644 --- a/compiler/src/dotty/tools/dotc/ast/untpd.scala +++ b/compiler/src/dotty/tools/dotc/ast/untpd.scala @@ -69,7 +69,10 @@ object untpd extends Trees.Instance[Untyped] with UntypedTreeInfo { case class InfixOp(left: Tree, op: Ident, right: Tree) extends OpTree case class PostfixOp(od: Tree, op: Ident) extends OpTree - case class PrefixOp(op: Ident, od: Tree) extends OpTree + case class PrefixOp(op: Ident, od: Tree) extends OpTree { + override def isType = op.isType + override def isTerm = op.isTerm + } case class Parens(t: Tree) extends ProxyTree { def forwardTo = t } diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index da28195c1a5d..eb468c5216f8 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -877,7 +877,7 @@ object Parsers { typeBounds().withPos(Position(start, in.lastOffset, start)) } else if (isIdent && nme.raw.isUnary(in.name)) - atPos(in.offset) { PrefixOp(termIdent(), path(thisOK = true)) } + atPos(in.offset) { PrefixOp(typeIdent(), path(thisOK = true)) } else path(thisOK = false, handleSingletonType) match { case r @ SingletonTypeTree(_) => r case r => convertToTypeId(r) From a53c2fff41c1ca340e1fbfbd75dfda0af996e9cb Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 15:04:10 +0100 Subject: [PATCH 17/62] Fix missing core libs test Rearrange initialization sequence so that we fail with a missing core libs exception before any other symbols are forced. --- compiler/src/dotty/tools/dotc/core/Definitions.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index d380d3e0648c..b4f3497ad327 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -1103,13 +1103,15 @@ class Definitions { def init()(implicit ctx: Context) = { this.ctx = ctx if (!_isInitialized) { - // force initialization of every symbol that is synthesized or hijacked by the compiler - val forced = syntheticCoreClasses ++ syntheticCoreMethods ++ ScalaValueClasses() - // Enter all symbols from the scalaShadowing package in the scala package for (m <- ScalaShadowingPackageClass.info.decls) ScalaPackageClass.enter(m) + // force initialization of every symbol that is synthesized or hijacked by the compiler + // Placed here so that an abort with a MissingCoreLibraryException because scalaShadowing + // is not found comes before any additional errors are reported. + val forced = syntheticCoreClasses ++ syntheticCoreMethods ++ ScalaValueClasses() + _isInitialized = true } } From 1a7803cfe08a93778c03ca5e2fe978813bae5762 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 15:05:28 +0100 Subject: [PATCH 18/62] Change reify for types We now keep the (type-) trees and proceed as with normal reification. This saves us the trouble of having to define a separate pickler for types with splices in them. --- compiler/src/dotty/tools/dotc/core/Mode.scala | 3 + .../tools/dotc/transform/FirstTransform.scala | 11 +-- .../tools/dotc/transform/ReifyQuotes.scala | 71 ++++++++++--------- library/src/scala/meta/Reifier.scala | 2 +- 4 files changed, 46 insertions(+), 41 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Mode.scala b/compiler/src/dotty/tools/dotc/core/Mode.scala index c97e20a88d49..8c5cf3fb67a5 100644 --- a/compiler/src/dotty/tools/dotc/core/Mode.scala +++ b/compiler/src/dotty/tools/dotc/core/Mode.scala @@ -93,4 +93,7 @@ object Mode { /** We are in the IDE */ val Interactive = newMode(20, "Interactive") + + /** We are in a quoted type during the ReifyQuotes phase */ + val InQuotedType = newMode(21, "InQuotedType") } diff --git a/compiler/src/dotty/tools/dotc/transform/FirstTransform.scala b/compiler/src/dotty/tools/dotc/transform/FirstTransform.scala index a3e877a3c292..398bd54d9978 100644 --- a/compiler/src/dotty/tools/dotc/transform/FirstTransform.scala +++ b/compiler/src/dotty/tools/dotc/transform/FirstTransform.scala @@ -188,19 +188,20 @@ class FirstTransform extends MiniPhase with InfoTransformer { thisPhase => override def transformStats(trees: List[Tree])(implicit ctx: Context): List[Tree] = ast.Trees.flatten(reorderAndComplete(trees)(ctx.withPhase(thisPhase.next))) + private def toTypeTree(tree: Tree)(implicit ctx: Context) = + if (ctx.mode.is(Mode.InQuotedType)) tree else TypeTree(tree.tpe).withPos(tree.pos) + override def transformOther(tree: Tree)(implicit ctx: Context) = tree match { case tree: Import => EmptyTree case tree: NamedArg => transformAllDeep(tree.arg) - case tree => if (tree.isType) TypeTree(tree.tpe).withPos(tree.pos) else tree + case tree => if (tree.isType) toTypeTree(tree) else tree } override def transformIdent(tree: Ident)(implicit ctx: Context) = - if (tree.isType) TypeTree(tree.tpe).withPos(tree.pos) - else constToLiteral(tree) + if (tree.isType) toTypeTree(tree) else constToLiteral(tree) override def transformSelect(tree: Select)(implicit ctx: Context) = - if (tree.isType) TypeTree(tree.tpe).withPos(tree.pos) - else constToLiteral(tree) + if (tree.isType) toTypeTree(tree) else constToLiteral(tree) override def transformTypeApply(tree: TypeApply)(implicit ctx: Context) = constToLiteral(tree) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index b8b7e54b77b4..84e9c73cd2fa 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -9,27 +9,31 @@ import MegaPhase.MiniPhase import scala.collection.mutable /** Translates quoted terms and types to reify method calls. + * The mini phase needs to run in the same group as FirstTransform. So far we lack + * the machinery to express this constraint in code. */ class ReifyQuotes extends MiniPhase { import ast.tpd._ override def phaseName: String = "reifyQuotes" - def reifyTree(tree: Tree)(implicit ctx: Context): String = i"($tree)" // TODO: replace with TASTY - def reifyType(tpe: Type)(implicit ctx: Context): String = i"[$tpe]" // TODO: replace with TASTY + /** Serialize `tree`. Embedded splices are represented as nodes of the form + * + * Select(qual, sym) + * + * where `sym` is either `defn.MetaExpr_~` or `defn.MetaType_~`. For any splice, + * the `qual` part will be of the form `Typed(EmptyTree, TypeTree())`, + * but that knowledge is not needed to uniquely identify a splice node. + */ + def reifyTree(tree: Tree)(implicit ctx: Context): String = tree.show // TODO: replace with TASTY - private def reifyCall(meth: Symbol, typeArg: Type, reified: String, splices: List[Tree])(implicit ctx: Context) = - ref(meth) - .appliedToType(typeArg) - .appliedTo( - Literal(Constant(reified)), - SeqLiteral(splices, TypeTree(defn.MetaExprType.appliedTo(TypeBounds.empty)))) + private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = { - override def transformApply(tree: Apply)(implicit ctx: Context): Tree = { object liftSplices extends TreeMap { val splices = new mutable.ListBuffer[Tree] override def transform(tree: Tree)(implicit ctx: Context) = tree match { - case tree @ Select(qual, name) if tree.symbol == defn.MetaExpr_~ => + case tree @ Select(qual, name) + if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ => splices += qual val placeHolder = Typed(EmptyTree, TypeTree(qual.tpe.widen)) cpy.Select(tree)(placeHolder, name) @@ -37,31 +41,28 @@ class ReifyQuotes extends MiniPhase { super.transform(tree) } } - if (tree.fun.symbol == defn.quoteMethod) { - val body = tree.args.head - val reified = reifyTree(liftSplices.transform(body)) - reifyCall(defn.Reifier_reifyExpr, body.tpe.widen, reified, liftSplices.splices.toList) - } - else tree + + val reified = reifyTree(liftSplices.transform(body)) + var splices = liftSplices.splices.toList + if (isType) // transform splices again because embedded type trees were not rewritten before + splices = splices.map(transformAllDeep(_)(ctx.retractMode(Mode.InQuotedType))) + val spliceType = if (isType) defn.MetaTypeType else defn.MetaExprType + + ref(if (isType) defn.Reifier_reifyType else defn.Reifier_reifyExpr) + .appliedToType(if (isType) body.tpe else body.tpe.widen) + .appliedTo( + Literal(Constant(reified)), + SeqLiteral(splices, TypeTree(spliceType.appliedTo(TypeBounds.empty)))) } - override def transformTypeApply(tree: TypeApply)(implicit ctx: Context): Tree = { - object liftSplices extends TypeMap { - val splices = new mutable.ListBuffer[Tree] - override def apply(t: Type) = t match { - case t @ TypeRef(pre: TermRef, _) if t.symbol == defn.MetaType_~ => - splices += ref(pre).withPos(tree.args.head.pos) - val placeHolder = SkolemType(pre.widen) - t.withPrefix(placeHolder) - case _ => - mapOver(t) - } - } - if (tree.fun.symbol == defn.typeQuoteMethod) { - val body = tree.args.head.tpe - val reified = reifyType(liftSplices(body)) - reifyCall(defn.Reifier_reifyType, body, reified, liftSplices.splices.toList) - } + override def transformApply(tree: Apply)(implicit ctx: Context): Tree = + if (tree.fun.symbol == defn.quoteMethod) reifyCall(tree.args.head, isType = false) else tree - } -} + + override def prepareForTypeApply(tree: TypeApply)(implicit ctx: Context): Context = + if (tree.symbol == defn.typeQuoteMethod) ctx.addMode(Mode.InQuotedType) else ctx + + override def transformTypeApply(tree: TypeApply)(implicit ctx: Context): Tree = + if (tree.fun.symbol == defn.typeQuoteMethod) reifyCall(tree.args.head, isType = true) + else tree +} \ No newline at end of file diff --git a/library/src/scala/meta/Reifier.scala b/library/src/scala/meta/Reifier.scala index e2fa4fe400f6..777d9b1decbf 100644 --- a/library/src/scala/meta/Reifier.scala +++ b/library/src/scala/meta/Reifier.scala @@ -4,6 +4,6 @@ object Reifier { def reifyExpr[T](repr: String, args: Seq[Expr[_]]): Expr[T] = ??? - def reifyType[T](repr: String, args: Seq[Expr[_]]): Type[T] = ??? + def reifyType[T](repr: String, args: Seq[Type[_]]): Type[T] = ??? } From 72c32c554182bc2d01c43eb0018a1b41a6266186 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 15:19:39 +0100 Subject: [PATCH 19/62] Rename reify -> unpickle Also, add test --- .../dotty/tools/dotc/core/Definitions.scala | 4 ++-- .../tools/dotc/transform/ReifyQuotes.scala | 17 +++++++++------- library/src/scala/meta/Reifier.scala | 9 --------- library/src/scala/meta/Unpickler.scala | 20 +++++++++++++++++++ tests/pos/quoteTest.scala | 15 ++++++++++++++ 5 files changed, 47 insertions(+), 18 deletions(-) delete mode 100644 library/src/scala/meta/Reifier.scala create mode 100644 library/src/scala/meta/Unpickler.scala create mode 100644 tests/pos/quoteTest.scala diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index b4f3497ad327..4e7cbd8780f6 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -603,8 +603,8 @@ class Definitions { def MetaType_~(implicit ctx: Context) = MetaTypeClass.info.member(tpnme.UNARY_~).symbol.asType - def Reifier_reifyExpr = ctx.requiredMethod("scala.meta.Reifier.reifyExpr") - def Reifier_reifyType = ctx.requiredMethod("scala.meta.Reifier.reifyType") + def Unpickler_unpickleExpr = ctx.requiredMethod("scala.meta.Unpickler.unpickleExpr") + def Unpickler_unpickleType = ctx.requiredMethod("scala.meta.Unpickler.unpickleType") lazy val EqType = ctx.requiredClassRef("scala.Eq") def EqClass(implicit ctx: Context) = EqType.symbol.asClass diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 84e9c73cd2fa..b3afa3665e06 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -25,7 +25,8 @@ class ReifyQuotes extends MiniPhase { * the `qual` part will be of the form `Typed(EmptyTree, TypeTree())`, * but that knowledge is not needed to uniquely identify a splice node. */ - def reifyTree(tree: Tree)(implicit ctx: Context): String = tree.show // TODO: replace with TASTY + def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String = + tree.show // TODO: replace with TASTY private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = { @@ -34,7 +35,11 @@ class ReifyQuotes extends MiniPhase { override def transform(tree: Tree)(implicit ctx: Context) = tree match { case tree @ Select(qual, name) if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ => - splices += qual + splices += { + if (isType) // transform splice again because embedded type trees were not rewritten before + transformAllDeep(qual)(ctx.retractMode(Mode.InQuotedType)) + else qual + } val placeHolder = Typed(EmptyTree, TypeTree(qual.tpe.widen)) cpy.Select(tree)(placeHolder, name) case _ => @@ -42,13 +47,11 @@ class ReifyQuotes extends MiniPhase { } } - val reified = reifyTree(liftSplices.transform(body)) - var splices = liftSplices.splices.toList - if (isType) // transform splices again because embedded type trees were not rewritten before - splices = splices.map(transformAllDeep(_)(ctx.retractMode(Mode.InQuotedType))) + val reified = pickleTree(liftSplices.transform(body), isType) + val splices = liftSplices.splices.toList val spliceType = if (isType) defn.MetaTypeType else defn.MetaExprType - ref(if (isType) defn.Reifier_reifyType else defn.Reifier_reifyExpr) + ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr) .appliedToType(if (isType) body.tpe else body.tpe.widen) .appliedTo( Literal(Constant(reified)), diff --git a/library/src/scala/meta/Reifier.scala b/library/src/scala/meta/Reifier.scala deleted file mode 100644 index 777d9b1decbf..000000000000 --- a/library/src/scala/meta/Reifier.scala +++ /dev/null @@ -1,9 +0,0 @@ -package scala.meta - -object Reifier { - - def reifyExpr[T](repr: String, args: Seq[Expr[_]]): Expr[T] = ??? - - def reifyType[T](repr: String, args: Seq[Type[_]]): Type[T] = ??? - -} diff --git a/library/src/scala/meta/Unpickler.scala b/library/src/scala/meta/Unpickler.scala new file mode 100644 index 000000000000..d3817df5463c --- /dev/null +++ b/library/src/scala/meta/Unpickler.scala @@ -0,0 +1,20 @@ +package scala.meta + +/** Provides methods to unpickle `Expr` and `Type` trees. */ +object Unpickler { + + /** Representation of pickled trees. For now it's String, but it + * should be changed to some kind of TASTY bundle. + */ + type Pickled = String + + /** Unpickle `repr` which represents a pickled `Expr` tree, + * replacing splice nodes with `args` + */ + def unpickleExpr[T](repr: Pickled, args: Seq[Expr[_]]): Expr[T] = ??? + + /** Unpickle `repr` which represents a pickled `Type` tree, + * replacing splice nodes with `args` + */ + def unpickleType[T](repr: Pickled, args: Seq[Type[_]]): Type[T] = ??? +} diff --git a/tests/pos/quoteTest.scala b/tests/pos/quoteTest.scala new file mode 100644 index 000000000000..4944897cf8c8 --- /dev/null +++ b/tests/pos/quoteTest.scala @@ -0,0 +1,15 @@ +import scala.meta._ + +object Test { + + def f[T](t: Type[T], x: Expr[T]) = { + val y: t.unary_~ = x.unary_~ + val z: ~t = ~x + } + + f('[Int], '(2)) + f('[Boolean], '{ true }) + + def g(es: Expr[String], t: Type[String]) = + f('[List[~t]], '{ (~es + "!") :: Nil }) +} From b35ff5f4d1c46e304c2accd9ab87fde63a59a65d Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 16:28:19 +0100 Subject: [PATCH 20/62] Make ReifyQuotes a macro phase The previous attempt to make it a miniphase in the FirstTransform group turned out to be too complicated. The implementation so far did not handle type splices in Expr trees correctly, and doing so would not have been easy. We now run ReifyQuotes as a macro phase immediately after pickler. This means trees are reified exactly as they are pickled. We address the problem of slowdowns due to extra tree traversal by keeping a flag in CompilationUnit which allows us to skip the traversal altogether for compilation units that do not contain quotes. --- .../dotty/tools/dotc/CompilationUnit.scala | 5 ++ compiler/src/dotty/tools/dotc/Compiler.scala | 4 +- .../dotty/tools/dotc/core/Definitions.scala | 3 + compiler/src/dotty/tools/dotc/core/Mode.scala | 3 - .../tools/dotc/transform/FirstTransform.scala | 4 +- .../tools/dotc/transform/ReifyQuotes.scala | 80 ++++++++++--------- .../src/dotty/tools/dotc/typer/Typer.scala | 1 + library/src/scala/meta/Expr.scala | 2 +- library/src/scala/meta/Quoted.scala | 6 ++ library/src/scala/meta/Type.scala | 2 +- library/src/scala/meta/Unpickler.scala | 4 +- 11 files changed, 65 insertions(+), 49 deletions(-) create mode 100644 library/src/scala/meta/Quoted.scala diff --git a/compiler/src/dotty/tools/dotc/CompilationUnit.scala b/compiler/src/dotty/tools/dotc/CompilationUnit.scala index 63458e8fb578..fb150b48bee8 100644 --- a/compiler/src/dotty/tools/dotc/CompilationUnit.scala +++ b/compiler/src/dotty/tools/dotc/CompilationUnit.scala @@ -21,6 +21,11 @@ class CompilationUnit(val source: SourceFile) { /** Pickled TASTY binaries, indexed by class. */ var pickled: Map[ClassSymbol, Array[Byte]] = Map() + + /** Will be reset to `true` if `untpdTree` contains `Quote` trees. The information + * is used in phase ReifyQuotes in order to avoid traversing a quote-less tree. + */ + var containsQuotes: Boolean = false } object CompilationUnit { diff --git a/compiler/src/dotty/tools/dotc/Compiler.scala b/compiler/src/dotty/tools/dotc/Compiler.scala index f5fcc8c4c61e..914272f5249c 100644 --- a/compiler/src/dotty/tools/dotc/Compiler.scala +++ b/compiler/src/dotty/tools/dotc/Compiler.scala @@ -47,10 +47,10 @@ class Compiler { List(new sbt.ExtractAPI), // Sends a representation of the API of classes to sbt via callbacks List(new Pickler), // Generate TASTY info List(new LinkAll), // Reload compilation units from TASTY for library code (if needed) + List(new ReifyQuotes), // Turn quoted trees into explicit run-time data structures List(new FirstTransform, // Some transformations to put trees into a canonical form new CheckReentrant, // Internal use only: Check that compiled program has no data races involving global vars - new ElimPackagePrefixes, // Eliminate syntactic references to Java packages - new ReifyQuotes), // Eliminate syntactic references to Java packages + new ElimPackagePrefixes), // Eliminate references to package prefixes in Select nodes List(new CheckStatic, // Check restrictions that apply to @static members new ElimRepeated, // Rewrite vararg parameters and arguments new NormalizeFlags, // Rewrite some definition flags diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 4e7cbd8780f6..e48762722cbd 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -592,6 +592,9 @@ class Definitions { def ClassTagClass(implicit ctx: Context) = ClassTagType.symbol.asClass def ClassTagModule(implicit ctx: Context) = ClassTagClass.companionModule + lazy val MetaQuotedType = ctx.requiredClassRef("scala.meta.Quoted") + def MetaQuotedClass(implicit ctx: Context) = MetaQuotedType.symbol.asClass + lazy val MetaExprType = ctx.requiredClassRef("scala.meta.Expr") def MetaExprClass(implicit ctx: Context) = MetaExprType.symbol.asClass diff --git a/compiler/src/dotty/tools/dotc/core/Mode.scala b/compiler/src/dotty/tools/dotc/core/Mode.scala index 8c5cf3fb67a5..c97e20a88d49 100644 --- a/compiler/src/dotty/tools/dotc/core/Mode.scala +++ b/compiler/src/dotty/tools/dotc/core/Mode.scala @@ -93,7 +93,4 @@ object Mode { /** We are in the IDE */ val Interactive = newMode(20, "Interactive") - - /** We are in a quoted type during the ReifyQuotes phase */ - val InQuotedType = newMode(21, "InQuotedType") } diff --git a/compiler/src/dotty/tools/dotc/transform/FirstTransform.scala b/compiler/src/dotty/tools/dotc/transform/FirstTransform.scala index 398bd54d9978..af46afec3a6b 100644 --- a/compiler/src/dotty/tools/dotc/transform/FirstTransform.scala +++ b/compiler/src/dotty/tools/dotc/transform/FirstTransform.scala @@ -189,8 +189,8 @@ class FirstTransform extends MiniPhase with InfoTransformer { thisPhase => ast.Trees.flatten(reorderAndComplete(trees)(ctx.withPhase(thisPhase.next))) private def toTypeTree(tree: Tree)(implicit ctx: Context) = - if (ctx.mode.is(Mode.InQuotedType)) tree else TypeTree(tree.tpe).withPos(tree.pos) - + TypeTree(tree.tpe).withPos(tree.pos) + override def transformOther(tree: Tree)(implicit ctx: Context) = tree match { case tree: Import => EmptyTree case tree: NamedArg => transformAllDeep(tree.arg) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index b3afa3665e06..5ba95c2d8006 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -12,60 +12,64 @@ import scala.collection.mutable * The mini phase needs to run in the same group as FirstTransform. So far we lack * the machinery to express this constraint in code. */ -class ReifyQuotes extends MiniPhase { +class ReifyQuotes extends MacroTransform { import ast.tpd._ override def phaseName: String = "reifyQuotes" + override def run(implicit ctx: Context): Unit = + if (ctx.compilationUnit.containsQuotes) super.run + + protected def newTransformer(implicit ctx: Context): Transformer = new Reifier + /** Serialize `tree`. Embedded splices are represented as nodes of the form * * Select(qual, sym) * * where `sym` is either `defn.MetaExpr_~` or `defn.MetaType_~`. For any splice, - * the `qual` part will be of the form `Typed(EmptyTree, TypeTree())`, - * but that knowledge is not needed to uniquely identify a splice node. + * the `qual` part should not be pickled, since it will be added separately later + * as a splice. */ def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String = tree.show // TODO: replace with TASTY - private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = { + private class Reifier extends Transformer { + + /** Turn `body` of quote into a call of `scala.meta.Unpickler.unpickleType` or + * `scala.meta.Unpickler.unpickleExpr` depending onwhether `isType` is true or not. + * The arguments to the method are: + * + * - the serialized `body`, as returned from `pickleTree` + * - all splices found in `body` + */ + private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = { - object liftSplices extends TreeMap { - val splices = new mutable.ListBuffer[Tree] - override def transform(tree: Tree)(implicit ctx: Context) = tree match { - case tree @ Select(qual, name) - if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ => - splices += { - if (isType) // transform splice again because embedded type trees were not rewritten before - transformAllDeep(qual)(ctx.retractMode(Mode.InQuotedType)) - else qual - } - val placeHolder = Typed(EmptyTree, TypeTree(qual.tpe.widen)) - cpy.Select(tree)(placeHolder, name) - case _ => - super.transform(tree) + object collectSplices extends TreeAccumulator[mutable.ListBuffer[Tree]] { + override def apply(splices: mutable.ListBuffer[Tree], tree: Tree)(implicit ctx: Context) = tree match { + case tree @ Select(qual, _) + if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ => + splices += transform(qual) + case _ => + foldOver(splices, tree) + } } - } + val splices = collectSplices(new mutable.ListBuffer[Tree], body).toList + val reified = pickleTree(body, isType) - val reified = pickleTree(liftSplices.transform(body), isType) - val splices = liftSplices.splices.toList - val spliceType = if (isType) defn.MetaTypeType else defn.MetaExprType + ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr) + .appliedToType(if (isType) body.tpe else body.tpe.widen) + .appliedTo( + Literal(Constant(reified)), + SeqLiteral(splices, TypeTree(defn.MetaQuotedType))) + } - ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr) - .appliedToType(if (isType) body.tpe else body.tpe.widen) - .appliedTo( - Literal(Constant(reified)), - SeqLiteral(splices, TypeTree(spliceType.appliedTo(TypeBounds.empty)))) + override def transform(tree: Tree)(implicit ctx: Context): Tree = tree match { + case Apply(fn, arg :: Nil) if fn.symbol == defn.quoteMethod => + reifyCall(arg, isType = false) + case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod => + reifyCall(arg, isType = true) + case _ => + super.transform(tree) + } } - - override def transformApply(tree: Apply)(implicit ctx: Context): Tree = - if (tree.fun.symbol == defn.quoteMethod) reifyCall(tree.args.head, isType = false) - else tree - - override def prepareForTypeApply(tree: TypeApply)(implicit ctx: Context): Context = - if (tree.symbol == defn.typeQuoteMethod) ctx.addMode(Mode.InQuotedType) else ctx - - override def transformTypeApply(tree: TypeApply)(implicit ctx: Context): Tree = - if (tree.fun.symbol == defn.typeQuoteMethod) reifyCall(tree.args.head, isType = true) - else tree } \ No newline at end of file diff --git a/compiler/src/dotty/tools/dotc/typer/Typer.scala b/compiler/src/dotty/tools/dotc/typer/Typer.scala index 2fb9ccd3524e..d4c37dd95c04 100644 --- a/compiler/src/dotty/tools/dotc/typer/Typer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Typer.scala @@ -1073,6 +1073,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit } def typedQuote(tree: untpd.Quote, pt: Type)(implicit ctx: Context): Tree = track("typedQuote") { + ctx.compilationUnit.containsQuotes = true val untpd.Quote(body) = tree val isType = body.isType val resultClass = if (isType) defn.MetaTypeClass else defn.MetaExprClass diff --git a/library/src/scala/meta/Expr.scala b/library/src/scala/meta/Expr.scala index a61c6145b0d6..983818c32164 100644 --- a/library/src/scala/meta/Expr.scala +++ b/library/src/scala/meta/Expr.scala @@ -1,5 +1,5 @@ package scala.meta -class Expr[T] { +class Expr[T] extends Quoted { def unary_~ : T = ??? } diff --git a/library/src/scala/meta/Quoted.scala b/library/src/scala/meta/Quoted.scala new file mode 100644 index 000000000000..3040370c3af9 --- /dev/null +++ b/library/src/scala/meta/Quoted.scala @@ -0,0 +1,6 @@ +package scala.meta + +/** Common superclass of Expr and Type */ +class Quoted + + diff --git a/library/src/scala/meta/Type.scala b/library/src/scala/meta/Type.scala index dc19c0234b32..ebb20c6b3db5 100644 --- a/library/src/scala/meta/Type.scala +++ b/library/src/scala/meta/Type.scala @@ -1,5 +1,5 @@ package scala.meta -class Type[T] { +class Type[T] extends Quoted { type unary_~ = T } diff --git a/library/src/scala/meta/Unpickler.scala b/library/src/scala/meta/Unpickler.scala index d3817df5463c..d4344e96b392 100644 --- a/library/src/scala/meta/Unpickler.scala +++ b/library/src/scala/meta/Unpickler.scala @@ -11,10 +11,10 @@ object Unpickler { /** Unpickle `repr` which represents a pickled `Expr` tree, * replacing splice nodes with `args` */ - def unpickleExpr[T](repr: Pickled, args: Seq[Expr[_]]): Expr[T] = ??? + def unpickleExpr[T](repr: Pickled, args: Seq[Quoted]): Expr[T] = ??? /** Unpickle `repr` which represents a pickled `Type` tree, * replacing splice nodes with `args` */ - def unpickleType[T](repr: Pickled, args: Seq[Type[_]]): Type[T] = ??? + def unpickleType[T](repr: Pickled, args: Seq[Quoted]): Type[T] = ??? } From 0e0f31dd7c191310446eaf2f1095eb46513ca00f Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 16:49:39 +0100 Subject: [PATCH 21/62] Don't force quote methods --- compiler/src/dotty/tools/dotc/core/Definitions.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index e48762722cbd..8942a2f39127 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -1096,7 +1096,9 @@ class Definitions { /** Lists core methods that don't have underlying bytecode, but are synthesized on-the-fly in every reflection universe */ lazy val syntheticCoreMethods = - AnyMethods ++ ObjectMethods ++ List(String_+, throwMethod, quoteMethod, typeQuoteMethod) + AnyMethods ++ ObjectMethods ++ List(String_+, throwMethod + //, quoteMethod, typeQuoteMethod // we omit these because they force Expr and Type too early + ) lazy val reservedScalaClassNames: Set[Name] = syntheticScalaClasses.map(_.name).toSet @@ -1111,8 +1113,6 @@ class Definitions { ScalaPackageClass.enter(m) // force initialization of every symbol that is synthesized or hijacked by the compiler - // Placed here so that an abort with a MissingCoreLibraryException because scalaShadowing - // is not found comes before any additional errors are reported. val forced = syntheticCoreClasses ++ syntheticCoreMethods ++ ScalaValueClasses() _isInitialized = true From 1bdf8a0d76a21408b16e6779f24f3176eac9fb41 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 5 Dec 2017 17:16:59 +0100 Subject: [PATCH 22/62] Remove outdated comment --- compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 5ba95c2d8006..dcae205a9b43 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -9,8 +9,6 @@ import MegaPhase.MiniPhase import scala.collection.mutable /** Translates quoted terms and types to reify method calls. - * The mini phase needs to run in the same group as FirstTransform. So far we lack - * the machinery to express this constraint in code. */ class ReifyQuotes extends MacroTransform { import ast.tpd._ From 87ee12f1860b0797e5af21a1d8018930172a99fa Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 6 Dec 2017 11:32:06 +0100 Subject: [PATCH 23/62] Make '( and '{ tokens that can start an expression Fixes a bug in parsing quotes. --- compiler/src/dotty/tools/dotc/parsing/Tokens.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala index 507b7f250200..3c1b3b679944 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Tokens.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Tokens.scala @@ -211,7 +211,7 @@ object Tokens extends TokensCommon { USCORE, NULL, THIS, SUPER, TRUE, FALSE, RETURN, XMLSTART) final val canStartExpressionTokens = atomicExprTokens | BitSet( - LBRACE, LPAREN, IF, DO, WHILE, FOR, NEW, TRY, THROW) + LBRACE, LPAREN, QBRACE, QPAREN, IF, DO, WHILE, FOR, NEW, TRY, THROW) final val canStartTypeTokens = literalTokens | identifierTokens | BitSet( THIS, SUPER, USCORE, LPAREN, AT) From 4fb2eb0decd4dadaa78f5a7a1475514e21a4a273 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 6 Dec 2017 11:32:59 +0100 Subject: [PATCH 24/62] Revamp of ReifyQuotes - now supports multi-staging - and checks that PCP holds --- .../tools/dotc/transform/ReifyQuotes.scala | 108 +++++++++++++++--- 1 file changed, 89 insertions(+), 19 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index dcae205a9b43..b889ab368753 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -8,7 +8,8 @@ import ast.Trees._ import MegaPhase.MiniPhase import scala.collection.mutable -/** Translates quoted terms and types to reify method calls. +/** Translates quoted terms and types to `unpickle` method calls. + * Checks that the phase consistency principle (PCP) holds. */ class ReifyQuotes extends MacroTransform { import ast.tpd._ @@ -33,6 +34,48 @@ class ReifyQuotes extends MacroTransform { private class Reifier extends Transformer { + /** The current staging level */ + private var currentLevel = 0 + + /** The splices encountered so far, indexed by staging level */ + private val splicesAtLevel = mutable.ArrayBuffer(new mutable.ListBuffer[Tree]) + + // Invariant: -1 <= currentLevel <= splicesAtLevel.length + + /** A map from locally defined symbol's to the staging levels of their definitions */ + private val levelOf = new mutable.HashMap[Symbol, Int] + + /** A stack of entered symbol's, to be unwound after block exit */ + private var enteredSyms: List[Symbol] = Nil + + /** Enter staging level of symbol defined by `tree`, if applicable. */ + def markDef(tree: Tree)(implicit ctx: Context) = tree match { + case tree: MemberDef if !levelOf.contains(tree.symbol) => + levelOf(tree.symbol) = currentLevel + enteredSyms = tree.symbol :: enteredSyms + case _ => + } + + /** If reference is to a locally defined symbol, check that its staging level + * matches the current level. + */ + def checkLevel(tree: Tree)(implicit ctx: Context): Unit = { + + def check(sym: Symbol, show: Symbol => String): Unit = + if (levelOf.getOrElse(sym, currentLevel) != currentLevel) + ctx.error(em"""access to ${show(sym)} from wrong staging level: + | - the definition is at level ${levelOf(sym)}, + | - but the access is at level $currentLevel.""", tree.pos) + + def showThis(sym: Symbol) = i"${sym.name}.this" + + val sym = tree.symbol + if (sym.exists) + if (tree.isInstanceOf[This]) check(sym, showThis) + else if (sym.owner.isType) check(sym.owner, showThis) + else check(sym, _.show) + } + /** Turn `body` of quote into a call of `scala.meta.Unpickler.unpickleType` or * `scala.meta.Unpickler.unpickleExpr` depending onwhether `isType` is true or not. * The arguments to the method are: @@ -40,32 +83,59 @@ class ReifyQuotes extends MacroTransform { * - the serialized `body`, as returned from `pickleTree` * - all splices found in `body` */ - private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = { - - object collectSplices extends TreeAccumulator[mutable.ListBuffer[Tree]] { - override def apply(splices: mutable.ListBuffer[Tree], tree: Tree)(implicit ctx: Context) = tree match { - case tree @ Select(qual, _) - if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ => - splices += transform(qual) - case _ => - foldOver(splices, tree) - } - } - val splices = collectSplices(new mutable.ListBuffer[Tree], body).toList - val reified = pickleTree(body, isType) - + private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr) .appliedToType(if (isType) body.tpe else body.tpe.widen) .appliedTo( - Literal(Constant(reified)), - SeqLiteral(splices, TypeTree(defn.MetaQuotedType))) + Literal(Constant(pickleTree(body, isType))), + SeqLiteral(splicesAtLevel(currentLevel).toList, TypeTree(defn.MetaQuotedType))) + + /** Perform operation `op` in quoted context */ + private def inQuote(op: => Tree)(implicit ctx: Context) = { + currentLevel += 1 + if (currentLevel == splicesAtLevel.length) splicesAtLevel += null + val savedSplices = splicesAtLevel(currentLevel) + splicesAtLevel(currentLevel) = new mutable.ListBuffer[Tree] + try op + finally { + splicesAtLevel(currentLevel) = savedSplices + currentLevel -= 1 + } } override def transform(tree: Tree)(implicit ctx: Context): Tree = tree match { case Apply(fn, arg :: Nil) if fn.symbol == defn.quoteMethod => - reifyCall(arg, isType = false) + inQuote(reifyCall(transform(arg), isType = false)) case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod => - reifyCall(arg, isType = true) + inQuote(reifyCall(transform(arg), isType = true)) + case Select(body, name) + if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ => + currentLevel -= 1 + val body1 = try transform(body) finally currentLevel += 1 + if (currentLevel > 0) { + splicesAtLevel(currentLevel) += body1 + tree + } + else { + if (currentLevel < 0) + ctx.error(i"splice ~ not allowed under toplevel splice", tree.pos) + cpy.Select(tree)(body1, name) + } + case (_: Ident) | (_: This) => + checkLevel(tree) + super.transform(tree) + case _: MemberDef => + markDef(tree) + super.transform(tree) + case Block(stats, _) => + val last = enteredSyms + stats.foreach(markDef) + try super.transform(tree) + finally + while (enteredSyms ne last) { + levelOf -= enteredSyms.head + enteredSyms = enteredSyms.tail + } case _ => super.transform(tree) } From 11345070ade8dba0caa3ea074d801ccd5d7f7a2c Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 6 Dec 2017 12:17:51 +0100 Subject: [PATCH 25/62] Rename meta -> quoted --- .../dotty/tools/dotc/core/Definitions.scala | 27 +++++++++---------- .../tools/dotc/transform/ReifyQuotes.scala | 10 +++---- .../src/dotty/tools/dotc/typer/Typer.scala | 2 +- library/src/scala/{meta => quoted}/Expr.scala | 3 ++- .../src/scala/{meta => quoted}/Quoted.scala | 2 +- library/src/scala/{meta => quoted}/Type.scala | 3 ++- .../scala/{meta => quoted}/Unpickler.scala | 2 +- tests/neg/quoteTest.scala | 16 +++++++++++ tests/pos/quoteTest.scala | 4 +-- 9 files changed, 43 insertions(+), 26 deletions(-) rename library/src/scala/{meta => quoted}/Expr.scala (59%) rename library/src/scala/{meta => quoted}/Quoted.scala (73%) rename library/src/scala/{meta => quoted}/Type.scala (70%) rename library/src/scala/{meta => quoted}/Unpickler.scala (96%) create mode 100644 tests/neg/quoteTest.scala diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 8942a2f39127..3f926aebd54c 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -5,7 +5,6 @@ package core import Types._, Contexts._, Symbols._, Denotations._, SymDenotations._, StdNames._, Names._ import Flags._, Scopes._, Decorators._, NameOps._, util.Positions._, Periods._ import unpickleScala2.Scala2Unpickler.ensureConstructor -import scala.annotation.{ switch, meta } import scala.collection.{ mutable, immutable } import PartialFunction._ import collection.mutable @@ -299,11 +298,11 @@ class Definitions { /** Method representing a term quote */ lazy val quoteMethod = enterPolyMethod(OpsPackageClass, nme.QUOTE, 1, - pt => MethodType(pt.paramRefs(0) :: Nil, MetaExprType.appliedTo(pt.paramRefs(0) :: Nil))) + pt => MethodType(pt.paramRefs(0) :: Nil, QuotedExprType.appliedTo(pt.paramRefs(0) :: Nil))) /** Method representing a type quote */ lazy val typeQuoteMethod = enterPolyMethod(OpsPackageClass, nme.QUOTE, 1, - pt => MetaTypeType.appliedTo(pt.paramRefs(0) :: Nil)) + pt => QuotedTypeType.appliedTo(pt.paramRefs(0) :: Nil)) lazy val NothingClass: ClassSymbol = enterCompleteClassSymbol( ScalaPackageClass, tpnme.Nothing, AbstractFinal, List(AnyClass.typeRef)) @@ -592,22 +591,22 @@ class Definitions { def ClassTagClass(implicit ctx: Context) = ClassTagType.symbol.asClass def ClassTagModule(implicit ctx: Context) = ClassTagClass.companionModule - lazy val MetaQuotedType = ctx.requiredClassRef("scala.meta.Quoted") - def MetaQuotedClass(implicit ctx: Context) = MetaQuotedType.symbol.asClass + lazy val QuotedType = ctx.requiredClassRef("scala.quoted.Quoted") + def QuotedClass(implicit ctx: Context) = QuotedType.symbol.asClass - lazy val MetaExprType = ctx.requiredClassRef("scala.meta.Expr") - def MetaExprClass(implicit ctx: Context) = MetaExprType.symbol.asClass + lazy val QuotedExprType = ctx.requiredClassRef("scala.quoted.Expr") + def QuotedExprClass(implicit ctx: Context) = QuotedExprType.symbol.asClass - def MetaExpr_~(implicit ctx: Context) = MetaExprClass.requiredMethod(nme.UNARY_~) + def QuotedExpr_~(implicit ctx: Context) = QuotedExprClass.requiredMethod(nme.UNARY_~) - lazy val MetaTypeType = ctx.requiredClassRef("scala.meta.Type") - def MetaTypeClass(implicit ctx: Context) = MetaTypeType.symbol.asClass + lazy val QuotedTypeType = ctx.requiredClassRef("scala.quoted.Type") + def QuotedTypeClass(implicit ctx: Context) = QuotedTypeType.symbol.asClass - def MetaType_~(implicit ctx: Context) = - MetaTypeClass.info.member(tpnme.UNARY_~).symbol.asType + def QuotedType_~(implicit ctx: Context) = + QuotedTypeClass.info.member(tpnme.UNARY_~).symbol.asType - def Unpickler_unpickleExpr = ctx.requiredMethod("scala.meta.Unpickler.unpickleExpr") - def Unpickler_unpickleType = ctx.requiredMethod("scala.meta.Unpickler.unpickleType") + def Unpickler_unpickleExpr = ctx.requiredMethod("scala.quoted.Unpickler.unpickleExpr") + def Unpickler_unpickleType = ctx.requiredMethod("scala.quoted.Unpickler.unpickleType") lazy val EqType = ctx.requiredClassRef("scala.Eq") def EqClass(implicit ctx: Context) = EqType.symbol.asClass diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index b889ab368753..22c531cfc794 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -25,7 +25,7 @@ class ReifyQuotes extends MacroTransform { * * Select(qual, sym) * - * where `sym` is either `defn.MetaExpr_~` or `defn.MetaType_~`. For any splice, + * where `sym` is either `defn.QuotedExpr_~` or `defn.QuotedType_~`. For any splice, * the `qual` part should not be pickled, since it will be added separately later * as a splice. */ @@ -76,8 +76,8 @@ class ReifyQuotes extends MacroTransform { else check(sym, _.show) } - /** Turn `body` of quote into a call of `scala.meta.Unpickler.unpickleType` or - * `scala.meta.Unpickler.unpickleExpr` depending onwhether `isType` is true or not. + /** Turn `body` of quote into a call of `scala.quoted.Unpickler.unpickleType` or + * `scala.quoted.Unpickler.unpickleExpr` depending onwhether `isType` is true or not. * The arguments to the method are: * * - the serialized `body`, as returned from `pickleTree` @@ -88,7 +88,7 @@ class ReifyQuotes extends MacroTransform { .appliedToType(if (isType) body.tpe else body.tpe.widen) .appliedTo( Literal(Constant(pickleTree(body, isType))), - SeqLiteral(splicesAtLevel(currentLevel).toList, TypeTree(defn.MetaQuotedType))) + SeqLiteral(splicesAtLevel(currentLevel).toList, TypeTree(defn.QuotedType))) /** Perform operation `op` in quoted context */ private def inQuote(op: => Tree)(implicit ctx: Context) = { @@ -109,7 +109,7 @@ class ReifyQuotes extends MacroTransform { case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod => inQuote(reifyCall(transform(arg), isType = true)) case Select(body, name) - if tree.symbol == defn.MetaExpr_~ || tree.symbol == defn.MetaType_~ => + if tree.symbol == defn.QuotedExpr_~ || tree.symbol == defn.QuotedType_~ => currentLevel -= 1 val body1 = try transform(body) finally currentLevel += 1 if (currentLevel > 0) { diff --git a/compiler/src/dotty/tools/dotc/typer/Typer.scala b/compiler/src/dotty/tools/dotc/typer/Typer.scala index d4c37dd95c04..740d0de017c1 100644 --- a/compiler/src/dotty/tools/dotc/typer/Typer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Typer.scala @@ -1076,7 +1076,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit ctx.compilationUnit.containsQuotes = true val untpd.Quote(body) = tree val isType = body.isType - val resultClass = if (isType) defn.MetaTypeClass else defn.MetaExprClass + val resultClass = if (isType) defn.QuotedTypeClass else defn.QuotedExprClass val proto1 = pt.baseType(resultClass) match { case AppliedType(_, argType :: Nil) => argType case _ => WildcardType diff --git a/library/src/scala/meta/Expr.scala b/library/src/scala/quoted/Expr.scala similarity index 59% rename from library/src/scala/meta/Expr.scala rename to library/src/scala/quoted/Expr.scala index 983818c32164..8e6282b24e06 100644 --- a/library/src/scala/meta/Expr.scala +++ b/library/src/scala/quoted/Expr.scala @@ -1,5 +1,6 @@ -package scala.meta +package scala.quoted class Expr[T] extends Quoted { def unary_~ : T = ??? + def run: T = ??? } diff --git a/library/src/scala/meta/Quoted.scala b/library/src/scala/quoted/Quoted.scala similarity index 73% rename from library/src/scala/meta/Quoted.scala rename to library/src/scala/quoted/Quoted.scala index 3040370c3af9..b425c0bfb874 100644 --- a/library/src/scala/meta/Quoted.scala +++ b/library/src/scala/quoted/Quoted.scala @@ -1,4 +1,4 @@ -package scala.meta +package scala.quoted /** Common superclass of Expr and Type */ class Quoted diff --git a/library/src/scala/meta/Type.scala b/library/src/scala/quoted/Type.scala similarity index 70% rename from library/src/scala/meta/Type.scala rename to library/src/scala/quoted/Type.scala index ebb20c6b3db5..67c604399580 100644 --- a/library/src/scala/meta/Type.scala +++ b/library/src/scala/quoted/Type.scala @@ -1,5 +1,6 @@ -package scala.meta +package scala.quoted class Type[T] extends Quoted { type unary_~ = T } + diff --git a/library/src/scala/meta/Unpickler.scala b/library/src/scala/quoted/Unpickler.scala similarity index 96% rename from library/src/scala/meta/Unpickler.scala rename to library/src/scala/quoted/Unpickler.scala index d4344e96b392..76fa0efbbf31 100644 --- a/library/src/scala/meta/Unpickler.scala +++ b/library/src/scala/quoted/Unpickler.scala @@ -1,4 +1,4 @@ -package scala.meta +package scala.quoted /** Provides methods to unpickle `Expr` and `Type` trees. */ object Unpickler { diff --git a/tests/neg/quoteTest.scala b/tests/neg/quoteTest.scala new file mode 100644 index 000000000000..da56595bfed4 --- /dev/null +++ b/tests/neg/quoteTest.scala @@ -0,0 +1,16 @@ +import scala.quoted._ + +class Test { + + val x: Int = 0 + + '{ '(x + 1) // error: wrong staging level + + '((y: Expr[Int]) => ~y ) // error: wrong staging level + + } + + '(x + 1) // error: wrong staging level + + '((y: Expr[Int]) => ~y ) // error: wrong staging level +} diff --git a/tests/pos/quoteTest.scala b/tests/pos/quoteTest.scala index 4944897cf8c8..e969ab49a5f6 100644 --- a/tests/pos/quoteTest.scala +++ b/tests/pos/quoteTest.scala @@ -1,8 +1,8 @@ -import scala.meta._ +import scala.quoted._ object Test { - def f[T](t: Type[T], x: Expr[T]) = { + def f[T](t: Type[T], x: Expr[T]) = '{ val y: t.unary_~ = x.unary_~ val z: ~t = ~x } From d87db6cd6129c72fc147b0d064275bd6c8c816c4 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 6 Dec 2017 15:29:44 +0100 Subject: [PATCH 26/62] Suppress some staging level errors Don't issue errors in imports or inline methods. Also, improve the way objects are referred to in error messages. --- .../src/dotty/tools/dotc/transform/ReifyQuotes.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 22c531cfc794..1e1516e800a6 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -62,12 +62,16 @@ class ReifyQuotes extends MacroTransform { def checkLevel(tree: Tree)(implicit ctx: Context): Unit = { def check(sym: Symbol, show: Symbol => String): Unit = - if (levelOf.getOrElse(sym, currentLevel) != currentLevel) + if (!sym.isStaticOwner && + !ctx.owner.ownersIterator.exists(_.isInlineMethod) && + levelOf.getOrElse(sym, currentLevel) != currentLevel) ctx.error(em"""access to ${show(sym)} from wrong staging level: | - the definition is at level ${levelOf(sym)}, | - but the access is at level $currentLevel.""", tree.pos) - def showThis(sym: Symbol) = i"${sym.name}.this" + def showThis(sym: Symbol) = + if (sym.is(ModuleClass)) sym.sourceModule.show + else i"${sym.name}.this" val sym = tree.symbol if (sym.exists) @@ -136,6 +140,8 @@ class ReifyQuotes extends MacroTransform { levelOf -= enteredSyms.head enteredSyms = enteredSyms.tail } + case _: Import => + tree case _ => super.transform(tree) } From dedf6711c1f4e225d714af455508d3cf3d7456b4 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Fri, 8 Dec 2017 13:36:17 +0100 Subject: [PATCH 27/62] Remove printing inefficiencies Two major changes - Remove the recursive `show` when homogenizing Or-types. This one caused a combinatorial explosion when test-pickling Compiler.scala, because the phases list is an OrType of many element types. - Make `provided` by name in its text prefix. --- .../dotty/tools/dotc/printing/PlainPrinter.scala | 14 ++++++-------- .../dotty/tools/dotc/printing/RefinedPrinter.scala | 12 ++++++------ compiler/src/dotty/tools/dotc/printing/Texts.scala | 6 ++++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala b/compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala index 6cd933a48dc9..f3cfb622ba17 100644 --- a/compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala +++ b/compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala @@ -53,10 +53,7 @@ class PlainPrinter(_ctx: Context) extends Printer { case AndType(tp1, tp2) => homogenize(tp1) & homogenize(tp2) case OrType(tp1, tp2) => - if (tp1.show > tp2.show) - homogenize(tp1) | homogenize(tp2) - else - homogenize(tp2) | homogenize(tp1) + homogenize(tp1) | homogenize(tp2) case tp: SkolemType => homogenize(tp.info) case tp: LazyRef => @@ -199,20 +196,20 @@ class PlainPrinter(_ctx: Context) extends Printer { case tp: TypeLambda => changePrec(GlobalPrec) { "[" ~ paramsText(tp) ~ "]" ~ lambdaHash(tp) ~ - (" => " provided !tp.resultType.isInstanceOf[MethodType]) ~ + (Str(" => ") provided !tp.resultType.isInstanceOf[MethodType]) ~ toTextGlobal(tp.resultType) } case AnnotatedType(tpe, annot) => toTextLocal(tpe) ~ " " ~ toText(annot) case tp: TypeVar => if (tp.isInstantiated) - toTextLocal(tp.instanceOpt) ~ ("^" provided ctx.settings.YprintDebug.value) + toTextLocal(tp.instanceOpt) ~ (Str("^") provided ctx.settings.YprintDebug.value) else { val constr = ctx.typerState.constraint val bounds = if (constr.contains(tp)) constr.fullBounds(tp.origin)(ctx.addMode(Mode.Printing)) else TypeBounds.empty - if (bounds.isAlias) toText(bounds.lo) ~ ("^" provided ctx.settings.YprintDebug.value) + if (bounds.isAlias) toText(bounds.lo) ~ (Str("^") provided ctx.settings.YprintDebug.value) else if (ctx.settings.YshowVarBounds.value) "(" ~ toText(tp.origin) ~ "?" ~ toText(bounds) ~ ")" else toText(tp.origin) } @@ -498,7 +495,7 @@ class PlainPrinter(_ctx: Context) extends Printer { else Text() - nodeName ~ "(" ~ elems ~ tpSuffix ~ ")" ~ (node.pos.toString provided ctx.settings.YprintPos.value) + nodeName ~ "(" ~ elems ~ tpSuffix ~ ")" ~ (Str(node.pos.toString) provided ctx.settings.YprintPos.value) case _ => tree.fallbackToText(this) } @@ -544,6 +541,7 @@ class PlainPrinter(_ctx: Context) extends Printer { def plain = this protected def keywordStr(text: String): String = coloredStr(text, SyntaxHighlighting.KeywordColor) + protected def keywordText(text: String): Text = coloredStr(text, SyntaxHighlighting.KeywordColor) protected def valDefText(text: Text): Text = coloredText(text, SyntaxHighlighting.ValDefColor) protected def typeText(text: Text): Text = coloredText(text, SyntaxHighlighting.TypeColor) protected def literalText(text: Text): Text = coloredText(text, SyntaxHighlighting.LiteralColor) diff --git a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala index 2251870900c2..c1805a5f1f4d 100644 --- a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala +++ b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala @@ -125,11 +125,11 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { atPrec(InfixPrec) { argText(args.head) } else toTextTuple(args.init) - (keywordStr("implicit ") provided isImplicit) ~ argStr ~ " => " ~ argText(args.last) + (keywordText("implicit ") provided isImplicit) ~ argStr ~ " => " ~ argText(args.last) } def toTextDependentFunction(appType: MethodType): Text = { - ("implicit " provided appType.isImplicitMethod) ~ + (keywordText("implicit ") provided appType.isImplicitMethod) ~ "(" ~ paramsText(appType) ~ ") => " ~ toText(appType.resultType) } @@ -186,7 +186,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { withoutPos(super.toText(tp)) case tp: SelectionProto => return "?{ " ~ toText(tp.name) ~ - (" " provided !tp.name.toSimpleName.last.isLetterOrDigit) ~ + (Str(" ") provided !tp.name.toSimpleName.last.isLetterOrDigit) ~ ": " ~ toText(tp.memberProto) ~ " }" case tp: ViewProto => return toText(tp.argType) ~ " ?=>? " ~ toText(tp.resultType) @@ -272,7 +272,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { if (homogenizedView && mods.flags.isTypeFlags) flagMask &~= Implicit // drop implicit from classes val flags = mods.flags & flagMask val flagsText = if (flags.isEmpty) "" else keywordStr((mods.flags & flagMask).toString) - Text(mods.annotations.map(annotText), " ") ~~ flagsText ~~ (kw provided !suppressKw) + Text(mods.annotations.map(annotText), " ") ~~ flagsText ~~ (Str(kw) provided !suppressKw) } def varianceText(mods: untpd.Modifiers) = @@ -343,7 +343,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { val bodyText = "{" ~~ selfText ~~ toTextGlobal(primaryConstrs ::: body, "\n") ~ "}" - prefix ~ (keywordStr(" extends") provided !ofNew) ~~ parentsText ~~ bodyText + prefix ~ (keywordText(" extends") provided !ofNew) ~~ parentsText ~~ bodyText } def toTextPackageId(pid: Tree): Text = @@ -412,7 +412,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { blockText(stats :+ expr) case If(cond, thenp, elsep) => changePrec(GlobalPrec) { - keywordStr("if ") ~ toText(cond) ~ (keywordStr(" then") provided !cond.isInstanceOf[Parens]) ~~ toText(thenp) ~ optText(elsep)(keywordStr(" else ") ~ _) + keywordStr("if ") ~ toText(cond) ~ (keywordText(" then") provided !cond.isInstanceOf[Parens]) ~~ toText(thenp) ~ optText(elsep)(keywordStr(" else ") ~ _) } case Closure(env, ref, target) => "closure(" ~ (toTextGlobal(env, ", ") ~ " | " provided env.nonEmpty) ~ diff --git a/compiler/src/dotty/tools/dotc/printing/Texts.scala b/compiler/src/dotty/tools/dotc/printing/Texts.scala index 6745355c9ac6..41e91a5957a1 100644 --- a/compiler/src/dotty/tools/dotc/printing/Texts.scala +++ b/compiler/src/dotty/tools/dotc/printing/Texts.scala @@ -146,8 +146,6 @@ object Texts { def over (that: Text) = if (this.isVertical) Vertical(that :: this.relems) else Vertical(that :: this :: Nil) - - def provided(pred: Boolean) = if (pred) this else Str("") } object Text { @@ -169,6 +167,10 @@ object Texts { /** The given texts `xs`, each on a separate line */ def lines(xs: Traversable[Text]) = Vertical(xs.toList.reverse) + + implicit class textDeco(text: => Text) { + def provided(cond: Boolean): Text = if (cond) text else Str("") + } } case class Str(s: String, lineRange: LineRange = EmptyLineRange) extends Text { From bc6c690f75291c2640ad43d67db57152b5ea6267 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Fri, 8 Dec 2017 13:37:10 +0100 Subject: [PATCH 28/62] Pull inlined bindings under a splice --- .../src/dotty/tools/dotc/transform/ReifyQuotes.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 1e1516e800a6..216ca49747b0 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -21,6 +21,10 @@ class ReifyQuotes extends MacroTransform { protected def newTransformer(implicit ctx: Context): Transformer = new Reifier + /** is tree splice operation? */ + def isSplice(tree: Select)(implicit ctx: Context) = + tree.symbol == defn.QuotedExpr_~ || tree.symbol == defn.QuotedType_~ + /** Serialize `tree`. Embedded splices are represented as nodes of the form * * Select(qual, sym) @@ -112,8 +116,7 @@ class ReifyQuotes extends MacroTransform { inQuote(reifyCall(transform(arg), isType = false)) case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod => inQuote(reifyCall(transform(arg), isType = true)) - case Select(body, name) - if tree.symbol == defn.QuotedExpr_~ || tree.symbol == defn.QuotedType_~ => + case tree @ Select(body, name) if isSplice(tree) => currentLevel -= 1 val body1 = try transform(body) finally currentLevel += 1 if (currentLevel > 0) { @@ -140,6 +143,10 @@ class ReifyQuotes extends MacroTransform { levelOf -= enteredSyms.head enteredSyms = enteredSyms.tail } + case Inlined(call, bindings, expansion @ Select(body, name)) if isSplice(expansion) => + // To maintain phase consistency, convert inlined expressions of the form + // `{ bindings; ~expansion }` to `~{ bindings; expansion }` + cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name) case _: Import => tree case _ => From f2c09d404bfa9d97d17cf773f297371a8a6d8f7e Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Fri, 8 Dec 2017 13:39:30 +0100 Subject: [PATCH 29/62] Inline by-name parameters directly Do not bind them to an auxiliary method. This has the advantage that we pass the actual quoted arguments to a macro implementation instead of passing proxies. To avoid code duplication, inline methods using an argument more than once should create a local auxiliary method instead. This is done in `trace.scala` and `Stats.scala`. --- .../dotty/tools/dotc/reporting/trace.scala | 16 +++--- .../src/dotty/tools/dotc/typer/Inliner.scala | 54 +++++++++---------- .../src/dotty/tools/dotc/util/Stats.scala | 8 +-- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/reporting/trace.scala b/compiler/src/dotty/tools/dotc/reporting/trace.scala index 4d4c0578ffb9..d0184c4c9e67 100644 --- a/compiler/src/dotty/tools/dotc/reporting/trace.scala +++ b/compiler/src/dotty/tools/dotc/reporting/trace.scala @@ -14,14 +14,18 @@ object trace { conditionally(ctx.settings.YdebugTrace.value, question, false)(op) @inline - def conditionally[TC](cond: Boolean, question: => String, show: Boolean)(op: => TC)(implicit ctx: Context): TC = - if (Config.tracingEnabled && cond) apply[TC](question, Printers.default, show)(op) - else op + def conditionally[TC](cond: Boolean, question: => String, show: Boolean)(op: => TC)(implicit ctx: Context): TC = { + def op1 = op + if (Config.tracingEnabled && cond) apply[TC](question, Printers.default, show)(op1) + else op1 + } @inline - def apply[T](question: => String, printer: Printers.Printer, show: Boolean)(op: => T)(implicit ctx: Context): T = - if (!Config.tracingEnabled || printer.eq(config.Printers.noPrinter)) op - else doTrace[T](question, printer, show)(op) + def apply[T](question: => String, printer: Printers.Printer, show: Boolean)(op: => T)(implicit ctx: Context): T = { + def op1 = op + if (!Config.tracingEnabled || printer.eq(config.Printers.noPrinter)) op1 + else doTrace[T](question, printer, show)(op1) + } @inline def apply[T](question: => String, printer: Printers.Printer)(op: => T)(implicit ctx: Context): T = diff --git a/compiler/src/dotty/tools/dotc/typer/Inliner.scala b/compiler/src/dotty/tools/dotc/typer/Inliner.scala index 3240b3556e10..368b7383d794 100644 --- a/compiler/src/dotty/tools/dotc/typer/Inliner.scala +++ b/compiler/src/dotty/tools/dotc/typer/Inliner.scala @@ -496,32 +496,34 @@ class Inliner(call: tpd.Tree, rhs: tpd.Tree)(implicit ctx: Context) { val expansion1 = InlineTyper.typed(expansion, pt)(inlineCtx) /** Does given definition bind a closure that will be inlined? */ - def bindsDeadClosure(defn: ValOrDefDef) = Ident(defn.symbol.termRef) match { - case InlineableClosure(_) => !InlineTyper.retainedClosures.contains(defn.symbol) + def bindsDeadInlineable(defn: ValOrDefDef) = Ident(defn.symbol.termRef) match { + case InlineableArg(_) => !InlineTyper.retainedInlineables.contains(defn.symbol) case _ => false } /** All bindings in `bindingsBuf` except bindings of inlineable closures */ - val bindings = bindingsBuf.toList.filterNot(bindsDeadClosure).map(_.withPos(call.pos)) + val bindings = bindingsBuf.toList.filterNot(bindsDeadInlineable).map(_.withPos(call.pos)) tpd.Inlined(call, bindings, expansion1) } } - /** An extractor for references to closure arguments that refer to `@inline` methods */ - private object InlineableClosure { + /** An extractor for references to inlineable arguments. These are : + * - by-value arguments marked with `inline` + * - all by-name arguments + */ + private object InlineableArg { lazy val paramProxies = paramProxy.values.toSet def unapply(tree: Ident)(implicit ctx: Context): Option[Tree] = - if (paramProxies.contains(tree.tpe)) { + if (paramProxies.contains(tree.tpe)) bindingsBuf.find(_.name == tree.name) match { - case Some(ddef: ValDef) if ddef.symbol.is(Inline) => - ddef.rhs match { - case closure(_, meth, _) => Some(meth) - case _ => None - } + case Some(vdef: ValDef) if vdef.symbol.is(Inline) => + Some(vdef.rhs.changeOwner(vdef.symbol, ctx.owner)) + case Some(ddef: DefDef) => + Some(ddef.rhs.changeOwner(ddef.symbol, ctx.owner)) case _ => None } - } else None + else None } /** A typer for inlined code. Its purpose is: @@ -533,16 +535,13 @@ class Inliner(call: tpd.Tree, rhs: tpd.Tree)(implicit ctx: Context) { */ private object InlineTyper extends ReTyper { - var retainedClosures = Set[Symbol]() + var retainedInlineables = Set[Symbol]() - override def typedIdent(tree: untpd.Ident, pt: Type)(implicit ctx: Context) = { - val tree1 = super.typedIdent(tree, pt) - tree1 match { - case InlineableClosure(_) => retainedClosures += tree.symbol - case _ => + override def typedIdent(tree: untpd.Ident, pt: Type)(implicit ctx: Context) = + tree.asInstanceOf[tpd.Tree] match { + case InlineableArg(rhs) => inlining.println(i"inline arg $tree -> $rhs"); rhs + case _ => super.typedIdent(tree, pt) } - tree1 - } override def typedSelect(tree: untpd.Select, pt: Type)(implicit ctx: Context): Tree = { assert(tree.hasType, tree) @@ -565,12 +564,13 @@ class Inliner(call: tpd.Tree, rhs: tpd.Tree)(implicit ctx: Context) { } } - override def typedApply(tree: untpd.Apply, pt: Type)(implicit ctx: Context) = tree.asInstanceOf[tpd.Tree] match { - case Apply(Select(InlineableClosure(fn), nme.apply), args) => - inlining.println(i"reducing $tree with closure $fn") - typed(fn.appliedToArgs(args), pt) - case _ => - super.typedApply(tree, pt) - } + override def typedApply(tree: untpd.Apply, pt: Type)(implicit ctx: Context) = + tree.asInstanceOf[tpd.Tree] match { + case Apply(Select(InlineableArg(closure(_, fn, _)), nme.apply), args) => + inlining.println(i"reducing $tree with closure $fn") + typed(fn.appliedToArgs(args), pt) + case _ => + super.typedApply(tree, pt) + } } } diff --git a/compiler/src/dotty/tools/dotc/util/Stats.scala b/compiler/src/dotty/tools/dotc/util/Stats.scala index c69796f84896..55684a8850bc 100644 --- a/compiler/src/dotty/tools/dotc/util/Stats.scala +++ b/compiler/src/dotty/tools/dotc/util/Stats.scala @@ -46,11 +46,13 @@ import collection.mutable def trackTime[T](fn: String)(op: => T) = if (enabled) doTrackTime(fn)(op) else op - def doTrackTime[T](fn: String)(op: => T) = + def doTrackTime[T](fn: String)(op: => T) = { + def op1 = op if (monitored) { val start = System.nanoTime - try op finally record(fn, ((System.nanoTime - start) / 1000).toInt) - } else op + try op1 finally record(fn, ((System.nanoTime - start) / 1000).toInt) + } else op1 + } class HeartBeat extends Thread() { @volatile private[Stats] var continue = true From db7e42ac0ca6eccebb67d45657f0d9eec67a2400 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Fri, 8 Dec 2017 13:39:40 +0100 Subject: [PATCH 30/62] Add test case --- tests/pos/quoted.scala | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/pos/quoted.scala diff --git a/tests/pos/quoted.scala b/tests/pos/quoted.scala new file mode 100644 index 000000000000..a9874180b803 --- /dev/null +++ b/tests/pos/quoted.scala @@ -0,0 +1,25 @@ +import scala.quoted._ + +class Test { + + object Macros { + + inline def assert(expr: => Boolean): Unit = + ~ assertImpl('(expr)) + + def assertImpl(expr: Expr[Boolean]) = + '{ if !(~expr) then throw new AssertionError(s"failed assertion: ${~expr}") } + + } + + val program = '{ + import Macros._ + + val x = 1 + assert(x != 0) + + ~assertImpl('(x != 0)) + } + + program.run +} From a9c3b5f455345cae546dc6568b7c5950e9988d09 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 11 Dec 2017 14:12:33 +0100 Subject: [PATCH 31/62] Fix layout of implicit error messages There were extraneous whitespaces and prepositions. --- compiler/src/dotty/tools/dotc/typer/Implicits.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/typer/Implicits.scala b/compiler/src/dotty/tools/dotc/typer/Implicits.scala index 302d19924d13..19a074a126a0 100644 --- a/compiler/src/dotty/tools/dotc/typer/Implicits.scala +++ b/compiler/src/dotty/tools/dotc/typer/Implicits.scala @@ -672,13 +672,14 @@ trait Implicits { self: Typer => | | ${arg.show.replace("\n", "\n ")} | - |But $tpe.explanation}.""" + |But ${tpe.explanation}.""" } } + def location(preposition: String) = if (where.isEmpty) "" else s" $preposition $where" arg.tpe match { case ambi: AmbiguousImplicits => - msg(s"ambiguous implicit arguments: ${ambi.explanation} of $where")( - s"ambiguous implicit arguments of type ${pt.show} found for $where") + msg(s"ambiguous implicit arguments: ${ambi.explanation}${location("of")}")( + s"ambiguous implicit arguments of type ${pt.show} found${location("for")}") case _ => val userDefined = for { @@ -691,7 +692,7 @@ trait Implicits { self: Typer => pt.typeSymbol.typeParams.map(_.name.unexpandedName.toString), pt.argInfos) } - msg(userDefined.getOrElse(em"no implicit argument of type $pt was found for $where"))() + msg(userDefined.getOrElse(em"no implicit argument of type $pt was found${location("for")}"))() } } From 90acfedaf85c8110ef45a46c8ea340298cd4d927 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 11 Dec 2017 14:17:35 +0100 Subject: [PATCH 32/62] Polishings to TreePickler --- compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala index 37e82f8fdf37..43131954a6bf 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala @@ -329,10 +329,10 @@ class TreePickler(pickler: TastyPickler) { case tp: TermRef if name != nme.WILDCARD => // wildcards are pattern bound, need to be preserved as ids. pickleType(tp) - case _ => + case tp => writeByte(if (tree.isType) IDENTtpt else IDENT) pickleName(name) - pickleType(tree.tpe) + pickleType(tp) } case This(qual) => if (qual.isEmpty) pickleType(tree.tpe) @@ -483,7 +483,7 @@ class TreePickler(pickler: TastyPickler) { else { if (!tree.self.isEmpty) registerTreeAddr(tree.self) pickleType { - cinfo.selfInfo match { + selfInfo match { case sym: Symbol => sym.info case tp: Type => tp } From 205b4cd80af69e29162ec61f8e6c456886618b6d Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 11 Dec 2017 14:18:24 +0100 Subject: [PATCH 33/62] Add MacroTransform that allows implicit search Also, drop redundant clause in existing MacroTransform --- .../src/dotty/tools/dotc/core/Contexts.scala | 9 ++ .../src/dotty/tools/dotc/core/Phases.scala | 4 + .../tools/dotc/transform/MacroTransform.scala | 1 - .../MacroTransformWithImplicits.scala | 90 +++++++++++++++++++ .../src/dotty/tools/dotc/typer/FrontEnd.scala | 2 + .../dotty/tools/dotc/typer/Implicits.scala | 2 +- .../src/dotty/tools/dotc/typer/Namer.scala | 11 +-- .../src/dotty/tools/dotc/typer/Typer.scala | 2 +- 8 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/transform/MacroTransformWithImplicits.scala diff --git a/compiler/src/dotty/tools/dotc/core/Contexts.scala b/compiler/src/dotty/tools/dotc/core/Contexts.scala index fe5f5b6d4c62..629794efb74a 100644 --- a/compiler/src/dotty/tools/dotc/core/Contexts.scala +++ b/compiler/src/dotty/tools/dotc/core/Contexts.scala @@ -359,6 +359,15 @@ object Contexts { else if (untpd.isSuperConstrCall(stat) && this.owner.isClass) superCallContext else ctx.fresh.setOwner(exprOwner) + /** A new context that summarizes an import statement */ + def importContext(imp: Import[_], sym: Symbol) = { + val impNameOpt = imp.expr match { + case ref: RefTree[_] => Some(ref.name.asTermName) + case _ => None + } + ctx.fresh.setImportInfo(new ImportInfo(implicit ctx => sym, imp.selectors, impNameOpt)) + } + /** The current source file; will be derived from current * compilation unit. */ diff --git a/compiler/src/dotty/tools/dotc/core/Phases.scala b/compiler/src/dotty/tools/dotc/core/Phases.scala index 61732763addc..0abc540448a0 100644 --- a/compiler/src/dotty/tools/dotc/core/Phases.scala +++ b/compiler/src/dotty/tools/dotc/core/Phases.scala @@ -276,6 +276,10 @@ object Phases { * and type applications. */ def relaxedTyping: Boolean = false + + /** If set, implicit search is enabled */ + def allowsImplicitSearch: Boolean = false + /** List of names of phases that should precede this phase */ def runsAfter: Set[Class[_ <: Phase]] = Set.empty diff --git a/compiler/src/dotty/tools/dotc/transform/MacroTransform.scala b/compiler/src/dotty/tools/dotc/transform/MacroTransform.scala index 5ef7b4d9d28c..9d84a2adb66c 100644 --- a/compiler/src/dotty/tools/dotc/transform/MacroTransform.scala +++ b/compiler/src/dotty/tools/dotc/transform/MacroTransform.scala @@ -41,7 +41,6 @@ abstract class MacroTransform extends Phase { def transformStats(trees: List[Tree], exprOwner: Symbol)(implicit ctx: Context): List[Tree] = { def transformStat(stat: Tree): Tree = stat match { case _: Import | _: DefTree => transform(stat) - case Thicket(stats) => cpy.Thicket(stat)(stats mapConserve transformStat) case _ => transform(stat)(ctx.exprContext(stat, exprOwner)) } flatten(trees.mapconserve(transformStat(_))) diff --git a/compiler/src/dotty/tools/dotc/transform/MacroTransformWithImplicits.scala b/compiler/src/dotty/tools/dotc/transform/MacroTransformWithImplicits.scala new file mode 100644 index 000000000000..e6c8e255a265 --- /dev/null +++ b/compiler/src/dotty/tools/dotc/transform/MacroTransformWithImplicits.scala @@ -0,0 +1,90 @@ +package dotty.tools.dotc +package transform + +import core._ +import typer._ +import ast.Trees._ +import Contexts._ +import Symbols._ +import Decorators._ +import collection.mutable +import annotation.tailrec + +/** A Macrotransform that maintains the necessary infrastructore to support + * contxtual implicit searches (type-scope implicits are supported anyway). + */ +abstract class MacroTransformWithImplicits extends MacroTransform { + import ast.tpd._ + + override def allowsImplicitSearch = true + + class ImplicitsTransformer extends Transformer { + + /** Transform statements, while maintaining import contexts and expression contexts + * in the same way as Typer does. The code addresses additional concerns: + * - be tail-recursive where possible + * - don't re-allocate trees where nothing has changed + */ + override def transformStats(stats: List[Tree], exprOwner: Symbol)(implicit ctx: Context): List[Tree] = { + + @tailrec def traverse(curStats: List[Tree])(implicit ctx: Context): List[Tree] = { + + def recur(stats: List[Tree], changed: Tree, rest: List[Tree])(implicit ctx: Context): List[Tree] = { + if (stats eq curStats) { + val rest1 = transformStats(rest, exprOwner) + changed match { + case Thicket(trees) => trees ::: rest1 + case tree => tree :: rest1 + } + } + else stats.head :: recur(stats.tail, changed, rest) + } + + curStats match { + case stat :: rest => + val statCtx = stat match { + case stat: DefTree => ctx + case _ => ctx.exprContext(stat, exprOwner) + } + val restCtx = stat match { + case stat: Import => ctx.importContext(stat, stat.symbol) + case _ => ctx + } + val stat1 = transform(stat)(statCtx) + if (stat1 ne stat) recur(stats, stat1, rest)(restCtx) + else traverse(rest)(restCtx) + case nil => + stats + } + } + traverse(stats) + } + + private def nestedScopeCtx(defs: List[Tree])(implicit ctx: Context): Context = { + val nestedCtx = ctx.fresh.setNewScope + defs foreach { + case d: DefTree => nestedCtx.enter(d.symbol) + case _ => + } + nestedCtx + } + + override def transform(tree: Tree)(implicit ctx: Context): Tree = { + def localCtx = ctx.withOwner(tree.symbol) + tree match { + case tree: Block => + super.transform(tree)(nestedScopeCtx(tree.stats)) + case tree: DefDef => + implicit val ctx = localCtx + cpy.DefDef(tree)( + tree.name, + transformSub(tree.tparams), + tree.vparamss mapConserve (transformSub(_)), + transform(tree.tpt), + transform(tree.rhs)(nestedScopeCtx(tree.vparamss.flatten))) + case _ => + super.transform(tree) + } + } + } +} \ No newline at end of file diff --git a/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala b/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala index 6cb045853ad1..19eb297ff652 100644 --- a/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala +++ b/compiler/src/dotty/tools/dotc/typer/FrontEnd.scala @@ -19,6 +19,8 @@ class FrontEnd extends Phase { override def isTyper = true import ast.tpd + override def allowsImplicitSearch = true + /** The contexts for compilation units that are parsed but not yet entered */ private[this] var remaining: List[Context] = Nil diff --git a/compiler/src/dotty/tools/dotc/typer/Implicits.scala b/compiler/src/dotty/tools/dotc/typer/Implicits.scala index 19a074a126a0..d35852a2b2e8 100644 --- a/compiler/src/dotty/tools/dotc/typer/Implicits.scala +++ b/compiler/src/dotty/tools/dotc/typer/Implicits.scala @@ -749,7 +749,7 @@ trait Implicits { self: Typer => * !!! todo: catch potential cycles */ def inferImplicit(pt: Type, argument: Tree, pos: Position)(implicit ctx: Context): SearchResult = track("inferImplicit") { - assert(!ctx.isAfterTyper, + assert(ctx.phase.allowsImplicitSearch, if (argument.isEmpty) i"missing implicit parameter of type $pt after typer" else i"type error: ${argument.tpe} does not conform to $pt${err.whyNoMatchStr(argument.tpe, pt)}") trace(s"search implicit ${pt.show}, arg = ${argument.show}: ${argument.tpe.show}", implicits, show = true) { diff --git a/compiler/src/dotty/tools/dotc/typer/Namer.scala b/compiler/src/dotty/tools/dotc/typer/Namer.scala index 09ffc9f0d0e4..0e7cecb0f360 100644 --- a/compiler/src/dotty/tools/dotc/typer/Namer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Namer.scala @@ -385,15 +385,6 @@ class Namer { typer: Typer => case _ => tree } - /** A new context that summarizes an import statement */ - def importContext(imp: Import, sym: Symbol)(implicit ctx: Context) = { - val impNameOpt = imp.expr match { - case ref: RefTree => Some(ref.name.asTermName) - case _ => None - } - ctx.fresh.setImportInfo(new ImportInfo(implicit ctx => sym, imp.selectors, impNameOpt)) - } - /** A new context for the interior of a class */ def inClassContext(selfInfo: DotClass /* Should be Type | Symbol*/)(implicit ctx: Context): Context = { val localCtx: Context = ctx.fresh.setNewScope @@ -441,7 +432,7 @@ class Namer { typer: Typer => setDocstring(pkg, stat) ctx case imp: Import => - importContext(imp, createSymbol(imp)) + ctx.importContext(imp, createSymbol(imp)) case mdef: DefTree => val sym = enterSymbol(createSymbol(mdef)) setDocstring(sym, origStat) diff --git a/compiler/src/dotty/tools/dotc/typer/Typer.scala b/compiler/src/dotty/tools/dotc/typer/Typer.scala index 740d0de017c1..25d70a898e47 100644 --- a/compiler/src/dotty/tools/dotc/typer/Typer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Typer.scala @@ -1775,7 +1775,7 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit case (imp: untpd.Import) :: rest => val imp1 = typed(imp) buf += imp1 - traverse(rest)(importContext(imp, imp1.symbol)) + traverse(rest)(ctx.importContext(imp, imp1.symbol)) case (mdef: untpd.DefTree) :: rest => mdef.removeAttachment(ExpandedTree) match { case Some(xtree) => From b466e0afa6ea1908fcef8dc0fe2b30fa2887949a Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Mon, 11 Dec 2017 14:58:17 +0100 Subject: [PATCH 34/62] Allow cross-stage persistence for types ... provided they are backed by type tags, i.e. implicit values of type `quoted.Type`. --- .../tools/dotc/transform/ReifyQuotes.scala | 280 +++++++++++++----- tests/neg/quoteTest.scala | 5 + tests/pos/quoteTest.scala | 19 +- 3 files changed, 220 insertions(+), 84 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 216ca49747b0..99f538ed76f3 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -5,13 +5,18 @@ import core._ import Decorators._, Flags._, Types._, Contexts._, Symbols._, Constants._ import Flags._ import ast.Trees._ +import util.Positions._ +import StdNames._ +import ast.untpd import MegaPhase.MiniPhase +import typer.Implicits._ +import NameKinds.OuterSelectName import scala.collection.mutable /** Translates quoted terms and types to `unpickle` method calls. * Checks that the phase consistency principle (PCP) holds. */ -class ReifyQuotes extends MacroTransform { +class ReifyQuotes extends MacroTransformWithImplicits { import ast.tpd._ override def phaseName: String = "reifyQuotes" @@ -21,9 +26,9 @@ class ReifyQuotes extends MacroTransform { protected def newTransformer(implicit ctx: Context): Transformer = new Reifier - /** is tree splice operation? */ - def isSplice(tree: Select)(implicit ctx: Context) = - tree.symbol == defn.QuotedExpr_~ || tree.symbol == defn.QuotedType_~ + /** Is symbol a splice operation? */ + def isSplice(sym: Symbol)(implicit ctx: Context) = + sym == defn.QuotedExpr_~ || sym == defn.QuotedType_~ /** Serialize `tree`. Embedded splices are represented as nodes of the form * @@ -36,13 +41,57 @@ class ReifyQuotes extends MacroTransform { def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String = tree.show // TODO: replace with TASTY - private class Reifier extends Transformer { + private class Reifier extends ImplicitsTransformer { + + /** A class for collecting the splices of some quoted expression */ + private class Splices { + + /** A listbuffer collecting splices */ + val buf = new mutable.ListBuffer[Tree] + + /** A map from type ref T to "expression of type quoted.Type[T]". + * These will be turned into splices using `addTags` + */ + val typeTagOfRef = new mutable.LinkedHashMap[TypeRef, Tree]() + + /** Assuming typeTagOfRef = `Type1 -> tag1, ..., TypeN -> tagN`, the expression + * + * { type = .unary_~ + * ... + * type = .unary.~ + * + * } + * + * where all references to `TypeI` in `expr` are rewired to point to the locally + * defined versions. As a side effect, append the expressions `tag1, ..., `tagN` + * as splices to `buf`. + */ + def addTags(expr: Tree)(implicit ctx: Context): Tree = + if (typeTagOfRef.isEmpty) expr + else { + val assocs = typeTagOfRef.toList + val typeDefs = for ((tp, tag) <- assocs) yield { + val original = tp.symbol.asType + val rhs = tag.select(tpnme.UNARY_~) + val alias = ctx.typeAssigner.assignType(untpd.TypeBoundsTree(rhs, rhs), rhs, rhs) + val local = original.copy( + owner = ctx.owner, + flags = Synthetic, + info = TypeAlias(tag.tpe.select(tpnme.UNARY_~))) + ctx.typeAssigner.assignType(untpd.TypeDef(original.name, alias), local) + } + val (trefs, tags) = assocs.unzip + tags ++=: buf + typeTagOfRef.clear() + Block(typeDefs, expr).subst(trefs.map(_.symbol), typeDefs.map(_.symbol)) + } + } /** The current staging level */ private var currentLevel = 0 /** The splices encountered so far, indexed by staging level */ - private val splicesAtLevel = mutable.ArrayBuffer(new mutable.ListBuffer[Tree]) + private val splicesAtLevel = mutable.ArrayBuffer(new Splices) // Invariant: -1 <= currentLevel <= splicesAtLevel.length @@ -54,34 +103,109 @@ class ReifyQuotes extends MacroTransform { /** Enter staging level of symbol defined by `tree`, if applicable. */ def markDef(tree: Tree)(implicit ctx: Context) = tree match { - case tree: MemberDef if !levelOf.contains(tree.symbol) => + case tree: DefTree if !levelOf.contains(tree.symbol) => levelOf(tree.symbol) = currentLevel enteredSyms = tree.symbol :: enteredSyms case _ => } - /** If reference is to a locally defined symbol, check that its staging level - * matches the current level. + /** If `tree` refers to a locally defined symbol (either directly, or in a pickled type), + * check that its staging level matches the current level. References to types + * that are phase-incorrect can still be healed as follows. + * + * If `T` is a reference to a type at the wrong level, and there is an implicit value `tag` + * of type `quoted.Type[T]`, transform `tag` yielding `tag1` and add the binding `T -> tag1` + * to the `typeTagOfRef` map of the current `Splices` structure. These entries will be turned + * info additional type definitions in method `addTags`. */ - def checkLevel(tree: Tree)(implicit ctx: Context): Unit = { - - def check(sym: Symbol, show: Symbol => String): Unit = - if (!sym.isStaticOwner && - !ctx.owner.ownersIterator.exists(_.isInlineMethod) && - levelOf.getOrElse(sym, currentLevel) != currentLevel) - ctx.error(em"""access to ${show(sym)} from wrong staging level: - | - the definition is at level ${levelOf(sym)}, - | - but the access is at level $currentLevel.""", tree.pos) - - def showThis(sym: Symbol) = - if (sym.is(ModuleClass)) sym.sourceModule.show - else i"${sym.name}.this" - - val sym = tree.symbol - if (sym.exists) - if (tree.isInstanceOf[This]) check(sym, showThis) - else if (sym.owner.isType) check(sym.owner, showThis) - else check(sym, _.show) + private def checkLevel(tree: Tree)(implicit ctx: Context): Tree = { + + /** Try to heal phase-inconsistent reference to type `T` using a local type definition. + * @return None if successful + * @return Some(msg) if unsuccessful where `msg` is a potentially empty error message + * to be added to the "inconsistent phase" message. + */ + def heal(tp: Type): Option[String] = tp match { + case tp: TypeRef => + val reqType = defn.QuotedTypeType.appliedTo(tp) + val tag = ctx.typer.inferImplicitArg(reqType, tree.pos) + tag.tpe match { + case fail: SearchFailureType => + Some(i""" + | + | The access would be accepted with the right type tag, but + | ${ctx.typer.missingArgMsg(tag, reqType, "")}""") + case _ => + splicesAtLevel(currentLevel).typeTagOfRef(tp) = { + currentLevel -= 1 + try transform(tag) finally currentLevel += 1 + } + None + } + case _ => + Some("") + } + + /** Check reference to `sym` for phase consistency, where `tp` is the underlying type + * by which we refer to `sym`. + */ + def check(sym: Symbol, tp: Type): Unit = { + val isThis = tp.isInstanceOf[ThisType] + def symStr = + if (!isThis) sym.show + else if (sym.is(ModuleClass)) sym.sourceModule.show + else i"${sym.name}.this" + if (!isThis && sym.maybeOwner.isType) + check(sym.owner, sym.owner.thisType) + else if (sym.exists && !sym.isStaticOwner && + !ctx.owner.ownersIterator.exists(_.isInlineMethod) && + levelOf.getOrElse(sym, currentLevel) != currentLevel) + heal(tp) match { + case Some(errMsg) => + ctx.error(em"""access to $symStr from wrong staging level: + | - the definition is at level ${levelOf(sym)}, + | - but the access is at level $currentLevel.$errMsg""", tree.pos) + case None => + } + } + + /** Check all named types and this types in a given type for phase consistency */ + object checkType extends TypeAccumulator[Unit] { + /** Check that all NamedType and ThisType parts of `tp` are level-correct. + * If they are not, try to heal with a local binding to a typetag splice + */ + def apply(tp: Type): Unit = apply((), tp) + def apply(acc: Unit, tp: Type): Unit = reporting.trace(i"check type level $tp at $currentLevel") { + tp match { + case tp: NamedType if isSplice(tp.symbol) => + currentLevel -= 1 + try foldOver(acc, tp) finally currentLevel += 1 + case tp: NamedType => + check(tp.symbol, tp) + foldOver(acc, tp) + case tp: ThisType => + check(tp.cls, tp) + foldOver(acc, tp) + case _ => + foldOver(acc, tp) + } + } + } + + tree match { + case (_: Ident) | (_: This) => + check(tree.symbol, tree.tpe) + case (_: UnApply) | (_: TypeTree) => + checkType(tree.tpe) + case Select(qual, OuterSelectName(_, levels)) => + checkType(tree.tpe.widen) + case _: Bind => + checkType(tree.symbol.info) + case _: Template => + checkType(tree.symbol.owner.asClass.givenSelfType) + case _ => + } + tree } /** Turn `body` of quote into a call of `scala.quoted.Unpickler.unpickleType` or @@ -91,66 +215,64 @@ class ReifyQuotes extends MacroTransform { * - the serialized `body`, as returned from `pickleTree` * - all splices found in `body` */ - private def reifyCall(body: Tree, isType: Boolean)(implicit ctx: Context) = - ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr) - .appliedToType(if (isType) body.tpe else body.tpe.widen) - .appliedTo( - Literal(Constant(pickleTree(body, isType))), - SeqLiteral(splicesAtLevel(currentLevel).toList, TypeTree(defn.QuotedType))) - - /** Perform operation `op` in quoted context */ - private def inQuote(op: => Tree)(implicit ctx: Context) = { + private def reify(body: Tree, isType: Boolean)(implicit ctx: Context) = { currentLevel += 1 if (currentLevel == splicesAtLevel.length) splicesAtLevel += null + val splices = new Splices val savedSplices = splicesAtLevel(currentLevel) - splicesAtLevel(currentLevel) = new mutable.ListBuffer[Tree] - try op + splicesAtLevel(currentLevel) = splices + try { + val body1 = splices.addTags(transform(body)) + ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr) + .appliedToType(if (isType) body1.tpe else body1.tpe.widen) + .appliedTo( + Literal(Constant(pickleTree(body1, isType))), + SeqLiteral(splices.buf.toList, TypeTree(defn.QuotedType))) + } finally { splicesAtLevel(currentLevel) = savedSplices currentLevel -= 1 } } - override def transform(tree: Tree)(implicit ctx: Context): Tree = tree match { - case Apply(fn, arg :: Nil) if fn.symbol == defn.quoteMethod => - inQuote(reifyCall(transform(arg), isType = false)) - case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod => - inQuote(reifyCall(transform(arg), isType = true)) - case tree @ Select(body, name) if isSplice(tree) => - currentLevel -= 1 - val body1 = try transform(body) finally currentLevel += 1 - if (currentLevel > 0) { - splicesAtLevel(currentLevel) += body1 - tree - } - else { - if (currentLevel < 0) - ctx.error(i"splice ~ not allowed under toplevel splice", tree.pos) - cpy.Select(tree)(body1, name) + override def transform(tree: Tree)(implicit ctx: Context): Tree = + reporting.trace(i"reify $tree at $currentLevel", show = true) { + tree match { + case Apply(fn, arg :: Nil) if fn.symbol == defn.quoteMethod => + reify(arg, isType = false) + case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod => + reify(arg, isType = true) + case tree @ Select(body, name) if isSplice(tree.symbol) => + currentLevel -= 1 + val body1 = try transform(body) finally currentLevel += 1 + if (currentLevel > 0) { + splicesAtLevel(currentLevel).buf += body1 + tree + } + else { + if (currentLevel < 0) + ctx.error(i"splice ~ not allowed under toplevel splice", tree.pos) + cpy.Select(tree)(body1, name) + } + case Block(stats, _) => + val last = enteredSyms + stats.foreach(markDef) + try super.transform(tree) + finally + while (enteredSyms ne last) { + levelOf -= enteredSyms.head + enteredSyms = enteredSyms.tail + } + case Inlined(call, bindings, expansion @ Select(body, name)) if isSplice(expansion.symbol) => + // To maintain phase consistency, convert inlined expressions of the form + // `{ bindings; ~expansion }` to `~{ bindings; expansion }` + cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name) + case _: Import => + tree + case _ => + markDef(tree) + checkLevel(super.transform(tree)) } - case (_: Ident) | (_: This) => - checkLevel(tree) - super.transform(tree) - case _: MemberDef => - markDef(tree) - super.transform(tree) - case Block(stats, _) => - val last = enteredSyms - stats.foreach(markDef) - try super.transform(tree) - finally - while (enteredSyms ne last) { - levelOf -= enteredSyms.head - enteredSyms = enteredSyms.tail - } - case Inlined(call, bindings, expansion @ Select(body, name)) if isSplice(expansion) => - // To maintain phase consistency, convert inlined expressions of the form - // `{ bindings; ~expansion }` to `~{ bindings; expansion }` - cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name) - case _: Import => - tree - case _ => - super.transform(tree) - } + } } } \ No newline at end of file diff --git a/tests/neg/quoteTest.scala b/tests/neg/quoteTest.scala index da56595bfed4..19611a62b172 100644 --- a/tests/neg/quoteTest.scala +++ b/tests/neg/quoteTest.scala @@ -13,4 +13,9 @@ class Test { '(x + 1) // error: wrong staging level '((y: Expr[Int]) => ~y ) // error: wrong staging level + + def f[T](t: Type[T], x: Expr[T]) = '{ + val z2 = ~x // error: wrong staging level for type T + } + } diff --git a/tests/pos/quoteTest.scala b/tests/pos/quoteTest.scala index e969ab49a5f6..020a8d342185 100644 --- a/tests/pos/quoteTest.scala +++ b/tests/pos/quoteTest.scala @@ -2,14 +2,23 @@ import scala.quoted._ object Test { - def f[T](t: Type[T], x: Expr[T]) = '{ + def f[T](x: Expr[T])(t0: Type[T]) = { + implicit val t: Type[T] = t0 + '{ + val y: t.unary_~ = x.unary_~ + val z = ~x + } + } + + def f2[T](x: Expr[T])(implicit t: Type[T]) = '{ val y: t.unary_~ = x.unary_~ - val z: ~t = ~x + val z = ~x } - f('[Int], '(2)) - f('[Boolean], '{ true }) + f('(2))('[Int]) + f('{ true })('[Boolean]) def g(es: Expr[String], t: Type[String]) = - f('[List[~t]], '{ (~es + "!") :: Nil }) + f('{ (~es + "!") :: Nil })('[List[~t]]) } + From 22d74294972a2029479f233c96854ad46bdf2de5 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 13 Dec 2017 17:17:13 +0100 Subject: [PATCH 35/62] Add Quotable type class Quotable is a way to lift expressions from T to Expr[T] without having to go through quotes. This makes writing staged interpreters more pleasant. --- library/src/scala/quoted/Expr.scala | 5 +++++ library/src/scala/quoted/Quotable.scala | 20 ++++++++++++++++++ library/src/scala/quoted/Type.scala | 5 +++++ tests/pos/quotable.scala | 28 +++++++++++++++++++++++++ tests/pos/stagedInterpreter.scala | 22 +++++++++++++++++++ 5 files changed, 80 insertions(+) create mode 100644 library/src/scala/quoted/Quotable.scala create mode 100644 tests/pos/quotable.scala create mode 100644 tests/pos/stagedInterpreter.scala diff --git a/library/src/scala/quoted/Expr.scala b/library/src/scala/quoted/Expr.scala index 8e6282b24e06..5cbdcbde4122 100644 --- a/library/src/scala/quoted/Expr.scala +++ b/library/src/scala/quoted/Expr.scala @@ -4,3 +4,8 @@ class Expr[T] extends Quoted { def unary_~ : T = ??? def run: T = ??? } + +object Expr { + implicit def toExpr[T](x: T)(implicit ev: Quotable[T]): Expr[T] = + ev.toExpr(x) +} diff --git a/library/src/scala/quoted/Quotable.scala b/library/src/scala/quoted/Quotable.scala new file mode 100644 index 000000000000..46a84aba2b33 --- /dev/null +++ b/library/src/scala/quoted/Quotable.scala @@ -0,0 +1,20 @@ +package scala.quoted +import scala.math + +/** A typeclass for types that can be turned to `quoted.Expr[T]` + * without going through an explicit `'(...)` operation. + */ +abstract class Quotable[T] { + implicit def toExpr(x: T): Expr[T] +} + +/** Some base quotable types. To be completed with at least all types + * that are valid Scala literals. The actual implementation of these + * typed could be in terms of `ast.tpd.Literal`; the test `quotable.scala` + * gives an alternative implementation using just the basic staging system. + */ +object Quotable { + + implicit def IntIsQuotable: Quotable[Int] = ??? + implicit def BooleanIsQuotable: Quotable[Boolean] = ??? +} diff --git a/library/src/scala/quoted/Type.scala b/library/src/scala/quoted/Type.scala index 67c604399580..ad22a6da90e0 100644 --- a/library/src/scala/quoted/Type.scala +++ b/library/src/scala/quoted/Type.scala @@ -4,3 +4,8 @@ class Type[T] extends Quoted { type unary_~ = T } +/** Some basic type tags, currently incomplete */ +object Type { + implicit def IntTag: Type[Int] = new Type[Int] + implicit def BooleanTag: Type[Boolean] = new Type[Boolean] +} diff --git a/tests/pos/quotable.scala b/tests/pos/quotable.scala new file mode 100644 index 000000000000..f49eabf95c2d --- /dev/null +++ b/tests/pos/quotable.scala @@ -0,0 +1,28 @@ +import scala.quoted._ + +object Test { + + implicit def IntIsQuotable: Quotable[Int] = new { + def toExpr(n: Int): Expr[Int] = n match { + case Int.MinValue => '(Int.MinValue) + case _ if n < 0 => '(-(~toExpr(n))) + case 0 => '(0) + case _ if n % 2 == 0 => '( ~toExpr(n / 2) * 2) + case _ => '( ~toExpr(n / 2) * 2 + 1) + } + } + + implicit def BooleanIsQuotable: Quotable[Boolean] = new { + implicit def toExpr(b: Boolean) = + if (b) '(true) else '(false) + } + + implicit def ListIsQuotable[T: Type: Quotable]: Quotable[List[T]] = new { + def toExpr(xs: List[T]): Expr[List[T]] = xs match { + case x :: xs1 => '{ ~implicitly[Quotable[T]].toExpr(x) :: ~toExpr(xs1) } + case Nil => '(Nil: List[T]) + } + } + + val xs: Expr[List[Int]] = 1 :: 2 :: 3 :: Nil +} diff --git a/tests/pos/stagedInterpreter.scala b/tests/pos/stagedInterpreter.scala new file mode 100644 index 000000000000..70012fc861f6 --- /dev/null +++ b/tests/pos/stagedInterpreter.scala @@ -0,0 +1,22 @@ +import scala.quoted._ + +enum Exp { + case Num(n: Int) + case Plus(e1: Exp, e2: Exp) + case Var(x: String) +} + +object Test { + import Exp._ + + val exp = Plus(Plus(Num(2), Var("x")), Num(4)) + + def compile(e: Exp, env: Map[String, Expr[Int]]): Expr[Int] = e match { + case Num(n) => n + case Plus(e1, e2) => '(~compile(e1, env) + ~compile(e2, env)) + case Var(x) => env(x) + } + + val res = (x: Int) => ~compile(exp, Map("x" -> '(x))) + +} From dbb7c8100bbfb1b72e41fc70bed772f611fa0b90 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 13 Dec 2017 17:31:25 +0100 Subject: [PATCH 36/62] Add Let to staged interpreter test --- tests/pos/stagedInterpreter.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/pos/stagedInterpreter.scala b/tests/pos/stagedInterpreter.scala index 70012fc861f6..db7f55eb78fb 100644 --- a/tests/pos/stagedInterpreter.scala +++ b/tests/pos/stagedInterpreter.scala @@ -4,19 +4,30 @@ enum Exp { case Num(n: Int) case Plus(e1: Exp, e2: Exp) case Var(x: String) + case Let(x: String, e: Exp, in: Exp) } object Test { import Exp._ + val keepLets = true + val exp = Plus(Plus(Num(2), Var("x")), Num(4)) + val letExp = Let("x", Num(3), exp) + def compile(e: Exp, env: Map[String, Expr[Int]]): Expr[Int] = e match { case Num(n) => n case Plus(e1, e2) => '(~compile(e1, env) + ~compile(e2, env)) case Var(x) => env(x) + case Let(x, e, body) => + if (keepLets) + '{ val y = ~compile(e, env); ~compile(body, env + (x -> '(y))) } + else + compile(body, env + (x -> compile(e, env))) } - val res = (x: Int) => ~compile(exp, Map("x" -> '(x))) + val res1 = (x: Int) => ~compile(exp, Map("x" -> '(x))) + val res2 = compile(letExp, Map()) } From 300c37ecad09be019f18d08d0c72e860ff9141b4 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Thu, 14 Dec 2017 10:33:20 +0100 Subject: [PATCH 37/62] Don't require implicits to make type refs persist stages It turns out that we can always make type references cross-stage persistent because we can create a `quoted.Type[T]` from any type `T`. So implicit search is not needed, after all. This means we don't need anymore the infrastructure of allowing implicit searches in macro transforms. But we don't remove it because it might become useful later. --- .../tools/dotc/transform/ReifyQuotes.scala | 59 +++++++------------ tests/neg/quoteTest.scala | 2 +- tests/pos/quoteTest.scala | 10 +--- tests/pos/stagedInterpreter.scala | 4 +- 4 files changed, 26 insertions(+), 49 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 99f538ed76f3..dcc75ebeb138 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -9,14 +9,13 @@ import util.Positions._ import StdNames._ import ast.untpd import MegaPhase.MiniPhase -import typer.Implicits._ import NameKinds.OuterSelectName import scala.collection.mutable /** Translates quoted terms and types to `unpickle` method calls. * Checks that the phase consistency principle (PCP) holds. */ -class ReifyQuotes extends MacroTransformWithImplicits { +class ReifyQuotes extends MacroTransform { import ast.tpd._ override def phaseName: String = "reifyQuotes" @@ -41,7 +40,7 @@ class ReifyQuotes extends MacroTransformWithImplicits { def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String = tree.show // TODO: replace with TASTY - private class Reifier extends ImplicitsTransformer { + private class Reifier extends Transformer { /** A class for collecting the splices of some quoted expression */ private class Splices { @@ -83,7 +82,7 @@ class ReifyQuotes extends MacroTransformWithImplicits { val (trefs, tags) = assocs.unzip tags ++=: buf typeTagOfRef.clear() - Block(typeDefs, expr).subst(trefs.map(_.symbol), typeDefs.map(_.symbol)) + Block(typeDefs, expr.subst(trefs.map(_.symbol), typeDefs.map(_.symbol))) } } @@ -113,39 +112,17 @@ class ReifyQuotes extends MacroTransformWithImplicits { * check that its staging level matches the current level. References to types * that are phase-incorrect can still be healed as follows. * - * If `T` is a reference to a type at the wrong level, and there is an implicit value `tag` - * of type `quoted.Type[T]`, transform `tag` yielding `tag1` and add the binding `T -> tag1` - * to the `typeTagOfRef` map of the current `Splices` structure. These entries will be turned - * info additional type definitions in method `addTags`. + * If `T` is a reference to a type at the wrong level, heal it by setting things up + * so that we later add a type definition + * + * type T' = ~quoted.Type[T] + * + * to the quoted text and rename T to T' in it. This is done later in `reify` via + * `Splice#addTags`. checkLevel itself only records what needs to be done in the + * `typeTagOfRef` field of the current `Splice` structure. */ private def checkLevel(tree: Tree)(implicit ctx: Context): Tree = { - /** Try to heal phase-inconsistent reference to type `T` using a local type definition. - * @return None if successful - * @return Some(msg) if unsuccessful where `msg` is a potentially empty error message - * to be added to the "inconsistent phase" message. - */ - def heal(tp: Type): Option[String] = tp match { - case tp: TypeRef => - val reqType = defn.QuotedTypeType.appliedTo(tp) - val tag = ctx.typer.inferImplicitArg(reqType, tree.pos) - tag.tpe match { - case fail: SearchFailureType => - Some(i""" - | - | The access would be accepted with the right type tag, but - | ${ctx.typer.missingArgMsg(tag, reqType, "")}""") - case _ => - splicesAtLevel(currentLevel).typeTagOfRef(tp) = { - currentLevel -= 1 - try transform(tag) finally currentLevel += 1 - } - None - } - case _ => - Some("") - } - /** Check reference to `sym` for phase consistency, where `tp` is the underlying type * by which we refer to `sym`. */ @@ -160,12 +137,18 @@ class ReifyQuotes extends MacroTransformWithImplicits { else if (sym.exists && !sym.isStaticOwner && !ctx.owner.ownersIterator.exists(_.isInlineMethod) && levelOf.getOrElse(sym, currentLevel) != currentLevel) - heal(tp) match { - case Some(errMsg) => + tp match { + case tp: TypeRef => + // Legalize reference to phase-inconstent type + splicesAtLevel(currentLevel).typeTagOfRef(tp) = { + currentLevel -= 1 + val tag = New(defn.QuotedTypeType.appliedTo(tp), Nil) + try transform(tag) finally currentLevel += 1 + } + case _ => ctx.error(em"""access to $symStr from wrong staging level: | - the definition is at level ${levelOf(sym)}, - | - but the access is at level $currentLevel.$errMsg""", tree.pos) - case None => + | - but the access is at level $currentLevel.""", tree.pos) } } diff --git a/tests/neg/quoteTest.scala b/tests/neg/quoteTest.scala index 19611a62b172..37e2fe02c141 100644 --- a/tests/neg/quoteTest.scala +++ b/tests/neg/quoteTest.scala @@ -15,7 +15,7 @@ class Test { '((y: Expr[Int]) => ~y ) // error: wrong staging level def f[T](t: Type[T], x: Expr[T]) = '{ - val z2 = ~x // error: wrong staging level for type T + val z2 = ~x // OK } } diff --git a/tests/pos/quoteTest.scala b/tests/pos/quoteTest.scala index 020a8d342185..e3671a2ea3e3 100644 --- a/tests/pos/quoteTest.scala +++ b/tests/pos/quoteTest.scala @@ -2,15 +2,7 @@ import scala.quoted._ object Test { - def f[T](x: Expr[T])(t0: Type[T]) = { - implicit val t: Type[T] = t0 - '{ - val y: t.unary_~ = x.unary_~ - val z = ~x - } - } - - def f2[T](x: Expr[T])(implicit t: Type[T]) = '{ + def f[T](x: Expr[T])(t: Type[T]) = '{ val y: t.unary_~ = x.unary_~ val z = ~x } diff --git a/tests/pos/stagedInterpreter.scala b/tests/pos/stagedInterpreter.scala index db7f55eb78fb..2ba57be2b004 100644 --- a/tests/pos/stagedInterpreter.scala +++ b/tests/pos/stagedInterpreter.scala @@ -27,7 +27,9 @@ object Test { compile(body, env + (x -> compile(e, env))) } - val res1 = (x: Int) => ~compile(exp, Map("x" -> '(x))) + val res1 = '{ (x: Int) => ~compile(exp, Map("x" -> '(x))) } val res2 = compile(letExp, Map()) + + res1.run } From 646453ac625ac56d2cb7df3dace90dd482eea82b Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Thu, 14 Dec 2017 14:51:39 +0100 Subject: [PATCH 38/62] Add infrastructure for pickling/unpickling with holes and splices --- .../tools/dotc/core/tasty/DottyUnpickler.scala | 2 +- .../src/dotty/tools/dotc/core/tasty/Splicing.scala | 14 ++++++++++++++ .../dotty/tools/dotc/core/tasty/TastyFormat.scala | 6 +++++- .../dotty/tools/dotc/core/tasty/TreePickler.scala | 10 ++++++++++ .../tools/dotc/core/tasty/TreeUnpickler.scala | 10 +++++++++- 5 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/core/tasty/Splicing.scala diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala index 8178019aa033..a4847e644765 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala @@ -20,7 +20,7 @@ object DottyUnpickler { class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler]) extends SectionUnpickler[TreeUnpickler]("ASTs") { def unpickle(reader: TastyReader, nameAtRef: NameTable) = - new TreeUnpickler(reader, nameAtRef, posUnpickler) + new TreeUnpickler(reader, nameAtRef, posUnpickler, Seq.empty) } class PositionsSectionUnpickler extends SectionUnpickler[PositionUnpickler]("Positions") { diff --git a/compiler/src/dotty/tools/dotc/core/tasty/Splicing.scala b/compiler/src/dotty/tools/dotc/core/tasty/Splicing.scala new file mode 100644 index 000000000000..57a5be686f9e --- /dev/null +++ b/compiler/src/dotty/tools/dotc/core/tasty/Splicing.scala @@ -0,0 +1,14 @@ +package dotty.tools +package dotc +package core +package tasty + +import ast.Trees._ + +object Splicing { + import ast.tpd._ + + type Splice = AnyRef /* tpd.Tree | tpd.Tree => tpd.Tree */ + + case class Hole(args: List[Tree]) extends TermTree +} \ No newline at end of file diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala index 82824495513d..12e053e128f5 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala @@ -109,6 +109,7 @@ Standard-Section: "ASTs" TopLevelStat* BYNAMEtpt underlying_Term EMPTYTREE SHARED term_ASTRef + HOLE Length idx_Nat arg_Tree* Application = APPLY Length fn_Term arg_Term* TYPEAPPLY Length fn_Term arg_Type* @@ -396,6 +397,7 @@ object TastyFormat { final val ANNOTATION = 173 final val TERMREFin = 174 final val TYPEREFin = 175 + final val HOLE = 255 final val firstSimpleTreeTag = UNITconst final val firstNatTreeTag = SHARED @@ -555,6 +557,7 @@ object TastyFormat { case SUPERtype => "SUPERtype" case TERMREFin => "TERMREFin" case TYPEREFin => "TYPEREFin" + case REFINEDtype => "REFINEDtype" case REFINEDtpt => "REFINEDtpt" case APPLIEDtype => "APPLIEDtype" @@ -576,6 +579,7 @@ object TastyFormat { case ANNOTATION => "ANNOTATION" case PRIVATEqualified => "PRIVATEqualified" case PROTECTEDqualified => "PROTECTEDqualified" + case HOLE => "HOLE" } /** @return If non-negative, the number of leading references (represented as nats) of a length/trees entry. @@ -583,7 +587,7 @@ object TastyFormat { */ def numRefs(tag: Int) = tag match { case VALDEF | DEFDEF | TYPEDEF | TYPEPARAM | PARAM | NAMEDARG | RETURN | BIND | - SELFDEF | REFINEDtype | TERMREFin | TYPEREFin => 1 + SELFDEF | REFINEDtype | TERMREFin | TYPEREFin | HOLE => 1 case RENAMED | PARAMtype => 2 case POLYtype | METHODtype | TYPELAMBDAtype => -1 case _ => 0 diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala index 43131954a6bf..ebd8f41ad1ca 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala @@ -14,6 +14,7 @@ import StdNames.nme import TastyBuffer._ import TypeApplications._ import transform.SymUtils._ +import Splicing.Hole import config.Config class TreePickler(pickler: TastyPickler) { @@ -27,6 +28,8 @@ class TreePickler(pickler: TastyPickler) { private val forwardSymRefs = Symbols.newMutableSymbolMap[List[Addr]] private val pickledTypes = new java.util.IdentityHashMap[Type, Any] // Value type is really Addr, but that's not compatible with null + private var holeCount = 0 + private def withLength(op: => Unit) = { val lengthAddr = reserveRef(relative = true) op @@ -542,6 +545,13 @@ class TreePickler(pickler: TastyPickler) { case TypeBoundsTree(lo, hi) => writeByte(TYPEBOUNDStpt) withLength { pickleTree(lo); pickleTree(hi) } + case Hole(args) => + writeByte(HOLE) + withLength { + writeNat(holeCount) + holeCount += 1 + args.foreach(pickleTree) + } } catch { case ex: AssertionError => diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index 8052deda72e5..cc681f0401f7 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -11,6 +11,7 @@ import util.Positions._ import ast.{tpd, Trees, untpd} import Trees._ import Decorators._ +import Splicing.Splice import transform.SymUtils._ import TastyUnpickler._, TastyBuffer._ import scala.annotation.{tailrec, switch} @@ -25,7 +26,10 @@ import config.Config * @param tastyName the nametable * @param posUNpicklerOpt the unpickler for positions, if it exists */ -class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpicklerOpt: Option[PositionUnpickler]) { +class TreeUnpickler(reader: TastyReader, + nameAtRef: NameRef => TermName, + posUnpicklerOpt: Option[PositionUnpickler], + splices: Seq[Splice]) { import TastyFormat._ import TreeUnpickler._ import tpd._ @@ -1031,6 +1035,10 @@ class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpi LambdaTypeTree(tparams, body) case TYPEBOUNDStpt => TypeBoundsTree(readTpt(), readTpt()) + case HOLE => + val idx = readNat() + val args = until(end)(readTerm()) + (splices(idx) /: args)(_.asInstanceOf[Tree => Tree](_)).asInstanceOf[Tree] case _ => readPathTerm() } From 376d3254a1e586b7eb3116b7ed95354ab8ccbfa1 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 16 Dec 2017 12:37:04 +0100 Subject: [PATCH 39/62] Print TypeDefs from trees As is already the case for the other member defs, use the right hand side instead of the symbol's type. --- .../src/dotty/tools/dotc/printing/RefinedPrinter.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala index c1805a5f1f4d..57587541d036 100644 --- a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala +++ b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala @@ -458,7 +458,8 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { case ByNameTypeTree(tpt) => "=> " ~ toTextLocal(tpt) case TypeBoundsTree(lo, hi) => - optText(lo)(" >: " ~ _) ~ optText(hi)(" <: " ~ _) + if (lo eq hi) optText(lo)(" = " ~ _) + else optText(lo)(" >: " ~ _) ~ optText(hi)(" <: " ~ _) case Bind(name, body) => changePrec(InfixPrec) { toText(name) ~ " @ " ~ toText(body) } case Alternative(trees) => @@ -489,10 +490,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { def typeDefText(tparamsText: => Text, rhsText: => Text) = dclTextOr { modText(tree.mods, keywordStr("type")) ~~ (varianceText(tree.mods) ~ typeText(nameIdText(tree))) ~ - withEnclosingDef(tree) { - if (tree.hasType) toText(tree.symbol.info) // TODO: always print RHS, once we pickle/unpickle type trees - else tparamsText ~ rhsText - } + withEnclosingDef(tree) { tparamsText ~ rhsText } } def recur(rhs: Tree, tparamsTxt: => Text): Text = rhs match { case impl: Template => @@ -503,6 +501,8 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) { typeDefText(tparamsTxt, toText(rhs)) case LambdaTypeTree(tparams, body) => recur(body, tparamsText(tparams)) + case rhs: TypeTree if rhs.tpe.isInstanceOf[TypeBounds] => + typeDefText(tparamsTxt, toText(rhs)) case rhs => typeDefText(tparamsTxt, optText(rhs)(" = " ~ _)) } From a1bff5f99d4c4c870ea328174addc16a8f5e9d65 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 16 Dec 2017 12:39:30 +0100 Subject: [PATCH 40/62] Make TreeTypeMap non-final We'll need to create a subclass in order to support substitutions with holes. To enable this, add a method `newMap` that creates a tree map of the same kind as the current one. --- .../src/dotty/tools/dotc/ast/TreeTypeMap.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala b/compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala index 68bd65696e64..a0b580118157 100644 --- a/compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala +++ b/compiler/src/dotty/tools/dotc/ast/TreeTypeMap.scala @@ -31,7 +31,7 @@ import dotty.tools.dotc.transform.SymUtils._ * gets two different denotations in the same period. Hence, if -Yno-double-bindings is * set, we would get a data race assertion error. */ -final class TreeTypeMap( +class TreeTypeMap( val typeMap: Type => Type = IdentityTypeMap, val treeMap: tpd.Tree => tpd.Tree = identity _, val oldOwners: List[Symbol] = Nil, @@ -154,7 +154,7 @@ final class TreeTypeMap( assert(!to.exists(substFrom contains _)) assert(!from.exists(newOwners contains _)) assert(!to.exists(oldOwners contains _)) - new TreeTypeMap( + newMap( typeMap, treeMap, from ++ oldOwners, @@ -163,6 +163,16 @@ final class TreeTypeMap( to ++ substTo) } + /** A new map of the same class this one */ + protected def newMap( + typeMap: Type => Type, + treeMap: Tree => Tree, + oldOwners: List[Symbol], + newOwners: List[Symbol], + substFrom: List[Symbol], + substTo: List[Symbol])(implicit ctx: Context) = + new TreeTypeMap(typeMap, treeMap, oldOwners, newOwners, substFrom, substTo) + /** Apply `typeMap` and `ownerMap` to given symbols `syms` * and return a treemap that contains the substitution * between original and mapped symbols. From 6a252e01cd18f183d6aca3addb975e0840ab7aa2 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 16 Dec 2017 12:40:08 +0100 Subject: [PATCH 41/62] Move Unpickler to runtime.quoted It's internal only, no need to put in a public package. --- compiler/src/dotty/tools/dotc/core/Definitions.scala | 4 ++-- library/src/scala/{ => runtime}/quoted/Unpickler.scala | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) rename library/src/scala/{ => runtime}/quoted/Unpickler.scala (70%) diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 3f926aebd54c..88299624cf5d 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -605,8 +605,8 @@ class Definitions { def QuotedType_~(implicit ctx: Context) = QuotedTypeClass.info.member(tpnme.UNARY_~).symbol.asType - def Unpickler_unpickleExpr = ctx.requiredMethod("scala.quoted.Unpickler.unpickleExpr") - def Unpickler_unpickleType = ctx.requiredMethod("scala.quoted.Unpickler.unpickleType") + def Unpickler_unpickleExpr = ctx.requiredMethod("scala.runtime.quoted.Unpickler.unpickleExpr") + def Unpickler_unpickleType = ctx.requiredMethod("scala.runtime.quoted.Unpickler.unpickleType") lazy val EqType = ctx.requiredClassRef("scala.Eq") def EqClass(implicit ctx: Context) = EqType.symbol.asClass diff --git a/library/src/scala/quoted/Unpickler.scala b/library/src/scala/runtime/quoted/Unpickler.scala similarity index 70% rename from library/src/scala/quoted/Unpickler.scala rename to library/src/scala/runtime/quoted/Unpickler.scala index 76fa0efbbf31..2073f7516f0c 100644 --- a/library/src/scala/quoted/Unpickler.scala +++ b/library/src/scala/runtime/quoted/Unpickler.scala @@ -1,4 +1,6 @@ -package scala.quoted +package scala.runtime.quoted + +import scala.quoted._ /** Provides methods to unpickle `Expr` and `Type` trees. */ object Unpickler { @@ -11,10 +13,10 @@ object Unpickler { /** Unpickle `repr` which represents a pickled `Expr` tree, * replacing splice nodes with `args` */ - def unpickleExpr[T](repr: Pickled, args: Seq[Quoted]): Expr[T] = ??? + def unpickleExpr[T](repr: Pickled, args: Seq[Any]): Expr[T] = ??? /** Unpickle `repr` which represents a pickled `Type` tree, * replacing splice nodes with `args` */ - def unpickleType[T](repr: Pickled, args: Seq[Quoted]): Type[T] = ??? + def unpickleType[T](repr: Pickled, args: Seq[Any]): Type[T] = ??? } From bc6dde8a00b1a12c579fbc4aaf70dfd60560dcc8 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 16 Dec 2017 12:41:26 +0100 Subject: [PATCH 42/62] Export more printing methods from Printer The added methods are useful to have when using Printers. --- .../tools/dotc/printing/PlainPrinter.scala | 18 --------------- .../dotty/tools/dotc/printing/Printer.scala | 22 ++++++++++++++++++- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala b/compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala index f3cfb622ba17..ea0f0ef0b3a0 100644 --- a/compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala +++ b/compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala @@ -76,24 +76,6 @@ class PlainPrinter(_ctx: Context) extends Printer { private def selfRecName(n: Int) = s"z$n" - /** Render elements alternating with `sep` string */ - protected def toText(elems: Traversable[Showable], sep: String) = - Text(elems map (_ toText this), sep) - - /** Render element within highest precedence */ - protected def toTextLocal(elem: Showable): Text = - atPrec(DotPrec) { elem.toText(this) } - - /** Render element within lowest precedence */ - protected def toTextGlobal(elem: Showable): Text = - atPrec(GlobalPrec) { elem.toText(this) } - - protected def toTextLocal(elems: Traversable[Showable], sep: String) = - atPrec(DotPrec) { toText(elems, sep) } - - protected def toTextGlobal(elems: Traversable[Showable], sep: String) = - atPrec(GlobalPrec) { toText(elems, sep) } - /** If the name of the symbol's owner should be used when you care about * seeing an interesting name: in such cases this symbol is e.g. a method * parameter with a synthetic name, a constructor named "this", an object diff --git a/compiler/src/dotty/tools/dotc/printing/Printer.scala b/compiler/src/dotty/tools/dotc/printing/Printer.scala index e0794627425a..f054a3dcb357 100644 --- a/compiler/src/dotty/tools/dotc/printing/Printer.scala +++ b/compiler/src/dotty/tools/dotc/printing/Printer.scala @@ -103,7 +103,27 @@ abstract class Printer { /** Textual representation of info relating to an import clause */ def toText(result: ImportInfo): Text - /** Perform string or text-producing operation `op` so that only a + /** Render element within highest precedence */ + def toTextLocal(elem: Showable): Text = + atPrec(DotPrec) { elem.toText(this) } + + /** Render element within lowest precedence */ + def toTextGlobal(elem: Showable): Text = + atPrec(GlobalPrec) { elem.toText(this) } + + /** Render elements alternating with `sep` string */ + def toText(elems: Traversable[Showable], sep: String) = + Text(elems map (_ toText this), sep) + + /** Render elements within highest precedence */ + def toTextLocal(elems: Traversable[Showable], sep: String) = + atPrec(DotPrec) { toText(elems, sep) } + + /** Render elements within lowest precedence */ + def toTextGlobal(elems: Traversable[Showable], sep: String) = + atPrec(GlobalPrec) { toText(elems, sep) } + + /** Perform string or text-producing operation `op` so that only a * summarized text with given recursion depth is shown */ def summarized[T](depth: Int)(op: => T): T From d8dd7392f082d879c1a5aef714fa25326828bbaa Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 16 Dec 2017 12:43:11 +0100 Subject: [PATCH 43/62] Add Lambda method to tpd A useful helper method to build a closure without having to define a symbol. --- compiler/src/dotty/tools/dotc/ast/tpd.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/ast/tpd.scala b/compiler/src/dotty/tools/dotc/ast/tpd.scala index b27c16d0eb00..eecac47fe704 100644 --- a/compiler/src/dotty/tools/dotc/ast/tpd.scala +++ b/compiler/src/dotty/tools/dotc/ast/tpd.scala @@ -104,6 +104,12 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo { Closure(Nil, call, targetTpt)) } + /** A closure whole anonymous function has the given method type */ + def Lambda(tpe: MethodType, rhsFn: List[Tree] => Tree)(implicit ctx: Context): Block = { + val meth = ctx.newSymbol(ctx.owner, nme.ANON_FUN, Synthetic | Method, tpe) + Closure(meth, tss => rhsFn(tss.head).changeOwner(ctx.owner, meth)) + } + def CaseDef(pat: Tree, guard: Tree, body: Tree)(implicit ctx: Context): CaseDef = ta.assignType(untpd.CaseDef(pat, guard, body), body) From 1d52ab7da17051eb6029562b1a51cc8587945616 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 16 Dec 2017 12:45:11 +0100 Subject: [PATCH 44/62] ReifyQuotes v2 A rewrite to handle embedded holes correctly --- .../dotty/tools/dotc/core/Definitions.scala | 4 +- .../src/dotty/tools/dotc/core/StdNames.scala | 1 + .../tools/dotc/core/tasty/Splicing.scala | 14 - .../tools/dotc/core/tasty/TreePickler.scala | 23 +- .../tools/dotc/core/tasty/TreeUnpickler.scala | 9 +- .../tools/dotc/transform/ReifyQuotes.scala | 410 +++++++++++------- 6 files changed, 271 insertions(+), 190 deletions(-) delete mode 100644 compiler/src/dotty/tools/dotc/core/tasty/Splicing.scala diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 88299624cf5d..5058e3f3cf7a 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -591,13 +591,11 @@ class Definitions { def ClassTagClass(implicit ctx: Context) = ClassTagType.symbol.asClass def ClassTagModule(implicit ctx: Context) = ClassTagClass.companionModule - lazy val QuotedType = ctx.requiredClassRef("scala.quoted.Quoted") - def QuotedClass(implicit ctx: Context) = QuotedType.symbol.asClass - lazy val QuotedExprType = ctx.requiredClassRef("scala.quoted.Expr") def QuotedExprClass(implicit ctx: Context) = QuotedExprType.symbol.asClass def QuotedExpr_~(implicit ctx: Context) = QuotedExprClass.requiredMethod(nme.UNARY_~) + def QuotedExpr_run(implicit ctx: Context) = QuotedExprClass.requiredMethod(nme.run) lazy val QuotedTypeType = ctx.requiredClassRef("scala.quoted.Type") def QuotedTypeClass(implicit ctx: Context) = QuotedTypeType.symbol.asClass diff --git a/compiler/src/dotty/tools/dotc/core/StdNames.scala b/compiler/src/dotty/tools/dotc/core/StdNames.scala index 10b0d0d896af..21ba10f18553 100644 --- a/compiler/src/dotty/tools/dotc/core/StdNames.scala +++ b/compiler/src/dotty/tools/dotc/core/StdNames.scala @@ -485,6 +485,7 @@ object StdNames { val reflect : N = "reflect" val reify : N = "reify" val rootMirror : N = "rootMirror" + val run: N = "run" val runOrElse: N = "runOrElse" val runtime: N = "runtime" val runtimeClass: N = "runtimeClass" diff --git a/compiler/src/dotty/tools/dotc/core/tasty/Splicing.scala b/compiler/src/dotty/tools/dotc/core/tasty/Splicing.scala deleted file mode 100644 index 57a5be686f9e..000000000000 --- a/compiler/src/dotty/tools/dotc/core/tasty/Splicing.scala +++ /dev/null @@ -1,14 +0,0 @@ -package dotty.tools -package dotc -package core -package tasty - -import ast.Trees._ - -object Splicing { - import ast.tpd._ - - type Splice = AnyRef /* tpd.Tree | tpd.Tree => tpd.Tree */ - - case class Hole(args: List[Tree]) extends TermTree -} \ No newline at end of file diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala index ebd8f41ad1ca..36aad0b67655 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala @@ -4,7 +4,7 @@ package core package tasty import ast.Trees._ -import ast.untpd +import ast.{untpd, tpd} import TastyFormat._ import Contexts._, Symbols._, Types._, Names._, Constants._, Decorators._, Annotations._, StdNames.tpnme, NameOps._ import collection.mutable @@ -14,22 +14,30 @@ import StdNames.nme import TastyBuffer._ import TypeApplications._ import transform.SymUtils._ -import Splicing.Hole +import printing.Printer +import printing.Texts._ import config.Config +object TreePickler { + + case class Hole(idx: Int, args: List[tpd.Tree]) extends tpd.TermTree { + override def fallbackToText(printer: Printer): Text = + s"[[$idx|" ~~ printer.toTextGlobal(args, ", ") ~~ "]]" + } +} + class TreePickler(pickler: TastyPickler) { val buf = new TreeBuffer pickler.newSection("ASTs", buf) + import TreePickler._ import buf._ import pickler.nameBuffer.nameIndex - import ast.tpd._ + import tpd._ private val symRefs = Symbols.newMutableSymbolMap[Addr] private val forwardSymRefs = Symbols.newMutableSymbolMap[List[Addr]] private val pickledTypes = new java.util.IdentityHashMap[Type, Any] // Value type is really Addr, but that's not compatible with null - private var holeCount = 0 - private def withLength(op: => Unit) = { val lengthAddr = reserveRef(relative = true) op @@ -545,11 +553,10 @@ class TreePickler(pickler: TastyPickler) { case TypeBoundsTree(lo, hi) => writeByte(TYPEBOUNDStpt) withLength { pickleTree(lo); pickleTree(hi) } - case Hole(args) => + case Hole(idx, args) => writeByte(HOLE) withLength { - writeNat(holeCount) - holeCount += 1 + writeNat(idx) args.foreach(pickleTree) } } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index cc681f0401f7..d8e54b635f8c 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -8,10 +8,9 @@ import StdNames._, Denotations._, Flags._, Constants._, Annotations._ import NameKinds._ import typer.Checking.checkNonCyclic import util.Positions._ -import ast.{tpd, Trees, untpd} +import ast.{tpd, untpd, Trees} import Trees._ import Decorators._ -import Splicing.Splice import transform.SymUtils._ import TastyUnpickler._, TastyBuffer._ import scala.annotation.{tailrec, switch} @@ -29,7 +28,7 @@ import config.Config class TreeUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName, posUnpicklerOpt: Option[PositionUnpickler], - splices: Seq[Splice]) { + splices: Seq[Any]) { import TastyFormat._ import TreeUnpickler._ import tpd._ @@ -1038,7 +1037,9 @@ class TreeUnpickler(reader: TastyReader, case HOLE => val idx = readNat() val args = until(end)(readTerm()) - (splices(idx) /: args)(_.asInstanceOf[Tree => Tree](_)).asInstanceOf[Tree] + val splice = splices(idx) + if (args.isEmpty) splice.asInstanceOf[Tree] + else splice.asInstanceOf[Seq[Any] => Tree](args) case _ => readPathTerm() } diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index dcc75ebeb138..ba3457046d2e 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -5,13 +5,21 @@ import core._ import Decorators._, Flags._, Types._, Contexts._, Symbols._, Constants._ import Flags._ import ast.Trees._ +import ast.TreeTypeMap import util.Positions._ import StdNames._ import ast.untpd +import tasty.TreePickler.Hole import MegaPhase.MiniPhase import NameKinds.OuterSelectName import scala.collection.mutable +// TODO +// implement hole substitution directly +// check that inline methods override nothing +// drop inline methods +// adapt to Expr/Type when passing arguments to splices + /** Translates quoted terms and types to `unpickle` method calls. * Checks that the phase consistency principle (PCP) holds. */ @@ -23,11 +31,8 @@ class ReifyQuotes extends MacroTransform { override def run(implicit ctx: Context): Unit = if (ctx.compilationUnit.containsQuotes) super.run - protected def newTransformer(implicit ctx: Context): Transformer = new Reifier - - /** Is symbol a splice operation? */ - def isSplice(sym: Symbol)(implicit ctx: Context) = - sym == defn.QuotedExpr_~ || sym == defn.QuotedType_~ + protected def newTransformer(implicit ctx: Context): Transformer = + new Reifier(inQuote = false, null, 0, new LevelInfo) /** Serialize `tree`. Embedded splices are represented as nodes of the form * @@ -40,77 +45,195 @@ class ReifyQuotes extends MacroTransform { def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String = tree.show // TODO: replace with TASTY - private class Reifier extends Transformer { - - /** A class for collecting the splices of some quoted expression */ - private class Splices { - - /** A listbuffer collecting splices */ - val buf = new mutable.ListBuffer[Tree] - - /** A map from type ref T to "expression of type quoted.Type[T]". - * These will be turned into splices using `addTags` - */ - val typeTagOfRef = new mutable.LinkedHashMap[TypeRef, Tree]() - - /** Assuming typeTagOfRef = `Type1 -> tag1, ..., TypeN -> tagN`, the expression - * - * { type = .unary_~ - * ... - * type = .unary.~ - * - * } - * - * where all references to `TypeI` in `expr` are rewired to point to the locally - * defined versions. As a side effect, append the expressions `tag1, ..., `tagN` - * as splices to `buf`. - */ - def addTags(expr: Tree)(implicit ctx: Context): Tree = - if (typeTagOfRef.isEmpty) expr - else { - val assocs = typeTagOfRef.toList - val typeDefs = for ((tp, tag) <- assocs) yield { - val original = tp.symbol.asType - val rhs = tag.select(tpnme.UNARY_~) - val alias = ctx.typeAssigner.assignType(untpd.TypeBoundsTree(rhs, rhs), rhs, rhs) - val local = original.copy( - owner = ctx.owner, - flags = Synthetic, - info = TypeAlias(tag.tpe.select(tpnme.UNARY_~))) - ctx.typeAssigner.assignType(untpd.TypeDef(original.name, alias), local) - } - val (trefs, tags) = assocs.unzip - tags ++=: buf - typeTagOfRef.clear() - Block(typeDefs, expr.subst(trefs.map(_.symbol), typeDefs.map(_.symbol))) - } + private class LevelInfo { + /** A map from locally defined symbols to the staging levels of their definitions */ + val levelOf = new mutable.HashMap[Symbol, Int] + + /** A stack of entered symbols, to be unwound after scope exit */ + var enteredSyms: List[Symbol] = Nil + } + + /** A tree substituter that also works for holes */ + class SubstMap( + typeMap: Type => Type = IdentityTypeMap, + treeMap: Tree => Tree = identity _, + oldOwners: List[Symbol] = Nil, + newOwners: List[Symbol] = Nil, + substFrom: List[Symbol], + substTo: List[Symbol])(implicit ctx: Context) + extends TreeTypeMap(typeMap, treeMap, oldOwners, newOwners, substFrom, substTo) { + + override def transform(tree: Tree)(implicit ctx: Context): Tree = tree match { + case Hole(n, args) => + Hole(n, args.mapConserve(transform)).withPos(tree.pos).withType(mapType(tree.tpe)) + case _ => + super.transform(tree) } - /** The current staging level */ - private var currentLevel = 0 + override def newMap( + typeMap: Type => Type, + treeMap: Tree => Tree, + oldOwners: List[Symbol], + newOwners: List[Symbol], + substFrom: List[Symbol], + substTo: List[Symbol])(implicit ctx: Context) = + new SubstMap(typeMap, treeMap, oldOwners, newOwners, substFrom, substTo) + } + + /** Requiring that `paramRefs` consists of a single reference `seq` to a Seq[Any], + * a tree map that replaces each hole with index `n` with `seq(n)`, applied + * to any arguments in the hole. + */ + private def replaceHoles(paramRefs: List[Tree]) = new TreeMap { + val seq :: Nil = paramRefs + override def transform(tree: Tree)(implicit ctx: Context): Tree = tree match { + case Hole(n, args) => + val arg = + seq.select(nme.apply).appliedTo(Literal(Constant(n))).ensureConforms(tree.tpe) + if (args.isEmpty) arg + else arg.select(nme.apply).appliedTo(SeqLiteral(args, TypeTree(defn.AnyType))) + case _ => + super.transform(tree) + } + } + + /** If `tree` has holes, convert it to a function taking a `Seq` of elements as arguments + * where each hole is replaced by the corresponding sequence element. + */ + private def elimHoles(tree: Tree)(implicit ctx: Context): Tree = + if (tree.existsSubTree(_.isInstanceOf[Hole])) + Lambda( + MethodType(defn.SeqType.appliedTo(defn.AnyType) :: Nil, tree.tpe), + replaceHoles(_).transform(tree)) + else tree + + /** The main transformer class + * @param inQuote we are within a `'(...)` context that is not shadowed by a nested `~(...)` + * @param outer the next outer reifier, null is this is the topmost transformer + * @param level the current level, where quotes add one and splices subtract one level + * @param levels a stacked map from symbols to the levels in which they were defined + */ + private class Reifier(inQuote: Boolean, val outer: Reifier, val level: Int, levels: LevelInfo) extends Transformer { + import levels._ - /** The splices encountered so far, indexed by staging level */ - private val splicesAtLevel = mutable.ArrayBuffer(new Splices) + /** A nested reifier for a quote (if `isQuote = true`) or a splice (if not) */ + def nested(isQuote: Boolean): Reifier = + new Reifier(isQuote, this, if (isQuote) level + 1 else level - 1, levels) - // Invariant: -1 <= currentLevel <= splicesAtLevel.length + /** We are in a `~(...)` context that is not shadowed by a nested `'(...)` */ + def inSplice = outer != null && !inQuote - /** A map from locally defined symbol's to the staging levels of their definitions */ - private val levelOf = new mutable.HashMap[Symbol, Int] + /** A list of embedded quotes (if `inSplice = true`) or splices (if `inQuote = true`) */ + val embedded = new mutable.ListBuffer[Tree] - /** A stack of entered symbol's, to be unwound after block exit */ - private var enteredSyms: List[Symbol] = Nil + /** A map from type ref T to "expression of type `quoted.Type[T]`". + * These will be turned into splices using `addTags` + */ + val importedTypes = new mutable.LinkedHashSet[TypeRef]() + + /** Assuming typeTagOfRef = `Type1 -> tag1, ..., TypeN -> tagN`, the expression + * + * { type = .unary_~ + * ... + * type = .unary.~ + * + * } + * + * where all references to `TypeI` in `expr` are rewired to point to the locally + * defined versions. As a side effect, prepend the expressions `tag1, ..., `tagN` + * as splices to `buf`. + */ + def addTags(expr: Tree)(implicit ctx: Context): Tree = + if (importedTypes.isEmpty) expr + else { + val trefs = importedTypes.toList + val typeDefs = for (tref <- trefs) yield { + val tag = New(defn.QuotedTypeType.appliedTo(tref), Nil) + val rhs = transform(tag.select(tpnme.UNARY_~)) + val alias = ctx.typeAssigner.assignType(untpd.TypeBoundsTree(rhs, rhs), rhs, rhs) + val original = tref.symbol.asType + val local = original.copy( + owner = ctx.owner, + flags = Synthetic, + info = TypeAlias(tag.tpe.select(tpnme.UNARY_~))) + ctx.typeAssigner.assignType(untpd.TypeDef(original.name, alias), local) + } + importedTypes.clear() + Block(typeDefs, + new SubstMap(substFrom = trefs.map(_.symbol), substTo = typeDefs.map(_.symbol)) + .apply(expr)) + } /** Enter staging level of symbol defined by `tree`, if applicable. */ def markDef(tree: Tree)(implicit ctx: Context) = tree match { - case tree: DefTree if !levelOf.contains(tree.symbol) => - levelOf(tree.symbol) = currentLevel - enteredSyms = tree.symbol :: enteredSyms + case tree: DefTree => + val sym = tree.symbol + if ((sym.isClass || !sym.maybeOwner.isType) && !levelOf.contains(sym)) { + levelOf(sym) = level + enteredSyms = sym :: enteredSyms + } case _ => } + /** Is symbol a splice operation? */ + def isSplice(sym: Symbol)(implicit ctx: Context) = + sym == defn.QuotedExpr_~ || sym == defn.QuotedType_~ + + /** Are we in the body of an inline method? */ + def inInline(implicit ctx: Context) = ctx.owner.ownersIterator.exists(_.isInlineMethod) + + /** Issue a "splice outside quote" error unless we ar in the body of an inline method */ + def spliceOutsideQuotes(pos: Position)(implicit ctx: Context) = + if (!inInline) ctx.error(i"splice outside quotes", pos) + + /** Check reference to `sym` for phase consistency, where `tp` is the underlying type + * by which we refer to `sym`. + */ + def check(sym: Symbol, tp: Type, pos: Position)(implicit ctx: Context): Unit = { + val isThis = tp.isInstanceOf[ThisType] + def symStr = + if (!isThis) sym.show + else if (sym.is(ModuleClass)) sym.sourceModule.show + else i"${sym.name}.this" + if (!isThis && sym.maybeOwner.isType) + check(sym.owner, sym.owner.thisType, pos) + else if (sym.exists && !sym.isStaticOwner && !inInline && + levelOf.getOrElse(sym, level) != level) + tp match { + case tp: TypeRef => + importedTypes += tp + case _ => + ctx.error(em"""access to $symStr from wrong staging level: + | - the definition is at level ${levelOf(sym)}, + | - but the access is at level $level.""", pos) + } + } + + /** Check all named types and this-types in a given type for phase consistency. */ + def checkType(pos: Position)(implicit ctx: Context): TypeAccumulator[Unit] = new TypeAccumulator[Unit] { + def apply(acc: Unit, tp: Type): Unit = reporting.trace(i"check type level $tp at $level") { + tp match { + case tp: NamedType if isSplice(tp.symbol) => + if (inQuote) outer.checkType(pos).foldOver(acc, tp) + else { + spliceOutsideQuotes(pos) + tp + } + case tp: NamedType => + check(tp.symbol, tp, pos) + foldOver(acc, tp) + case tp: ThisType => + check(tp.cls, tp, pos) + foldOver(acc, tp) + case _ => + foldOver(acc, tp) + } + } + } + /** If `tree` refers to a locally defined symbol (either directly, or in a pickled type), * check that its staging level matches the current level. References to types - * that are phase-incorrect can still be healed as follows. + * that are phase-incorrect can still be healed as follows: * * If `T` is a reference to a type at the wrong level, heal it by setting things up * so that we later add a type definition @@ -118,134 +241,99 @@ class ReifyQuotes extends MacroTransform { * type T' = ~quoted.Type[T] * * to the quoted text and rename T to T' in it. This is done later in `reify` via - * `Splice#addTags`. checkLevel itself only records what needs to be done in the + * `addTags`. `checkLevel` itself only records what needs to be done in the * `typeTagOfRef` field of the current `Splice` structure. */ private def checkLevel(tree: Tree)(implicit ctx: Context): Tree = { - - /** Check reference to `sym` for phase consistency, where `tp` is the underlying type - * by which we refer to `sym`. - */ - def check(sym: Symbol, tp: Type): Unit = { - val isThis = tp.isInstanceOf[ThisType] - def symStr = - if (!isThis) sym.show - else if (sym.is(ModuleClass)) sym.sourceModule.show - else i"${sym.name}.this" - if (!isThis && sym.maybeOwner.isType) - check(sym.owner, sym.owner.thisType) - else if (sym.exists && !sym.isStaticOwner && - !ctx.owner.ownersIterator.exists(_.isInlineMethod) && - levelOf.getOrElse(sym, currentLevel) != currentLevel) - tp match { - case tp: TypeRef => - // Legalize reference to phase-inconstent type - splicesAtLevel(currentLevel).typeTagOfRef(tp) = { - currentLevel -= 1 - val tag = New(defn.QuotedTypeType.appliedTo(tp), Nil) - try transform(tag) finally currentLevel += 1 - } - case _ => - ctx.error(em"""access to $symStr from wrong staging level: - | - the definition is at level ${levelOf(sym)}, - | - but the access is at level $currentLevel.""", tree.pos) - } - } - - /** Check all named types and this types in a given type for phase consistency */ - object checkType extends TypeAccumulator[Unit] { - /** Check that all NamedType and ThisType parts of `tp` are level-correct. - * If they are not, try to heal with a local binding to a typetag splice - */ - def apply(tp: Type): Unit = apply((), tp) - def apply(acc: Unit, tp: Type): Unit = reporting.trace(i"check type level $tp at $currentLevel") { - tp match { - case tp: NamedType if isSplice(tp.symbol) => - currentLevel -= 1 - try foldOver(acc, tp) finally currentLevel += 1 - case tp: NamedType => - check(tp.symbol, tp) - foldOver(acc, tp) - case tp: ThisType => - check(tp.cls, tp) - foldOver(acc, tp) - case _ => - foldOver(acc, tp) - } - } - } - tree match { case (_: Ident) | (_: This) => - check(tree.symbol, tree.tpe) + check(tree.symbol, tree.tpe, tree.pos) case (_: UnApply) | (_: TypeTree) => - checkType(tree.tpe) + checkType(tree.pos).apply((), tree.tpe) case Select(qual, OuterSelectName(_, levels)) => - checkType(tree.tpe.widen) + checkType(tree.pos).apply((), tree.tpe.widen) case _: Bind => - checkType(tree.symbol.info) + checkType(tree.pos).apply((), tree.symbol.info) case _: Template => - checkType(tree.symbol.owner.asClass.givenSelfType) + checkType(tree.pos).apply((), tree.symbol.owner.asClass.givenSelfType) case _ => } tree } - /** Turn `body` of quote into a call of `scala.quoted.Unpickler.unpickleType` or - * `scala.quoted.Unpickler.unpickleExpr` depending onwhether `isType` is true or not. - * The arguments to the method are: - * - * - the serialized `body`, as returned from `pickleTree` - * - all splices found in `body` + /** Split `body` into a core and a list of embedded splices. + * Then if inside a splice, make a hole from these parts. + * If outside a splice, generate a call tp `scala.quoted.Unpickler.unpickleType` or + * `scala.quoted.Unpickler.unpickleExpr` that matches `tpe` with + * core and splices as arguments. */ - private def reify(body: Tree, isType: Boolean)(implicit ctx: Context) = { - currentLevel += 1 - if (currentLevel == splicesAtLevel.length) splicesAtLevel += null - val splices = new Splices - val savedSplices = splicesAtLevel(currentLevel) - splicesAtLevel(currentLevel) = splices - try { - val body1 = splices.addTags(transform(body)) + private def quotation(body: Tree, quote: Tree)(implicit ctx: Context) = { + val (body1, splices) = nested(isQuote = true).split(body) + if (inSplice) + makeHole(body1, splices, quote.tpe) + else { + val isType = quote.tpe.isRef(defn.QuotedTypeClass) ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr) .appliedToType(if (isType) body1.tpe else body1.tpe.widen) .appliedTo( Literal(Constant(pickleTree(body1, isType))), - SeqLiteral(splices.buf.toList, TypeTree(defn.QuotedType))) + SeqLiteral(splices, TypeTree(defn.AnyType))) + } + }.withPos(quote.pos) + + /** If inside a quote, split `body` into a core and a list of embedded quotes + * and make a hole from these parts. Otherwise issue an error, unless we + * are in the body of an inline method. + */ + private def splice(body: Tree, splice: Tree)(implicit ctx: Context): Tree = { + if (inQuote) { + val (body1, quotes) = nested(isQuote = false).split(body) + makeHole(body1, quotes, splice.tpe) } - finally { - splicesAtLevel(currentLevel) = savedSplices - currentLevel -= 1 + else { + spliceOutsideQuotes(splice.pos) + if (splice.isType) TypeTree(splice.tpe.dealias) + else transform(body).select(defn.QuotedExpr_run) } + }.withPos(splice.pos) + + /** Transform `tree` and return the resulting tree and all `embedded` quotes + * or splices as a pair, after performing the `addTags` transform. + */ + private def split(tree: Tree)(implicit ctx: Context): (Tree, List[Tree]) = { + val tree1 = addTags(transform(tree)) + (tree1, embedded.toList.map(elimHoles)) + } + + /** Register `body` as an `embedded` quote or splice + * and return a hole with `splices` as arguments and the given type `tpe`. + */ + private def makeHole(body: Tree, splices: List[Tree], tpe: Type)(implicit ctx: Context): Hole = { + val idx = embedded.length + embedded += body + Hole(idx, splices).withType(tpe).asInstanceOf[Hole] } override def transform(tree: Tree)(implicit ctx: Context): Tree = - reporting.trace(i"reify $tree at $currentLevel", show = true) { + reporting.trace(i"reify $tree at $level", show = true) { + def mapOverTree(lastEntered: List[Symbol]) = + try super.transform(tree) + finally + while (enteredSyms ne lastEntered) { + levelOf -= enteredSyms.head + enteredSyms = enteredSyms.tail + } tree match { case Apply(fn, arg :: Nil) if fn.symbol == defn.quoteMethod => - reify(arg, isType = false) + quotation(arg, tree) case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod => - reify(arg, isType = true) - case tree @ Select(body, name) if isSplice(tree.symbol) => - currentLevel -= 1 - val body1 = try transform(body) finally currentLevel += 1 - if (currentLevel > 0) { - splicesAtLevel(currentLevel).buf += body1 - tree - } - else { - if (currentLevel < 0) - ctx.error(i"splice ~ not allowed under toplevel splice", tree.pos) - cpy.Select(tree)(body1, name) - } + quotation(arg, tree) + case Select(body, _) if isSplice(tree.symbol) => + splice(body, tree) case Block(stats, _) => val last = enteredSyms stats.foreach(markDef) - try super.transform(tree) - finally - while (enteredSyms ne last) { - levelOf -= enteredSyms.head - enteredSyms = enteredSyms.tail - } + mapOverTree(last) case Inlined(call, bindings, expansion @ Select(body, name)) if isSplice(expansion.symbol) => // To maintain phase consistency, convert inlined expressions of the form // `{ bindings; ~expansion }` to `~{ bindings; expansion }` @@ -254,7 +342,7 @@ class ReifyQuotes extends MacroTransform { tree case _ => markDef(tree) - checkLevel(super.transform(tree)) + checkLevel(mapOverTree(enteredSyms)) } } } From a8b9ba727d295a3297be75bfad507d4c31de2575 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 16 Dec 2017 18:53:57 +0100 Subject: [PATCH 45/62] Mark inline methods containing splices as macros If an inline method contains a toplevel splice, mark it s a macro. Furthermore, allow for serialization of the Macro flag in Tasty. --- .../src/dotty/tools/dotc/core/Flags.scala | 2 +- .../tools/dotc/core/tasty/TastyFormat.scala | 8 +++++-- .../tools/dotc/core/tasty/TreePickler.scala | 1 + .../tools/dotc/core/tasty/TreeUnpickler.scala | 1 + .../tools/dotc/transform/ReifyQuotes.scala | 12 ++++------ .../dotty/tools/dotc/transform/SymUtils.scala | 4 ++++ .../src/dotty/tools/dotc/typer/Typer.scala | 24 +++++++++++++++---- 7 files changed, 37 insertions(+), 15 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Flags.scala b/compiler/src/dotty/tools/dotc/core/Flags.scala index 60438a125334..beaefdecc7af 100644 --- a/compiler/src/dotty/tools/dotc/core/Flags.scala +++ b/compiler/src/dotty/tools/dotc/core/Flags.scala @@ -414,7 +414,7 @@ object Flags { /** A Scala 2.12 or higher trait */ final val Scala_2_12_Trait = typeFlag(58, "") - /** A macro (Scala 2.x only) */ + /** A macro */ final val Macro = commonFlag(59, "") /** A method that is known to have inherited default parameters */ diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala index 12e053e128f5..a3ae450313ec 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyFormat.scala @@ -181,7 +181,8 @@ Standard-Section: "ASTs" TopLevelStat* IMPLICIT LAZY OVERRIDE - INLINE // macro + INLINE // inline method + MACRO // inline method containing toplevel splices STATIC // mapped to static Java member OBJECT // an object or its class TRAIT // a trait @@ -226,7 +227,7 @@ object TastyFormat { final val header = Array(0x5C, 0xA1, 0xAB, 0x1F) val MajorVersion = 2 - val MinorVersion = 0 + val MinorVersion = 1 /** Tags used to serialize names */ class NameTags { @@ -297,6 +298,7 @@ object TastyFormat { final val SCALA2X = 29 final val DEFAULTparameterized = 30 final val STABLE = 31 + final val MACRO = 32 // Cat. 2: tag Nat @@ -419,6 +421,7 @@ object TastyFormat { | LAZY | OVERRIDE | INLINE + | MACRO | STATIC | OBJECT | TRAIT @@ -472,6 +475,7 @@ object TastyFormat { case LAZY => "LAZY" case OVERRIDE => "OVERRIDE" case INLINE => "INLINE" + case MACRO => "MACRO" case STATIC => "STATIC" case OBJECT => "OBJECT" case TRAIT => "TRAIT" diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala index 36aad0b67655..87c3cbb554b7 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala @@ -587,6 +587,7 @@ class TreePickler(pickler: TastyPickler) { if (flags is Case) writeByte(CASE) if (flags is Override) writeByte(OVERRIDE) if (flags is Inline) writeByte(INLINE) + if (flags is Macro) writeByte(MACRO) if (flags is JavaStatic) writeByte(STATIC) if (flags is Module) writeByte(OBJECT) if (flags is Local) writeByte(LOCAL) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index d8e54b635f8c..265437adf4a6 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -524,6 +524,7 @@ class TreeUnpickler(reader: TastyReader, case LAZY => addFlag(Lazy) case OVERRIDE => addFlag(Override) case INLINE => addFlag(Inline) + case MACRO => addFlag(Macro) case STATIC => addFlag(JavaStatic) case OBJECT => addFlag(Module) case TRAIT => addFlag(Trait) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index ba3457046d2e..4828d9ebfef4 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -11,11 +11,11 @@ import StdNames._ import ast.untpd import tasty.TreePickler.Hole import MegaPhase.MiniPhase +import SymUtils._ import NameKinds.OuterSelectName import scala.collection.mutable // TODO -// implement hole substitution directly // check that inline methods override nothing // drop inline methods // adapt to Expr/Type when passing arguments to splices @@ -175,10 +175,6 @@ class ReifyQuotes extends MacroTransform { case _ => } - /** Is symbol a splice operation? */ - def isSplice(sym: Symbol)(implicit ctx: Context) = - sym == defn.QuotedExpr_~ || sym == defn.QuotedType_~ - /** Are we in the body of an inline method? */ def inInline(implicit ctx: Context) = ctx.owner.ownersIterator.exists(_.isInlineMethod) @@ -213,7 +209,7 @@ class ReifyQuotes extends MacroTransform { def checkType(pos: Position)(implicit ctx: Context): TypeAccumulator[Unit] = new TypeAccumulator[Unit] { def apply(acc: Unit, tp: Type): Unit = reporting.trace(i"check type level $tp at $level") { tp match { - case tp: NamedType if isSplice(tp.symbol) => + case tp: NamedType if tp.symbol.isSplice => if (inQuote) outer.checkType(pos).foldOver(acc, tp) else { spliceOutsideQuotes(pos) @@ -328,13 +324,13 @@ class ReifyQuotes extends MacroTransform { quotation(arg, tree) case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod => quotation(arg, tree) - case Select(body, _) if isSplice(tree.symbol) => + case Select(body, _) if tree.symbol.isSplice => splice(body, tree) case Block(stats, _) => val last = enteredSyms stats.foreach(markDef) mapOverTree(last) - case Inlined(call, bindings, expansion @ Select(body, name)) if isSplice(expansion.symbol) => + case Inlined(call, bindings, expansion @ Select(body, name)) if expansion.symbol.isSplice => // To maintain phase consistency, convert inlined expressions of the form // `{ bindings; ~expansion }` to `~{ bindings; expansion }` cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name) diff --git a/compiler/src/dotty/tools/dotc/transform/SymUtils.scala b/compiler/src/dotty/tools/dotc/transform/SymUtils.scala index 776ecb057892..aa881fde2194 100644 --- a/compiler/src/dotty/tools/dotc/transform/SymUtils.scala +++ b/compiler/src/dotty/tools/dotc/transform/SymUtils.scala @@ -175,4 +175,8 @@ class SymUtils(val self: Symbol) extends AnyVal { else if (owner.is(Package)) false else owner.isLocal } + + /** Is symbol a splice operation? */ + def isSplice(implicit ctx: Context): Boolean = + self == defn.QuotedExpr_~ || self == defn.QuotedType_~ } diff --git a/compiler/src/dotty/tools/dotc/typer/Typer.scala b/compiler/src/dotty/tools/dotc/typer/Typer.scala index 25d70a898e47..4b0890cc6c31 100644 --- a/compiler/src/dotty/tools/dotc/typer/Typer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Typer.scala @@ -393,11 +393,24 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit checkValue(assignType(cpy.Select(tree)(qual, tree.name), qual), pt) def typedSelect(tree: untpd.Select, pt: Type)(implicit ctx: Context): Tree = track("typedSelect") { + + /** If we are an an inline method but not in a nested quote, mark the inline method + * as a macro. + */ + def markAsMacro(c: Context): Unit = + if (!c.tree.isInstanceOf[untpd.Quote]) + if (c.owner eq c.outer.owner) markAsMacro(c.outer) + else if (c.owner.isInlineMethod) c.owner.setFlag(Macro) + else if (!c.outer.owner.is(Package)) markAsMacro(c.outer) + def typeSelectOnTerm(implicit ctx: Context): Tree = { val qual1 = typedExpr(tree.qualifier, selectionProto(tree.name, pt, this)) if (tree.name.isTypeName) checkStable(qual1.tpe, qual1.pos) val select = typedSelect(tree, pt, qual1) - if (select.tpe ne TryDynamicCallType) select + if (select.tpe ne TryDynamicCallType) { + if (select.symbol.isSplice) markAsMacro(ctx) + select + } else if (pt.isInstanceOf[PolyProto] || pt.isInstanceOf[FunProto] || pt == AssignProto) select else typedDynamicSelect(tree, Nil, pt) } @@ -1081,10 +1094,13 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit case AppliedType(_, argType :: Nil) => argType case _ => WildcardType } - if (isType) - ref(defn.typeQuoteMethod).appliedToTypeTrees(typedType(body, proto1) :: Nil) + val nestedCtx = ctx.fresh.setTree(tree) + if (isType) { + val body1 = typedType(body, proto1)(nestedCtx) + ref(defn.typeQuoteMethod).appliedToTypeTrees(body1 :: Nil) + } else { - val body1 = typed(body, proto1) + val body1 = typed(body, proto1)(nestedCtx) ref(defn.quoteMethod).appliedToType(body1.tpe.widen).appliedTo(body1) } } From 40235024900959fb353d2bf067dfad479ed04bc2 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sat, 16 Dec 2017 20:00:28 +0100 Subject: [PATCH 46/62] Eliminate macros in RefChecks ... after checking that they do not override anything. This means we do not need to transform their bodies in ReifyQuotes. --- compiler/src/dotty/tools/dotc/core/Flags.scala | 3 +++ .../dotty/tools/dotc/transform/ReifyQuotes.scala | 15 +++++---------- .../dotty/tools/dotc/transform/TreeChecker.scala | 3 ++- .../src/dotty/tools/dotc/typer/RefChecks.scala | 13 ++++++++----- tests/neg/quotedMacroOverride.scala | 13 +++++++++++++ 5 files changed, 31 insertions(+), 16 deletions(-) create mode 100644 tests/neg/quotedMacroOverride.scala diff --git a/compiler/src/dotty/tools/dotc/core/Flags.scala b/compiler/src/dotty/tools/dotc/core/Flags.scala index beaefdecc7af..f1cd9db376a3 100644 --- a/compiler/src/dotty/tools/dotc/core/Flags.scala +++ b/compiler/src/dotty/tools/dotc/core/Flags.scala @@ -598,6 +598,9 @@ object Flags { /** Is a default parameter in Scala 2*/ final val DefaultParameter = allOf(Param, DefaultParameterized) + /** A Scala 2 Macro */ + final val Scala2Macro = allOf(Macro, Scala2x) + /** A trait that does not need to be initialized */ final val NoInitsTrait = allOf(Trait, NoInits) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 4828d9ebfef4..83196b36a8fd 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -16,8 +16,6 @@ import NameKinds.OuterSelectName import scala.collection.mutable // TODO -// check that inline methods override nothing -// drop inline methods // adapt to Expr/Type when passing arguments to splices /** Translates quoted terms and types to `unpickle` method calls. @@ -175,12 +173,9 @@ class ReifyQuotes extends MacroTransform { case _ => } - /** Are we in the body of an inline method? */ - def inInline(implicit ctx: Context) = ctx.owner.ownersIterator.exists(_.isInlineMethod) - /** Issue a "splice outside quote" error unless we ar in the body of an inline method */ def spliceOutsideQuotes(pos: Position)(implicit ctx: Context) = - if (!inInline) ctx.error(i"splice outside quotes", pos) + ctx.error(i"splice outside quotes", pos) /** Check reference to `sym` for phase consistency, where `tp` is the underlying type * by which we refer to `sym`. @@ -193,8 +188,7 @@ class ReifyQuotes extends MacroTransform { else i"${sym.name}.this" if (!isThis && sym.maybeOwner.isType) check(sym.owner, sym.owner.thisType, pos) - else if (sym.exists && !sym.isStaticOwner && !inInline && - levelOf.getOrElse(sym, level) != level) + else if (sym.exists && !sym.isStaticOwner && levelOf.getOrElse(sym, level) != level) tp match { case tp: TypeRef => importedTypes += tp @@ -288,8 +282,7 @@ class ReifyQuotes extends MacroTransform { } else { spliceOutsideQuotes(splice.pos) - if (splice.isType) TypeTree(splice.tpe.dealias) - else transform(body).select(defn.QuotedExpr_run) + splice } }.withPos(splice.pos) @@ -336,6 +329,8 @@ class ReifyQuotes extends MacroTransform { cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name) case _: Import => tree + case tree: DefDef if tree.symbol.is(Macro) => + cpy.DefDef(tree)(rhs = EmptyTree) case _ => markDef(tree) checkLevel(mapOverTree(enteredSyms)) diff --git a/compiler/src/dotty/tools/dotc/transform/TreeChecker.scala b/compiler/src/dotty/tools/dotc/transform/TreeChecker.scala index 237bc2d2efad..f59d418d110d 100644 --- a/compiler/src/dotty/tools/dotc/transform/TreeChecker.scala +++ b/compiler/src/dotty/tools/dotc/transform/TreeChecker.scala @@ -375,7 +375,8 @@ class TreeChecker extends Phase with SymTransformer { def isNonMagicalMethod(x: Symbol) = x.is(Method) && !x.isCompanionMethod && - !x.isValueClassConvertMethod + !x.isValueClassConvertMethod && + !(x.is(Macro) && ctx.phase.refChecked) val symbolsNotDefined = cls.classInfo.decls.toList.toSet.filter(isNonMagicalMethod) -- impl.body.map(_.symbol) - constr.symbol diff --git a/compiler/src/dotty/tools/dotc/typer/RefChecks.scala b/compiler/src/dotty/tools/dotc/typer/RefChecks.scala index fcbf7c6ff5e4..aaadbb66ec12 100644 --- a/compiler/src/dotty/tools/dotc/typer/RefChecks.scala +++ b/compiler/src/dotty/tools/dotc/typer/RefChecks.scala @@ -130,7 +130,7 @@ object RefChecks { * That is for overriding member M and overridden member O: * * 1.1. M must have the same or stronger access privileges as O. - * 1.2. O must not be final. + * 1.2. O must not be effectively final. * 1.3. O is deferred, or M has `override` modifier. * 1.4. If O is stable, then so is M. * // @M: LIFTED 1.5. Neither M nor O are a parameterized type alias @@ -144,8 +144,9 @@ object RefChecks { * 1.8.1 M's type is a subtype of O's type, or * 1.8.2 M is of type []S, O is of type ()T and S <: T, or * 1.8.3 M is of type ()S, O is of type []T and S <: T, or - * 1.9. If M is a macro def, O cannot be deferred unless there's a concrete method overriding O. - * 1.10. If M is not a macro def, O cannot be a macro def. + * 1.9 M must not be a Dotty macro def + * 1.10. If M is a 2.x macro def, O cannot be deferred unless there's a concrete method overriding O. + * 1.11. If M is not a macro def, O cannot be a macro def. * 2. Check that only abstract classes have deferred members * 3. Check that concrete classes do not have deferred definitions * that are not implemented in a subclass. @@ -372,9 +373,11 @@ object RefChecks { overrideError("may not override a non-lazy value") } else if (other.is(Lazy) && !other.isRealMethod && !member.is(Lazy)) { overrideError("must be declared lazy to override a lazy value") - } else if (other.is(Deferred) && member.is(Macro) && member.extendedOverriddenSymbols.forall(_.is(Deferred))) { // (1.9) + } else if (member.is(Macro, butNot = Scala2x)) { // (1.9) + overrideError("is a macro, may not override anything") + } else if (other.is(Deferred) && member.is(Scala2Macro) && member.extendedOverriddenSymbols.forall(_.is(Deferred))) { // (1.10) overrideError("cannot be used here - term macros cannot override abstract methods") - } else if (other.is(Macro) && !member.is(Macro)) { // (1.10) + } else if (other.is(Macro) && !member.is(Macro)) { // (1.11) overrideError("cannot be used here - only term macros can override term macros") } else if (!compatibleTypes(memberTp(self), otherTp(self)) && !compatibleTypes(memberTp(upwardsSelf), otherTp(upwardsSelf))) { diff --git a/tests/neg/quotedMacroOverride.scala b/tests/neg/quotedMacroOverride.scala new file mode 100644 index 000000000000..56a14fe304c8 --- /dev/null +++ b/tests/neg/quotedMacroOverride.scala @@ -0,0 +1,13 @@ +object Test { + + abstract class A { + def f(): Unit + inline def g(): Unit = () + } + + class B extends A { + inline def f() = ~('()) // error: may not override + override def g() = () // error: may not override + } + +} From d1abf405981d7d37ff11c05aeccc124b12c41b42 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 17 Dec 2017 12:59:51 +0100 Subject: [PATCH 47/62] Typecheck macros as if they were in a quoted context A macro is intended to be used in a quoted context, so should be checked in such a context as well. --- compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 83196b36a8fd..8604d9a558aa 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -330,6 +330,8 @@ class ReifyQuotes extends MacroTransform { case _: Import => tree case tree: DefDef if tree.symbol.is(Macro) => + val tree1 = nested(isQuote = true).transform(tree) + // check macro code as it if appeared in a quoted context cpy.DefDef(tree)(rhs = EmptyTree) case _ => markDef(tree) From 04d3a58b2a07a34f4b637be374eaaecf8376e88a Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Sun, 17 Dec 2017 14:19:59 +0100 Subject: [PATCH 48/62] Refine handling of inline parameters of macros These are at runtime trees with constant types. We can extract the constant from the tree, so they can also be used in a splicing context. --- .../tools/dotc/transform/ReifyQuotes.scala | 21 ++++++++++++++----- tests/neg/inlinevals.scala | 6 ++++++ tests/pos/quoted.scala | 13 ++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 8604d9a558aa..858e5f0430e7 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -15,9 +15,6 @@ import SymUtils._ import NameKinds.OuterSelectName import scala.collection.mutable -// TODO -// adapt to Expr/Type when passing arguments to splices - /** Translates quoted terms and types to `unpickle` method calls. * Checks that the phase consistency principle (PCP) holds. */ @@ -173,6 +170,19 @@ class ReifyQuotes extends MacroTransform { case _ => } + /** Does the level of `sym` match the current level? + * An exception is made for inline vals in macros. These are also OK if their level + * is one higher than the current level, because on execution such values + * are constant expression trees and we can pull out the constant from the tree. + */ + def levelOK(sym: Symbol)(implicit ctx: Context): Boolean = levelOf.get(sym) match { + case Some(l) => + l == level || + sym.is(Inline) && sym.owner.is(Macro) && sym.info.isValueType && l - 1 == level + case None => + true + } + /** Issue a "splice outside quote" error unless we ar in the body of an inline method */ def spliceOutsideQuotes(pos: Position)(implicit ctx: Context) = ctx.error(i"splice outside quotes", pos) @@ -188,7 +198,7 @@ class ReifyQuotes extends MacroTransform { else i"${sym.name}.this" if (!isThis && sym.maybeOwner.isType) check(sym.owner, sym.owner.thisType, pos) - else if (sym.exists && !sym.isStaticOwner && levelOf.getOrElse(sym, level) != level) + else if (sym.exists && !sym.isStaticOwner && !levelOK(sym)) tp match { case tp: TypeRef => importedTypes += tp @@ -329,7 +339,8 @@ class ReifyQuotes extends MacroTransform { cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name) case _: Import => tree - case tree: DefDef if tree.symbol.is(Macro) => + case tree: DefDef if tree.symbol.is(Macro) && level == 0 => + markDef(tree) val tree1 = nested(isQuote = true).transform(tree) // check macro code as it if appeared in a quoted context cpy.DefDef(tree)(rhs = EmptyTree) diff --git a/tests/neg/inlinevals.scala b/tests/neg/inlinevals.scala index 184aa2168772..d7d45e248a74 100644 --- a/tests/neg/inlinevals.scala +++ b/tests/neg/inlinevals.scala @@ -17,6 +17,12 @@ object Test { inline val M = X // error: rhs must be constant expression + inline val xs = List(1, 2, 3) // error: must be a constant expression + + def f(inline xs: List[Int]) = xs + + f(List(1, 2, 3)) // error: must be a constant expression + def byname(inline f: => String): Int = ??? // ok byname("hello" ++ " world") diff --git a/tests/pos/quoted.scala b/tests/pos/quoted.scala index a9874180b803..3155c353ca33 100644 --- a/tests/pos/quoted.scala +++ b/tests/pos/quoted.scala @@ -10,6 +10,13 @@ class Test { def assertImpl(expr: Expr[Boolean]) = '{ if !(~expr) then throw new AssertionError(s"failed assertion: ${~expr}") } + inline def power(inline n: Int, x: Double) = ~powerCode(n, '(x)) + + def powerCode(n: Int, x: Expr[Double]): Expr[Double] = + if (n == 0) '(1.0) + else if (n == 1) x + else if (n % 2 == 0) '{ { val y = ~x * ~x; ~powerCode(n / 2, '(y)) } } + else '{ ~x * ~powerCode(n - 1, x) } } val program = '{ @@ -19,6 +26,12 @@ class Test { assert(x != 0) ~assertImpl('(x != 0)) + + val y = math.sqrt(2.0) + + power(3, y) + + ~powerCode(3, '{math.sqrt(2.0)}) } program.run From 94e7c83581411f75ad54a9445ff173145af9d45a Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 12:42:16 +0100 Subject: [PATCH 49/62] Force definitions of quote methods when initializing Definitions The methods have to be known when unpickling. Their type needs to be evaluated lazily, however, as otherwise `Expr` would be forced too early. --- .../dotty/tools/dotc/core/Definitions.scala | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 5058e3f3cf7a..263db5b842da 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -145,11 +145,20 @@ class Definitions { } private def enterPolyMethod(cls: ClassSymbol, name: TermName, typeParamCount: Int, - resultTypeFn: PolyType => Type, flags: FlagSet = EmptyFlags) = { + resultTypeFn: PolyType => Type, flags: FlagSet = EmptyFlags, + useCompleter: Boolean = false) = { val tparamNames = PolyType.syntheticParamNames(typeParamCount) val tparamInfos = tparamNames map (_ => TypeBounds.empty) - val ptype = PolyType(tparamNames)(_ => tparamInfos, resultTypeFn) - enterMethod(cls, name, ptype, flags) + def ptype = PolyType(tparamNames)(_ => tparamInfos, resultTypeFn) + val info = + if (useCompleter) + new LazyType { + def complete(denot: SymDenotation)(implicit ctx: Context): Unit = { + denot.info = ptype + } + } + else ptype + enterMethod(cls, name, info, flags) } private def enterT1ParameterlessMethod(cls: ClassSymbol, name: TermName, resultTypeFn: PolyType => Type, flags: FlagSet) = @@ -298,11 +307,13 @@ class Definitions { /** Method representing a term quote */ lazy val quoteMethod = enterPolyMethod(OpsPackageClass, nme.QUOTE, 1, - pt => MethodType(pt.paramRefs(0) :: Nil, QuotedExprType.appliedTo(pt.paramRefs(0) :: Nil))) + pt => MethodType(pt.paramRefs(0) :: Nil, QuotedExprType.appliedTo(pt.paramRefs(0) :: Nil)), + useCompleter = true) /** Method representing a type quote */ lazy val typeQuoteMethod = enterPolyMethod(OpsPackageClass, nme.QUOTE, 1, - pt => QuotedTypeType.appliedTo(pt.paramRefs(0) :: Nil)) + pt => QuotedTypeType.appliedTo(pt.paramRefs(0) :: Nil), + useCompleter = true) lazy val NothingClass: ClassSymbol = enterCompleteClassSymbol( ScalaPackageClass, tpnme.Nothing, AbstractFinal, List(AnyClass.typeRef)) @@ -1093,9 +1104,7 @@ class Definitions { /** Lists core methods that don't have underlying bytecode, but are synthesized on-the-fly in every reflection universe */ lazy val syntheticCoreMethods = - AnyMethods ++ ObjectMethods ++ List(String_+, throwMethod - //, quoteMethod, typeQuoteMethod // we omit these because they force Expr and Type too early - ) + AnyMethods ++ ObjectMethods ++ List(String_+, throwMethod, quoteMethod, typeQuoteMethod) lazy val reservedScalaClassNames: Set[Name] = syntheticScalaClasses.map(_.name).toSet From 2e3c56bafec91330a3c1cab41a32d3ba1d93e10e Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 12:55:52 +0100 Subject: [PATCH 50/62] Syntax polishing Complete the description in `syntax.md` and align it with the parser. --- .../src/dotty/tools/dotc/parsing/Parsers.scala | 9 +++++---- docs/docs/internals/syntax.md | 16 ++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index eb468c5216f8..9e75ade74971 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -1285,9 +1285,9 @@ object Parsers { /** SimpleExpr ::= new Template * | BlockExpr - * | ‘'’ BlockExpr - * | ‘'’ ‘(’ ExprsInParens ‘)’ - * | ‘'’ ‘[’ Type ‘]’ + * | ‘'{’ BlockExprContents ‘}’ + * | ‘'(’ ExprsInParens ‘)’ + * | ‘'[’ Type ‘]’ * | SimpleExpr1 [`_'] * SimpleExpr1 ::= literal * | xmlLiteral @@ -1467,7 +1467,8 @@ object Parsers { } else fn - /** BlockExpr ::= `{' (CaseClauses | Block) `}' + /** BlockExpr ::= `{' BlockExprContents `}' + * BlockExprContents ::= CaseClauses | Block */ def blockExpr(): Tree = atPos(in.offset) { inDefScopeBraces { diff --git a/docs/docs/internals/syntax.md b/docs/docs/internals/syntax.md index 34bf8c390da2..ddc70855ff1d 100644 --- a/docs/docs/internals/syntax.md +++ b/docs/docs/internals/syntax.md @@ -26,7 +26,7 @@ upper ::= ‘A’ | … | ‘Z’ | ‘\$’ | ‘_’ “… and U lower ::= ‘a’ | … | ‘z’ “… and Unicode category Ll” letter ::= upper | lower “… and Unicode categories Lo, Lt, Nl” digit ::= ‘0’ | … | ‘9’ -paren ::= ‘(’ | ‘)’ | ‘[’ | ‘]’ | ‘{’ | ‘}’ +paren ::= ‘(’ | ‘)’ | ‘[’ | ‘]’ | ‘{’ | ‘}’ | ‘'(’ | ‘'[’ | ‘'{’ delim ::= ‘`’ | ‘'’ | ‘"’ | ‘.’ | ‘;’ | ‘,’ opchar ::= “printableChar not matched by (whiteSpace | upper | lower | letter | digit | paren | delim | opchar | Unicode_Sm | @@ -183,9 +183,9 @@ InfixExpr ::= PrefixExpr PrefixExpr ::= [‘-’ | ‘+’ | ‘~’ | ‘!’] SimpleExpr PrefixOp(expr, op) SimpleExpr ::= ‘new’ Template New(templ) | BlockExpr - | ‘'’ BlockExpr - | ‘'’ ‘(’ ExprsInParens ‘)’ - | ‘'’ ‘[’ Type ‘]’ + | ''{’ BlockExprContents ‘}’ + | ‘'(’ ExprsInParens ‘)’ + | ‘'[’ Type ‘]’ | SimpleExpr1 [‘_’] PostfixOp(expr, _) SimpleExpr1 ::= Literal | Path @@ -202,8 +202,8 @@ ParArgumentExprs ::= ‘(’ ExprsInParens ‘)’ | ‘(’ [ExprsInParens] PostfixExpr ‘:’ ‘_’ ‘*’ ‘)’ exprs :+ Typed(expr, Ident(wildcardStar)) ArgumentExprs ::= ParArgumentExprs | [nl] BlockExpr -BlockExpr ::= ‘{’ CaseClauses ‘}’ Match(EmptyTree, cases) - | ‘{’ Block ‘}’ block // starts at { +BlockExpr ::= ‘{’ BlockExprContents ‘}’ +BlockExprContents ::= CaseClauses | Block Block ::= {BlockStat semi} [BlockResult] Block(stats, expr?) BlockStat ::= Import | {Annotation} [‘implicit’ | ‘lazy’] Def @@ -220,8 +220,8 @@ Enumerator ::= Generator Generator ::= Pattern1 ‘<-’ Expr GenFrom(pat, expr) Guard ::= ‘if’ PostfixExpr -CaseClauses ::= CaseClause { CaseClause } CaseDef(pat, guard?, block) // block starts at => -CaseClause ::= ‘case’ (Pattern [Guard] ‘=>’ Block | INT) +CaseClauses ::= CaseClause { CaseClause } Match(EmptyTree, cases) +CaseClause ::= ‘case’ (Pattern [Guard] ‘=>’ Block | INT) CaseDef(pat, guard?, block) // block starts at => Pattern ::= Pattern1 { ‘|’ Pattern1 } Alternative(pats) Pattern1 ::= PatVar ‘:’ RefinedType Bind(name, Typed(Ident(wildcard), tpe)) From 0c6cbc37f858df3905473ce6dff935f7796c0427 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 12:57:00 +0100 Subject: [PATCH 51/62] Rename Quotable -> Liftable --- library/src/scala/quoted/Expr.scala | 2 +- .../scala/quoted/{Quotable.scala => Liftable.scala} | 12 +++++------- tests/pos/{quotable.scala => liftable.scala} | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) rename library/src/scala/quoted/{Quotable.scala => Liftable.scala} (64%) rename tests/pos/{quotable.scala => liftable.scala} (90%) diff --git a/library/src/scala/quoted/Expr.scala b/library/src/scala/quoted/Expr.scala index 5cbdcbde4122..7e11aac7b477 100644 --- a/library/src/scala/quoted/Expr.scala +++ b/library/src/scala/quoted/Expr.scala @@ -6,6 +6,6 @@ class Expr[T] extends Quoted { } object Expr { - implicit def toExpr[T](x: T)(implicit ev: Quotable[T]): Expr[T] = + implicit def toExpr[T](x: T)(implicit ev: Liftable[T]): Expr[T] = ev.toExpr(x) } diff --git a/library/src/scala/quoted/Quotable.scala b/library/src/scala/quoted/Liftable.scala similarity index 64% rename from library/src/scala/quoted/Quotable.scala rename to library/src/scala/quoted/Liftable.scala index 46a84aba2b33..9e64a7c1ed8e 100644 --- a/library/src/scala/quoted/Quotable.scala +++ b/library/src/scala/quoted/Liftable.scala @@ -1,20 +1,18 @@ package scala.quoted -import scala.math /** A typeclass for types that can be turned to `quoted.Expr[T]` * without going through an explicit `'(...)` operation. */ -abstract class Quotable[T] { +abstract class Liftable[T] { implicit def toExpr(x: T): Expr[T] } -/** Some base quotable types. To be completed with at least all types +/** Some liftable base types. To be completed with at least all types * that are valid Scala literals. The actual implementation of these * typed could be in terms of `ast.tpd.Literal`; the test `quotable.scala` * gives an alternative implementation using just the basic staging system. */ -object Quotable { - - implicit def IntIsQuotable: Quotable[Int] = ??? - implicit def BooleanIsQuotable: Quotable[Boolean] = ??? +object Liftable { + implicit def IntIsLiftable: Liftable[Int] = ??? + implicit def BooleanIsLiftable: Liftable[Boolean] = ??? } diff --git a/tests/pos/quotable.scala b/tests/pos/liftable.scala similarity index 90% rename from tests/pos/quotable.scala rename to tests/pos/liftable.scala index f49eabf95c2d..779c85b8cb2c 100644 --- a/tests/pos/quotable.scala +++ b/tests/pos/liftable.scala @@ -17,7 +17,7 @@ object Test { if (b) '(true) else '(false) } - implicit def ListIsQuotable[T: Type: Quotable]: Quotable[List[T]] = new { + implicit def ListIsQuotable[T: Quotable]: Quotable[List[T]] = new { def toExpr(xs: List[T]): Expr[List[T]] = xs match { case x :: xs1 => '{ ~implicitly[Quotable[T]].toExpr(x) :: ~toExpr(xs1) } case Nil => '(Nil: List[T]) From b90ed13da9ea466c6c0c45d7c32307d9bb20ef56 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 13:01:49 +0100 Subject: [PATCH 52/62] Add reference documentation This is still somewhere between a paper draft and true reference docs. But since it's the only thing we have right now, it's worthwhile to integrate it now and change it later. --- .../reference/symmetric-meta-programming.md | 745 ++++++++++++++++++ docs/sidebar.yml | 6 +- 2 files changed, 749 insertions(+), 2 deletions(-) create mode 100644 docs/docs/reference/symmetric-meta-programming.md diff --git a/docs/docs/reference/symmetric-meta-programming.md b/docs/docs/reference/symmetric-meta-programming.md new file mode 100644 index 000000000000..cc3abf91dcd4 --- /dev/null +++ b/docs/docs/reference/symmetric-meta-programming.md @@ -0,0 +1,745 @@ +# Symmetric Meta Programming + +Symmetric meta programming is a new framework for staging and certain +forms of macros. It is is expressed as strongly and statically typed +code using two fundamental operations: quotations and splicing. A +novel aspect of the approach is that these two operations are +regulated by a phase consistency principle that treats splices and +quotes in exactly the same way. + +## Overview + +### Quotes and Splices + +Symmetric meta programming is built on two well-known fundamental +operations: quotation and splicing. Quotation is expressed as +`'(...)` or `'{...}` for expressions (both forms are equivalent) and +as `'[...]` for types. Splicing is expressed as a prefix `~` operator. + +For example, the code below presents an inline function `assert` +which calls at compile-time a method `assertImpl` with a boolean +expression tree as argument. `assertImpl` evaluates the expression and +prints it again in an error message if it evaluates to `false`. + + import scala.quoted._ + + inline def assert(expr: => Boolean): Unit = + ~ assertImpl('(expr)) + + def assertImpl(expr: Expr[Boolean]) = + '{ if !(~expr) then throw new AssertionError(s"failed assertion: ${~expr}") } + + +If `e` is an expression, then `'(e)` or `'{e}` represent the typed +abstract syntax tree representing `e`. If `T` is a type, then `'[T]` +represents the type structure representing `T`. The precise +definitions of "typed abstract syntax tree" or "type-structure" do not +matter for now, the terms are used only to give some +intuition. Conversely, `~ e` evaluates the expression `e`, which must +yield a typed abstract syntax tree or type structure, and embeds the +result as an expression (respectively, type) in the enclosing program. + +Quotations can have spliced parts in them; in this case the embedded +splices are evaluated and embedded as part of the formation of the +quotation. + +Quotes and splices are duals of each other. For arbitrary +expressions `e` and types `T` we have: + + ~'(e) = e + '(~e) = e + ~'[T] = T + '[~T] = T + +### Types for Quotations + +The type signatures of quotes and splices can be described using +two fundamental types: + + - `Expr[T]`: abstract syntax trees representing expressions of type `T` + - `Type[T]`: type structures representing type `T`. + +Quoting takes expressions of type `T` to expressions of type `Expr[T]` +and it takes types `T` to expressions of type `Type[T]`. Splicing +takes expressions of type `Expr[T]` to expressions of type `T` and it +takes expressions of type `Type[T]` to types `T`. + +The two types can be are defined in package `scala.quoted` as follows: + + package scala.quoted + + abstract class Expr[T] { + def unary_~: T // splice operation + } + class Type[T] { + type unary_~ = T // splice type + } + +The `Expr` class has a `private` constructor, so only + +### The Phase Consistency Principle + +A fundamental *phase consistency principle* (PCP) regulates accesses +to free variables in quoted and spliced code: + + - _For any free variable reference `x`, the number of quoted scopes and the number of spliced scopes between the reference to `x` and the definition of `x` must be equal_. + +Here, `this`-references count as free variables. On the other +hand, we assume that all imports are fully expanded and that `_root_` is +not a free variable. So references to global definitions are +allowed everywhere. + +The phase consistency principle can be motivated as follows: First, +suppose the result of a program `P` is some quoted text `'{ ... x +... }` that refers to a free variable `x` in `P` This can be +represented only by referring to original the variable `x`. Hence, the +result of the program will need to persist the program state itself as +one of its parts. We don't want to do this, hence this situation +should be made illegal. Dually, suppose a top-level part of a program +is a spliced text `~{ ... x ... }` that refers to a free variable `x` +in `P`. This would mean that we refer during _construction_ of `P` to +a value that is available only during _execution_ of `P`. This is of +course impossible and therefore needs to be ruled out. Now, the +small-step evaluation of a program will reduce quotes and splices in +equal measure using the cancellation rules above. But it will neither +create nor remove quotes or splices individually. So the PCP ensures +that program elaboration will lead to neither of the two unwanted +situations described above. + +In the the range of features it covers, symmetric meta programming is +quite close to the MetaML family. One difference is that MetaML does +not have an equivalent of the PCP - quoted code in MetaML _can_ access +variables in its immediately enclosing environment, with some +restrictions and caveats since such accesses involve serialization. +However, this does not constitute a fundamental gain in +expressiveness. Symmetric meta programming allows to define a `Liftable` +type-class which can implement such accesses within the confines of the +PCP. This is explained further in a later section. + +## Details + +### `Expr` as an Applicative + +We postulate an implicit "Apply" decorator that turns a tree +describing a function into a function mapping trees to trees. + + implicit class AsApplicative[T, U](f: Expr[T => U]) extends AnyVal { + def apply(x: Expr[T]): Expr[U] = ??? + } + +This decorator turns `Expr` into an applicative functor, where `Expr`s +over function types can be applied to `Expr` arguments. The definition +of `AsApplicative(f).apply(x)` is assumed to be functionally the same as +`'((~f)(~x))`, however it should optimize this call by returning the +result of beta-reducing `f(x)` if `f` is a known lambda expression + +The `AsApplicative` decorator distributes applications of `Expr` over function +arrows: + + AsApplicative(_).apply: Expr[S => T] => (Expr[S] => Expr[T]) + +The dual of expansion, let's call it `reflect`, can be defined as follows: + + def reflect[T, U](f: Expr[T] => Expr[U]): Expr[T => U] = '{ + (x: T) => ~f('(x)) + } + +Note how the fundamental phase consistency principle works in two +different directions here for `f` and `x`. The reference to `f` is +legal because it is quoted, then spliced, whereas the reference to `x` +is legal because it is spliced, then quoted. + +### Types and the PCP + +In principle, The phase consistency principle applies to types as well +as for expressions. This might seem too restrictive. Indeed, the +definition of `reflect` above is not phase correct since there is a +quote but no splice between the parameter binding of `T` and its +usage. But the code can be made phase correct by adding a binding +of a `Type[T]` tag: + + def reflect[T, U](f: Expr[T] => Expr[U]): Expr[T => U] = { + val Ttag = new Type[T] + '{ (x: ~Ttag) => ~f('(x)) + } + +To avoid clutter, the Scala implementation will add these tags +automatically in the case of a PCP violation involving types. As a consequence, +types can be effectively ignored for phase consistency checking. + +### Example Expansion + +Assume an `Array` class with an inline `map` method that forwards to a macro implementation. + + class Array[T] { + inline def map[U](f: T => U): Array[U] = ~ Macros.mapImpl[T, U]('[U], '(this), '(f)) + } + +Here's the definition of the `mapImpl` macro, which takes quoted types and expressions to a quoted expression: + + object Macros { + + def mapImpl[T, U](u: Type[U], arr: Expr[Array[T]], op: Expr[T => U])(implicit ctx: Context): Expr[Array[U]] = '{ + var i = 0 + val xs = ~arr + var len = xs.length + val ys = new Array[~u] + while (i < len) { + ys(i) = ~op('(xs(i))) + i += 1 + } + ys + } + } + +Here's an application of `map` and how it rewrites to optimized code: + + genSeq[Int]().map(x => x + 1) + +==> (inline) + + val $this: Seq[Int] = genSeq[Int]() + val f: Int => Int = x => x * x + ~ _root_.Macros.mapImpl[Int, Int]('[Int], '($this), '(f)) + +==> (splice) + + val $this: Seq[Int] = genSeq[Int]() + val f: Int => Int = x => x * x + + { + var i = 0 + val xs = ~'($this) + var len = xs.length + val ys = new Array[~'[Int]] + while (i < len) { + ys(i) = ~('(f)('(xs(i)))) + i += 1 + } + ys + } + +==> (expand and splice inside quotes) + + val $this: Seq[Int] = genSeq[Int]() + val f: Int => Int = x => x * x + + { + var i = 0 + val xs = $this + var len = xs.length + val ys = new Array[Int] + while (i < len) { + ys(i) = xs(i) + 1 + i += 1 + } + ys + } + +==> (elim dead code) + + val $this: Seq[Int] = genSeq[Int]() + + { + var i = 0 + val xs = $this + var len = xs.length + val ys = new Array[Int] + while (i < len) { + ys(i) = xs(i) + 1 + i += 1 + } + ys + } + + +### Relationship with Inline and Macros + +Seen by itself, principled meta-programming looks more like a +framework for staging than one for compile-time meta programming with +macros. But combined with Dotty's `inline` it can be turned into a +compile-time system. The idea is that macro elaboration can be +understood as a combination of a macro library and a quoted +program. For instance, here's the `assert` macro again together with a +program that calls `assert`. + + object Macros { + + inline def assert(expr: => Boolean): Unit = + ~ assertImpl('(expr)) + + def assertImpl(expr: Expr[Boolean]) = + '{ if !(~expr) then throw new AssertionError(s"failed assertion: ${~expr}") } + } + + val program = { + val x = 1 + Macros.assert(x != 0) + } + +Inlining the `assert` function would give the following program: + + val program = { + val x = 1 + ~Macros.assertImpl('(x != 0)) + } + +The example is only phase correct because Macros is a global value and +as such not subject to phase consistency checking. Conceptually that's +a bit unsatisfactory. If the PCP is so fundamental, it should be +applicable without the global value exception. But in the example as +given this does not hold since both `assert` and `program` call +`assertImpl` with a splice but no quote. + +However, one can could argue that the example is really missing +an important aspect: The macro library has to be compiled in a phase +prior to the program using it, but in the code above, macro +and program are defined together. A more accurate view of +macros would be to have the user program be in a phase after the macro +definitions, reflecting the fact that macros have to be defined and +compiled before they are used. Hence, conceptually the program part +should be treated by the compiler as if it was quoted: + + val program = '{ + val x = 1 + ~Macros.assertImpl('(x != 0)) + } + +If `program` is treated as a quoted expression, the call to +`Macro.assertImpl` becomes phase correct even if macro library and +program are conceptualized as local definitions. + +But what about the call from `assert` to `assertImpl? Here, we need a +tweak of the typing rules. An inline function such as `assert` that +contains a splice operation outside an enclosing quote is called a +_macro_. Macros are supposed to be expanded in a subsequent phase, +i.e. in a quoted context. Therefore, they are also type checked as if +they were in a quoted context, For instance, the definition of +`assert` is typechecked as if it appeared inside quotes. This makes +the call from `assert` to `assertImpl` phase-correct, even if we +assume that both definitions are local. + +The second role of `inline` in Dotty is to mark a `val` that is +constant or a parameter that will be constant when instantiated. This +aspect is also important for macro expansion. To illustrate this, +consider an implementation of the `power` function that makes use of a +statically known exponent: + + inline def power(inline n: Int, x: Double) = ~powerCode(n, '(x)) + + private def powerCode(n: Int, x: Expr[Double]): Expr[Double] = + if (n == 0) '(1.0) + else if (n == 1) x + else if (n % 2 == 0) '{ { val y = ~x * ~x; ~powerCode(n / 2, '(y)) } } + else '{ ~x * ~powerCode(n - 1, x) } + +The usage of `n` as an argument in `~powerCode(n, '(x))` is not +phase-consistent, since `n` appears in a splice without an enclosing +quote. Normally that would be a problem because it means that we need +the _value_ of `n` at compile time, which is not available for general +parameters. But since `n` is an inline parameter of a macro, we know +that at the macro's expansion point `n` will be instantiated to a +constant, so the value of `n` will in fact be known at this +point. To reflect this, we loosen the phase consistency requirements +as follows: + + - If `x` is an inline value (or an inline parameter of an inline +function), it can be accessed in all contexts where the number of +splices minus the number of quotes between use and definition is +either 0 or 1. + +### Relationship with Staging + +The framework expresses at the same time compile-time meta-programming +and staging. The phase in which code is run is determined by the +difference between the number of splice scopes and quote scopes in +which it is embedded. + + - If there are more splices than quotes, the code is run at + "compile-time" i.e. as a macro. In the general case, this means + running an interpreter that evaluates the code, which is + represented as a typed abstract syntax tree. The interpreter can + fall back to reflective calls when evaluating an application of a + previously compiled method. If the splice excess is more than one, + it would mean that a macro's implementation code (as opposed to the + code it expands to) invokes other macros. If macros are realized by + interpretation, this would lead to towers of interpreters, where + the first interpreter would itself interpret an interpreter code + that possibly interprets another interpreter and so on. + + - If the number of splices equals the number of quotes, the code is + compiled and run as usual. + + - If the number of quotes exceeds the number of splices, the code is + staged. That is, it produces a typed abstract syntax tree or type + structure at run-time. A quote excess of more than one corresponds + to multi-staged programming. + +Providing an interpreter for the full language is quite difficult, and +it is even more difficult to make that interpreter run efficiently. So +we currently impose the following restrictions on the use of splices. + + 1. A top-level splice must appear in an inline function (turning that function + into a macro) + + 2. The splice must call a previously compiled method. + + 3. Splices inside splices (but no intervening quotes) are not allowed. + + 4. A macro method is effectively final and it may override no other method. + +The framework as discussed so far allows code to be staged, i.e. be prepared +to be executed at a later stage. To run that code, there is another method +in class `Expr` called `run`. Note that `~` and `run` both map from `Expr[T]` +to `T` but only `~` is subject to the PCP, whereas `run` is just a normal method. + + abstract class Expr[T] { + def unary_~: T + def run: T // run staged code + } + +### The `Liftable` type-class + +Consider the following implementation of a staged interpreter that implements +a compiler through staging. + + import scala.quoted._ + + enum Exp { + case Num(n: Int) + case Plus(e1: Exp, e2: Exp) + case Var(x: String) + case Let(x: String, e: Exp, in: Exp) + } + +The interpreted language consists of numbers `Num`, addition `Plus`, and variables +`Var` which are bound by `Let`. Here are two sample expressions in the language: + + val exp = Plus(Plus(Num(2), Var("x")), Num(4)) + val letExp = Let("x", Num(3), exp) + +Here's a compiler that maps an expression given in the interpreted +language to quoted Scala code of type `Expr[Int]`. +The compiler takes an environment that maps variable names to Scala `Expr`s. + + def compile(e: Exp, env: Map[String, Expr[Int]]): Expr[Int] = e match { + case Num(n) => + n + case Plus(e1, e2) => + '(~compile(e1, env) + ~compile(e2, env)) + case Var(x) => + env(x) + case Let(x, e, body) => + '{ val y = ~compile(e, env); ~compile(body, env + (x -> '(y))) } + } + +Running `compile(letExp, Map())` would yield the following Scala code: + + '{ val y = 3; (2 + y) + 4 } + +The body the first clause (`case Num(n) => n`) looks suspicious. `n` +is declared as an `Int`, yet the result of `compile` is declared to be +`Expr[Int]`. Shouldn't `n be quoted? The answer is that this would not +work since replacing `n by `'n` in the clause would not be phase +correct. + +What happens instead "under the hood" is an implicit conversion: `n` +is expanded to `scala.quoted.Expr.toExpr(n)`. The `toExpr` conversion +is defined in the companion object of class `Expr` as follows: + + object Expr { + implicit def toExpr[T](x: T)(implicit ev: Liftable[T]): Expr[T] = + ev.toExpr(x) + } + +The conversion says that values of types implementing the `Liftable` +type class can be converted ("lifted") automatically to `Expr` +values. Dotty comes with instance definitions of `Liftable` for +several types including all underlying types of literals. For instance +`Int` values can be converted to `Expr[Int]` values by wrapping the +value in a `Literal` tree node. This makes use of the underlying tree +representation in the compiler for efficiency. But the `Liftable` +instances are nevertheless not "magic" in the sense that they could +all be defined in a user program without knowing anything about the +representation of `Expr` trees. For instance, here is a possible +instance of `Liftable[Boolean]`: + + implicit def BooleanIsLiftable: Liftable[Boolean] = new { + implicit def toExpr(b: Boolean) = if (b) '(true) else '(false) + } + +Once we can lift bits, we can work our way up. For instance, here is a +possible implementation of `Liftable[Int]` that does not use the underlying +tree machinery: + + implicit def IntIsLiftable: Liftable[Int] = new { + def toExpr(n: Int): Expr[Int] = n match { + case Int.MinValue => '(Int.MinValue) + case _ if n < 0 => '(-(~toExpr(n))) + case 0 => '(0) + case _ if n % 2 == 0 => '(~toExpr(n / 2) * 2) + case _ => '(~toExpr(n / 2) * 2 + 1) + } + } + +Since `Liftable` is a type class, instances can be conditional. For instance +a `List` is liftable if its element type is: + + implicit def ListIsLiftable[T: Liftable]: Liftable[List[T]] = new { + def toExpr(xs: List[T]): Expr[List[T]] = xs match { + case x :: xs1 => '(~implicitly[Liftable[T]].toExpr(x) :: ~toExpr(xs1)) + case Nil => '(Nil: List[T]) + } + } + +In the end, `Liftable` resembles very much a serialization +framework. Like the latter it can be derived systematically for all +collections, case classes and enums. + +## Implementation + +### Syntax changes: + +A splice `~e` on an expression of type `Expr[T]` is a normal prefix +operator. To make it work as a type operator on `Type[T]` as well, we +need a syntax change that introduces prefix operators as types. + + SimpleType ::= ... + [‘-’ | ‘+’ | ‘~’ | ‘!’] StableId + +Analogously to the situation with expressions a prefix type operator +such as `~ e` is treated as a shorthand for the type `e.unary_~`. + +Quotes are supported by introducing new tokens `'(`, `'{`, and `'[` +and adding quoted variants `'(...)`, `'{...}` and `'[...]` to the +`SimpleExpr` productions. + + SimpleExpr ::= ... + | ‘'{’ BlockExprContents ‘}’ + | ‘'’ ‘(’ ExprsInParens ‘)’ + | ‘'’ ‘[’ Type ‘]’ + +Syntax changes are given relative to the [Dotty reference +grammar](../internal/syntax.md). + +An alternative syntax would treat `'` as a separate operator. This +would be attractive since it enables quoting single identifies as +e.g. `'x` instead of `'(x)`. But it would clash with symbol +literals. So it could be done only if symbol literals were abolished. + +### Implementation in `dotc` + +Quotes and splices are primitive forms in the generated abstract +syntax trees. They are eliminated in an expansion phase +`ReifyQuotes`. This phase runs after typing and pickling. + +Macro-expansion works outside-in. If the outermost scope is a splice, +the spliced AST will be evaluated in an interpreter. A call to a +previously compiled method can be implemented as a reflective call to +that method. With the restrictions on splices that are currently in +place that's all that's needed. We might allow more interpretation in +splices in the future, which would allow us to loosen the +restriction. Quotes in spliced, interpreted code are kept as they +are, after splices nested in the quotes are expanded recursively. + +If the outermost scope is a quote, we need to generate code that +constructs the quoted tree at run-time. We implement this by +serializing the tree as a Tasty structure, which is stored +in a string literal. At runtime, an unpickler method is called to +deserialize the string into a tree. + +Splices inside quoted code insert the spliced tree as is, after +expanding any quotes in the spliced code recursively. + +## Formalization + +The phase consistency principle can be formalized in a calculus that +extends simply-typed lambda calculus with quotes and splices. + +### Syntax + +The syntax of terms, values, and types is given as follows: + + Terms t ::= x variable + (x: T) => t lambda + t t application + 't quote + ~t splice + + Values v ::= (x: T) => t lambda + 'q pure quote + + Quoted q ::= x | (x: T) => q | q q | 't + + Types T ::= A base type + T -> T function type + expr T quoted + +Typing rules are formulated using a stack of environments +`Es`. Individual environments `E` consist as usual of variable +bindings `x: T`. Environments can be combined using one of two +combinators `'` and `~`. + + Environment E ::= () empty + E, x: T + + Env. stack Es ::= () empty + E simple + Es * Es combined + + Separator * ::= ' + ~ + +The two environment combinators are both associative with left and +right identity `()`. The initial environment contains among other +predefined operations also a "run" operation `! : expr T => T`. That +is, `!` is treated just like any other function. + +### Operational semantics: + +We define a small step reduction relation `-->` with the following rules: + + ((x: T) => t) v --> [x := v]t + + ~('t) --> t + + t --> t' + ---------------- + e[t] --> e[t'] + +The first rule is standard call-by-value beta-reduction. The second +rule says that splice and quotes cancel each other out. The third rule +is a context rule; it says that reduction is allowed in the hole `[ ]` +position of an evaluation contexts. Evaluation contexts `e` and +splice evaluation context `e_s` are defined syntactically as follows: + + Eval context e ::= [ ] | e t | v e | 'e_s[~e] + Splice context e_s ::= [ ] | (x: T) => e_s | e_s t | q e_s + +### Typing rules + +Typing judgments are of the form `Es |- t: T`. There are two +substructural rules which express the fact that quotes and splices +cancel each other out: + + Es1 * Es2 |- t: T + --------------------------- + Es1 ~ E1 ' E2 * Es2 |- t: T + + + Es1 * Es2 |- t: T + --------------------------- + Es1 ' E1 ~ E2 * Es2 |- t: T + +The lambda calculus fragment of the rules is standard, except that we +use a stack of environments. The rules only interact with the topmost +environment of the stack. + + x: T in E + -------------- + Es * E |- x: T + + + Es * E, x: T |- t: T' + ------------------------------ + Es * E |- (x: T) => t: T -> T' + + + Es |- t: T' -> T Es |- t': T' + -------------------------------- + Es |- t t': T + + +The rules for quotes and splices map between `expr T` and `T` by trading `'` and `~` between +environments and terms. + + Es ~ () |- t: expr T + -------------------- + Es |- ~t: T + + + Es ' () |- t: T + ---------------- + Es |- 't: expr T + +## Going Further + +The presented meta-programming framework is so far quite restrictive +in that it does not allow for the inspection of quoted expressions and +types. It's possible to work around this by providing all necessary +information as normal, unquoted inline parameters. But we would gain +more flexibility by allowing for the inspection of quoted code with +pattern matching. This opens new possibilities. For instance, here is a +version of `power` that generates the multiplications directly if the +exponent is statically known and falls back to the dynamic +implementation of power otherwise. + + inline def power(n: Int, x: Double): Double = ~{ + '(n) match { + case Constant(n1) => powerCode(n1, '(x)) + case _ => '{ dynamicPower(n, x) } + } + } + + private def dynamicPower(n: Int, x: Double): Double = + if (n == 0) 1.0 + else if (n % 2 == 0) dynamicPower(n / 2, x * x) + else x * dynamicPower(n - 1, x) + +This assumes a `Constant` extractor that maps tree nodes representing +constants to their values. + +Once we allow for inspection of code, the "AsApplicative" operation +that maps expressions over functions to functions over expressions can +be implemented in user code: + + implicit class AsApplicative[T, U](f: Expr[T => U]) extends AnyVal { + def apply(x: Expr[T]): Expr[U] = + f match { + case Lambda(g) => g(x) + case _ => '((~f)(~x)) + } + +This assumes an extractor + + object Lambda { + def unapply[T, U](x: Expr[T => U]): Option[Expr[T] => Expr[U]] + } + +Once we allow inspection of code via extractors, it's tempting to also +add constructors that construct typed trees directly without going +through quotes. Most likely, those constructors would work over `Expr` +types which lack a known type argument. For instance, an `Apply` +constructor could be typed as follows: + + def Apply(fn: Expr[_], args: List[Expr[_]]): Expr[_] + +This would allow constructing applications from lists of arguments +without having to match the arguments one-by-one with the +corresponding formal parameter types of the function. We need then "at +the end" a method to convert an `Expr[_]` to an `Expr[T]` where `T` is +given from the outside. E.g. if `code` yields a `Expr[_]`, then +`code.atType[T]` yields an `Expr[T]`. The `atType` method has to be +implemented as a primitive; it would check that the computed type +structure of `Expr` is a subtype of the type structure representing +`T`. + +Before going down that route, we should carefully evaluate its +tradeoffs. Constructing trees that are only verified _a posteriori_ +to be type correct loses a lot of guidance for constructing the right +trees. So we should wait with this addition until we have more +use-cases that help us decide whether the loss in type-safety is worth +the gain in flexibility. + + +## Conclusion + +Meta-programming always made my head hurt (I believe that's the same +for many people). But with explicit `Expr/Type` types and quotes and +splices it has become downright pleasant. The method I was following +was to first define the underlying quoted or unquoted types using +`Expr` and `Type` and then insert quotes and splices to make the types +line up. Phase consistency was at the same time a great guideline +where to insert a splice or a quote and a vital sanity check that I +did the right thing. + diff --git a/docs/sidebar.yml b/docs/sidebar.yml index 09228fc8d360..baca398e3e89 100644 --- a/docs/sidebar.yml +++ b/docs/sidebar.yml @@ -43,10 +43,12 @@ sidebar: subsection: - title: Multiversal Equality url: docs/reference/multiversal-equality.html - - title: Inline - url: docs/reference/inline.html - title: Trait Parameters url: docs/reference/trait-parameters.html + - title: Inline + url: docs/reference/inline.html + - title: Symmetric Meta Programming + url: docs/reference/symmetric-meta-programming.html - title: By-Name Implicits url: docs/reference/implicit-by-name-parameters.html - title: Auto Parameter Tupling From 21ad0dbb8978e3c147f14fed8639c537d8f53ac7 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 14:08:14 +0100 Subject: [PATCH 53/62] Fix test --- tests/pos/liftable.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pos/liftable.scala b/tests/pos/liftable.scala index 779c85b8cb2c..02023994d9e7 100644 --- a/tests/pos/liftable.scala +++ b/tests/pos/liftable.scala @@ -2,7 +2,7 @@ import scala.quoted._ object Test { - implicit def IntIsQuotable: Quotable[Int] = new { + implicit def IntIsLiftable: Liftable[Int] = new { def toExpr(n: Int): Expr[Int] = n match { case Int.MinValue => '(Int.MinValue) case _ if n < 0 => '(-(~toExpr(n))) @@ -12,14 +12,14 @@ object Test { } } - implicit def BooleanIsQuotable: Quotable[Boolean] = new { + implicit def BooleanIsLiftable: Liftable[Boolean] = new { implicit def toExpr(b: Boolean) = if (b) '(true) else '(false) } - implicit def ListIsQuotable[T: Quotable]: Quotable[List[T]] = new { + implicit def ListIsLiftable[T: Liftable]: Liftable[List[T]] = new { def toExpr(xs: List[T]): Expr[List[T]] = xs match { - case x :: xs1 => '{ ~implicitly[Quotable[T]].toExpr(x) :: ~toExpr(xs1) } + case x :: xs1 => '{ ~implicitly[Liftable[T]].toExpr(x) :: ~toExpr(xs1) } case Nil => '(Nil: List[T]) } } From b0437ceda697a8ddc655973f3461a0d1622c5670 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 14:26:48 +0100 Subject: [PATCH 54/62] Fix syntax highlighting in docs Rename ' to a unicode quote character so that syntax highlighters don't get confused. --- .../reference/symmetric-meta-programming.md | 176 +++++++++--------- 1 file changed, 87 insertions(+), 89 deletions(-) diff --git a/docs/docs/reference/symmetric-meta-programming.md b/docs/docs/reference/symmetric-meta-programming.md index cc3abf91dcd4..4e8f42f39feb 100644 --- a/docs/docs/reference/symmetric-meta-programming.md +++ b/docs/docs/reference/symmetric-meta-programming.md @@ -24,14 +24,14 @@ prints it again in an error message if it evaluates to `false`. import scala.quoted._ inline def assert(expr: => Boolean): Unit = - ~ assertImpl('(expr)) + ~ assertImpl(’(expr)) def assertImpl(expr: Expr[Boolean]) = - '{ if !(~expr) then throw new AssertionError(s"failed assertion: ${~expr}") } + ’{ if !(~expr) then throw new AssertionError(s"failed assertion: ${~expr}") } -If `e` is an expression, then `'(e)` or `'{e}` represent the typed -abstract syntax tree representing `e`. If `T` is a type, then `'[T]` +If `e` is an expression, then `’(e)` or `’{e}` represent the typed +abstract syntax tree representing `e`. If `T` is a type, then `’[T]` represents the type structure representing `T`. The precise definitions of "typed abstract syntax tree" or "type-structure" do not matter for now, the terms are used only to give some @@ -46,10 +46,10 @@ quotation. Quotes and splices are duals of each other. For arbitrary expressions `e` and types `T` we have: - ~'(e) = e - '(~e) = e - ~'[T] = T - '[~T] = T + ~’(e) = e + ’(~e) = e + ~’[T] = T + ’[~T] = T ### Types for Quotations @@ -75,8 +75,6 @@ The two types can be are defined in package `scala.quoted` as follows: type unary_~ = T // splice type } -The `Expr` class has a `private` constructor, so only - ### The Phase Consistency Principle A fundamental *phase consistency principle* (PCP) regulates accesses @@ -90,11 +88,11 @@ not a free variable. So references to global definitions are allowed everywhere. The phase consistency principle can be motivated as follows: First, -suppose the result of a program `P` is some quoted text `'{ ... x +suppose the result of a program `P` is some quoted text `’{ ... x ... }` that refers to a free variable `x` in `P` This can be represented only by referring to original the variable `x`. Hence, the result of the program will need to persist the program state itself as -one of its parts. We don't want to do this, hence this situation +one of its parts. We don’t want to do this, hence this situation should be made illegal. Dually, suppose a top-level part of a program is a spliced text `~{ ... x ... }` that refers to a free variable `x` in `P`. This would mean that we refer during _construction_ of `P` to @@ -130,7 +128,7 @@ describing a function into a function mapping trees to trees. This decorator turns `Expr` into an applicative functor, where `Expr`s over function types can be applied to `Expr` arguments. The definition of `AsApplicative(f).apply(x)` is assumed to be functionally the same as -`'((~f)(~x))`, however it should optimize this call by returning the +`’((~f)(~x))`, however it should optimize this call by returning the result of beta-reducing `f(x)` if `f` is a known lambda expression The `AsApplicative` decorator distributes applications of `Expr` over function @@ -138,10 +136,10 @@ arrows: AsApplicative(_).apply: Expr[S => T] => (Expr[S] => Expr[T]) -The dual of expansion, let's call it `reflect`, can be defined as follows: +The dual of expansion, let’s call it `reflect`, can be defined as follows: - def reflect[T, U](f: Expr[T] => Expr[U]): Expr[T => U] = '{ - (x: T) => ~f('(x)) + def reflect[T, U](f: Expr[T] => Expr[U]): Expr[T => U] = ’{ + (x: T) => ~f(’(x)) } Note how the fundamental phase consistency principle works in two @@ -160,7 +158,7 @@ of a `Type[T]` tag: def reflect[T, U](f: Expr[T] => Expr[U]): Expr[T => U] = { val Ttag = new Type[T] - '{ (x: ~Ttag) => ~f('(x)) + ’{ (x: ~Ttag) => ~f(’(x)) } To avoid clutter, the Scala implementation will add these tags @@ -172,27 +170,27 @@ types can be effectively ignored for phase consistency checking. Assume an `Array` class with an inline `map` method that forwards to a macro implementation. class Array[T] { - inline def map[U](f: T => U): Array[U] = ~ Macros.mapImpl[T, U]('[U], '(this), '(f)) + inline def map[U](f: T => U): Array[U] = ~ Macros.mapImpl[T, U](’[U], ’(this), ’(f)) } -Here's the definition of the `mapImpl` macro, which takes quoted types and expressions to a quoted expression: +Here’s the definition of the `mapImpl` macro, which takes quoted types and expressions to a quoted expression: object Macros { - def mapImpl[T, U](u: Type[U], arr: Expr[Array[T]], op: Expr[T => U])(implicit ctx: Context): Expr[Array[U]] = '{ + def mapImpl[T, U](u: Type[U], arr: Expr[Array[T]], op: Expr[T => U])(implicit ctx: Context): Expr[Array[U]] = ’{ var i = 0 val xs = ~arr var len = xs.length val ys = new Array[~u] while (i < len) { - ys(i) = ~op('(xs(i))) + ys(i) = ~op(’(xs(i))) i += 1 } ys } } -Here's an application of `map` and how it rewrites to optimized code: +Here’s an application of `map` and how it rewrites to optimized code: genSeq[Int]().map(x => x + 1) @@ -200,7 +198,7 @@ Here's an application of `map` and how it rewrites to optimized code: val $this: Seq[Int] = genSeq[Int]() val f: Int => Int = x => x * x - ~ _root_.Macros.mapImpl[Int, Int]('[Int], '($this), '(f)) + ~ _root_.Macros.mapImpl[Int, Int](’[Int], ’($this), ’(f)) ==> (splice) @@ -209,11 +207,11 @@ Here's an application of `map` and how it rewrites to optimized code: { var i = 0 - val xs = ~'($this) + val xs = ~’($this) var len = xs.length - val ys = new Array[~'[Int]] + val ys = new Array[~’[Int]] while (i < len) { - ys(i) = ~('(f)('(xs(i)))) + ys(i) = ~(’(f)(’(xs(i)))) i += 1 } ys @@ -257,19 +255,19 @@ Here's an application of `map` and how it rewrites to optimized code: Seen by itself, principled meta-programming looks more like a framework for staging than one for compile-time meta programming with -macros. But combined with Dotty's `inline` it can be turned into a +macros. But combined with Dotty’s `inline` it can be turned into a compile-time system. The idea is that macro elaboration can be understood as a combination of a macro library and a quoted -program. For instance, here's the `assert` macro again together with a +program. For instance, here’s the `assert` macro again together with a program that calls `assert`. object Macros { inline def assert(expr: => Boolean): Unit = - ~ assertImpl('(expr)) + ~ assertImpl(’(expr)) def assertImpl(expr: Expr[Boolean]) = - '{ if !(~expr) then throw new AssertionError(s"failed assertion: ${~expr}") } + ’{ if !(~expr) then throw new AssertionError(s"failed assertion: ${~expr}") } } val program = { @@ -281,11 +279,11 @@ Inlining the `assert` function would give the following program: val program = { val x = 1 - ~Macros.assertImpl('(x != 0)) + ~Macros.assertImpl(’(x != 0)) } The example is only phase correct because Macros is a global value and -as such not subject to phase consistency checking. Conceptually that's +as such not subject to phase consistency checking. Conceptually that’s a bit unsatisfactory. If the PCP is so fundamental, it should be applicable without the global value exception. But in the example as given this does not hold since both `assert` and `program` call @@ -300,9 +298,9 @@ definitions, reflecting the fact that macros have to be defined and compiled before they are used. Hence, conceptually the program part should be treated by the compiler as if it was quoted: - val program = '{ + val program = ’{ val x = 1 - ~Macros.assertImpl('(x != 0)) + ~Macros.assertImpl(’(x != 0)) } If `program` is treated as a quoted expression, the call to @@ -325,20 +323,20 @@ aspect is also important for macro expansion. To illustrate this, consider an implementation of the `power` function that makes use of a statically known exponent: - inline def power(inline n: Int, x: Double) = ~powerCode(n, '(x)) + inline def power(inline n: Int, x: Double) = ~powerCode(n, ’(x)) private def powerCode(n: Int, x: Expr[Double]): Expr[Double] = - if (n == 0) '(1.0) + if (n == 0) ’(1.0) else if (n == 1) x - else if (n % 2 == 0) '{ { val y = ~x * ~x; ~powerCode(n / 2, '(y)) } } - else '{ ~x * ~powerCode(n - 1, x) } + else if (n % 2 == 0) ’{ { val y = ~x * ~x; ~powerCode(n / 2, ’(y)) } } + else ’{ ~x * ~powerCode(n - 1, x) } -The usage of `n` as an argument in `~powerCode(n, '(x))` is not +The usage of `n` as an argument in `~powerCode(n, ’(x))` is not phase-consistent, since `n` appears in a splice without an enclosing quote. Normally that would be a problem because it means that we need the _value_ of `n` at compile time, which is not available for general parameters. But since `n` is an inline parameter of a macro, we know -that at the macro's expansion point `n` will be instantiated to a +that at the macro’s expansion point `n` will be instantiated to a constant, so the value of `n` will in fact be known at this point. To reflect this, we loosen the phase consistency requirements as follows: @@ -361,7 +359,7 @@ which it is embedded. represented as a typed abstract syntax tree. The interpreter can fall back to reflective calls when evaluating an application of a previously compiled method. If the splice excess is more than one, - it would mean that a macro's implementation code (as opposed to the + it would mean that a macro’s implementation code (as opposed to the code it expands to) invokes other macros. If macros are realized by interpretation, this would lead to towers of interpreters, where the first interpreter would itself interpret an interpreter code @@ -418,7 +416,7 @@ The interpreted language consists of numbers `Num`, addition `Plus`, and variabl val exp = Plus(Plus(Num(2), Var("x")), Num(4)) val letExp = Let("x", Num(3), exp) -Here's a compiler that maps an expression given in the interpreted +Here’s a compiler that maps an expression given in the interpreted language to quoted Scala code of type `Expr[Int]`. The compiler takes an environment that maps variable names to Scala `Expr`s. @@ -426,21 +424,21 @@ The compiler takes an environment that maps variable names to Scala `Expr`s. case Num(n) => n case Plus(e1, e2) => - '(~compile(e1, env) + ~compile(e2, env)) + ’(~compile(e1, env) + ~compile(e2, env)) case Var(x) => env(x) case Let(x, e, body) => - '{ val y = ~compile(e, env); ~compile(body, env + (x -> '(y))) } + ’{ val y = ~compile(e, env); ~compile(body, env + (x -> ’(y))) } } Running `compile(letExp, Map())` would yield the following Scala code: - '{ val y = 3; (2 + y) + 4 } + ’{ val y = 3; (2 + y) + 4 } The body the first clause (`case Num(n) => n`) looks suspicious. `n` is declared as an `Int`, yet the result of `compile` is declared to be -`Expr[Int]`. Shouldn't `n be quoted? The answer is that this would not -work since replacing `n by `'n` in the clause would not be phase +`Expr[Int]`. Shouldn’t `n be quoted? The answer is that this would not +work since replacing `n by `’n` in the clause would not be phase correct. What happens instead "under the hood" is an implicit conversion: `n` @@ -465,7 +463,7 @@ representation of `Expr` trees. For instance, here is a possible instance of `Liftable[Boolean]`: implicit def BooleanIsLiftable: Liftable[Boolean] = new { - implicit def toExpr(b: Boolean) = if (b) '(true) else '(false) + implicit def toExpr(b: Boolean) = if (b) ’(true) else ’(false) } Once we can lift bits, we can work our way up. For instance, here is a @@ -474,11 +472,11 @@ tree machinery: implicit def IntIsLiftable: Liftable[Int] = new { def toExpr(n: Int): Expr[Int] = n match { - case Int.MinValue => '(Int.MinValue) - case _ if n < 0 => '(-(~toExpr(n))) - case 0 => '(0) - case _ if n % 2 == 0 => '(~toExpr(n / 2) * 2) - case _ => '(~toExpr(n / 2) * 2 + 1) + case Int.MinValue => ’(Int.MinValue) + case _ if n < 0 => ’(-(~toExpr(n))) + case 0 => ’(0) + case _ if n % 2 == 0 => ’(~toExpr(n / 2) * 2) + case _ => ’(~toExpr(n / 2) * 2 + 1) } } @@ -487,8 +485,8 @@ a `List` is liftable if its element type is: implicit def ListIsLiftable[T: Liftable]: Liftable[List[T]] = new { def toExpr(xs: List[T]): Expr[List[T]] = xs match { - case x :: xs1 => '(~implicitly[Liftable[T]].toExpr(x) :: ~toExpr(xs1)) - case Nil => '(Nil: List[T]) + case x :: xs1 => ’(~implicitly[Liftable[T]].toExpr(x) :: ~toExpr(xs1)) + case Nil => ’(Nil: List[T]) } } @@ -498,7 +496,7 @@ collections, case classes and enums. ## Implementation -### Syntax changes: +### Syntax changes A splice `~e` on an expression of type `Expr[T]` is a normal prefix operator. To make it work as a type operator on `Type[T]` as well, we @@ -510,21 +508,21 @@ need a syntax change that introduces prefix operators as types. Analogously to the situation with expressions a prefix type operator such as `~ e` is treated as a shorthand for the type `e.unary_~`. -Quotes are supported by introducing new tokens `'(`, `'{`, and `'[` -and adding quoted variants `'(...)`, `'{...}` and `'[...]` to the +Quotes are supported by introducing new tokens `’(`, `’{`, and `’[` +and adding quoted variants `’(...)`, `’{...}` and `’[...]` to the `SimpleExpr` productions. SimpleExpr ::= ... - | ‘'{’ BlockExprContents ‘}’ - | ‘'’ ‘(’ ExprsInParens ‘)’ - | ‘'’ ‘[’ Type ‘]’ + | ‘’{’ BlockExprContents ‘}’ + | ‘’’ ‘(’ ExprsInParens ‘)’ + | ‘’’ ‘[’ Type ‘]’ Syntax changes are given relative to the [Dotty reference grammar](../internal/syntax.md). -An alternative syntax would treat `'` as a separate operator. This +An alternative syntax would treat `’` as a separate operator. This would be attractive since it enables quoting single identifies as -e.g. `'x` instead of `'(x)`. But it would clash with symbol +e.g. `’x` instead of `’(x)`. But it would clash with symbol literals. So it could be done only if symbol literals were abolished. ### Implementation in `dotc` @@ -537,7 +535,7 @@ Macro-expansion works outside-in. If the outermost scope is a splice, the spliced AST will be evaluated in an interpreter. A call to a previously compiled method can be implemented as a reflective call to that method. With the restrictions on splices that are currently in -place that's all that's needed. We might allow more interpretation in +place that’s all that’s needed. We might allow more interpretation in splices in the future, which would allow us to loosen the restriction. Quotes in spliced, interpreted code are kept as they are, after splices nested in the quotes are expanded recursively. @@ -563,13 +561,13 @@ The syntax of terms, values, and types is given as follows: Terms t ::= x variable (x: T) => t lambda t t application - 't quote + ’t quote ~t splice Values v ::= (x: T) => t lambda - 'q pure quote + ’q pure quote - Quoted q ::= x | (x: T) => q | q q | 't + Quoted q ::= x | (x: T) => q | q q | ’t Types T ::= A base type T -> T function type @@ -578,7 +576,7 @@ The syntax of terms, values, and types is given as follows: Typing rules are formulated using a stack of environments `Es`. Individual environments `E` consist as usual of variable bindings `x: T`. Environments can be combined using one of two -combinators `'` and `~`. +combinators `’` and `~`. Environment E ::= () empty E, x: T @@ -587,7 +585,7 @@ combinators `'` and `~`. E simple Es * Es combined - Separator * ::= ' + Separator * ::= ’ ~ The two environment combinators are both associative with left and @@ -601,11 +599,11 @@ We define a small step reduction relation `-->` with the following rules: ((x: T) => t) v --> [x := v]t - ~('t) --> t + ~(’t) --> t - t --> t' + t --> t’ ---------------- - e[t] --> e[t'] + e[t] --> e[t’] The first rule is standard call-by-value beta-reduction. The second rule says that splice and quotes cancel each other out. The third rule @@ -613,7 +611,7 @@ is a context rule; it says that reduction is allowed in the hole `[ ]` position of an evaluation contexts. Evaluation contexts `e` and splice evaluation context `e_s` are defined syntactically as follows: - Eval context e ::= [ ] | e t | v e | 'e_s[~e] + Eval context e ::= [ ] | e t | v e | ’e_s[~e] Splice context e_s ::= [ ] | (x: T) => e_s | e_s t | q e_s ### Typing rules @@ -624,12 +622,12 @@ cancel each other out: Es1 * Es2 |- t: T --------------------------- - Es1 ~ E1 ' E2 * Es2 |- t: T + Es1 ~ E1 ’ E2 * Es2 |- t: T Es1 * Es2 |- t: T --------------------------- - Es1 ' E1 ~ E2 * Es2 |- t: T + Es1 ’ E1 ~ E2 * Es2 |- t: T The lambda calculus fragment of the rules is standard, except that we use a stack of environments. The rules only interact with the topmost @@ -640,17 +638,17 @@ environment of the stack. Es * E |- x: T - Es * E, x: T |- t: T' + Es * E, x: T |- t: T’ ------------------------------ - Es * E |- (x: T) => t: T -> T' + Es * E |- (x: T) => t: T -> T’ - Es |- t: T' -> T Es |- t': T' + Es |- t: T’ -> T Es |- t’: T’ -------------------------------- - Es |- t t': T + Es |- t t’: T -The rules for quotes and splices map between `expr T` and `T` by trading `'` and `~` between +The rules for quotes and splices map between `expr T` and `T` by trading `’` and `~` between environments and terms. Es ~ () |- t: expr T @@ -658,15 +656,15 @@ environments and terms. Es |- ~t: T - Es ' () |- t: T + Es ’ () |- t: T ---------------- - Es |- 't: expr T + Es |- ’t: expr T ## Going Further The presented meta-programming framework is so far quite restrictive in that it does not allow for the inspection of quoted expressions and -types. It's possible to work around this by providing all necessary +types. It’s possible to work around this by providing all necessary information as normal, unquoted inline parameters. But we would gain more flexibility by allowing for the inspection of quoted code with pattern matching. This opens new possibilities. For instance, here is a @@ -675,9 +673,9 @@ exponent is statically known and falls back to the dynamic implementation of power otherwise. inline def power(n: Int, x: Double): Double = ~{ - '(n) match { - case Constant(n1) => powerCode(n1, '(x)) - case _ => '{ dynamicPower(n, x) } + ’(n) match { + case Constant(n1) => powerCode(n1, ’(x)) + case _ => ’{ dynamicPower(n, x) } } } @@ -697,7 +695,7 @@ be implemented in user code: def apply(x: Expr[T]): Expr[U] = f match { case Lambda(g) => g(x) - case _ => '((~f)(~x)) + case _ => ’((~f)(~x)) } This assumes an extractor @@ -706,7 +704,7 @@ This assumes an extractor def unapply[T, U](x: Expr[T => U]): Option[Expr[T] => Expr[U]] } -Once we allow inspection of code via extractors, it's tempting to also +Once we allow inspection of code via extractors, it’s tempting to also add constructors that construct typed trees directly without going through quotes. Most likely, those constructors would work over `Expr` types which lack a known type argument. For instance, an `Apply` @@ -734,7 +732,7 @@ the gain in flexibility. ## Conclusion -Meta-programming always made my head hurt (I believe that's the same +Meta-programming always made my head hurt (I believe that’s the same for many people). But with explicit `Expr/Type` types and quotes and splices it has become downright pleasant. The method I was following was to first define the underlying quoted or unquoted types using From 5fe6d15fc53bc61d78e9107c47d2fc85c2c1260f Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 15:12:05 +0100 Subject: [PATCH 55/62] Doecumentation fixes Also, add AsFunction implicit class to Expr. --- .../reference/symmetric-meta-programming.md | 66 ++++++++++--------- library/src/scala/quoted/Expr.scala | 4 ++ 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/docs/docs/reference/symmetric-meta-programming.md b/docs/docs/reference/symmetric-meta-programming.md index 4e8f42f39feb..124de4bea088 100644 --- a/docs/docs/reference/symmetric-meta-programming.md +++ b/docs/docs/reference/symmetric-meta-programming.md @@ -1,6 +1,6 @@ # Symmetric Meta Programming -Symmetric meta programming is a new framework for staging and certain +Symmetric meta programming is a new framework for staging and for some forms of macros. It is is expressed as strongly and statically typed code using two fundamental operations: quotations and splicing. A novel aspect of the approach is that these two operations are @@ -116,27 +116,30 @@ PCP. This is explained further in a later section. ## Details -### `Expr` as an Applicative +### From `Expr`s to Functions and Back -We postulate an implicit "Apply" decorator that turns a tree +The `Expr` companion object contains an "AsFunction" decorator that turns a tree describing a function into a function mapping trees to trees. - implicit class AsApplicative[T, U](f: Expr[T => U]) extends AnyVal { - def apply(x: Expr[T]): Expr[U] = ??? + object Expr { + ... + implicit class AsFunction[T, U](f: Expr[T => U]) extends AnyVal { + def apply(x: Expr[T]): Expr[U] = ??? + } } -This decorator turns `Expr` into an applicative functor, where `Expr`s +This decorator gives `Expr` the `apply` operation of an applicative functor, where `Expr`s over function types can be applied to `Expr` arguments. The definition -of `AsApplicative(f).apply(x)` is assumed to be functionally the same as +of `AsFunction(f).apply(x)` is assumed to be functionally the same as `’((~f)(~x))`, however it should optimize this call by returning the result of beta-reducing `f(x)` if `f` is a known lambda expression -The `AsApplicative` decorator distributes applications of `Expr` over function +The `AsFunction` decorator distributes applications of `Expr` over function arrows: - AsApplicative(_).apply: Expr[S => T] => (Expr[S] => Expr[T]) + AsFunction(_).apply: Expr[S => T] => (Expr[S] => Expr[T]) -The dual of expansion, let’s call it `reflect`, can be defined as follows: +Its dual, let’s call it `reflect`, can be defined as follows: def reflect[T, U](f: Expr[T] => Expr[U]): Expr[T => U] = ’{ (x: T) => ~f(’(x)) @@ -253,7 +256,7 @@ Here’s an application of `map` and how it rewrites to optimized code: ### Relationship with Inline and Macros -Seen by itself, principled meta-programming looks more like a +Seen by itself, symmetric meta-programming looks more like a framework for staging than one for compile-time meta programming with macros. But combined with Dotty’s `inline` it can be turned into a compile-time system. The idea is that macro elaboration can be @@ -307,7 +310,7 @@ If `program` is treated as a quoted expression, the call to `Macro.assertImpl` becomes phase correct even if macro library and program are conceptualized as local definitions. -But what about the call from `assert` to `assertImpl? Here, we need a +But what about the call from `assert` to `assertImpl`? Here, we need a tweak of the typing rules. An inline function such as `assert` that contains a splice operation outside an enclosing quote is called a _macro_. Macros are supposed to be expanded in a subsequent phase, @@ -318,7 +321,7 @@ the call from `assert` to `assertImpl` phase-correct, even if we assume that both definitions are local. The second role of `inline` in Dotty is to mark a `val` that is -constant or a parameter that will be constant when instantiated. This +either a constant or is a parameter that will be a constant when instantiated. This aspect is also important for macro expansion. To illustrate this, consider an implementation of the `power` function that makes use of a statically known exponent: @@ -331,7 +334,7 @@ statically known exponent: else if (n % 2 == 0) ’{ { val y = ~x * ~x; ~powerCode(n / 2, ’(y)) } } else ’{ ~x * ~powerCode(n - 1, x) } -The usage of `n` as an argument in `~powerCode(n, ’(x))` is not +The reference to `n` as an argument in `~powerCode(n, ’(x))` is not phase-consistent, since `n` appears in a splice without an enclosing quote. Normally that would be a problem because it means that we need the _value_ of `n` at compile time, which is not available for general @@ -435,10 +438,10 @@ Running `compile(letExp, Map())` would yield the following Scala code: ’{ val y = 3; (2 + y) + 4 } -The body the first clause (`case Num(n) => n`) looks suspicious. `n` +The body of the first clause, `case Num(n) => n`, looks suspicious. `n` is declared as an `Int`, yet the result of `compile` is declared to be -`Expr[Int]`. Shouldn’t `n be quoted? The answer is that this would not -work since replacing `n by `’n` in the clause would not be phase +`Expr[Int]`. Shouldn’t `n` be quoted? In fact this would not +work since replacing `n` by `’n` in the clause would not be phase correct. What happens instead "under the hood" is an implicit conversion: `n` @@ -480,7 +483,7 @@ tree machinery: } } -Since `Liftable` is a type class, instances can be conditional. For instance +Since `Liftable` is a type class, its instances can be conditional. For example, a `List` is liftable if its element type is: implicit def ListIsLiftable[T: Liftable]: Liftable[List[T]] = new { @@ -505,7 +508,7 @@ need a syntax change that introduces prefix operators as types. SimpleType ::= ... [‘-’ | ‘+’ | ‘~’ | ‘!’] StableId -Analogously to the situation with expressions a prefix type operator +Analogously to the situation with expressions, a prefix type operator such as `~ e` is treated as a shorthand for the type `e.unary_~`. Quotes are supported by introducing new tokens `’(`, `’{`, and `’[` @@ -521,7 +524,7 @@ Syntax changes are given relative to the [Dotty reference grammar](../internal/syntax.md). An alternative syntax would treat `’` as a separate operator. This -would be attractive since it enables quoting single identifies as +would be attractive since it enables quoting single identifiers as e.g. `’x` instead of `’(x)`. But it would clash with symbol literals. So it could be done only if symbol literals were abolished. @@ -538,7 +541,7 @@ that method. With the restrictions on splices that are currently in place that’s all that’s needed. We might allow more interpretation in splices in the future, which would allow us to loosen the restriction. Quotes in spliced, interpreted code are kept as they -are, after splices nested in the quotes are expanded recursively. +are, after splices nested in the quotes are expanded. If the outermost scope is a quote, we need to generate code that constructs the quoted tree at run-time. We implement this by @@ -575,7 +578,7 @@ The syntax of terms, values, and types is given as follows: Typing rules are formulated using a stack of environments `Es`. Individual environments `E` consist as usual of variable -bindings `x: T`. Environments can be combined using one of two +bindings `x: T`. Environments can be combined using the two combinators `’` and `~`. Environment E ::= () empty @@ -608,7 +611,7 @@ We define a small step reduction relation `-->` with the following rules: The first rule is standard call-by-value beta-reduction. The second rule says that splice and quotes cancel each other out. The third rule is a context rule; it says that reduction is allowed in the hole `[ ]` -position of an evaluation contexts. Evaluation contexts `e` and +position of an evaluation context. Evaluation contexts `e` and splice evaluation context `e_s` are defined syntactically as follows: Eval context e ::= [ ] | e t | v e | ’e_s[~e] @@ -662,7 +665,7 @@ environments and terms. ## Going Further -The presented meta-programming framework is so far quite restrictive +The meta-programming framework as presented and currently implemented is quite restrictive in that it does not allow for the inspection of quoted expressions and types. It’s possible to work around this by providing all necessary information as normal, unquoted inline parameters. But we would gain @@ -687,11 +690,11 @@ implementation of power otherwise. This assumes a `Constant` extractor that maps tree nodes representing constants to their values. -Once we allow for inspection of code, the "AsApplicative" operation +With the right extractors the "AsFunction" operation that maps expressions over functions to functions over expressions can be implemented in user code: - implicit class AsApplicative[T, U](f: Expr[T => U]) extends AnyVal { + implicit class AsFunction[T, U](f: Expr[T => U]) extends AnyVal { def apply(x: Expr[T]): Expr[U] = f match { case Lambda(g) => g(x) @@ -705,7 +708,7 @@ This assumes an extractor } Once we allow inspection of code via extractors, it’s tempting to also -add constructors that construct typed trees directly without going +add constructors that create typed trees directly without going through quotes. Most likely, those constructors would work over `Expr` types which lack a known type argument. For instance, an `Apply` constructor could be typed as follows: @@ -722,13 +725,14 @@ implemented as a primitive; it would check that the computed type structure of `Expr` is a subtype of the type structure representing `T`. -Before going down that route, we should carefully evaluate its -tradeoffs. Constructing trees that are only verified _a posteriori_ +Before going down that route, we should evaluate in detail the tradeoffs it +presents. Constructing trees that are only verified _a posteriori_ to be type correct loses a lot of guidance for constructing the right trees. So we should wait with this addition until we have more use-cases that help us decide whether the loss in type-safety is worth -the gain in flexibility. - +the gain in flexibility. In this context, it seems that deconstructing types is +less error-prone than deconstructing tersm, so one might also +envisage a solution that allows the former but not the latter. ## Conclusion diff --git a/library/src/scala/quoted/Expr.scala b/library/src/scala/quoted/Expr.scala index 7e11aac7b477..ef6652782dc7 100644 --- a/library/src/scala/quoted/Expr.scala +++ b/library/src/scala/quoted/Expr.scala @@ -8,4 +8,8 @@ class Expr[T] extends Quoted { object Expr { implicit def toExpr[T](x: T)(implicit ev: Liftable[T]): Expr[T] = ev.toExpr(x) + + implicit class AsFunction[T, U](private val f: Expr[T => U]) extends AnyVal { + def apply(x: Expr[T]): Expr[U] = ??? + } } From 07bbb2b54ab3ce345a98139668e15a92f8a37122 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 15:58:56 +0100 Subject: [PATCH 56/62] More doc fixes --- .../reference/symmetric-meta-programming.md | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/docs/docs/reference/symmetric-meta-programming.md b/docs/docs/reference/symmetric-meta-programming.md index 124de4bea088..5fac003c4fe8 100644 --- a/docs/docs/reference/symmetric-meta-programming.md +++ b/docs/docs/reference/symmetric-meta-programming.md @@ -4,8 +4,8 @@ Symmetric meta programming is a new framework for staging and for some forms of macros. It is is expressed as strongly and statically typed code using two fundamental operations: quotations and splicing. A novel aspect of the approach is that these two operations are -regulated by a phase consistency principle that treats splices and -quotes in exactly the same way. +regulated by a phase consistency principle that treats quotes and +splices in exactly the same way. ## Overview @@ -104,8 +104,8 @@ create nor remove quotes or splices individually. So the PCP ensures that program elaboration will lead to neither of the two unwanted situations described above. -In the the range of features it covers, symmetric meta programming is -quite close to the MetaML family. One difference is that MetaML does +In what concerns the range of features it covers, symmetric meta programming is +quite close to the MetaML family of languages. One difference is that MetaML does not have an equivalent of the PCP - quoted code in MetaML _can_ access variables in its immediately enclosing environment, with some restrictions and caveats since such accesses involve serialization. @@ -456,7 +456,7 @@ is defined in the companion object of class `Expr` as follows: The conversion says that values of types implementing the `Liftable` type class can be converted ("lifted") automatically to `Expr` values. Dotty comes with instance definitions of `Liftable` for -several types including all underlying types of literals. For instance +several types including all underlying types of literals. For example, `Int` values can be converted to `Expr[Int]` values by wrapping the value in a `Literal` tree node. This makes use of the underlying tree representation in the compiler for efficiency. But the `Liftable` @@ -592,9 +592,7 @@ combinators `’` and `~`. ~ The two environment combinators are both associative with left and -right identity `()`. The initial environment contains among other -predefined operations also a "run" operation `! : expr T => T`. That -is, `!` is treated just like any other function. +right identity `()`. ### Operational semantics: @@ -617,6 +615,14 @@ splice evaluation context `e_s` are defined syntactically as follows: Eval context e ::= [ ] | e t | v e | ’e_s[~e] Splice context e_s ::= [ ] | (x: T) => e_s | e_s t | q e_s +A _run_ operation can be added to the calculus by adding a binding +`run : expr T -> T` to the initial environment, together with the evaluation rule + + run('t) --> t + +That is, `run` reduces in the same way as `~` when in evaluation position. But unlike splices, +`run` operations are only evaluated outside of quotes, which means they are not affected by the PCP. + ### Typing rules Typing judgments are of the form `Es |- t: T`. There are two @@ -650,7 +656,6 @@ environment of the stack. -------------------------------- Es |- t t’: T - The rules for quotes and splices map between `expr T` and `T` by trading `’` and `~` between environments and terms. @@ -736,12 +741,10 @@ envisage a solution that allows the former but not the latter. ## Conclusion -Meta-programming always made my head hurt (I believe that’s the same -for many people). But with explicit `Expr/Type` types and quotes and -splices it has become downright pleasant. The method I was following -was to first define the underlying quoted or unquoted types using -`Expr` and `Type` and then insert quotes and splices to make the types -line up. Phase consistency was at the same time a great guideline -where to insert a splice or a quote and a vital sanity check that I -did the right thing. - +Meta-programming has a reputation of being difficult and confusing. +But with explicit `Expr/Type` types and quotes and splices it can become +downright pleasant. A simple strategy first defines the underlying quoted or unquoted +values using `Expr` and `Type` and then inserts quotes and splices to make the types +line up. Phase consistency is at the same time a great guideline +where to insert a splice or a quote and a vital sanity check that +the result makes sense. From 8656be3d384d98cc450a43dff7615e99eb639ebc Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 18:00:30 +0100 Subject: [PATCH 57/62] Add section "Limitations to Splicing" --- .../reference/symmetric-meta-programming.md | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/docs/reference/symmetric-meta-programming.md b/docs/docs/reference/symmetric-meta-programming.md index 5fac003c4fe8..65606711b3a8 100644 --- a/docs/docs/reference/symmetric-meta-programming.md +++ b/docs/docs/reference/symmetric-meta-programming.md @@ -253,7 +253,6 @@ Here’s an application of `map` and how it rewrites to optimized code: ys } - ### Relationship with Inline and Macros Seen by itself, symmetric meta-programming looks more like a @@ -399,6 +398,36 @@ to `T` but only `~` is subject to the PCP, whereas `run` is just a normal method def run: T // run staged code } +### Limitations to Splicing + +Quotes and splices are duals as far as the PCP is concerned. But there is an additional +restriction that needs to be imposed on splices to guarantee soundness: +code in splices must be free of side effects. The restriction prevents code like this: + + var x: Expr[T] + ’{ (y: T) => ~{ x = ’(y); 1 } } + +This code, if it was accepted, would "extrude" a reference to a quoted variable `y` from its scope. +This means we an subsequently access a variable outside the scope where it is defined, which is +likely problematic. The code is clearly phase consistent, so we cannot use PCP to +rule it out. Instead we postulate a future effect system that can guarantee that splices +are pure. In the absence of such a system we simply demand that spliced expressions are +pure by convention, and allow for undefined compiler behavior if they are not. This is analogous +to the status of pattern guards in Scala, which are also required, but not verified, to be pure. + +There is also a problem with `run` in splices. Consider the following expression: + + ’{ (x: Int) => ~{ {’(x)}.run; 1 } } + +This is again phase correct, but will lead us into trouble. Indeed, evaluating the splice will reduce the +expression `{’(x)}.run` to `x`. But then the result + + ’{ (x: Int) => ~{ x; 1 } } + +is no longer phase correct. To prevent this soundness hole it seems easiest to classify `run` as a side-effecting +operation. It would thus be prevented from appearing in splices. In a base language with side-effects we'd have to +do this anyway: Since `run` runs arbitrary code it can always produce a side effect if the code it runs produces one. + ### The `Liftable` type-class Consider the following implementation of a staged interpreter that implements @@ -615,14 +644,6 @@ splice evaluation context `e_s` are defined syntactically as follows: Eval context e ::= [ ] | e t | v e | ’e_s[~e] Splice context e_s ::= [ ] | (x: T) => e_s | e_s t | q e_s -A _run_ operation can be added to the calculus by adding a binding -`run : expr T -> T` to the initial environment, together with the evaluation rule - - run('t) --> t - -That is, `run` reduces in the same way as `~` when in evaluation position. But unlike splices, -`run` operations are only evaluated outside of quotes, which means they are not affected by the PCP. - ### Typing rules Typing judgments are of the form `Es |- t: T`. There are two From 833775fb4e13f751812640f1d329be82f9ac0a49 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 19 Dec 2017 18:08:22 +0100 Subject: [PATCH 58/62] Address review comments on reference section --- .../reference/symmetric-meta-programming.md | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/docs/reference/symmetric-meta-programming.md b/docs/docs/reference/symmetric-meta-programming.md index 65606711b3a8..fdf33b2964f8 100644 --- a/docs/docs/reference/symmetric-meta-programming.md +++ b/docs/docs/reference/symmetric-meta-programming.md @@ -1,7 +1,7 @@ # Symmetric Meta Programming Symmetric meta programming is a new framework for staging and for some -forms of macros. It is is expressed as strongly and statically typed +forms of macros. It is expressed as strongly and statically typed code using two fundamental operations: quotations and splicing. A novel aspect of the approach is that these two operations are regulated by a phase consistency principle that treats quotes and @@ -90,7 +90,7 @@ allowed everywhere. The phase consistency principle can be motivated as follows: First, suppose the result of a program `P` is some quoted text `’{ ... x ... }` that refers to a free variable `x` in `P` This can be -represented only by referring to original the variable `x`. Hence, the +represented only by referring to the original variable `x`. Hence, the result of the program will need to persist the program state itself as one of its parts. We don’t want to do this, hence this situation should be made illegal. Dually, suppose a top-level part of a program @@ -180,7 +180,7 @@ Here’s the definition of the `mapImpl` macro, which takes quoted types and exp object Macros { - def mapImpl[T, U](u: Type[U], arr: Expr[Array[T]], op: Expr[T => U])(implicit ctx: Context): Expr[Array[U]] = ’{ + def mapImpl[T, U](u: Type[U], arr: Expr[Array[T]], op: Expr[T => U]): Expr[Array[U]] = ’{ var i = 0 val xs = ~arr var len = xs.length @@ -314,7 +314,7 @@ tweak of the typing rules. An inline function such as `assert` that contains a splice operation outside an enclosing quote is called a _macro_. Macros are supposed to be expanded in a subsequent phase, i.e. in a quoted context. Therefore, they are also type checked as if -they were in a quoted context, For instance, the definition of +they were in a quoted context. For instance, the definition of `assert` is typechecked as if it appeared inside quotes. This makes the call from `assert` to `assertImpl` phase-correct, even if we assume that both definitions are local. @@ -631,9 +631,9 @@ We define a small step reduction relation `-->` with the following rules: ~(’t) --> t - t --> t’ - ---------------- - e[t] --> e[t’] + t1 --> t2 + ----------------- + e[t1] --> e[t2] The first rule is standard call-by-value beta-reduction. The second rule says that splice and quotes cancel each other out. The third rule @@ -668,14 +668,14 @@ environment of the stack. Es * E |- x: T - Es * E, x: T |- t: T’ - ------------------------------ - Es * E |- (x: T) => t: T -> T’ + Es * E, x: T1 |- t: T2 + ------------------------------- + Es * E |- (x: T1) => t: T -> T2 - Es |- t: T’ -> T Es |- t’: T’ - -------------------------------- - Es |- t t’: T + Es |- t1: T2 -> T Es |- t2: T2 + --------------------------------- + Es |- t1 t2: T The rules for quotes and splices map between `expr T` and `T` by trading `’` and `~` between environments and terms. @@ -743,7 +743,7 @@ constructor could be typed as follows: This would allow constructing applications from lists of arguments without having to match the arguments one-by-one with the -corresponding formal parameter types of the function. We need then "at +corresponding formal parameter types of the function. We then need "at the end" a method to convert an `Expr[_]` to an `Expr[T]` where `T` is given from the outside. E.g. if `code` yields a `Expr[_]`, then `code.atType[T]` yields an `Expr[T]`. The `atType` method has to be @@ -757,7 +757,7 @@ to be type correct loses a lot of guidance for constructing the right trees. So we should wait with this addition until we have more use-cases that help us decide whether the loss in type-safety is worth the gain in flexibility. In this context, it seems that deconstructing types is -less error-prone than deconstructing tersm, so one might also +less error-prone than deconstructing terms, so one might also envisage a solution that allows the former but not the latter. ## Conclusion From 33a93a65ad0c8f6f5b2e4b202010006e7bbe7c6b Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 20 Dec 2017 15:38:11 +0100 Subject: [PATCH 59/62] Use different names for quote and type quote methods Otherwise unpickler gets confused when it tries to resolve references to these methods. --- compiler/src/dotty/tools/dotc/core/Definitions.scala | 2 +- compiler/src/dotty/tools/dotc/core/StdNames.scala | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/core/Definitions.scala b/compiler/src/dotty/tools/dotc/core/Definitions.scala index 263db5b842da..19dab5bac91a 100644 --- a/compiler/src/dotty/tools/dotc/core/Definitions.scala +++ b/compiler/src/dotty/tools/dotc/core/Definitions.scala @@ -311,7 +311,7 @@ class Definitions { useCompleter = true) /** Method representing a type quote */ - lazy val typeQuoteMethod = enterPolyMethod(OpsPackageClass, nme.QUOTE, 1, + lazy val typeQuoteMethod = enterPolyMethod(OpsPackageClass, nme.TYPE_QUOTE, 1, pt => QuotedTypeType.appliedTo(pt.paramRefs(0) :: Nil), useCompleter = true) diff --git a/compiler/src/dotty/tools/dotc/core/StdNames.scala b/compiler/src/dotty/tools/dotc/core/StdNames.scala index 21ba10f18553..aab111a3fdf8 100644 --- a/compiler/src/dotty/tools/dotc/core/StdNames.scala +++ b/compiler/src/dotty/tools/dotc/core/StdNames.scala @@ -144,6 +144,7 @@ object StdNames { val COMPANION_MODULE_METHOD: N = "companion$module" val COMPANION_CLASS_METHOD: N = "companion$class" val QUOTE: N = "'" + val TYPE_QUOTE: N = "type_'" val TRAIT_SETTER_SEPARATOR: N = str.TRAIT_SETTER_SEPARATOR // value types (and AnyRef) are all used as terms as well From 39ea6fb1fd32182d083a4add13056c2ae54b8d36 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 20 Dec 2017 15:40:42 +0100 Subject: [PATCH 60/62] Set Macro flag and containsQuotesOrSplices in PostTyper It's easier to handle this in PostTyper since then we do not need to distinguish between inlined and normal code. Also, need to record quotes as well as splices since ReifyQuotes should also do macro expansion. Also, add a missing recursive call to `transform` in ReifyQuotes. --- .../dotty/tools/dotc/CompilationUnit.scala | 2 +- .../tools/dotc/transform/PostTyper.scala | 19 +++++++++++++++++++ .../tools/dotc/transform/ReifyQuotes.scala | 4 ++-- .../dotty/tools/dotc/transform/SymUtils.scala | 4 ++++ .../src/dotty/tools/dotc/typer/Typer.scala | 15 +-------------- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/CompilationUnit.scala b/compiler/src/dotty/tools/dotc/CompilationUnit.scala index fb150b48bee8..c3d9326f180c 100644 --- a/compiler/src/dotty/tools/dotc/CompilationUnit.scala +++ b/compiler/src/dotty/tools/dotc/CompilationUnit.scala @@ -25,7 +25,7 @@ class CompilationUnit(val source: SourceFile) { /** Will be reset to `true` if `untpdTree` contains `Quote` trees. The information * is used in phase ReifyQuotes in order to avoid traversing a quote-less tree. */ - var containsQuotes: Boolean = false + var containsQuotesOrSplices: Boolean = false } object CompilationUnit { diff --git a/compiler/src/dotty/tools/dotc/transform/PostTyper.scala b/compiler/src/dotty/tools/dotc/transform/PostTyper.scala index 0bec80c49cb9..56d5a011ec4b 100644 --- a/compiler/src/dotty/tools/dotc/transform/PostTyper.scala +++ b/compiler/src/dotty/tools/dotc/transform/PostTyper.scala @@ -159,6 +159,24 @@ class PostTyper extends MacroTransform with IdentityDenotTransformer { thisPhase } } + /** 1. If we are an an inline method but not in a nested quote, mark the inline method + * as a macro. + * + * 2. If selection is a quote or splice node, record that fact in the current compilation unit. + */ + private def handleMeta(sym: Symbol)(implicit ctx: Context): Unit = { + + def markAsMacro(c: Context): Unit = + if (c.owner eq c.outer.owner) markAsMacro(c.outer) + else if (c.owner.isInlineMethod) c.owner.setFlag(Macro) + else if (!c.outer.owner.is(Package)) markAsMacro(c.outer) + + if (sym.isSplice || sym.isQuote) { + markAsMacro(ctx) + ctx.compilationUnit.containsQuotesOrSplices = true + } + } + override def transform(tree: Tree)(implicit ctx: Context): Tree = try tree match { case tree: Ident if !tree.isType => @@ -167,6 +185,7 @@ class PostTyper extends MacroTransform with IdentityDenotTransformer { thisPhase case _ => tree } case tree @ Select(qual, name) => + handleMeta(tree.symbol) if (name.isTypeName) { Checking.checkRealizable(qual.tpe, qual.pos.focus) super.transform(tree) diff --git a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala index 858e5f0430e7..49decaf67949 100644 --- a/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala +++ b/compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala @@ -24,7 +24,7 @@ class ReifyQuotes extends MacroTransform { override def phaseName: String = "reifyQuotes" override def run(implicit ctx: Context): Unit = - if (ctx.compilationUnit.containsQuotes) super.run + if (ctx.compilationUnit.containsQuotesOrSplices) super.run protected def newTransformer(implicit ctx: Context): Transformer = new Reifier(inQuote = false, null, 0, new LevelInfo) @@ -336,7 +336,7 @@ class ReifyQuotes extends MacroTransform { case Inlined(call, bindings, expansion @ Select(body, name)) if expansion.symbol.isSplice => // To maintain phase consistency, convert inlined expressions of the form // `{ bindings; ~expansion }` to `~{ bindings; expansion }` - cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name) + transform(cpy.Select(expansion)(cpy.Inlined(tree)(call, bindings, body), name)) case _: Import => tree case tree: DefDef if tree.symbol.is(Macro) && level == 0 => diff --git a/compiler/src/dotty/tools/dotc/transform/SymUtils.scala b/compiler/src/dotty/tools/dotc/transform/SymUtils.scala index aa881fde2194..c30fa18a807e 100644 --- a/compiler/src/dotty/tools/dotc/transform/SymUtils.scala +++ b/compiler/src/dotty/tools/dotc/transform/SymUtils.scala @@ -176,6 +176,10 @@ class SymUtils(val self: Symbol) extends AnyVal { else owner.isLocal } + /** Is symbol a quote operation? */ + def isQuote(implicit ctx: Context): Boolean = + self == defn.quoteMethod || self == defn.typeQuoteMethod + /** Is symbol a splice operation? */ def isSplice(implicit ctx: Context): Boolean = self == defn.QuotedExpr_~ || self == defn.QuotedType_~ diff --git a/compiler/src/dotty/tools/dotc/typer/Typer.scala b/compiler/src/dotty/tools/dotc/typer/Typer.scala index 4b0890cc6c31..2648c2d183d3 100644 --- a/compiler/src/dotty/tools/dotc/typer/Typer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Typer.scala @@ -394,23 +394,11 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit def typedSelect(tree: untpd.Select, pt: Type)(implicit ctx: Context): Tree = track("typedSelect") { - /** If we are an an inline method but not in a nested quote, mark the inline method - * as a macro. - */ - def markAsMacro(c: Context): Unit = - if (!c.tree.isInstanceOf[untpd.Quote]) - if (c.owner eq c.outer.owner) markAsMacro(c.outer) - else if (c.owner.isInlineMethod) c.owner.setFlag(Macro) - else if (!c.outer.owner.is(Package)) markAsMacro(c.outer) - def typeSelectOnTerm(implicit ctx: Context): Tree = { val qual1 = typedExpr(tree.qualifier, selectionProto(tree.name, pt, this)) if (tree.name.isTypeName) checkStable(qual1.tpe, qual1.pos) val select = typedSelect(tree, pt, qual1) - if (select.tpe ne TryDynamicCallType) { - if (select.symbol.isSplice) markAsMacro(ctx) - select - } + if (select.tpe ne TryDynamicCallType) select else if (pt.isInstanceOf[PolyProto] || pt.isInstanceOf[FunProto] || pt == AssignProto) select else typedDynamicSelect(tree, Nil, pt) } @@ -1086,7 +1074,6 @@ class Typer extends Namer with TypeAssigner with Applications with Implicits wit } def typedQuote(tree: untpd.Quote, pt: Type)(implicit ctx: Context): Tree = track("typedQuote") { - ctx.compilationUnit.containsQuotes = true val untpd.Quote(body) = tree val isType = body.isType val resultClass = if (isType) defn.QuotedTypeClass else defn.QuotedExprClass From aa252fc5d804d21d211a215d4f7c27c218f5c562 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 20 Dec 2017 15:48:23 +0100 Subject: [PATCH 61/62] Reorder printers alphabetically --- .../dotty/tools/dotc/config/Printers.scala | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/config/Printers.scala b/compiler/src/dotty/tools/dotc/config/Printers.scala index b0936864c426..d914d4b6d885 100644 --- a/compiler/src/dotty/tools/dotc/config/Printers.scala +++ b/compiler/src/dotty/tools/dotc/config/Printers.scala @@ -11,26 +11,27 @@ object Printers { } val default: Printer = new Printer - val dottydoc: Printer = noPrinter - val core: Printer = noPrinter - val typr: Printer = noPrinter + val constr: Printer = noPrinter + val core: Printer = noPrinter val checks: Printer = noPrinter - val overload: Printer = noPrinter - val implicits: Printer = noPrinter - val implicitsDetailed: Printer = noPrinter - val subtyping: Printer = noPrinter - val unapp: Printer = noPrinter - val gadts: Printer = noPrinter - val hk: Printer = noPrinter - val variances: Printer = noPrinter - val incremental: Printer = noPrinter val config: Printer = noPrinter - val transforms: Printer = noPrinter val cyclicErrors: Printer = noPrinter - val pickling: Printer = noPrinter - val inlining: Printer = noPrinter + val dottydoc: Printer = noPrinter val exhaustivity: Printer = noPrinter + val incremental: Printer = noPrinter + val gadts: Printer = noPrinter + val hk: Printer = noPrinter + val implicits: Printer = noPrinter + val implicitsDetailed: Printer = noPrinter + val inlining: Printer = noPrinter + val overload: Printer = noPrinter val patmatch: Printer = noPrinter + val pickling: Printer = noPrinter val simplify: Printer = noPrinter + val subtyping: Printer = noPrinter + val transforms: Printer = noPrinter + val typr: Printer = noPrinter + val unapp: Printer = noPrinter + val variances: Printer = noPrinter } From 8f048a70cab943d71da946b6aa0f9c1d10605d2f Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 20 Dec 2017 15:50:25 +0100 Subject: [PATCH 62/62] Add test to pending This currently fails with "splice outside quote". Once macro expansion is implemented it should compile. --- tests/pending/pos/quotedSepComp/Macro_1.scala | 5 +++++ tests/pending/pos/quotedSepComp/Test_2.scala | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 tests/pending/pos/quotedSepComp/Macro_1.scala create mode 100644 tests/pending/pos/quotedSepComp/Test_2.scala diff --git a/tests/pending/pos/quotedSepComp/Macro_1.scala b/tests/pending/pos/quotedSepComp/Macro_1.scala new file mode 100644 index 000000000000..205f63c937d8 --- /dev/null +++ b/tests/pending/pos/quotedSepComp/Macro_1.scala @@ -0,0 +1,5 @@ +import scala.quoted._ +object Macros { + inline def assert(expr: => Boolean): Unit = ~ assertImpl('(expr)) + def assertImpl(expr: Expr[Boolean]) = '{ () } +} diff --git a/tests/pending/pos/quotedSepComp/Test_2.scala b/tests/pending/pos/quotedSepComp/Test_2.scala new file mode 100644 index 000000000000..42a3748830a0 --- /dev/null +++ b/tests/pending/pos/quotedSepComp/Test_2.scala @@ -0,0 +1,5 @@ +class Test { + import Macros._ + val x = 1 + assert(x != 0) +}