From a9e513bec2e147d7d9a028da324034dd3621940e Mon Sep 17 00:00:00 2001 From: "Paolo G. Giarrusso" Date: Sun, 29 Jul 2018 10:07:08 +0200 Subject: [PATCH 1/3] Fix #4306: Implement -scansource for Java files Also document `OutlineParser` with what I needed to discover. --- compiler/src/dotty/tools/dotc/Run.scala | 7 ++++-- .../dotty/tools/dotc/core/SymbolLoaders.scala | 6 ++++- .../tools/dotc/parsing/JavaParsers.scala | 23 +++++++++++++++++++ .../dotty/tools/dotc/parsing/Parsers.scala | 3 +++ .../dotty/tools/dotc/CompilationTests.scala | 4 +++- .../completeFromSource/nested/D.java | 18 +++++++++++++++ .../completeFromSource/nested/Test1.scala | 10 ++++++++ .../completeFromSource/nested/D.java | 18 +++++++++++++++ .../completeFromSource/nested/Test4.scala | 10 ++++++++ 9 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 tests/neg-custom-args/completeFromSource/nested/D.java create mode 100644 tests/neg-custom-args/completeFromSource/nested/Test1.scala create mode 100644 tests/pos-special/completeFromSource/nested/D.java create mode 100644 tests/pos-special/completeFromSource/nested/Test4.scala diff --git a/compiler/src/dotty/tools/dotc/Run.scala b/compiler/src/dotty/tools/dotc/Run.scala index bcab1c50db65..85e253cadcd8 100644 --- a/compiler/src/dotty/tools/dotc/Run.scala +++ b/compiler/src/dotty/tools/dotc/Run.scala @@ -19,9 +19,10 @@ import transform.TreeChecker import rewrite.Rewrites import java.io.{BufferedWriter, OutputStreamWriter} -import dotty.tools.dotc.profile.Profiler +import profile.Profiler import printing.XprintMode import parsing.Parsers.Parser +import parsing.JavaParsers.JavaParser import typer.ImplicitRunInfo import collection.mutable @@ -209,7 +210,9 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint lateFiles += file val unit = new CompilationUnit(getSource(file.path)) def process()(implicit ctx: Context) = { - unit.untpdTree = new Parser(unit.source).parse() + unit.untpdTree = + if (unit.isJava) new JavaParser(unit.source).parse() + else new Parser(unit.source).parse() ctx.typer.lateEnter(unit.untpdTree) def typeCheckUnit() = unit.tpdTree = ctx.typer.typedExpr(unit.untpdTree) if (typeCheck) diff --git a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala index fc3acef8c826..87d810e9cb5e 100644 --- a/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala +++ b/compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala @@ -16,6 +16,7 @@ import Decorators._ import scala.util.control.NonFatal import ast.Trees._ import ast.tpd +import parsing.JavaParsers.OutlineJavaParser import parsing.Parsers.OutlineParser import reporting.trace @@ -154,7 +155,10 @@ object SymbolLoaders { case _ => } - traverse(new OutlineParser(unit.source).parse(), Nil) + traverse( + if (unit.isJava) new OutlineJavaParser(unit.source).parse() + else new OutlineParser(unit.source).parse(), + Nil) } val unit = new CompilationUnit(ctx.run.getSource(src.path)) diff --git a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala index 5c203d123f4a..f2dd65cbf02a 100644 --- a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala @@ -907,4 +907,27 @@ object JavaParsers { unit } } + + + /** OutlineJavaParser parses top-level declarations in `source` to find declared classes, ignoring their bodies (which + * must only have balanced braces). This is used to map class names to defining sources. + * This is necessary even for Java, because the filename defining a non-public classes cannot be determined from the + * classname alone. + */ + class OutlineJavaParser(source: SourceFile)(implicit ctx: Context) extends JavaParser(source) { + + def skipBraces[T](body: T): T = { + accept(LBRACE) + var openBraces = 1 + while (in.token != EOF && openBraces > 0) { + if (in.token == LBRACE) openBraces += 1 + else if (in.token == RBRACE) openBraces -= 1 + in.nextToken() + } + body + } + + override def typeBody(leadingToken: Int, parentName: Name, parentTParams: List[TypeDef]): (List[Tree], List[Tree]) = + skipBraces((List(EmptyValDef), List(EmptyTree))) + } } diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index ee83866d48c9..3bd9ae2119e4 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -2638,6 +2638,9 @@ object Parsers { } + /** OutlineParser parses top-level declarations in `source` to find declared classes, ignoring their bodies (which + * must only have balanced braces). This is used to map class names to defining sources. + */ class OutlineParser(source: SourceFile)(implicit ctx: Context) extends Parser(source) { def skipBraces[T](body: T): T = { diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 90521d3acca4..7dc338710a6c 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -47,6 +47,7 @@ class CompilationTests extends ParallelTesting { compileFile("tests/pos-special/completeFromSource/Test.scala", defaultOptions.and("-sourcepath", "tests/pos-special")) + compileFile("tests/pos-special/completeFromSource/Test2.scala", defaultOptions.and("-sourcepath", "tests/pos-special")) + compileFile("tests/pos-special/completeFromSource/Test3.scala", defaultOptions.and("-sourcepath", "tests/pos-special", "-scansource")) + + compileFile("tests/pos-special/completeFromSource/nested/Test4.scala", defaultOptions.and("-sourcepath", "tests/pos-special", "-scansource")) + compileFilesInDir("tests/pos-special/fatal-warnings", defaultOptions.and("-Xfatal-warnings")) + compileList( "compileMixed", @@ -158,7 +159,8 @@ class CompilationTests extends ParallelTesting { compileFile("tests/neg-custom-args/i4372.scala", allowDeepSubtypes) + compileFile("tests/neg-custom-args/i1754.scala", allowDeepSubtypes) + compileFilesInDir("tests/neg-custom-args/isInstanceOf", allowDeepSubtypes and "-Xfatal-warnings") + - compileFile("tests/neg-custom-args/i3627.scala", allowDeepSubtypes) + compileFile("tests/neg-custom-args/i3627.scala", allowDeepSubtypes) + + compileFile("tests/neg-custom-args/completeFromSource/nested/Test1.scala", defaultOptions.and("-sourcepath", "tests/neg-custom-args", "-scansource")) }.checkExpectedErrors() // Run tests ----------------------------------------------------------------- diff --git a/tests/neg-custom-args/completeFromSource/nested/D.java b/tests/neg-custom-args/completeFromSource/nested/D.java new file mode 100644 index 000000000000..e437b32ed0b2 --- /dev/null +++ b/tests/neg-custom-args/completeFromSource/nested/D.java @@ -0,0 +1,18 @@ +package completeFromSource.nested; + +public class D { + public D(int i) { + + } + public String thisIsD(int i) { + return java.lang.Integer.toString(i); + } +} + +class E { + public E(String s) {} + + public String thisIsE(int i) { + return java.lang.Integer.toString(i); + } +} diff --git a/tests/neg-custom-args/completeFromSource/nested/Test1.scala b/tests/neg-custom-args/completeFromSource/nested/Test1.scala new file mode 100644 index 000000000000..baac82e3234c --- /dev/null +++ b/tests/neg-custom-args/completeFromSource/nested/Test1.scala @@ -0,0 +1,10 @@ +package completeFromSource.nested + +class Test4 { + + val d = new D(1) + val e = new E("xx") + + d.thisIsD(1) + e.thisIsE() // error +} diff --git a/tests/pos-special/completeFromSource/nested/D.java b/tests/pos-special/completeFromSource/nested/D.java new file mode 100644 index 000000000000..e437b32ed0b2 --- /dev/null +++ b/tests/pos-special/completeFromSource/nested/D.java @@ -0,0 +1,18 @@ +package completeFromSource.nested; + +public class D { + public D(int i) { + + } + public String thisIsD(int i) { + return java.lang.Integer.toString(i); + } +} + +class E { + public E(String s) {} + + public String thisIsE(int i) { + return java.lang.Integer.toString(i); + } +} diff --git a/tests/pos-special/completeFromSource/nested/Test4.scala b/tests/pos-special/completeFromSource/nested/Test4.scala new file mode 100644 index 000000000000..daa06d12c426 --- /dev/null +++ b/tests/pos-special/completeFromSource/nested/Test4.scala @@ -0,0 +1,10 @@ +package completeFromSource.nested + +class Test4 { + + val d = new D(1) + val e = new E("xx") + + d.thisIsD(1) + e.thisIsE(2) +} From 53b89c5859aaeed2cf5b7910aba8e01edc120507 Mon Sep 17 00:00:00 2001 From: "Paolo G. Giarrusso" Date: Mon, 30 Jul 2018 14:45:23 +0200 Subject: [PATCH 2/3] Streamline skipBraces as suggested --- .../src/dotty/tools/dotc/parsing/JavaParsers.scala | 9 +++++---- compiler/src/dotty/tools/dotc/parsing/Parsers.scala | 13 +++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala index f2dd65cbf02a..e0f9df427172 100644 --- a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala @@ -916,7 +916,7 @@ object JavaParsers { */ class OutlineJavaParser(source: SourceFile)(implicit ctx: Context) extends JavaParser(source) { - def skipBraces[T](body: T): T = { + def skipBraces(): Unit = { accept(LBRACE) var openBraces = 1 while (in.token != EOF && openBraces > 0) { @@ -924,10 +924,11 @@ object JavaParsers { else if (in.token == RBRACE) openBraces -= 1 in.nextToken() } - body } - override def typeBody(leadingToken: Int, parentName: Name, parentTParams: List[TypeDef]): (List[Tree], List[Tree]) = - skipBraces((List(EmptyValDef), List(EmptyTree))) + override def typeBody(leadingToken: Int, parentName: Name, parentTParams: List[TypeDef]): (List[Tree], List[Tree]) = { + skipBraces() + (List(EmptyValDef), List(EmptyTree))) + } } } diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 3bd9ae2119e4..fed5872dc40d 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -2643,7 +2643,7 @@ object Parsers { */ class OutlineParser(source: SourceFile)(implicit ctx: Context) extends Parser(source) { - def skipBraces[T](body: T): T = { + def skipBraces(): Unit = { accept(LBRACE) var openBraces = 1 while (in.token != EOF && openBraces > 0) { @@ -2654,11 +2654,16 @@ object Parsers { in.nextToken() } } - body } - override def blockExpr(): Tree = skipBraces(EmptyTree) + override def blockExpr(): Tree = { + skipBraces() + EmptyTree + } - override def templateBody() = skipBraces((EmptyValDef, List(EmptyTree))) + override def templateBody() = { + skipBraces() + (EmptyValDef, List(EmptyTree))) + } } } From 138779512b9e1b15bd0eeaeddd2ebb75d5e4cb4d Mon Sep 17 00:00:00 2001 From: "Paolo G. Giarrusso" Date: Mon, 30 Jul 2018 20:36:46 +0200 Subject: [PATCH 3/3] Abstract skipBraces across parsers --- .../tools/dotc/parsing/JavaParsers.scala | 16 ++------- .../dotty/tools/dotc/parsing/Parsers.scala | 35 +++++++++++-------- .../dotty/tools/dotc/parsing/Scanners.scala | 1 + 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala index e0f9df427172..f22afb941c73 100644 --- a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala @@ -914,21 +914,11 @@ object JavaParsers { * This is necessary even for Java, because the filename defining a non-public classes cannot be determined from the * classname alone. */ - class OutlineJavaParser(source: SourceFile)(implicit ctx: Context) extends JavaParser(source) { - - def skipBraces(): Unit = { - accept(LBRACE) - var openBraces = 1 - while (in.token != EOF && openBraces > 0) { - if (in.token == LBRACE) openBraces += 1 - else if (in.token == RBRACE) openBraces -= 1 - in.nextToken() - } - } - + class OutlineJavaParser(source: SourceFile)(implicit ctx: Context) extends JavaParser(source) with OutlineParserCommon { + override def skipBracesHook() = None override def typeBody(leadingToken: Int, parentName: Name, parentTParams: List[TypeDef]): (List[Tree], List[Tree]) = { skipBraces() - (List(EmptyValDef), List(EmptyTree))) + (List(EmptyValDef), List(EmptyTree)) } } } diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index fed5872dc40d..991255e913fa 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -130,6 +130,23 @@ object Parsers { ctx.error(msg, source atPos pos) } + trait OutlineParserCommon extends ParserCommon { + def accept(token: Int): Int + + def skipBracesHook(): Option[Tree] + def skipBraces(): Unit = { + accept(LBRACE) + var openBraces = 1 + while (in.token != EOF && openBraces > 0) { + skipBracesHook() getOrElse { + if (in.token == LBRACE) openBraces += 1 + else if (in.token == RBRACE) openBraces -= 1 + in.nextToken() + } + } + } + } + class Parser(source: SourceFile)(implicit ctx: Context) extends ParserCommon(source) { val in: Scanner = new Scanner(source) @@ -2641,20 +2658,10 @@ object Parsers { /** OutlineParser parses top-level declarations in `source` to find declared classes, ignoring their bodies (which * must only have balanced braces). This is used to map class names to defining sources. */ - class OutlineParser(source: SourceFile)(implicit ctx: Context) extends Parser(source) { + class OutlineParser(source: SourceFile)(implicit ctx: Context) extends Parser(source) with OutlineParserCommon { - def skipBraces(): Unit = { - accept(LBRACE) - var openBraces = 1 - while (in.token != EOF && openBraces > 0) { - if (in.token == XMLSTART) xmlLiteral() - else { - if (in.token == LBRACE) openBraces += 1 - else if (in.token == RBRACE) openBraces -= 1 - in.nextToken() - } - } - } + def skipBracesHook(): Option[Tree] = + if (in.token == XMLSTART) Some(xmlLiteral()) else None override def blockExpr(): Tree = { skipBraces() @@ -2663,7 +2670,7 @@ object Parsers { override def templateBody() = { skipBraces() - (EmptyValDef, List(EmptyTree))) + (EmptyValDef, List(EmptyTree)) } } } diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index 7aa2dc4d36eb..694386cef3b3 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -57,6 +57,7 @@ object Scanners { abstract class ScannerCommon(source: SourceFile)(implicit ctx: Context) extends CharArrayReader with TokenData { val buf = source.content + def nextToken(): Unit // Errors -----------------------------------------------------------------