From 4215b153563c5c7845c51e84ac3416a6ab5e4938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93lafur=20P=C3=A1ll=20Geirsson?= Date: Fri, 12 Oct 2018 16:55:17 +0200 Subject: [PATCH] Setup SemanticDB tests This commit updates the SemanticDB tests to assert that the output from converting TASTy to SemanticDB is the same as the SemanticDB produced by the semanticdb-scalac compiler plugin for Scala 2.12.7. Test failures report a diff like this: ```diff Test dotty.semanticdb.Tests.testExample failed: java.lang.AssertionError: --- tasty +++ scala2 -class Example { - val a: String = "1" +class Example /*example/Example#*/ { + val a /*example/Example#a.*/ : String /*scala/Predef.String#*/ = "1" ``` The "tasty" output is currently identical to the original source file because the TASTy to SemanticDB converter doesn't do anything. Once the converter is updated to emit `SymbolOccurrence` (https://scalameta.org/docs/semanticdb/specification.html#symboloccurrence) then the "tasty" output will include inline comments like `class Example /* pkg/Example# */` that can be read as "the symbol `pkg/Example#` was resolved next to the identifier on the left". The spec for SemanticDB symbols can be found here: - Scala symbols: https://scalameta.org/docs/semanticdb/specification.html#scala-symbol - Java symbols: https://scalameta.org/docs/semanticdb/specification.html#java-symbol To run the tests you must first produce the SemanticDB files for the semanticdb/input project by running the following commands: ``` cd semanticdb/input sbt compile ``` It's possible to include the semanticdb/input project in the main Dotty build but I tried to limit the number of changes to the main build. We use the org.scalameta:semanticdb_2.12 dependency to construct SemanticDB data structures. If there is interest it's possible to replace that dependency with a pure-Java module. I don't estimate it's a lot of work, I have just never used the Java protobuf binding generator. --- project/Build.scala | 15 ++- semanticdb/input/build.sbt | 3 + semanticdb/input/project/build.properties | 1 + .../src/main/scala/example/Example.scala | 17 +++ semanticdb/src/dotty/semanticdb/Main.scala | 2 +- ...onsumer.scala => SemanticdbConsumer.scala} | 2 +- semanticdb/test/dotty/semanticdb/MD5.scala | 29 ++++ .../test/dotty/semanticdb/Semanticdbs.scala | 125 ++++++++++++++++++ semanticdb/test/dotty/semanticdb/Tests.scala | 64 +++++++-- 9 files changed, 239 insertions(+), 19 deletions(-) create mode 100644 semanticdb/input/build.sbt create mode 100644 semanticdb/input/project/build.properties create mode 100644 semanticdb/input/src/main/scala/example/Example.scala rename semanticdb/src/dotty/semanticdb/{DBConsumer.scala => SemanticdbConsumer.scala} (92%) create mode 100644 semanticdb/test/dotty/semanticdb/MD5.scala create mode 100644 semanticdb/test/dotty/semanticdb/Semanticdbs.scala diff --git a/project/Build.scala b/project/Build.scala index e5ddd91d0de4..0ace61ce3b29 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -381,10 +381,15 @@ object Build { dottyLib + File.pathSeparator + dottyInterfaces + File.pathSeparator + otherDeps } - lazy val semanticDBSettings = Seq( + lazy val semanticdbSettings = Seq( baseDirectory in (Compile, run) := baseDirectory.value / "..", baseDirectory in Test := baseDirectory.value / "..", - libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test + unmanagedSourceDirectories in Test += baseDirectory.value / "input" / "src" / "main" / "scala", + libraryDependencies ++= List( + ("org.scalameta" %% "semanticdb" % "4.0.0" % Test).withDottyCompat(scalaVersion.value), + "com.novocode" % "junit-interface" % "0.11" % Test, + "com.googlecode.java-diff-utils" % "diffutils" % "1.3.0" % Test + ) ) // Settings shared between dotty-doc and dotty-doc-bootstrapped @@ -910,7 +915,7 @@ object Build { lazy val `dotty-bench` = project.in(file("bench")).asDottyBench(NonBootstrapped) lazy val `dotty-bench-bootstrapped` = project.in(file("bench")).asDottyBench(Bootstrapped) - lazy val `dotty-semanticdb` = project.in(file("semanticdb")).asDottySemanticDB(Bootstrapped) + lazy val `dotty-semanticdb` = project.in(file("semanticdb")).asDottySemanticdb(Bootstrapped) // Depend on dotty-library so that sbt projects using dotty automatically // depend on the dotty-library @@ -1305,9 +1310,9 @@ object Build { settings(commonBenchmarkSettings). enablePlugins(JmhPlugin) - def asDottySemanticDB(implicit mode: Mode): Project = project.withCommonSettings. + def asDottySemanticdb(implicit mode: Mode): Project = project.withCommonSettings. dependsOn(dottyCompiler). - settings(semanticDBSettings) + settings(semanticdbSettings) def asDist(implicit mode: Mode): Project = project. enablePlugins(PackPlugin). diff --git a/semanticdb/input/build.sbt b/semanticdb/input/build.sbt new file mode 100644 index 000000000000..dc3665d9cda8 --- /dev/null +++ b/semanticdb/input/build.sbt @@ -0,0 +1,3 @@ +scalaVersion := "2.12.7" +scalacOptions += "-Yrangepos" +addCompilerPlugin("org.scalameta" % "semanticdb-scalac" % "4.0.0" cross CrossVersion.full) diff --git a/semanticdb/input/project/build.properties b/semanticdb/input/project/build.properties new file mode 100644 index 000000000000..0cd8b07982e1 --- /dev/null +++ b/semanticdb/input/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.2.3 diff --git a/semanticdb/input/src/main/scala/example/Example.scala b/semanticdb/input/src/main/scala/example/Example.scala new file mode 100644 index 000000000000..00b34748ae19 --- /dev/null +++ b/semanticdb/input/src/main/scala/example/Example.scala @@ -0,0 +1,17 @@ +package example + +class Example { + val a: String = "1" + def a( + x: Int + ): String = + x.toString + def a( + x: Int, + y: Int + ): String = + a( + x + + y + ) +} diff --git a/semanticdb/src/dotty/semanticdb/Main.scala b/semanticdb/src/dotty/semanticdb/Main.scala index 6f3607b2c9c2..987619e8af7a 100644 --- a/semanticdb/src/dotty/semanticdb/Main.scala +++ b/semanticdb/src/dotty/semanticdb/Main.scala @@ -12,7 +12,7 @@ object Main { println("Dotty Semantic DB: No classes where passed as argument") } else { println("Running Dotty Semantic DB on: " + args.mkString(" ")) - ConsumeTasty(extraClasspath, classes, new DBConsumer) + ConsumeTasty(extraClasspath, classes, new SemanticdbConsumer) } } } diff --git a/semanticdb/src/dotty/semanticdb/DBConsumer.scala b/semanticdb/src/dotty/semanticdb/SemanticdbConsumer.scala similarity index 92% rename from semanticdb/src/dotty/semanticdb/DBConsumer.scala rename to semanticdb/src/dotty/semanticdb/SemanticdbConsumer.scala index cc7f5a4f3268..02663dc48af7 100644 --- a/semanticdb/src/dotty/semanticdb/DBConsumer.scala +++ b/semanticdb/src/dotty/semanticdb/SemanticdbConsumer.scala @@ -4,7 +4,7 @@ import scala.tasty.Tasty import scala.tasty.file.TastyConsumer import scala.tasty.util.TreeTraverser -class DBConsumer extends TastyConsumer { +class SemanticdbConsumer extends TastyConsumer { final def apply(tasty: Tasty)(root: tasty.Tree): Unit = { import tasty._ diff --git a/semanticdb/test/dotty/semanticdb/MD5.scala b/semanticdb/test/dotty/semanticdb/MD5.scala new file mode 100644 index 000000000000..4cd0f508b3c5 --- /dev/null +++ b/semanticdb/test/dotty/semanticdb/MD5.scala @@ -0,0 +1,29 @@ +package dotty.semanticdb + +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets +import java.security.MessageDigest + +object MD5 { + /** Returns the MD5 finger print for this string */ + def compute(string: String): String = { + compute(ByteBuffer.wrap(string.getBytes(StandardCharsets.UTF_8))) + } + def compute(buffer: ByteBuffer): String = { + val md = MessageDigest.getInstance("MD5") + md.update(buffer) + bytesToHex(md.digest()) + } + private val hexArray = "0123456789ABCDEF".toCharArray + private def bytesToHex(bytes: Array[Byte]): String = { + val hexChars = new Array[Char](bytes.length * 2) + var j = 0 + while (j < bytes.length) { + val v: Int = bytes(j) & 0xFF + hexChars(j * 2) = hexArray(v >>> 4) + hexChars(j * 2 + 1) = hexArray(v & 0x0F) + j += 1 + } + new String(hexChars) + } +} \ No newline at end of file diff --git a/semanticdb/test/dotty/semanticdb/Semanticdbs.scala b/semanticdb/test/dotty/semanticdb/Semanticdbs.scala new file mode 100644 index 000000000000..7c766c6e7380 --- /dev/null +++ b/semanticdb/test/dotty/semanticdb/Semanticdbs.scala @@ -0,0 +1,125 @@ +package dotty.semanticdb + +import java.nio.file._ +import java.nio.charset.StandardCharsets +import scala.meta.internal.{semanticdb => s} +import scala.collection.JavaConverters._ +import dotty.tools.dotc.util.SourceFile + +object Semanticdbs { + + /** + * Utility to load SemanticDB for Scala source files. + * + * @param sourceroot The workspace root directory, by convention matches the directory of build.sbt + * @param classpath The classpath for this project, can be a combination of jars and directories. + * Matches the `fullClasspath` task key from sbt but can be only `classDirectory` + * if you only care about reading SemanticDB files from a single project. + */ + class Loader(sourceroot: Path, classpath: List[Path]) { + private val META_INF = Paths.get("META-INF", "semanticdb") + private val classLoader = new java.net.URLClassLoader(classpath.map(_.toUri.toURL).toArray) + /** Returns a SemanticDB for a single Scala source file, if any. The path must be absolute. */ + def resolve(scalaAbsolutePath: Path): Option[s.TextDocument] = { + val scalaRelativePath = sourceroot.relativize(scalaAbsolutePath) + val filename = scalaRelativePath.getFileName.toString + val semanticdbRelativePath = scalaRelativePath.resolveSibling(filename + ".semanticdb") + val metaInfPath = META_INF.resolve(semanticdbRelativePath).toString + Option(classLoader.findResource(metaInfPath)).map { url => + val semanticdbAbsolutePath = Paths.get(url.toURI) + Semanticdbs.loadTextDocument(scalaAbsolutePath, scalaRelativePath, semanticdbAbsolutePath) + } + } + } + + /** Load SemanticDB TextDocument for a single Scala source file + * + * @param scalaAbsolutePath Absolute path to a Scala source file. + * @param scalaRelativePath scalaAbsolutePath relativized by the sourceroot. + * @param semanticdbAbsolutePath Absolute path to the SemanticDB file. + */ + def loadTextDocument( + scalaAbsolutePath: Path, + scalaRelativePath: Path, + semanticdbAbsolutePath: Path + ): s.TextDocument = { + val reluri = scalaRelativePath.iterator.asScala.mkString("/") + val sdocs = parseTextDocuments(semanticdbAbsolutePath) + sdocs.documents.find(_.uri == reluri) match { + case None => throw new NoSuchElementException(reluri) + case Some(document) => + val text = new String(Files.readAllBytes(scalaAbsolutePath), StandardCharsets.UTF_8) + // Assert the SemanticDB payload is in-sync with the contents of the Scala file on disk. + val md5FingerprintOnDisk = MD5.compute(text) + if (document.md5 != md5FingerprintOnDisk) { + throw new IllegalArgumentException("stale semanticdb: " + reluri) + } else { + // Update text document to include full text contents of the file. + document.withText(text) + } + } + } + + /** Parses SemanticDB text documents from an absolute path to a `*.semanticdb` file. */ + def parseTextDocuments(path: Path): s.TextDocuments = { + // NOTE: a *.semanticdb file is of type s.TextDocuments, not s.TextDocument + val in = Files.newInputStream(path) + try s.TextDocuments.parseFrom(in) + finally in.close() + } + + + /** Prettyprint a text document with symbol occurrences next to each resolved identifier. + * + * Useful for testing purposes to ensure that SymbolOccurrence values make sense and are correct. + * Example output (NOTE, slightly modified to avoid "unclosed comment" errors): + * {{{ + * class Example *example/Example#* { + * val a *example/Example#a.* : String *scala/Predef.String#* = "1" + * } + * }}} + **/ + def printTextDocument(doc: s.TextDocument): String = { + val sb = new StringBuilder + val occurrences = doc.occurrences.sorted + val sourceFile = new SourceFile(doc.uri, doc.text) + var offset = 0 + occurrences.foreach { occ => + val range = occ.range.get + val end = sourceFile.lineToOffset(range.endLine) + range.endCharacter + sb.append(doc.text.substring(offset, end)) + sb.append(" /* ") + .append(occ.symbol) + .append(" */ ") + offset = end + } + sb.append(doc.text.substring(offset)) + sb.toString() + } + + /** Sort symbol occurrences by their start position. */ + implicit val occurrenceOrdering: Ordering[s.SymbolOccurrence] = + new Ordering[s.SymbolOccurrence] { + override def compare(x: s.SymbolOccurrence, y: s.SymbolOccurrence): Int = { + if (x.range.isEmpty) 0 + else if (y.range.isEmpty) 0 + else { + val a = x.range.get + val b = y.range.get + val byLine = Integer.compare( + a.startLine, + b.startLine + ) + if (byLine != 0) { + byLine + } else { + val byCharacter = Integer.compare( + a.startCharacter, + b.startCharacter + ) + byCharacter + } + } + } + } +} diff --git a/semanticdb/test/dotty/semanticdb/Tests.scala b/semanticdb/test/dotty/semanticdb/Tests.scala index b497af89c482..faa7e64ce70b 100644 --- a/semanticdb/test/dotty/semanticdb/Tests.scala +++ b/semanticdb/test/dotty/semanticdb/Tests.scala @@ -6,26 +6,66 @@ import scala.tasty.file._ import org.junit.Test import org.junit.Assert._ +import java.nio.file._ +import scala.meta.internal.{semanticdb => s} +import scala.collection.JavaConverters._ class Tests { // TODO: update scala-0.10 on version change (or resolve automatically) - final def testClasspath = "out/bootstrap/dotty-semanticdb/scala-0.10/test-classes" - - @Test def testMain(): Unit = { - testOutput( - "tests.SimpleClass", - "SimpleClass;;" - ) - testOutput( - "tests.SimpleDef", - "SimpleDef;;foo;" - ) + final def tastyClassDirectory = "out/bootstrap/dotty-semanticdb/scala-0.11/test-classes" + val sourceroot = Paths.get("semanticdb", "input").toAbsolutePath + val sourceDirectory = sourceroot.resolve("src/main/scala") + + val semanticdbClassDirectory = sourceroot.resolve("target/scala-2.12/classes") + val semanticdbLoader = new Semanticdbs.Loader(sourceroot, List(semanticdbClassDirectory)) + /** Returns the SemanticDB for this Scala source file. */ + def getScalacSemanticdb(scalaFile: Path): s.TextDocument = { + semanticdbLoader.resolve(scalaFile).get + } + + /** TODO: Produce semanticdb from TASTy for this Scala source file. */ + def getTastySemanticdb(scalaFile: Path): s.TextDocument = { + ??? + } + + /** Fails the test if the s.TextDocument from tasty and semanticdb-scalac are not the same. */ + def checkFile(filename: String): Unit = { + val path = sourceDirectory.resolve(filename) + val scalac = getScalacSemanticdb(path) + val tasty = s.TextDocument(text = scalac.text) // TODO: replace with `getTastySemanticdb(path)` + val obtained = Semanticdbs.printTextDocument(tasty) + val expected = Semanticdbs.printTextDocument(scalac) + assertNoDiff(obtained, expected) } + /** Fails the test with a pretty diff if there obtained is not the same as expected */ + def assertNoDiff(obtained: String, expected: String): Unit = { + if (obtained.isEmpty && !expected.isEmpty) fail("obtained empty output") + def splitLines(string: String): java.util.List[String] = + string.trim.replace("\r\n", "\n").split("\n").toSeq.asJava + val obtainedLines = splitLines(obtained) + val b = splitLines(expected) + val patch = difflib.DiffUtils.diff(obtainedLines, b) + val diff = + if (patch.getDeltas.isEmpty) "" + else { + difflib.DiffUtils.generateUnifiedDiff( + "tasty", "scala2", obtainedLines, patch, 1 + ).asScala.mkString("\n") + } + if (!diff.isEmpty) { + fail("\n" + diff) + } + } + + + @Test def testExample(): Unit = checkFile("example/Example.scala") + // TODO: add more tests + def testOutput(className: String, expected: String): Unit = { val out = new StringBuilder - ConsumeTasty(testClasspath, List(className), new DBConsumer { + ConsumeTasty(tastyClassDirectory, List(className), new SemanticdbConsumer { override def println(x: Any): Unit = out.append(x).append(";") }) assertEquals(expected, out.result())