Setup SemanticDB tests.

This commit updates the SemanticDB tests to assert that the output from
converting TASTy to SemanticDB is the same as the SemanticDB produced by
the semanticdb-scalac compiler plugin for Scala 2.12.7.  Test failures
report a diff like this:

```diff
Test dotty.semanticdb.Tests.testExample failed: java.lang.AssertionError:
--- tasty
+++ scala2
-class Example {
-  val a: String = "1"
+class Example /*example/Example#*/  {
+  val a /*example/Example#a.*/ : String /*scala/Predef.String#*/  = "1"
```

The "tasty" output is currently identical to the original source file
because the TASTy to SemanticDB converter doesn't do anything.
Once the converter is updated to emit `SymbolOccurrence`
(https://scalameta.org/docs/semanticdb/specification.html#symboloccurrence)
then the "tasty" output will include inline comments like
`class Example /* pkg/Example# */` that can be read as "the symbol
`pkg/Example#` was resolved next to the identifier on the left".
The spec for SemanticDB symbols can be found here:

- Scala symbols: https://scalameta.org/docs/semanticdb/specification.html#scala-symbol
- Java symbols: https://scalameta.org/docs/semanticdb/specification.html#java-symbol

To run the tests you must first produce the SemanticDB files for the
semanticdb/input project by running the following commands:

```
cd semanticdb/input
sbt compile
```

It's possible to include the semanticdb/input project in the main Dotty
build but I tried to limit the number of changes to the main build.

We use the org.scalameta:semanticdb_2.12 dependency to construct
SemanticDB data structures. If there is interest it's possible to
replace that dependency with a pure-Java module. I don't estimate it's a
lot of work, I have just never used the Java protobuf binding generator.

Author: olafurpg

project/Build.scala

@@ -381,10 +381,15 @@ object Build {
     dottyLib + File.pathSeparator + dottyInterfaces + File.pathSeparator + otherDeps
   }
 
-  lazy val semanticDBSettings = Seq(
+  lazy val semanticdbSettings = Seq(
     baseDirectory in (Compile, run) := baseDirectory.value / "..",
     baseDirectory in Test := baseDirectory.value / "..",
-    libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % Test
+    unmanagedSourceDirectories in Test += baseDirectory.value / "input" / "src" / "main" / "scala",
+    libraryDependencies ++= List(
+      ("org.scalameta" %% "semanticdb" % "4.0.0" % Test).withDottyCompat(scalaVersion.value),
+      "com.novocode" % "junit-interface" % "0.11" % Test,
+      "com.googlecode.java-diff-utils" % "diffutils" % "1.3.0" % Test
+    )
   )
 
   // Settings shared between dotty-doc and dotty-doc-bootstrapped
@@ -910,7 +915,7 @@ object Build {
   lazy val `dotty-bench` = project.in(file("bench")).asDottyBench(NonBootstrapped)
   lazy val `dotty-bench-bootstrapped` = project.in(file("bench")).asDottyBench(Bootstrapped)
 
-  lazy val `dotty-semanticdb` = project.in(file("semanticdb")).asDottySemanticDB(Bootstrapped)
+  lazy val `dotty-semanticdb` = project.in(file("semanticdb")).asDottySemanticdb(Bootstrapped)
 
   // Depend on dotty-library so that sbt projects using dotty automatically
   // depend on the dotty-library
@@ -1305,9 +1310,9 @@ object Build {
       settings(commonBenchmarkSettings).
       enablePlugins(JmhPlugin)
 
-    def asDottySemanticDB(implicit mode: Mode): Project = project.withCommonSettings.
+    def asDottySemanticdb(implicit mode: Mode): Project = project.withCommonSettings.
       dependsOn(dottyCompiler).
-      settings(semanticDBSettings)
+      settings(semanticdbSettings)
 
     def asDist(implicit mode: Mode): Project = project.
       enablePlugins(PackPlugin).

semanticdb/input/build.sbt

@@ -0,0 +1,3 @@
+scalaVersion := "2.12.7"
+scalacOptions += "-Yrangepos"
+addCompilerPlugin("org.scalameta" % "semanticdb-scalac" % "4.0.0" cross CrossVersion.full)

semanticdb/input/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.2.3

semanticdb/input/src/main/scala/example/Example.scala

@@ -0,0 +1,17 @@
+package example
+
+class Example {
+  val a: String = "1"
+  def a(
+    x: Int
+  ): String =
+    x.toString
+  def a(
+    x: Int,
+    y: Int
+  ): String =
+    a(
+      x +
+      y
+    )
+}

semanticdb/src/dotty/semanticdb/Main.scala

@@ -12,7 +12,7 @@ object Main {
       println("Dotty Semantic DB: No classes where passed as argument")
     } else {
       println("Running Dotty Semantic DB on: " + args.mkString(" "))
-      ConsumeTasty(extraClasspath, classes, new DBConsumer)
+      ConsumeTasty(extraClasspath, classes, new SemanticdbConsumer)
     }
   }
 }

semanticdb/src/dotty/semanticdb/DBConsumer.scala renamed to semanticdb/src/dotty/semanticdb/SemanticdbConsumer.scala

@@ -4,7 +4,7 @@ import scala.tasty.Tasty
 import scala.tasty.file.TastyConsumer
 import scala.tasty.util.TreeTraverser
 
-class DBConsumer extends TastyConsumer {
+class SemanticdbConsumer extends TastyConsumer {
 
   final def apply(tasty: Tasty)(root: tasty.Tree): Unit = {
     import tasty._

semanticdb/test/dotty/semanticdb/MD5.scala

@@ -0,0 +1,29 @@
+package dotty.semanticdb
+
+import java.nio.ByteBuffer
+import java.nio.charset.StandardCharsets
+import java.security.MessageDigest
+
+object MD5 {
+  /** Returns the MD5 finger print for this string */
+  def compute(string: String): String = {
+    compute(ByteBuffer.wrap(string.getBytes(StandardCharsets.UTF_8)))
+  }
+  def compute(buffer: ByteBuffer): String = {
+    val md = MessageDigest.getInstance("MD5")
+    md.update(buffer)
+    bytesToHex(md.digest())
+  }
+  private val hexArray = "0123456789ABCDEF".toCharArray
+  private def bytesToHex(bytes: Array[Byte]): String = {
+    val hexChars = new Array[Char](bytes.length * 2)
+    var j = 0
+    while (j < bytes.length) {
+      val v: Int = bytes(j) & 0xFF
+      hexChars(j * 2) = hexArray(v >>> 4)
+      hexChars(j * 2 + 1) = hexArray(v & 0x0F)
+      j += 1
+    }
+    new String(hexChars)
+  }
+}

semanticdb/test/dotty/semanticdb/Semanticdbs.scala

@@ -0,0 +1,125 @@
+package dotty.semanticdb
+
+import java.nio.file._
+import java.nio.charset.StandardCharsets
+import scala.meta.internal.{semanticdb => s}
+import scala.collection.JavaConverters._
+import dotty.tools.dotc.util.SourceFile
+
+object Semanticdbs {
+
+  /**
+   * Utility to load SemanticDB for Scala source files.
+   *
+   * @param sourceroot The workspace root directory, by convention matches the directory of build.sbt
+   * @param classpath The classpath for this project, can be a combination of jars and directories.
+   *                  Matches the `fullClasspath` task key from sbt but can be only `classDirectory`
+   *                  if you only care about reading SemanticDB files from a single project.
+   */
+  class Loader(sourceroot: Path, classpath: List[Path]) {
+    private val META_INF = Paths.get("META-INF", "semanticdb")
+    private val classLoader = new java.net.URLClassLoader(classpath.map(_.toUri.toURL).toArray)
+    /** Returns a SemanticDB for a single Scala source file, if any. The path must be absolute. */
+    def resolve(scalaAbsolutePath: Path): Option[s.TextDocument] = {
+      val scalaRelativePath = sourceroot.relativize(scalaAbsolutePath)
+      val filename = scalaRelativePath.getFileName.toString
+      val semanticdbRelativePath = scalaRelativePath.resolveSibling(filename + ".semanticdb")
+      val metaInfPath = META_INF.resolve(semanticdbRelativePath).toString
+      Option(classLoader.findResource(metaInfPath)).map { url =>
+        val semanticdbAbsolutePath = Paths.get(url.toURI)
+        Semanticdbs.loadTextDocument(scalaAbsolutePath, scalaRelativePath, semanticdbAbsolutePath)
+      }
+    }
+  }
+
+  /** Load SemanticDB TextDocument for a single Scala source file
+   *
+   * @param scalaAbsolutePath Absolute path to a Scala source file.
+   * @param scalaRelativePath scalaAbsolutePath relativized by the sourceroot.
+   * @param semanticdbAbsolutePath Absolute path to the SemanticDB file.
+   */
+  def loadTextDocument(
+      scalaAbsolutePath: Path,
+      scalaRelativePath: Path,
+      semanticdbAbsolutePath: Path
+  ): s.TextDocument = {
+    val reluri = scalaRelativePath.iterator.asScala.mkString("/")
+    val sdocs = parseTextDocuments(semanticdbAbsolutePath)
+    sdocs.documents.find(_.uri == reluri) match {
+      case None => throw new NoSuchElementException(reluri)
+      case Some(document) =>
+        val text = new String(Files.readAllBytes(scalaAbsolutePath), StandardCharsets.UTF_8)
+        // Assert the SemanticDB payload is in-sync with the contents of the Scala file on disk.
+        val md5FingerprintOnDisk = MD5.compute(text)
+        if (document.md5 != md5FingerprintOnDisk) {
+          throw new IllegalArgumentException("stale semanticdb: " + reluri)
+        } else {
+          // Update text document to include full text contents of the file.
+          document.withText(text)
+        }
+    }
+  }
+
+  /** Parses SemanticDB text documents from an absolute path to a `*.semanticdb` file. */
+  def parseTextDocuments(path: Path): s.TextDocuments = {
+    // NOTE: a *.semanticdb file is of type s.TextDocuments, not s.TextDocument
+    val in = Files.newInputStream(path)
+    try s.TextDocuments.parseFrom(in)
+    finally in.close()
+  }
+
+
+  /** Prettyprint a text document with symbol occurrences next to each resolved identifier.
+   *
+   * Useful for testing purposes to ensure that SymbolOccurrence values make sense and are correct.
+   * Example output (NOTE, slightly modified to avoid "unclosed comment" errors):
+   * {{{
+   *   class Example *example/Example#*  {
+   *     val a *example/Example#a.* : String *scala/Predef.String#*  = "1"
+   *   }
+   * }}}
+   **/
+  def printTextDocument(doc: s.TextDocument): String = {
+    val sb = new StringBuilder
+    val occurrences = doc.occurrences.sorted
+    val sourceFile = new SourceFile(doc.uri, doc.text)
+    var offset = 0
+    occurrences.foreach { occ =>
+      val range = occ.range.get
+      val end = sourceFile.lineToOffset(range.endLine) + range.endCharacter
+      sb.append(doc.text.substring(offset, end))
+      sb.append(" /* ")
+        .append(occ.symbol)
+        .append(" */ ")
+      offset = end
+    }
+    sb.append(doc.text.substring(offset))
+    sb.toString()
+  }
+
+  /** Sort symbol occurrences by their start position. */
+  implicit val occurrenceOrdering: Ordering[s.SymbolOccurrence] =
+    new Ordering[s.SymbolOccurrence] {
+      override def compare(x: s.SymbolOccurrence, y: s.SymbolOccurrence): Int = {
+        if (x.range.isEmpty) 0
+        else if (y.range.isEmpty) 0
+        else {
+          val a = x.range.get
+          val b = y.range.get
+          val byLine = Integer.compare(
+            a.startLine,
+            b.startLine
+          )
+          if (byLine != 0) {
+            byLine
+          } else {
+            val byCharacter = Integer.compare(
+              a.startCharacter,
+              b.startCharacter
+            )
+            byCharacter
+          }
+        }
+      }
+    }
+}

semanticdb/test/dotty/semanticdb/Tests.scala

@@ -6,26 +6,66 @@ import scala.tasty.file._
 
 import org.junit.Test
 import org.junit.Assert._
+import java.nio.file._
+import scala.meta.internal.{semanticdb => s}
+import scala.collection.JavaConverters._
 
 class Tests {
 
   // TODO: update scala-0.10 on version change (or resolve automatically)
-  final def testClasspath = "out/bootstrap/dotty-semanticdb/scala-0.10/test-classes"
-
-  @Test def testMain(): Unit = {
-    testOutput(
-      "tests.SimpleClass",
-      "SimpleClass;<init>;"
-    )
-    testOutput(
-      "tests.SimpleDef",
-      "SimpleDef;<init>;foo;"
-    )
+  final def tastyClassDirectory = "out/bootstrap/dotty-semanticdb/scala-0.11/test-classes"
+  val sourceroot = Paths.get("semanticdb", "input").toAbsolutePath
+  val sourceDirectory = sourceroot.resolve("src/main/scala")
+
+  val semanticdbClassDirectory = sourceroot.resolve("target/scala-2.12/classes")
+  val semanticdbLoader = new Semanticdbs.Loader(sourceroot, List(semanticdbClassDirectory))
+  /** Returns the SemanticDB for this Scala source file. */
+  def getScalacSemanticdb(scalaFile: Path): s.TextDocument = {
+    semanticdbLoader.resolve(scalaFile).get
+  }
+
+  /** TODO: Produce semanticdb from TASTy for this Scala source file. */
+  def getTastySemanticdb(scalaFile: Path): s.TextDocument = {
+    ???
+  }
+
+  /** Fails the test if the s.TextDocument from tasty and semanticdb-scalac are not the same. */
+  def checkFile(filename: String): Unit = {
+    val path = sourceDirectory.resolve(filename)
+    val scalac = getScalacSemanticdb(path)
+    val tasty = s.TextDocument(text = scalac.text) // TODO: replace with `getTastySemanticdb(path)`
+    val obtained = Semanticdbs.printTextDocument(tasty)
+    val expected = Semanticdbs.printTextDocument(scalac)
+    assertNoDiff(obtained, expected)
   }
 
+  /** Fails the test with a pretty diff if there obtained is not the same as expected */
+  def assertNoDiff(obtained: String, expected: String): Unit = {
+    if (obtained.isEmpty && !expected.isEmpty) fail("obtained empty output")
+    def splitLines(string: String): java.util.List[String] =
+      string.trim.replace("\r\n", "\n").split("\n").toSeq.asJava
+    val obtainedLines = splitLines(obtained)
+    val b = splitLines(expected)
+    val patch = difflib.DiffUtils.diff(obtainedLines, b)
+    val diff =
+      if (patch.getDeltas.isEmpty) ""
+      else {
+        difflib.DiffUtils.generateUnifiedDiff(
+          "tasty", "scala2", obtainedLines, patch, 1
+        ).asScala.mkString("\n")
+      }
+    if (!diff.isEmpty) {
+      fail("\n" + diff)
+    }
+  }
+
+
+  @Test def testExample(): Unit = checkFile("example/Example.scala")
+  // TODO: add more tests
+
   def testOutput(className: String, expected: String): Unit = {
     val out = new StringBuilder
-    ConsumeTasty(testClasspath, List(className), new DBConsumer {
+    ConsumeTasty(tastyClassDirectory, List(className), new SemanticdbConsumer {
       override def println(x: Any): Unit = out.append(x).append(";")
     })
     assertEquals(expected, out.result())