Faster and simpler Java 9 classpath implementation

  - Take advantage of the `/packages` index provided by the
    jrt file system to avoid (expensive) Files.exist for
    non-existent entries across the full list of modules.
  - Extends ClassPath directly which leads to a simpler
    implemnentation that using the base class.
  - Add a unit test that shows we can read classes and packages
    from the Java standard library.

Fixes scala/scala-dev#306

With this change bootstrap time under Java 9 was comparable to
Java 8. Before, it was about 40% slower.

Author: retronym

src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala

@@ -10,9 +10,10 @@ import java.util.function.IntFunction
 import java.util
 import java.util.Comparator
 
-import scala.reflect.io.{AbstractFile, PlainFile}
+import scala.reflect.io.{AbstractFile, PlainFile, PlainNioFile}
 import scala.tools.nsc.util.{ClassPath, ClassRepresentation}
 import FileUtils._
+import scala.collection.JavaConverters._
 
 /**
  * A trait allowing to look for classpath entries in directories. It provides common logic for
@@ -121,51 +122,78 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo
   def asClassPathStrings: Seq[String] = Seq(dir.getPath)
 }
 
-object JImageDirectoryLookup {
-  import java.nio.file._, java.net.URI, scala.collection.JavaConverters._
-  def apply(): List[ClassPath] = {
+object JrtClassPath {
+  import java.nio.file._, java.net.URI
+  def apply(): Option[ClassPath] = {
     try {
       val fs = FileSystems.getFileSystem(URI.create("jrt:/"))
-      val dir: Path = fs.getPath("/modules")
-      val modules = Files.list(dir).iterator().asScala.toList
-      modules.map(m => new JImageDirectoryLookup(fs, m.getFileName.toString))
+      Some(new JrtClassPath(fs))
     } catch {
       case _: ProviderNotFoundException | _: FileSystemNotFoundException =>
-        Nil
+        None
     }
   }
 }
-class JImageDirectoryLookup(fs: java.nio.file.FileSystem, module: String) extends DirectoryLookup[ClassFileEntryImpl] with NoSourcePaths {
+
+/**
+  * Implementation `ClassPath` based on the JDK 9 encapsulated runtime modules (JEP-220)
+  *
+  * https://bugs.openjdk.java.net/browse/JDK-8066492 is the most up to date reference
+  * for the structure of the jrt:// filesystem.
+  *
+  * The implementation assumes that no classes exist in the empty package.
+  */
+final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with NoSourcePaths {
   import java.nio.file.Path, java.nio.file._
   type F = Path
-  val dir: Path = fs.getPath("/modules/" + module)
+  private val dir: Path = fs.getPath("/packages")
 
-  protected def emptyFiles: Array[Path] = Array.empty
-  protected def getSubDir(packageDirName: String): Option[Path] = {
-    val packageDir = dir.resolve(packageDirName)
-    if (Files.exists(packageDir) && Files.isDirectory(packageDir)) Some(packageDir)
-    else None
+  // e.g. "java.lang" -> Seq("/modules/java.base")
+  private val packageToModuleBases: Map[String, Seq[Path]] = {
+    val ps = Files.newDirectoryStream(dir).iterator().asScala
+    def lookup(pack: Path): Seq[Path] = {
+      Files.list(pack).iterator().asScala.map(l => if (Files.isSymbolicLink(l)) Files.readSymbolicLink(l) else l).toList
+    }
+    ps.map(p => (p.toString.stripPrefix("/packages/"), lookup(p))).toMap
   }
-  protected def listChildren(dir: Path, filter: Option[Path => Boolean]): Array[Path] = {
-    import scala.collection.JavaConverters._
-    val f = filter.getOrElse((p: Path) => true)
-    Files.list(dir).iterator().asScala.filter(f).toArray[Path]
+
+  override private[nsc] def packages(inPackage: String): Seq[PackageEntry] = {
+    def matches(packageDottedName: String) =
+      if (packageDottedName.contains("."))
+        packageOf(packageDottedName) == inPackage
+      else inPackage == ""
+    packageToModuleBases.keysIterator.filter(matches).map(PackageEntryImpl(_)).toVector
+  }
+  private[nsc] def classes(inPackage: String): Seq[ClassFileEntry] = {
+    if (inPackage == "") Nil
+    else {
+      packageToModuleBases.getOrElse(inPackage, Nil).flatMap(x =>
+        Files.list(x.resolve(inPackage.replace('.', '/'))).iterator().asScala.filter(_.getFileName.toString.endsWith(".class"))).map(x =>
+        ClassFileEntryImpl(new PlainNioFile(x))).toVector
+    }
   }
-  protected def getName(f: Path): String = f.getFileName.toString
-  protected def toAbstractFile(f: Path): AbstractFile = new scala.reflect.io.PlainNioFile(f)
-  protected def isPackage(f: Path): Boolean = Files.isDirectory(f) && mayBeValidPackage(f.getFileName.toString)
+
+  override private[nsc] def list(inPackage: String): ClassPathEntries =
+    if (inPackage == "") ClassPathEntries(packages(inPackage), Nil)
+    else ClassPathEntries(packages(inPackage), classes(inPackage))
 
   def asURLs: Seq[URL] = Seq(dir.toUri.toURL)
-  def asClassPathStrings: Seq[String] = asURLs.map(_.toString)
+  // We don't yet have a scheme to represent the JDK modules in our `-classpath`.
+  // java models them as entries in the new "module path", we'll probably need to follow this.
+  def asClassPathStrings: Seq[String] = Nil
 
   def findClassFile(className: String): Option[AbstractFile] = {
-    val relativePath = FileUtils.dirPath(className) + ".class"
-    val classFile = dir.resolve(relativePath)
-    if (Files.exists(classFile)) Some(new scala.reflect.io.PlainNioFile(classFile)) else None
+    if (!className.contains(".")) None
+    else {
+      val inPackage = packageOf(className)
+      packageToModuleBases.getOrElse(inPackage, Nil).iterator.flatMap{x =>
+        val file = x.resolve(className.replace('.', '/') + ".class")
+        if (Files.exists(file)) new scala.reflect.io.PlainNioFile(file) :: Nil else Nil
+      }.take(1).toList.headOption
+    }
   }
-  override protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  override protected def isMatchingFile(f: Path): Boolean = Files.isRegularFile(f) && f.getFileName.toString.endsWith(".class")
-  override private[nsc] def classes(inPackage: String): Seq[ClassFileEntry] = files(inPackage)
+  private def packageOf(dottedClassName: String): String =
+    dottedClassName.substring(0, dottedClassName.lastIndexOf("."))
 }
 
 case class DirectoryClassPath(dir: File) extends JFileDirectoryLookup[ClassFileEntryImpl] with NoSourcePaths {

src/compiler/scala/tools/util/PathResolver.scala

@@ -234,7 +234,7 @@ final class PathResolver(settings: Settings) {
 
     // Assemble the elements!
     def basis = List[Traversable[ClassPath]](
-      JImageDirectoryLookup.apply(),                // 0. The Java 9 classpath (backed by the jrt:/ virtual system)
+      JrtClassPath.apply(),                         // 0. The Java 9 classpath (backed by the jrt:/ virtual system, if available)
       classesInPath(javaBootClassPath),             // 1. The Java bootstrap class path.
       contentsOfDirsInPath(javaExtDirs),            // 2. The Java extension class path.
       classesInExpandedPath(javaUserClassPath),     // 3. The Java application class path.

test/junit/scala/tools/nsc/classpath/JrtClassPathTest.scala

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2014 Contributor. All rights reserved.
+ */
+package scala.tools.nsc.classpath
+
+import org.junit.Assert._
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+
+import scala.tools.nsc.Settings
+import scala.tools.nsc.backend.jvm.AsmUtils
+import scala.tools.nsc.util.ClassPath
+import scala.tools.util.PathResolver
+
+@RunWith(classOf[JUnit4])
+class JrtClassPathTest {
+
+  @Test def lookupJavaClasses(): Unit = {
+    val specVersion = scala.util.Properties.javaSpecVersion
+    // Run the test using the JDK8 or 9 provider for rt.jar depending on the platform the test is running on.
+    val cp: ClassPath =
+      if (specVersion == "" || specVersion == "1.8") {
+        val settings = new Settings()
+        val resolver = new PathResolver(settings)
+        val elements = new ClassPathFactory(settings).classesInPath(resolver.Calculated.javaBootClassPath)
+        AggregateClassPath(elements)
+      }
+      else JrtClassPath().get
+
+    assertEquals(Nil, cp.classes(""))
+    assertTrue(cp.packages("java").toString, cp.packages("java").exists(_.name == "java.lang"))
+    assertTrue(cp.classes("java.lang").exists(_.name == "Object"))
+    val jl_Object = cp.classes("java.lang").find(_.name == "Object").get
+    assertEquals("java/lang/Object", AsmUtils.classFromBytes(jl_Object.file.toByteArray).name)
+    assertTrue(cp.list("java.lang").packages.exists(_.name == "java.lang.annotation"))
+    assertTrue(cp.list("java.lang").classesAndSources.exists(_.name == "Object"))
+    assertTrue(cp.findClass("java.lang.Object").isDefined)
+    assertTrue(cp.findClassFile("java.lang.Object").isDefined)
+  }
+}