From 5eb08456537801c51db79caf5adb318418a103a9 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Wed, 14 Feb 2024 15:29:55 +0100 Subject: [PATCH] Java TASTy: use new threadsafe writer implementation Also fix bug where Jar entries for -Yjava-tasty-output have backslash on Windows. --- .../dotty/tools/dotc/transform/Pickler.scala | 28 +- .../dotty/tools/io/ClassfileWriterOps.scala | 50 ---- compiler/src/dotty/tools/io/FileWriters.scala | 254 ++++++++++++++++++ 3 files changed, 274 insertions(+), 58 deletions(-) delete mode 100644 compiler/src/dotty/tools/io/ClassfileWriterOps.scala create mode 100644 compiler/src/dotty/tools/io/FileWriters.scala diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index 0be66828d58c..27b5d53e25dc 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -9,7 +9,7 @@ import tasty.* import config.Printers.{noPrinter, pickling} import config.Feature import java.io.PrintStream -import io.ClassfileWriterOps +import io.FileWriters.TastyWriter import StdNames.{str, nme} import Periods.* import Phases.* @@ -19,8 +19,9 @@ import reporting.{ThrowingReporter, Profile, Message} import collection.mutable import util.concurrent.{Executor, Future} import compiletime.uninitialized -import dotty.tools.io.JarArchive +import dotty.tools.io.{JarArchive, AbstractFile} import dotty.tools.dotc.printing.OutlinePrinter +import scala.annotation.constructorOnly object Pickler { val name: String = "pickler" @@ -32,8 +33,17 @@ object Pickler { */ inline val ParallelPickling = true - class EarlyFileWriter(writer: ClassfileWriterOps): - export writer.{writeTasty, close} + class EarlyFileWriter private (writer: TastyWriter, origin: AbstractFile): + def this(dest: AbstractFile)(using @constructorOnly ctx: Context) = this(TastyWriter(dest), dest) + + export writer.writeTasty + + def close(): Unit = + writer.close() + origin match { + case jar: JarArchive => jar.close() // also close the file system + case _ => + } } /** This phase pickles trees */ @@ -184,7 +194,7 @@ class Pickler extends Phase { override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] = { val sigWriter: Option[Pickler.EarlyFileWriter] = ctx.settings.YjavaTastyOutput.value match case jar: JarArchive if jar.exists => - Some(Pickler.EarlyFileWriter(ClassfileWriterOps(jar))) + Some(Pickler.EarlyFileWriter(jar)) case _ => None val units0 = @@ -225,9 +235,11 @@ class Pickler extends Phase { (cls, pickled) <- unit.pickled if cls.isDefinedInCurrentRun do - val binaryName = cls.binaryClassName.replace('.', java.io.File.separatorChar).nn - val binaryClassName = if (cls.is(Module)) binaryName.stripSuffix(str.MODULE_SUFFIX).nn else binaryName - writer.writeTasty(binaryClassName, pickled()) + val binaryClassName = cls.binaryClassName + val internalName = + if (cls.is(Module)) binaryClassName.stripSuffix(str.MODULE_SUFFIX).nn + else binaryClassName + val _ = writer.writeTasty(internalName, pickled()) count += 1 finally writer.close() diff --git a/compiler/src/dotty/tools/io/ClassfileWriterOps.scala b/compiler/src/dotty/tools/io/ClassfileWriterOps.scala deleted file mode 100644 index c2107ded6f51..000000000000 --- a/compiler/src/dotty/tools/io/ClassfileWriterOps.scala +++ /dev/null @@ -1,50 +0,0 @@ -package dotty.tools.io - -import dotty.tools.io.* -import dotty.tools.dotc.core.Decorators.* -import dotty.tools.dotc.core.Contexts.* -import dotty.tools.dotc.report -import scala.language.unsafeNulls -import scala.annotation.constructorOnly - - -/** Experimental usage - writes bytes to JarArchives */ -class ClassfileWriterOps(outputDir: JarArchive)(using @constructorOnly ictx: Context) { - - type InternalName = String - - // if non-null, classfiles are written to a jar instead of the output directory - private val jarWriter: JarWriter | Null = - val localCtx = ictx - outputDir.underlyingSource.map { source => - if outputDir.isEmpty then - new Jar(source.file).jarWriter() - else inContext(localCtx) { - // Writing to non-empty JAR might be an undefined behaviour, e.g. in case if other files where - // created using `AbstractFile.bufferedOutputStream`instead of JarWriter - report.warning(em"Tried to write to non-empty JAR: $source") - null - } - }.getOrElse( - inContext(localCtx) { - report.warning(em"tried to create a file writer for $outputDir, but it had no underlying source.") - null - } - ) - - def writeTasty(className: InternalName, bytes: Array[Byte]): Unit = - writeToJar(className, bytes, ".tasty") - - private def writeToJar(className: InternalName, bytes: Array[Byte], suffix: String): Unit = { - if (jarWriter == null) return - val path = className + suffix - val out = jarWriter.newOutputStream(path) - try out.write(bytes, 0, bytes.length) - finally out.flush() - } - - def close(): Unit = { - if (jarWriter != null) jarWriter.close() - outputDir.close() - } -} diff --git a/compiler/src/dotty/tools/io/FileWriters.scala b/compiler/src/dotty/tools/io/FileWriters.scala new file mode 100644 index 000000000000..4f03194fa4ce --- /dev/null +++ b/compiler/src/dotty/tools/io/FileWriters.scala @@ -0,0 +1,254 @@ +package dotty.tools.io + +import dotty.tools.dotc.core.Contexts.* +import dotty.tools.dotc.core.Decorators.em +import dotty.tools.dotc.report +import dotty.tools.io.AbstractFile +import dotty.tools.io.JarArchive +import dotty.tools.io.PlainFile + +import java.io.BufferedOutputStream +import java.io.DataOutputStream +import java.io.FileOutputStream +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.channels.ClosedByInterruptException +import java.nio.channels.FileChannel +import java.nio.file.FileAlreadyExistsException +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import java.nio.file.attribute.FileAttribute +import java.util +import java.util.concurrent.ConcurrentHashMap +import java.util.zip.CRC32 +import java.util.zip.Deflater +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import scala.language.unsafeNulls + +/** Copied from `dotty.tools.backend.jvm.ClassfileWriters` but no `PostProcessorFrontendAccess` needed */ +object FileWriters { + type InternalName = String + type NullableFile = AbstractFile | Null + + /** + * The interface to writing classfiles. GeneratedClassHandler calls these methods to generate the + * directory and files that are created, and eventually calls `close` when the writing is complete. + * + * The companion object is responsible for constructing a appropriate and optimal implementation for + * the supplied settings. + * + * Operations are threadsafe. + */ + sealed trait TastyWriter { + /** + * Write a `.tasty` file. + * + * @param name the internal name of the class, e.g. "scala.Option" + */ + def writeTasty(name: InternalName, bytes: Array[Byte])(using Context): NullableFile + + /** + * Close the writer. Behavior is undefined after a call to `close`. + */ + def close(): Unit + + protected def classToRelativePath(className: InternalName): String = + className.replace('.', '/').nn + ".tasty" + } + + object TastyWriter { + + def apply(output: AbstractFile)(using Context): TastyWriter = { + + // In Scala 2 depenening on cardinality of distinct output dirs MultiClassWriter could have been used + // In Dotty we always use single output directory + val basicTastyWriter = new SingleTastyWriter( + FileWriter(output, None) + ) + + basicTastyWriter + } + + private final class SingleTastyWriter(underlying: FileWriter) extends TastyWriter { + + override def writeTasty(className: InternalName, bytes: Array[Byte])(using Context): NullableFile = { + underlying.writeFile(classToRelativePath(className), bytes) + } + + override def close(): Unit = underlying.close() + } + + } + + sealed trait FileWriter { + def writeFile(relativePath: String, bytes: Array[Byte])(using Context): NullableFile + def close(): Unit + } + + object FileWriter { + def apply(file: AbstractFile, jarManifestMainClass: Option[String])(using Context): FileWriter = + if (file.isInstanceOf[JarArchive]) { + val jarCompressionLevel = ctx.settings.YjarCompressionLevel.value + // Writing to non-empty JAR might be an undefined behaviour, e.g. in case if other files where + // created using `AbstractFile.bufferedOutputStream`instead of JarWritter + val jarFile = file.underlyingSource.getOrElse{ + throw new IllegalStateException("No underlying source for jar") + } + assert(file.isEmpty, s"Unsafe writing to non-empty JAR: $jarFile") + new JarEntryWriter(jarFile, jarManifestMainClass, jarCompressionLevel) + } + else if (file.isVirtual) new VirtualFileWriter(file) + else if (file.isDirectory) new DirEntryWriter(file.file.toPath.nn) + else throw new IllegalStateException(s"don't know how to handle an output of $file [${file.getClass}]") + } + + private final class JarEntryWriter(file: AbstractFile, mainClass: Option[String], compressionLevel: Int) extends FileWriter { + //keep these imports local - avoid confusion with scala naming + import java.util.jar.Attributes.Name.{MANIFEST_VERSION, MAIN_CLASS} + import java.util.jar.{JarOutputStream, Manifest} + + val storeOnly = compressionLevel == Deflater.NO_COMPRESSION + + val jarWriter: JarOutputStream = { + import scala.util.Properties.* + val manifest = new Manifest + val attrs = manifest.getMainAttributes.nn + attrs.put(MANIFEST_VERSION, "1.0") + attrs.put(ScalaCompilerVersion, versionNumberString) + mainClass.foreach(c => attrs.put(MAIN_CLASS, c)) + + val jar = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(file.file), 64000), manifest) + jar.setLevel(compressionLevel) + if (storeOnly) jar.setMethod(ZipOutputStream.STORED) + jar + } + + lazy val crc = new CRC32 + + override def writeFile(relativePath: String, bytes: Array[Byte])(using Context): NullableFile = this.synchronized { + val entry = new ZipEntry(relativePath) + if (storeOnly) { + // When using compression method `STORED`, the ZIP spec requires the CRC and compressed/ + // uncompressed sizes to be written before the data. The JarOutputStream could compute the + // values while writing the data, but not patch them into the stream after the fact. So we + // need to pre-compute them here. The compressed size is taken from size. + // https://stackoverflow.com/questions/1206970/how-to-create-uncompressed-zip-archive-in-java/5868403 + // With compression method `DEFLATED` JarOutputStream computes and sets the values. + entry.setSize(bytes.length) + crc.reset() + crc.update(bytes) + entry.setCrc(crc.getValue) + } + jarWriter.putNextEntry(entry) + try jarWriter.write(bytes, 0, bytes.length) + finally jarWriter.flush() + null + } + + override def close(): Unit = this.synchronized(jarWriter.close()) + } + + private final class DirEntryWriter(base: Path) extends FileWriter { + val builtPaths = new ConcurrentHashMap[Path, java.lang.Boolean]() + val noAttributes = Array.empty[FileAttribute[?]] + private val isWindows = scala.util.Properties.isWin + + private def checkName(component: Path)(using Context): Unit = if (isWindows) { + val specials = raw"(?i)CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9]".r + val name = component.toString + def warnSpecial(): Unit = report.warning(em"path component is special Windows device: ${name}") + specials.findPrefixOf(name).foreach(prefix => if (prefix.length == name.length || name(prefix.length) == '.') warnSpecial()) + } + + def ensureDirForPath(baseDir: Path, filePath: Path)(using Context): Unit = { + import java.lang.Boolean.TRUE + val parent = filePath.getParent + if (!builtPaths.containsKey(parent)) { + parent.iterator.forEachRemaining(checkName) + try Files.createDirectories(parent, noAttributes*) + catch { + case e: FileAlreadyExistsException => + // `createDirectories` reports this exception if `parent` is an existing symlink to a directory + // but that's fine for us (and common enough, `scalac -d /tmp` on mac targets symlink). + if (!Files.isDirectory(parent)) + throw new FileConflictException(s"Can't create directory $parent; there is an existing (non-directory) file in its path", e) + } + builtPaths.put(baseDir, TRUE) + var current = parent + while ((current ne null) && (null ne builtPaths.put(current, TRUE))) { + current = current.getParent + } + } + checkName(filePath.getFileName()) + } + + // the common case is that we are are creating a new file, and on MS Windows the create and truncate is expensive + // because there is not an options in the windows API that corresponds to this so the truncate is applied as a separate call + // even if the file is new. + // as this is rare, its best to always try to create a new file, and it that fails, then open with truncate if that fails + + private val fastOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE) + private val fallbackOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING) + + override def writeFile(relativePath: String, bytes: Array[Byte])(using Context): NullableFile = { + val path = base.resolve(relativePath) + try { + ensureDirForPath(base, path) + val os = if (isWindows) { + try FileChannel.open(path, fastOpenOptions) + catch { + case _: FileAlreadyExistsException => FileChannel.open(path, fallbackOpenOptions) + } + } else FileChannel.open(path, fallbackOpenOptions) + + try os.write(ByteBuffer.wrap(bytes), 0L) + catch { + case ex: ClosedByInterruptException => + try Files.deleteIfExists(path) // don't leave a empty of half-written classfile around after an interrupt + catch { case _: Throwable => () } + throw ex + } + os.close() + } catch { + case e: FileConflictException => + report.error(em"error writing ${path.toString}: ${e.getMessage}") + case e: java.nio.file.FileSystemException => + if (ctx.settings.Ydebug.value) e.printStackTrace() + report.error(em"error writing ${path.toString}: ${e.getClass.getName} ${e.getMessage}") + } + AbstractFile.getFile(path) + } + + override def close(): Unit = () + } + + private final class VirtualFileWriter(base: AbstractFile) extends FileWriter { + private def getFile(base: AbstractFile, path: String): AbstractFile = { + def ensureDirectory(dir: AbstractFile): AbstractFile = + if (dir.isDirectory) dir + else throw new FileConflictException(s"${base.path}/${path}: ${dir.path} is not a directory") + val components = path.split('/') + var dir = base + for (i <- 0 until components.length - 1) dir = ensureDirectory(dir) subdirectoryNamed components(i).toString + ensureDirectory(dir) fileNamed components.last.toString + } + + private def writeBytes(outFile: AbstractFile, bytes: Array[Byte]): Unit = { + val out = new DataOutputStream(outFile.bufferedOutput) + try out.write(bytes, 0, bytes.length) + finally out.close() + } + + override def writeFile(relativePath: String, bytes: Array[Byte])(using Context):NullableFile = { + val outFile = getFile(base, relativePath) + writeBytes(outFile, bytes) + outFile + } + override def close(): Unit = () + } + + /** Can't output a file due to the state of the file system. */ + class FileConflictException(msg: String, cause: Throwable = null) extends IOException(msg, cause) +}