From 2882c56c6ef2e50cf00d1e5cc991cc0208a65073 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Thu, 19 Jul 2018 09:46:18 +0200 Subject: [PATCH 1/3] Check tasty file hash --- .../dotty/tools/backend/jvm/GenBCode.scala | 7 +++-- .../dotc/core/classfile/ClassfileParser.scala | 13 ++++++-- .../core/tasty/TastyHeaderUnpickler.scala | 31 +++++++++++++++++++ .../tools/dotc/core/tasty/TastyPickler.scala | 7 ++++- .../dotc/core/tasty/TastyUnpickler.scala | 17 +--------- 5 files changed, 52 insertions(+), 23 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/core/tasty/TastyHeaderUnpickler.scala diff --git a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala index 49f5ff79130b..d3b1cf57049e 100644 --- a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala +++ b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala @@ -26,6 +26,7 @@ import scala.tools.asm import scala.tools.asm.tree._ import tpd._ import StdNames._ +import dotty.tools.dotc.core.tasty.TastyPickler import dotty.tools.io._ class GenBCode extends Phase { @@ -221,9 +222,9 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter val outstream = new DataOutputStream(outTastyFile.bufferedOutput) try outstream.write(binary) finally outstream.close() - // TASTY attribute is created but 0 bytes are stored in it. - // A TASTY attribute has length 0 if and only if the .tasty file exists. - Array.empty[Byte] + // TASTY attribute is created but only the header bytes are stored in it. + // A TASTY attribute has length `headerSize` if and only if the .tasty file exists. + java.util.Arrays.copyOf(binary, TastyPickler.headerSize) } else { // Create an empty file to signal that a tasty section exist in the corresponding .class // This is much cheaper and simpler to check than doing classfile parsing diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala index ac6af4dfd2c3..14bfeff16011 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala @@ -6,6 +6,7 @@ package classfile import Contexts._, Symbols._, Types._, Names._, StdNames._, NameOps._, Scopes._, Decorators._ import SymDenotations._, unpickleScala2.Scala2Unpickler._, Constants._, Annotations._, util.Positions._ import NameKinds.{ModuleClassName, DefaultGetterName} +import dotty.tools.dotc.core.tasty.{TastyHeaderUnpickler, TastyPickler} import ast.tpd._ import java.io.{ ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, File, IOException } import java.nio @@ -785,7 +786,10 @@ class ClassfileParser( if (scan(tpnme.TASTYATTR)) { val attrLen = in.nextInt - if (attrLen == 0) { // A tasty attribute implies the existence of the .tasty file + val bytes = in.nextBytes(attrLen) + val headerUnpickler = new TastyHeaderUnpickler(bytes) + headerUnpickler.readHeader() + if (headerUnpickler.isAtEnd) { // A tasty attribute with that has only a header implies the existence of the .tasty file val tastyBytes: Array[Byte] = classfile.underlyingSource match { // TODO: simplify when #3552 is fixed case None => ctx.error("Could not load TASTY from .tasty for virtual file " + classfile) @@ -816,10 +820,13 @@ class ClassfileParser( Array.empty } } - if (tastyBytes.nonEmpty) + if (tastyBytes.nonEmpty) { + if (!tastyBytes.startsWith(bytes)) + ctx.error("Header of TASTY file did not correspond to header in classfile. One of the files might be outdated or corrupted.") return unpickleTASTY(tastyBytes) + } } - else return unpickleTASTY(in.nextBytes(attrLen)) + else return unpickleTASTY(bytes) } if (scan(tpnme.ScalaATTR) && !scalaUnpickleWhitelist.contains(classRoot.name)) { diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyHeaderUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyHeaderUnpickler.scala new file mode 100644 index 000000000000..d06436e5af77 --- /dev/null +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyHeaderUnpickler.scala @@ -0,0 +1,31 @@ +package dotty.tools.dotc +package core +package tasty + +import java.util.UUID + +import dotty.tools.dotc.core.tasty.TastyFormat.{MajorVersion, MinorVersion, header} +import dotty.tools.dotc.core.tasty.TastyUnpickler.UnpickleException + +class TastyHeaderUnpickler(reader: TastyReader) { + import reader._ + + def this(bytes: Array[Byte]) = this(new TastyReader(bytes)) + + def readHeader(): UUID = { + for (i <- 0 until header.length) + check(readByte() == header(i), "not a TASTy file") + val major = readNat() + val minor = readNat() + check(major == MajorVersion && minor <= MinorVersion, + s"""TASTy signature has wrong version. + | expected: $MajorVersion.$MinorVersion + | found : $major.$minor""".stripMargin) + new UUID(readUncompressedLong(), readUncompressedLong()) + } + + def isAtEnd: Boolean = reader.isAtEnd + + private def check(cond: Boolean, msg: => String): Unit = + if (!cond) throw new UnpickleException(msg) +} diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala index a15c8c861fec..9edfcf18366d 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala @@ -29,12 +29,13 @@ class TastyPickler(val rootCls: ClassSymbol) { val uuidHi: Long = sections.iterator.map(x => pjwHash64(x._2.bytes)).fold(0L)(_ ^ _) val headerBuffer = { - val buf = new TastyBuffer(header.length + 24) + val buf = new TastyBuffer(TastyPickler.headerSize) for (ch <- header) buf.writeByte(ch.toByte) buf.writeNat(MajorVersion) buf.writeNat(MinorVersion) buf.writeUncompressedLong(uuidLow) buf.writeUncompressedLong(uuidHi) + assert(buf.length == TastyPickler.headerSize) buf } @@ -90,3 +91,7 @@ class TastyPickler(val rootCls: ClassSymbol) { h } } + +object TastyPickler { + val headerSize = header.length + 18 +} \ No newline at end of file diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyUnpickler.scala index f3ae3292776f..94dd7f4b8120 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyUnpickler.scala @@ -35,9 +35,6 @@ class TastyUnpickler(reader: TastyReader) { private val sectionReader = new mutable.HashMap[String, TastyReader] val nameAtRef = new NameTable - private def check(cond: Boolean, msg: => String) = - if (!cond) throw new UnpickleException(msg) - private def readName(): TermName = nameAtRef(readNameRef()) private def readString(): String = readName().toString @@ -73,19 +70,7 @@ class TastyUnpickler(reader: TastyReader) { result } - private def readHeader(): UUID = { - for (i <- 0 until header.length) - check(readByte() == header(i), "not a TASTy file") - val major = readNat() - val minor = readNat() - check(major == MajorVersion && minor <= MinorVersion, - s"""TASTy signature has wrong version. - | expected: $MajorVersion.$MinorVersion - | found : $major.$minor""".stripMargin) - new UUID(readUncompressedLong(), readUncompressedLong()) - } - - private val uuid = readHeader() + new TastyHeaderUnpickler(reader).readHeader() locally { until(readEnd()) { nameAtRef.add(readNameContents()) } From bd042ac4340e504b5a8f1f956a683f08be318efc Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 20 Jul 2018 12:04:08 +0200 Subject: [PATCH 2/3] Only save UUID in classfile --- .../dotty/tools/backend/jvm/GenBCode.scala | 16 ++++++++++---- .../dotc/core/classfile/ClassfileParser.scala | 22 ++++++++++--------- .../tools/dotc/core/tasty/TastyPickler.scala | 7 +----- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala index d3b1cf57049e..ffae0c29ff7f 100644 --- a/compiler/src/dotty/tools/backend/jvm/GenBCode.scala +++ b/compiler/src/dotty/tools/backend/jvm/GenBCode.scala @@ -26,7 +26,7 @@ import scala.tools.asm import scala.tools.asm.tree._ import tpd._ import StdNames._ -import dotty.tools.dotc.core.tasty.TastyPickler +import dotty.tools.dotc.core.tasty.{TastyBuffer, TastyHeaderUnpickler, TastyPickler} import dotty.tools.io._ class GenBCode extends Phase { @@ -222,9 +222,17 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter val outstream = new DataOutputStream(outTastyFile.bufferedOutput) try outstream.write(binary) finally outstream.close() - // TASTY attribute is created but only the header bytes are stored in it. - // A TASTY attribute has length `headerSize` if and only if the .tasty file exists. - java.util.Arrays.copyOf(binary, TastyPickler.headerSize) + + val uuid = new TastyHeaderUnpickler(binary).readHeader() + val lo = uuid.getMostSignificantBits + val hi = uuid.getLeastSignificantBits + + // TASTY attribute is created but only the UUID bytes are stored in it. + // A TASTY attribute has length 16 if and only if the .tasty file exists. + val buffer = new TastyBuffer(16) + buffer.writeUncompressedLong(lo) + buffer.writeUncompressedLong(hi) + buffer.bytes } else { // Create an empty file to signal that a tasty section exist in the corresponding .class // This is much cheaper and simpler to check than doing classfile parsing diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala index 14bfeff16011..2dc1b26c4b6f 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala @@ -5,15 +5,16 @@ package classfile import Contexts._, Symbols._, Types._, Names._, StdNames._, NameOps._, Scopes._, Decorators._ import SymDenotations._, unpickleScala2.Scala2Unpickler._, Constants._, Annotations._, util.Positions._ -import NameKinds.{ModuleClassName, DefaultGetterName} -import dotty.tools.dotc.core.tasty.{TastyHeaderUnpickler, TastyPickler} +import NameKinds.DefaultGetterName +import dotty.tools.dotc.core.tasty.{TastyHeaderUnpickler, TastyReader} import ast.tpd._ -import java.io.{ ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, File, IOException } -import java.nio +import java.io.{ ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, IOException } + import java.lang.Integer.toHexString import java.net.URLClassLoader +import java.util.UUID -import scala.collection.{ mutable, immutable } +import scala.collection.immutable import scala.collection.mutable.{ ListBuffer, ArrayBuffer } import scala.annotation.switch import typer.Checking.checkNonCyclic @@ -787,9 +788,7 @@ class ClassfileParser( if (scan(tpnme.TASTYATTR)) { val attrLen = in.nextInt val bytes = in.nextBytes(attrLen) - val headerUnpickler = new TastyHeaderUnpickler(bytes) - headerUnpickler.readHeader() - if (headerUnpickler.isAtEnd) { // A tasty attribute with that has only a header implies the existence of the .tasty file + if (bytes.length == 16) { // A tasty attribute with that has only a UUID implies the existence of the .tasty file val tastyBytes: Array[Byte] = classfile.underlyingSource match { // TODO: simplify when #3552 is fixed case None => ctx.error("Could not load TASTY from .tasty for virtual file " + classfile) @@ -821,8 +820,11 @@ class ClassfileParser( } } if (tastyBytes.nonEmpty) { - if (!tastyBytes.startsWith(bytes)) - ctx.error("Header of TASTY file did not correspond to header in classfile. One of the files might be outdated or corrupted.") + val reader = new TastyReader(bytes, 0, 16) + val expectedUUID = new UUID(reader.readUncompressedLong(), reader.readUncompressedLong()) + val tastyUUID = new TastyHeaderUnpickler(tastyBytes).readHeader() + if (expectedUUID != tastyUUID) + ctx.error(s"Tasty UUID ($tastyUUID) file did not correspond the tasty UUID declared in the classfile ($expectedUUID).") return unpickleTASTY(tastyBytes) } } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala index 9edfcf18366d..a15c8c861fec 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPickler.scala @@ -29,13 +29,12 @@ class TastyPickler(val rootCls: ClassSymbol) { val uuidHi: Long = sections.iterator.map(x => pjwHash64(x._2.bytes)).fold(0L)(_ ^ _) val headerBuffer = { - val buf = new TastyBuffer(TastyPickler.headerSize) + val buf = new TastyBuffer(header.length + 24) for (ch <- header) buf.writeByte(ch.toByte) buf.writeNat(MajorVersion) buf.writeNat(MinorVersion) buf.writeUncompressedLong(uuidLow) buf.writeUncompressedLong(uuidHi) - assert(buf.length == TastyPickler.headerSize) buf } @@ -91,7 +90,3 @@ class TastyPickler(val rootCls: ClassSymbol) { h } } - -object TastyPickler { - val headerSize = header.length + 18 -} \ No newline at end of file From e39b726482415d06f072a15c977fbdb5be41a570 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 20 Jul 2018 15:57:29 +0200 Subject: [PATCH 3/3] Update comment --- .../src/dotty/tools/dotc/core/classfile/ClassfileParser.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala index 2dc1b26c4b6f..0602b770233a 100644 --- a/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala +++ b/compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala @@ -788,7 +788,7 @@ class ClassfileParser( if (scan(tpnme.TASTYATTR)) { val attrLen = in.nextInt val bytes = in.nextBytes(attrLen) - if (bytes.length == 16) { // A tasty attribute with that has only a UUID implies the existence of the .tasty file + if (attrLen == 16) { // A tasty attribute with that has only a UUID (16 bytes) implies the existence of the .tasty file val tastyBytes: Array[Byte] = classfile.underlyingSource match { // TODO: simplify when #3552 is fixed case None => ctx.error("Could not load TASTY from .tasty for virtual file " + classfile)