From a454a990cc1a760b129376f69392359f03e58d2f Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 25 Aug 2020 10:33:46 +0200 Subject: [PATCH 01/11] Cache all memberNamed results Use a full cache instead of an LRU cache. For `typer/*.scala`, this reduced computed member searches on normal ClassDenotations from 520K to 170K. Cache hit rate improves to 97.5%, from 92.5%. (Without member caching there are almost 7Mio computed members for the same code base). --- .../tools/dotc/core/SymDenotations.scala | 12 +-- .../dotty/tools/dotc/util/LinearTable.scala | 89 +++++++++++++++++++ 2 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/util/LinearTable.scala diff --git a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala index 57a90b81695a..2a638b64dca2 100644 --- a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala @@ -1564,14 +1564,14 @@ object SymDenotations { initPrivateWithin: Symbol) extends SymDenotation(symbol, maybeOwner, name, initFlags, initInfo, initPrivateWithin) { - import util.LRUCache + import util.LinearTable // ----- caches ------------------------------------------------------- private var myTypeParams: List[TypeSymbol] = null private var fullNameCache: SimpleIdentityMap[QualifiedNameKind, Name] = SimpleIdentityMap.Empty - private var myMemberCache: LRUCache[Name, PreDenotation] = null + private var myMemberCache: LinearTable[Name, PreDenotation] = null private var myMemberCachePeriod: Period = Nowhere /** A cache from types T to baseType(T, C) */ @@ -1582,9 +1582,9 @@ object SymDenotations { private var baseDataCache: BaseData = BaseData.None private var memberNamesCache: MemberNames = MemberNames.None - private def memberCache(using Context): LRUCache[Name, PreDenotation] = { + private def memberCache(using Context): LinearTable[Name, PreDenotation] = { if (myMemberCachePeriod != ctx.period) { - myMemberCache = new LRUCache + myMemberCache = LinearTable.empty myMemberCachePeriod = ctx.period } myMemberCache @@ -1868,10 +1868,10 @@ object SymDenotations { final def nonPrivateMembersNamed(name: Name)(using Context): PreDenotation = { Stats.record("nonPrivateMembersNamed") if (Config.cacheMembersNamed) { - var denots: PreDenotation = memberCache lookup name + var denots: PreDenotation = memberCache.lookup(name) if (denots == null) { denots = computeNPMembersNamed(name) - memberCache.enter(name, denots) + myMemberCache = memberCache.enter(name, denots) } else if (Config.checkCacheMembersNamed) { val denots1 = computeNPMembersNamed(name) diff --git a/compiler/src/dotty/tools/dotc/util/LinearTable.scala b/compiler/src/dotty/tools/dotc/util/LinearTable.scala new file mode 100644 index 000000000000..d4bd4db81eda --- /dev/null +++ b/compiler/src/dotty/tools/dotc/util/LinearTable.scala @@ -0,0 +1,89 @@ +package dotty.tools.dotc.util + +/** A table with an immutable API that must be used linearly since it uses + * mutation internally. A packed array representation is used for sizes + * up to 8, and a hash table is used for larger sizes. + */ +abstract class LinearTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef]: + def lookup(key: Key): Value + def enter(key: Key, value: Value): LinearTable[Key, Value] + def invalidate(key: Key): Unit + def size: Int + +object LinearTable: + def empty[Key >: Null <: AnyRef, Value >: Null <: AnyRef]: LinearTable[Key, Value] = + ArrayTable[Key, Value](8) + +class ArrayTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef](capacity: Int) extends LinearTable[Key, Value]: + + val elems = new Array[AnyRef](capacity * 2) + var size = 0 + + def lookup(key: Key): Value = + var i = 0 + while i < elems.length do + if elems(i) eq key then + return elems(i + 1).asInstanceOf[Value] + if elems(i) == null then + return null + i += 2 + null + + def enter(key: Key, value: Value): LinearTable[Key, Value] = + var i = 0 + while i < elems.length do + if elems(i) eq key then + elems(i + 1) = value + return this + if elems(i) == null then + elems(i) = key + elems(i + 1) = value + size += 1 + return this + i += 2 + val ht = HashTable[Key, Value](initialCapacity = 16) + i = 0 + while i < elems.length do + ht.enter(elems(i).asInstanceOf[Key], elems(i + 1).asInstanceOf[Value]) + i += 2 + ht.enter(key, value) + ht + + def invalidate(key: Key): Unit = + var i = 0 + while i < elems.length do + if elems(i) eq key then + size -= 1 + elems(i) = null + return + i += 2 + + override def toString: String = + val buf = new StringBuilder + var i = 0 + while i < elems.length do + buf.append(if i == 0 then "ArrayTable(" else ", ") + if elems(i) != null then + buf.append(elems(i)) + buf.append(" -> ") + buf.append(elems(i + 1)) + i += 2 + buf.append(")") + buf.toString +end ArrayTable + +class HashTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef](initialCapacity: Int) extends LinearTable[Key, Value]: + private val table = java.util.HashMap[Key, Value](initialCapacity) + + def lookup(key: Key): Value = + table.get(key) + def enter(key: Key, value: Value): LinearTable[Key, Value] = + table.put(key, value) + this + def invalidate(key: Key): Unit = + table.remove(key) + def size: Int = + table.size + + override def toString: String = table.toString +end HashTable \ No newline at end of file From bb30c7c9ba2128b50373fba6c73accf5e7db9081 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 25 Aug 2020 15:24:13 +0200 Subject: [PATCH 02/11] Use an optimized HashTable instead of a LinearTable The two implementation classes of LinearTable had so much in common that they could be combined. Since the implementation is now fixed, we can use the usual mutable interface for a table. --- .../tools/dotc/core/SymDenotations.scala | 10 +- .../src/dotty/tools/dotc/util/HashTable.scala | 139 ++++++++++++++++++ .../dotty/tools/dotc/util/LinearTable.scala | 89 ----------- 3 files changed, 144 insertions(+), 94 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/util/HashTable.scala delete mode 100644 compiler/src/dotty/tools/dotc/util/LinearTable.scala diff --git a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala index 2a638b64dca2..636bd6d18c7f 100644 --- a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala @@ -1564,14 +1564,14 @@ object SymDenotations { initPrivateWithin: Symbol) extends SymDenotation(symbol, maybeOwner, name, initFlags, initInfo, initPrivateWithin) { - import util.LinearTable + import util.HashTable // ----- caches ------------------------------------------------------- private var myTypeParams: List[TypeSymbol] = null private var fullNameCache: SimpleIdentityMap[QualifiedNameKind, Name] = SimpleIdentityMap.Empty - private var myMemberCache: LinearTable[Name, PreDenotation] = null + private var myMemberCache: HashTable[Name, PreDenotation] = null private var myMemberCachePeriod: Period = Nowhere /** A cache from types T to baseType(T, C) */ @@ -1582,9 +1582,9 @@ object SymDenotations { private var baseDataCache: BaseData = BaseData.None private var memberNamesCache: MemberNames = MemberNames.None - private def memberCache(using Context): LinearTable[Name, PreDenotation] = { + private def memberCache(using Context): HashTable[Name, PreDenotation] = { if (myMemberCachePeriod != ctx.period) { - myMemberCache = LinearTable.empty + myMemberCache = HashTable() myMemberCachePeriod = ctx.period } myMemberCache @@ -1871,7 +1871,7 @@ object SymDenotations { var denots: PreDenotation = memberCache.lookup(name) if (denots == null) { denots = computeNPMembersNamed(name) - myMemberCache = memberCache.enter(name, denots) + memberCache.enter(name, denots) } else if (Config.checkCacheMembersNamed) { val denots1 = computeNPMembersNamed(name) diff --git a/compiler/src/dotty/tools/dotc/util/HashTable.scala b/compiler/src/dotty/tools/dotc/util/HashTable.scala new file mode 100644 index 000000000000..86b40872676a --- /dev/null +++ b/compiler/src/dotty/tools/dotc/util/HashTable.scala @@ -0,0 +1,139 @@ +package dotty.tools.dotc.util + +object HashTable: + inline val MaxDense = 8 + +/** A hash table using open hashing with linear scan which is also very space efficient + * at small sizes. + * @param initialCapacity Indicates the initial number of slots in the hash table. + * The actual number of slots is always a power of 2, so the + * initial size of the table will be the smallest power of two + * that is equal or greater than the given `initialCapacity`. + * Minimum value is 4. + * @param loadFactor The maximum fraction of used elements relative to capacity. + * The hash table will be re-sized once the number of elements exceeds + * the current size of the hash table multiplied by loadFactor. + * However, a table of size up to MaxDense will be re-sized to only + * once the number of elements reaches the table's size. + */ +class HashTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef] + (initialCapacity: Int = 8: Int, loadFactor: Float = 0.33f): + import HashTable.MaxDense + private var used: Int = _ + private var limit: Int = _ + private var table: Array[AnyRef] = _ + clear() + + private def allocate(capacity: Int) = + table = new Array[AnyRef](capacity * 2) + limit = if capacity <= MaxDense then capacity - 1 else (capacity * loadFactor).toInt + + private def roundToPower(n: Int) = + if Integer.bitCount(n) == 1 then n + else + def recur(n: Int): Int = + if n == 1 then 2 + else recur(n >>> 1) << 1 + recur(n) + + /** Remove all elements from this table and set back to initial configuration */ + def clear(): Unit = + used = 0 + allocate(roundToPower(initialCapacity max 4)) + + /** The number of elements in the set */ + def size: Int = used + + private def isDense = limit < MaxDense + + /** Hashcode, by default `System.identityHashCode`, but can be overriden */ + protected def hash(x: Key): Int = System.identityHashCode(x) + + /** Equality, by default `eq`, but can be overridden */ + protected def isEqual(x: Key, y: Key): Boolean = x eq y + + /** Turn hashcode `x` into a table index */ + private def index(x: Int): Int = x & (table.length - 2) + + private def firstIndex(key: Key) = if isDense then 0 else index(hash(key)) + private def nextIndex(idx: Int) = index(idx + 2) + + private def keyAt(idx: Int): Key = table(idx).asInstanceOf[Key] + private def valueAt(idx: Int): Value = table(idx + 1).asInstanceOf[Value] + + /** Find entry such that `isEqual(x, entry)`. If it exists, return it. + * If not, enter `x` in set and return `x`. + */ + def lookup(key: Key): Value = + var idx = firstIndex(key) + var k = keyAt(idx) + while k != null do + if isEqual(k, key) then return valueAt(idx) + idx = nextIndex(idx) + k = keyAt(idx) + null + + def enter(key: Key, value: Value): Unit = + var idx = firstIndex(key) + var k = keyAt(idx) + while k != null do + if isEqual(k, key) then + table(idx + 1) = value + return + idx = nextIndex(idx) + k = keyAt(idx) + table(idx) = key + table(idx + 1) = value + used += 1 + if used > limit then growTable() + + def invalidate(key: Key): Unit = + var idx = firstIndex(key) + var k = keyAt(idx) + while k != null do + if isEqual(k, key) then + var hole = idx + if !isDense then + while + idx = nextIndex(idx) + k = keyAt(idx) + k != null && index(hash(k)) != idx + do + table(hole) = k + table(hole + 1) = valueAt(idx) + hole = idx + table(hole) = null + used -= 1 + return + idx = nextIndex(idx) + k = keyAt(idx) + + private def addOld(key: Key, value: AnyRef): Unit = + var idx = firstIndex(key) + var k = keyAt(idx) + while k != null do + idx = nextIndex(idx) + k = keyAt(idx) + table(idx) = key + table(idx + 1) = value + + private def growTable(): Unit = + val oldTable = table + allocate(table.length) + if isDense then + Array.copy(oldTable, 0, table, 0, oldTable.length) + else + var idx = 0 + while idx < oldTable.length do + val key = oldTable(idx).asInstanceOf[Key] + if key != null then addOld(key, oldTable(idx + 1)) + idx += 2 + + def iterator: Iterator[(Key, Value)] = + for idx <- (0 until table.length by 2).iterator + if keyAt(idx) != null + yield (keyAt(idx), valueAt(idx)) + + override def toString: String = + iterator.map((k, v) => s"$k -> $v").mkString("LinearTable(", ", ", ")") +end HashTable diff --git a/compiler/src/dotty/tools/dotc/util/LinearTable.scala b/compiler/src/dotty/tools/dotc/util/LinearTable.scala deleted file mode 100644 index d4bd4db81eda..000000000000 --- a/compiler/src/dotty/tools/dotc/util/LinearTable.scala +++ /dev/null @@ -1,89 +0,0 @@ -package dotty.tools.dotc.util - -/** A table with an immutable API that must be used linearly since it uses - * mutation internally. A packed array representation is used for sizes - * up to 8, and a hash table is used for larger sizes. - */ -abstract class LinearTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef]: - def lookup(key: Key): Value - def enter(key: Key, value: Value): LinearTable[Key, Value] - def invalidate(key: Key): Unit - def size: Int - -object LinearTable: - def empty[Key >: Null <: AnyRef, Value >: Null <: AnyRef]: LinearTable[Key, Value] = - ArrayTable[Key, Value](8) - -class ArrayTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef](capacity: Int) extends LinearTable[Key, Value]: - - val elems = new Array[AnyRef](capacity * 2) - var size = 0 - - def lookup(key: Key): Value = - var i = 0 - while i < elems.length do - if elems(i) eq key then - return elems(i + 1).asInstanceOf[Value] - if elems(i) == null then - return null - i += 2 - null - - def enter(key: Key, value: Value): LinearTable[Key, Value] = - var i = 0 - while i < elems.length do - if elems(i) eq key then - elems(i + 1) = value - return this - if elems(i) == null then - elems(i) = key - elems(i + 1) = value - size += 1 - return this - i += 2 - val ht = HashTable[Key, Value](initialCapacity = 16) - i = 0 - while i < elems.length do - ht.enter(elems(i).asInstanceOf[Key], elems(i + 1).asInstanceOf[Value]) - i += 2 - ht.enter(key, value) - ht - - def invalidate(key: Key): Unit = - var i = 0 - while i < elems.length do - if elems(i) eq key then - size -= 1 - elems(i) = null - return - i += 2 - - override def toString: String = - val buf = new StringBuilder - var i = 0 - while i < elems.length do - buf.append(if i == 0 then "ArrayTable(" else ", ") - if elems(i) != null then - buf.append(elems(i)) - buf.append(" -> ") - buf.append(elems(i + 1)) - i += 2 - buf.append(")") - buf.toString -end ArrayTable - -class HashTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef](initialCapacity: Int) extends LinearTable[Key, Value]: - private val table = java.util.HashMap[Key, Value](initialCapacity) - - def lookup(key: Key): Value = - table.get(key) - def enter(key: Key, value: Value): LinearTable[Key, Value] = - table.put(key, value) - this - def invalidate(key: Key): Unit = - table.remove(key) - def size: Int = - table.size - - override def toString: String = table.toString -end HashTable \ No newline at end of file From 53961f31879e6aec238c0784c37667ee73229259 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 25 Aug 2020 18:01:20 +0200 Subject: [PATCH 03/11] Improve re-allocation of hash tables When going from dense to hashed, we should expand by a multiple greater than two, or we will have to reallocate immediately afterwards. --- .../src/dotty/tools/dotc/util/HashTable.scala | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/util/HashTable.scala b/compiler/src/dotty/tools/dotc/util/HashTable.scala index 86b40872676a..42c65f53154e 100644 --- a/compiler/src/dotty/tools/dotc/util/HashTable.scala +++ b/compiler/src/dotty/tools/dotc/util/HashTable.scala @@ -1,7 +1,10 @@ package dotty.tools.dotc.util object HashTable: - inline val MaxDense = 8 + /** The number of elements up to which dense packing is used. + * If the number of elements reaches `DenseLimit` a hash table is used instead + */ + inline val DenseLimit = 8 /** A hash table using open hashing with linear scan which is also very space efficient * at small sizes. @@ -10,15 +13,16 @@ object HashTable: * initial size of the table will be the smallest power of two * that is equal or greater than the given `initialCapacity`. * Minimum value is 4. - * @param loadFactor The maximum fraction of used elements relative to capacity. - * The hash table will be re-sized once the number of elements exceeds - * the current size of the hash table multiplied by loadFactor. - * However, a table of size up to MaxDense will be re-sized to only + * @param capacityMultiple The minimum multiple of capacity relative to used elements. + * The hash table will be re-sized once the number of elements + * multiplied by capacityMultiple exceeds the current size of the hash table. + * However, a table of size up to DenseLimit will be re-sized only * once the number of elements reaches the table's size. */ class HashTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef] - (initialCapacity: Int = 8: Int, loadFactor: Float = 0.33f): - import HashTable.MaxDense + (initialCapacity: Int = 8, capacityMultiple: Int = 3): + import HashTable.DenseLimit + private var used: Int = _ private var limit: Int = _ private var table: Array[AnyRef] = _ @@ -26,15 +30,11 @@ class HashTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef] private def allocate(capacity: Int) = table = new Array[AnyRef](capacity * 2) - limit = if capacity <= MaxDense then capacity - 1 else (capacity * loadFactor).toInt + limit = if capacity <= DenseLimit then capacity - 1 else capacity / capacityMultiple private def roundToPower(n: Int) = if Integer.bitCount(n) == 1 then n - else - def recur(n: Int): Int = - if n == 1 then 2 - else recur(n >>> 1) << 1 - recur(n) + else 1 << (32 - Integer.numberOfLeadingZeros(n)) /** Remove all elements from this table and set back to initial configuration */ def clear(): Unit = @@ -44,7 +44,7 @@ class HashTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef] /** The number of elements in the set */ def size: Int = used - private def isDense = limit < MaxDense + private def isDense = limit < DenseLimit /** Hashcode, by default `System.identityHashCode`, but can be overriden */ protected def hash(x: Key): Int = System.identityHashCode(x) @@ -119,7 +119,10 @@ class HashTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef] private def growTable(): Unit = val oldTable = table - allocate(table.length) + val newLength = + if oldTable.length == DenseLimit then DenseLimit * 2 * roundToPower(capacityMultiple) + else table.length + allocate(newLength) if isDense then Array.copy(oldTable, 0, table, 0, oldTable.length) else From 72f24ed7977e17f0382ef6534f5414d77f8f54da Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Tue, 25 Aug 2020 17:51:42 +0200 Subject: [PATCH 04/11] Disable flakey test We get occasional failures of the form java.lang.AssertionError: assertion failed: asTerm called on not-a-Term val while compiling tests/pos/projections.scala Fatal compiler crash when compiling: tests/pos/projections.scala: assertion failed: asTerm called on not-a-Term val dotty.DottyPredef$.assertFail(DottyPredef.scala:17) dotty.tools.dotc.core.Symbols$Symbol.asTerm(Symbols.scala:156) when compiling pos/projections.scala with FromTasty tests. It was first observed when we parallelized position pickling, so it might have something to do with that. --- compiler/test/dotc/pos-from-tasty.blacklist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/test/dotc/pos-from-tasty.blacklist b/compiler/test/dotc/pos-from-tasty.blacklist index 1fac00b403ab..8a083db6d39d 100644 --- a/compiler/test/dotc/pos-from-tasty.blacklist +++ b/compiler/test/dotc/pos-from-tasty.blacklist @@ -13,3 +13,7 @@ i7087.scala # Nullability nullable.scala notNull.scala + +# Flakey behavior +projections.scala + From a8aa3de4aeca1c7e7b4572f2a93b71be49bc6303 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 26 Aug 2020 10:14:51 +0200 Subject: [PATCH 05/11] Special case PreDenotation ops for NoDenotation NoDenotation is a frequent result, so we should keep post processing it as simple as possible. As a side effect this now forces less, which makes some previously failing tests pass. I updated i3253.scala so that it fails again and moved i9052 to pos. --- compiler/src/dotty/tools/dotc/core/Denotations.scala | 10 +++++----- .../src/dotty/tools/dotc/core/SymDenotations.scala | 7 +++++++ tests/neg/i3253.scala | 4 ++-- tests/{neg => pos}/i9052/A.scala | 0 tests/{neg => pos}/i9052/B.scala | 0 5 files changed, 14 insertions(+), 7 deletions(-) rename tests/{neg => pos}/i9052/A.scala (100%) rename tests/{neg => pos}/i9052/B.scala (100%) diff --git a/compiler/src/dotty/tools/dotc/core/Denotations.scala b/compiler/src/dotty/tools/dotc/core/Denotations.scala index 004cd9e0ce30..719b4e0e68a2 100644 --- a/compiler/src/dotty/tools/dotc/core/Denotations.scala +++ b/compiler/src/dotty/tools/dotc/core/Denotations.scala @@ -982,7 +982,7 @@ object Denotations { final def first: SingleDenotation = this final def last: SingleDenotation = this - final def matches(other: SingleDenotation)(using Context): Boolean = + def matches(other: SingleDenotation)(using Context): Boolean = val d = signature.matchDegree(other.signature) d match @@ -1013,13 +1013,13 @@ object Denotations { end matches def mapInherited(ownDenots: PreDenotation, prevDenots: PreDenotation, pre: Type)(using Context): SingleDenotation = - if (hasUniqueSym && prevDenots.containsSym(symbol)) NoDenotation - else if (isType) filterDisjoint(ownDenots).asSeenFrom(pre) + if hasUniqueSym && prevDenots.containsSym(symbol) then NoDenotation + else if isType then filterDisjoint(ownDenots).asSeenFrom(pre) else asSeenFrom(pre).filterDisjoint(ownDenots) - final def filterWithPredicate(p: SingleDenotation => Boolean): SingleDenotation = + def filterWithPredicate(p: SingleDenotation => Boolean): SingleDenotation = if (p(this)) this else NoDenotation - final def filterDisjoint(denots: PreDenotation)(using Context): SingleDenotation = + def filterDisjoint(denots: PreDenotation)(using Context): SingleDenotation = if (denots.exists && denots.matches(this)) NoDenotation else this def filterWithFlags(required: FlagSet, excluded: FlagSet)(using Context): SingleDenotation = if (required.isEmpty && excluded.isEmpty || compatibleWith(required, excluded)) this else NoDenotation diff --git a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala index 636bd6d18c7f..1324a4c5db02 100644 --- a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala @@ -2325,6 +2325,13 @@ object SymDenotations { override def owner: Symbol = throw new AssertionError("NoDenotation.owner") override def computeAsSeenFrom(pre: Type)(using Context): SingleDenotation = this override def mapInfo(f: Type => Type)(using Context): SingleDenotation = this + + override def matches(other: SingleDenotation)(using Context): Boolean = false + override def mapInherited(ownDenots: PreDenotation, prevDenots: PreDenotation, pre: Type)(using Context): SingleDenotation = this + override def filterWithPredicate(p: SingleDenotation => Boolean): SingleDenotation = this + override def filterDisjoint(denots: PreDenotation)(using Context): SingleDenotation = this + override def filterWithFlags(required: FlagSet, excluded: FlagSet)(using Context): SingleDenotation = this + NoSymbol.denot = this validFor = Period.allInRun(NoRunId) } diff --git a/tests/neg/i3253.scala b/tests/neg/i3253.scala index 51acaad92706..04a79bb73ea0 100644 --- a/tests/neg/i3253.scala +++ b/tests/neg/i3253.scala @@ -1,5 +1,5 @@ import Test.test -object Test { +class A: def test = " " * 10 // error -} +object Test extends A diff --git a/tests/neg/i9052/A.scala b/tests/pos/i9052/A.scala similarity index 100% rename from tests/neg/i9052/A.scala rename to tests/pos/i9052/A.scala diff --git a/tests/neg/i9052/B.scala b/tests/pos/i9052/B.scala similarity index 100% rename from tests/neg/i9052/B.scala rename to tests/pos/i9052/B.scala From a0ea37e3d7857c30e882889ac18c62beaf9815bb Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 26 Aug 2020 15:52:17 +0200 Subject: [PATCH 06/11] Avoid recomputations of sourceModule and moduleClass --- .../src/dotty/tools/dotc/core/SymDenotations.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala index 1324a4c5db02..d75ec652980e 100644 --- a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala @@ -2455,6 +2455,8 @@ object SymDenotations { def apply(module: TermSymbol, modcls: ClassSymbol): LazyType = this private var myDecls: Scope = EmptyScope + private var mySourceModule: Symbol = null + private var myModuleClass: Symbol = null private var mySourceModuleFn: Context ?=> Symbol = LazyType.NoSymbolFn private var myModuleClassFn: Context ?=> Symbol = LazyType.NoSymbolFn @@ -2464,8 +2466,12 @@ object SymDenotations { else sym.info.typeParams def decls: Scope = myDecls - def sourceModule(using Context): Symbol = mySourceModuleFn - def moduleClass(using Context): Symbol = myModuleClassFn + def sourceModule(using Context): Symbol = + if mySourceModule == null then mySourceModule = mySourceModuleFn + mySourceModule + def moduleClass(using Context): Symbol = + if myModuleClass == null then myModuleClass = myModuleClassFn + myModuleClass def withDecls(decls: Scope): this.type = { myDecls = decls; this } def withSourceModule(sourceModuleFn: Context ?=> Symbol): this.type = { mySourceModuleFn = sourceModuleFn; this } From 3f01f783afbd08447e86cc387ab4dfa98afe39d2 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 26 Aug 2020 18:31:46 +0200 Subject: [PATCH 07/11] Cache package object members --- compiler/src/dotty/tools/dotc/core/SymDenotations.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala index d75ec652980e..f9dc85c33f48 100644 --- a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala @@ -2217,14 +2217,13 @@ object SymDenotations { def recur(pobjs: List[ClassDenotation], acc: PreDenotation): PreDenotation = pobjs match { case pcls :: pobjs1 => if (pcls.isCompleting) recur(pobjs1, acc) - else { - val pmembers = pcls.computeNPMembersNamed(name).filterWithPredicate { d => + else + val pobjMembers = pcls.nonPrivateMembersNamed(name).filterWithPredicate { d => // Drop members of `Any` and `Object` val owner = d.symbol.maybeOwner (owner ne defn.AnyClass) && (owner ne defn.ObjectClass) } - recur(pobjs1, acc.union(pmembers)) - } + recur(pobjs1, acc.union(pobjMembers)) case nil => val directMembers = super.computeNPMembersNamed(name) if !acc.exists then directMembers From 1343d8661eb6db1463fd68a65cd4755701737f32 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 26 Aug 2020 19:16:23 +0200 Subject: [PATCH 08/11] Cache also private members This means we have to do extra work for a nonPrivateMember that hits a private member. But that's probably not a very common case. On the other hand, every find member operation needs one less scope lookup this way. --- .../dotty/tools/dotc/core/Denotations.scala | 17 ++-- .../src/dotty/tools/dotc/core/Scopes.scala | 9 +- .../tools/dotc/core/SymDenotations.scala | 93 +++++++++---------- .../dotc/transform/Instrumentation.scala | 3 +- 4 files changed, 52 insertions(+), 70 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/Denotations.scala b/compiler/src/dotty/tools/dotc/core/Denotations.scala index 719b4e0e68a2..21afdcc85af6 100644 --- a/compiler/src/dotty/tools/dotc/core/Denotations.scala +++ b/compiler/src/dotty/tools/dotc/core/Denotations.scala @@ -1022,7 +1022,12 @@ object Denotations { def filterDisjoint(denots: PreDenotation)(using Context): SingleDenotation = if (denots.exists && denots.matches(this)) NoDenotation else this def filterWithFlags(required: FlagSet, excluded: FlagSet)(using Context): SingleDenotation = - if (required.isEmpty && excluded.isEmpty || compatibleWith(required, excluded)) this else NoDenotation + def symd: SymDenotation = this match + case symd: SymDenotation => symd + case _ => symbol.denot + if !required.isEmpty && !symd.isAllOf(required) + || !excluded.isEmpty && symd.isOneOf(excluded) then NoDenotation + else this def aggregate[T](f: SingleDenotation => T, g: (T, T) => T): T = f(this) type AsSeenFromResult = SingleDenotation @@ -1056,16 +1061,6 @@ object Denotations { if (!owner.membersNeedAsSeenFrom(pre) || symbol.is(NonMember)) this else derived(symbol.info) } - - /** Does this denotation have all the `required` flags but none of the `excluded` flags? - */ - private def compatibleWith(required: FlagSet, excluded: FlagSet)(using Context): Boolean = { - val symd: SymDenotation = this match { - case symd: SymDenotation => symd - case _ => symbol.denot - } - symd.isAllOf(required) && !symd.isOneOf(excluded) - } } abstract class NonSymSingleDenotation(symbol: Symbol, initInfo: Type, override val prefix: Type) extends SingleDenotation(symbol, initInfo) { diff --git a/compiler/src/dotty/tools/dotc/core/Scopes.scala b/compiler/src/dotty/tools/dotc/core/Scopes.scala index 90b27683cdf7..cb5f5c55e337 100644 --- a/compiler/src/dotty/tools/dotc/core/Scopes.scala +++ b/compiler/src/dotty/tools/dotc/core/Scopes.scala @@ -148,12 +148,11 @@ object Scopes { * Symbols occur in the result in reverse order relative to their occurrence * in `this.toList`. */ - final def denotsNamed(name: Name, select: SymDenotation => Boolean = selectAll)(using Context): PreDenotation = { + final def denotsNamed(name: Name)(using Context): PreDenotation = { var syms: PreDenotation = NoDenotation var e = lookupEntry(name) while (e != null) { - val d = e.sym.denot - if (select(d)) syms = syms union d + syms = syms union e.sym.denot e = lookupNextEntry(e) } syms @@ -458,10 +457,6 @@ object Scopes { */ def scopeTransform(owner: Symbol)(op: => MutableScope): MutableScope = op - val selectAll: SymDenotation => Boolean = alwaysTrue - val selectPrivate: SymDenotation => Boolean = d => (d.flagsUNSAFE is Flags.Private) - val selectNonPrivate: SymDenotation => Boolean = d => !(d.flagsUNSAFE is Flags.Private) - /** The empty scope (immutable). */ object EmptyScope extends Scope { diff --git a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala index f9dc85c33f48..917d70cf70f4 100644 --- a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala @@ -1855,61 +1855,53 @@ object SymDenotations { * The elements of the returned pre-denotation all * have existing symbols. */ - final def membersNamed(name: Name)(using Context): PreDenotation = { - val privates = info.decls.denotsNamed(name, selectPrivate) - privates union nonPrivateMembersNamed(name).filterDisjoint(privates) - } - - /** All non-private members of this class that have the given name. - * The elements of the returned pre-denotation all - * have existing symbols. - * @param inherited The method is called on a parent class from computeNPMembersNamed - */ - final def nonPrivateMembersNamed(name: Name)(using Context): PreDenotation = { - Stats.record("nonPrivateMembersNamed") - if (Config.cacheMembersNamed) { + final def membersNamed(name: Name)(using Context): PreDenotation = + Stats.record("membersNamed") + if Config.cacheMembersNamed then var denots: PreDenotation = memberCache.lookup(name) - if (denots == null) { - denots = computeNPMembersNamed(name) + if denots == null then + denots = computeMembersNamed(name) memberCache.enter(name, denots) - } - else if (Config.checkCacheMembersNamed) { - val denots1 = computeNPMembersNamed(name) + else if Config.checkCacheMembersNamed then + val denots1 = computeMembersNamed(name) assert(denots.exists == denots1.exists, s"cache inconsistency: cached: $denots, computed $denots1, name = $name, owner = $this") - } denots - } - else computeNPMembersNamed(name) - } + else computeMembersNamed(name) + - private[core] def computeNPMembersNamed(name: Name)(using Context): PreDenotation = { - Stats.record("computeNPMembersNamed after fingerprint") - ensureCompleted() - val ownDenots = info.decls.denotsNamed(name, selectNonPrivate) - if (debugTrace) // DEBUG + /** All non-private members of this class that have the given name. + * The elements of the returned pre-denotation all have existing symbols. + */ + final def nonPrivateMembersNamed(name: Name)(using Context): PreDenotation = + val mbr = membersNamed(name) + val nonPrivate = mbr.filterWithFlags(EmptyFlags, Private) + if nonPrivate eq mbr then mbr + else addInherited(name, nonPrivate) + + private[core] def computeMembersNamed(name: Name)(using Context): PreDenotation = + Stats.record("computeMembersNamed") + val ownDenots = info.decls.denotsNamed(name) + if debugTrace then println(s"$this.member($name), ownDenots = $ownDenots") - def collect(denots: PreDenotation, parents: List[Type]): PreDenotation = parents match { + if name.isConstructorName then ownDenots + else addInherited(name, ownDenots) + + private def addInherited(name: Name, ownDenots: PreDenotation)(using Context): PreDenotation = + def collect(denots: PreDenotation, parents: List[Type]): PreDenotation = parents match case p :: ps => val denots1 = collect(denots, ps) - p.classSymbol.denot match { + p.classSymbol.denot match case parentd: ClassDenotation => - denots1.union( - parentd.nonPrivateMembersNamed(name) - .mapInherited(ownDenots, denots1, thisType)) + val inherited = parentd.nonPrivateMembersNamed(name) + denots1.union(inherited.mapInherited(ownDenots, denots1, thisType)) case _ => denots1 - } - case nil => - denots - } - if (name.isConstructorName) ownDenots - else collect(ownDenots, classParents) - } + case nil => denots + collect(ownDenots, classParents) - override final def findMember(name: Name, pre: Type, required: FlagSet, excluded: FlagSet)(using Context): Denotation = { - val raw = if (excluded.is(Private)) nonPrivateMembersNamed(name) else membersNamed(name) + override final def findMember(name: Name, pre: Type, required: FlagSet, excluded: FlagSet)(using Context): Denotation = + val raw = if excluded.is(Private) then nonPrivateMembersNamed(name) else membersNamed(name) raw.filterWithFlags(required, excluded).asSeenFrom(pre).toDenot(pre) - } /** Compute tp.baseType(this) */ final def baseTypeOf(tp: Type)(using Context): Type = { @@ -2213,8 +2205,9 @@ object SymDenotations { * object that hides a class or object in the scala package of the same name, because * the behavior would then be unintuitive for such members. */ - override def computeNPMembersNamed(name: Name)(using Context): PreDenotation = { - def recur(pobjs: List[ClassDenotation], acc: PreDenotation): PreDenotation = pobjs match { + override def computeMembersNamed(name: Name)(using Context): PreDenotation = + + def recur(pobjs: List[ClassDenotation], acc: PreDenotation): PreDenotation = pobjs match case pcls :: pobjs1 => if (pcls.isCompleting) recur(pobjs1, acc) else @@ -2225,12 +2218,11 @@ object SymDenotations { } recur(pobjs1, acc.union(pobjMembers)) case nil => - val directMembers = super.computeNPMembersNamed(name) + val directMembers = super.computeMembersNamed(name) if !acc.exists then directMembers else acc.union(directMembers.filterWithPredicate(!_.symbol.isAbsent())) match case d: DenotUnion => dropStale(d) case d => d - } def dropStale(multi: DenotUnion): PreDenotation = val compiledNow = multi.filterWithPredicate(d => @@ -2272,13 +2264,12 @@ object SymDenotations { multi.filterWithPredicate(_.symbol.associatedFile == chosen) end dropStale - if (symbol `eq` defn.ScalaPackageClass) { - val denots = super.computeNPMembersNamed(name) - if (denots.exists || name == nme.CONSTRUCTOR) denots + if symbol eq defn.ScalaPackageClass then + val denots = super.computeMembersNamed(name) + if denots.exists || name == nme.CONSTRUCTOR then denots else recur(packageObjs, NoDenotation) - } else recur(packageObjs, NoDenotation) - } + end computeMembersNamed /** The union of the member names of the package and the package object */ override def memberNames(keepOnly: NameFilter)(implicit onBehalf: MemberNames, ctx: Context): Set[Name] = { diff --git a/compiler/src/dotty/tools/dotc/transform/Instrumentation.scala b/compiler/src/dotty/tools/dotc/transform/Instrumentation.scala index 848c6866b22a..97126df3a6d6 100644 --- a/compiler/src/dotty/tools/dotc/transform/Instrumentation.scala +++ b/compiler/src/dotty/tools/dotc/transform/Instrumentation.scala @@ -31,7 +31,8 @@ class Instrumentation extends MiniPhase { thisPhase => private val namesOfInterest = List( "::", "+=", "toString", "newArray", "box", "toCharArray", "map", "flatMap", "filter", "withFilter", "collect", "foldLeft", "foldRight", "take", - "reverse", "mapConserve", "mapconserve", "filterConserve", "zip") + "reverse", "mapConserve", "mapconserve", "filterConserve", "zip", + "denotsNamed", "lookup", "lookupEntry", "lookupAll", "toList") private var namesToRecord: Set[Name] = _ private var consName: TermName = _ From 44a07d6cdde262e062a1e60a9c1b414fe985e62e Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Wed, 26 Aug 2020 19:32:53 +0200 Subject: [PATCH 09/11] Revert "Disable flakey test" This reverts commit f8d6ac4c61673084b4591eab27f320ecc387c7b9. --- compiler/test/dotc/pos-from-tasty.blacklist | 4 ---- 1 file changed, 4 deletions(-) diff --git a/compiler/test/dotc/pos-from-tasty.blacklist b/compiler/test/dotc/pos-from-tasty.blacklist index 8a083db6d39d..1fac00b403ab 100644 --- a/compiler/test/dotc/pos-from-tasty.blacklist +++ b/compiler/test/dotc/pos-from-tasty.blacklist @@ -13,7 +13,3 @@ i7087.scala # Nullability nullable.scala notNull.scala - -# Flakey behavior -projections.scala - From 058e88d10655b6cf4e38d992989edfb96b268907 Mon Sep 17 00:00:00 2001 From: Martin Odersky Date: Thu, 27 Aug 2020 19:13:56 +0200 Subject: [PATCH 10/11] Fix handling of constructors --- compiler/src/dotty/tools/dotc/core/SymDenotations.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala index 917d70cf70f4..fc014ec6684f 100644 --- a/compiler/src/dotty/tools/dotc/core/SymDenotations.scala +++ b/compiler/src/dotty/tools/dotc/core/SymDenotations.scala @@ -1883,8 +1883,7 @@ object SymDenotations { val ownDenots = info.decls.denotsNamed(name) if debugTrace then println(s"$this.member($name), ownDenots = $ownDenots") - if name.isConstructorName then ownDenots - else addInherited(name, ownDenots) + addInherited(name, ownDenots) private def addInherited(name: Name, ownDenots: PreDenotation)(using Context): PreDenotation = def collect(denots: PreDenotation, parents: List[Type]): PreDenotation = parents match @@ -1897,7 +1896,8 @@ object SymDenotations { case _ => denots1 case nil => denots - collect(ownDenots, classParents) + if name.isConstructorName then ownDenots + else collect(ownDenots, classParents) override final def findMember(name: Name, pre: Type, required: FlagSet, excluded: FlagSet)(using Context): Denotation = val raw = if excluded.is(Private) then nonPrivateMembersNamed(name) else membersNamed(name) From 9e4dbe833c31ab4d26b51b06e02b5c06bf3fcac9 Mon Sep 17 00:00:00 2001 From: odersky Date: Tue, 1 Sep 2020 11:29:59 +0200 Subject: [PATCH 11/11] Update compiler/src/dotty/tools/dotc/util/HashTable.scala Co-authored-by: Fengyun Liu --- compiler/src/dotty/tools/dotc/util/HashTable.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/src/dotty/tools/dotc/util/HashTable.scala b/compiler/src/dotty/tools/dotc/util/HashTable.scala index 42c65f53154e..d91d6c5d1a9f 100644 --- a/compiler/src/dotty/tools/dotc/util/HashTable.scala +++ b/compiler/src/dotty/tools/dotc/util/HashTable.scala @@ -138,5 +138,5 @@ class HashTable[Key >: Null <: AnyRef, Value >: Null <: AnyRef] yield (keyAt(idx), valueAt(idx)) override def toString: String = - iterator.map((k, v) => s"$k -> $v").mkString("LinearTable(", ", ", ")") + iterator.map((k, v) => s"$k -> $v").mkString("HashTable(", ", ", ")") end HashTable