Add experimental flexible types feature on top of explicit nulls

Enabled by -Yflexible-types with -Yexplicit-nulls.
A flexible type T! is a non-denotable type such that T <: T! <: T|Null and T|Null <: T! <: T. Here we patch return types and parameter types of Java methods and fields to use flexible types.
This is unsound and kills subtyping transitivity but makes interop with Java play more nicely with the explicit nulls experimental feature (i.e. fewer nullability casts).
Also adds a few tests for flexible types, mostly lifted from the explicit nulls tests.

Author: evangirardin

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -381,6 +381,7 @@ private sealed trait YSettings:
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Disable kind polymorphism.")
   val YexplicitNulls: Setting[Boolean] = BooleanSetting("-Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")
+  val YflexibleTypes: Setting[Boolean] = BooleanSetting("-Yflexible-types", "Make Java return types and parameter types use flexible types. Flexible types essentially circumvent explicit nulls and force something resembling the old type system for Java interop.")
   val YcheckInit: Setting[Boolean] = BooleanSetting("-Ysafe-init", "Ensure safe initialization of objects.")
   val YcheckInitGlobal: Setting[Boolean] = BooleanSetting("-Ysafe-init-global", "Check safe initialization of global objects.")
   val YrequireTargetName: Setting[Boolean] = BooleanSetting("-Yrequire-targetName", "Warn if an operator is defined without a @targetName annotation.")

compiler/src/dotty/tools/dotc/core/CheckRealizable.scala

@@ -122,6 +122,7 @@ class CheckRealizable(using Context) {
         case tp: TypeProxy => isConcrete(tp.underlying)
         case tp: AndType => isConcrete(tp.tp1) && isConcrete(tp.tp2)
         case tp: OrType  => isConcrete(tp.tp1) && isConcrete(tp.tp2)
+        case tp: FlexibleType => isConcrete(tp.underlying)
         case _ => false
       }
       if (!isConcrete(tp)) NotConcrete

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -453,6 +453,9 @@ object Contexts {
     /** Is the explicit nulls option set? */
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
+    /** Is the flexible types option set? */
+    def flexibleTypes: Boolean = base.settings.YflexibleTypes.value
+
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
 

compiler/src/dotty/tools/dotc/core/JavaNullInterop.scala

@@ -63,10 +63,10 @@ object JavaNullInterop {
       // Don't nullify the return type of the `toString` method.
       // Don't nullify the return type of constructors.
       // Don't nullify the return type of methods with a not-null annotation.
-      nullifyExceptReturnType(tp)
+      nullifyExceptReturnType(tp, sym.owner.isClass)
     else
       // Otherwise, nullify everything
-      nullifyType(tp)
+      nullifyType(tp, sym.owner.isClass)
   }
 
   private def hasNotNullAnnot(sym: Symbol)(using Context): Boolean =
@@ -77,12 +77,12 @@ object JavaNullInterop {
    *  If tp is a type of a field, the inside of the type is nullified,
    *  but the result type is not nullable.
    */
-  private def nullifyExceptReturnType(tp: Type)(using Context): Type =
-    new JavaNullMap(true)(tp)
+  private def nullifyExceptReturnType(tp: Type, ownerIsClass: Boolean)(using Context): Type =
+    if ctx.flexibleTypes /*&& ownerIsClass*/ then new JavaFlexibleMap(true)(tp) else new JavaNullMap(true)(tp) // FLEX PARAMS
 
   /** Nullifies a Java type by adding `| Null` in the relevant places. */
-  private def nullifyType(tp: Type)(using Context): Type =
-    new JavaNullMap(false)(tp)
+  private def nullifyType(tp: Type, ownerIsClass: Boolean)(using Context): Type =
+    if ctx.flexibleTypes /*&& ownerIsClass*/ then new JavaFlexibleMap(false)(tp) else new JavaNullMap(false)(tp) // FLEX PARAMS
 
   /** A type map that implements the nullification function on types. Given a Java-sourced type, this adds `| Null`
    *  in the right places to make the nulls explicit in Scala.
@@ -146,4 +146,63 @@ object JavaNullInterop {
       case _ => tp
     }
   }
+
+  /**
+   *  Flexible types
+   */
+
+  private class JavaFlexibleMap(var outermostLevelAlreadyNullable: Boolean)(using Context) extends TypeMap {
+    /** Should we nullify `tp` at the outermost level? */
+    def needsFlexible(tp: Type): Boolean =
+      !outermostLevelAlreadyNullable && (tp match {
+        case tp: TypeRef =>
+          // We don't modify value types because they're non-nullable even in Java.
+          !tp.symbol.isValueClass &&
+          // We don't modify `Any` because it's already nullable.
+          !tp.isRef(defn.AnyClass) &&
+          // We don't nullify Java varargs at the top level.
+          // Example: if `setNames` is a Java method with signature `void setNames(String... names)`,
+          // then its Scala signature will be `def setNames(names: (String|Null)*): Unit`.
+          // This is because `setNames(null)` passes as argument a single-element array containing the value `null`,
+          // and not a `null` array.
+          !tp.isRef(defn.RepeatedParamClass)
+        case _ => true
+      })
+
+    override def apply(tp: Type): Type = tp match {
+      case tp: TypeRef if needsFlexible(tp) =>
+        //println(Thread.currentThread().getStackTrace()(3).getMethodName())
+        FlexibleType(tp)
+      case appTp @ AppliedType(tycon, targs) =>
+        val oldOutermostNullable = outermostLevelAlreadyNullable
+        // We don't make the outmost levels of type arguments nullable if tycon is Java-defined.
+        // This is because Java classes are _all_ nullified, so both `java.util.List[String]` and
+        // `java.util.List[String|Null]` contain nullable elements.
+        outermostLevelAlreadyNullable = tp.classSymbol.is(JavaDefined)
+        val targs2 = targs map this
+        outermostLevelAlreadyNullable = oldOutermostNullable
+        val appTp2 = derivedAppliedType(appTp, tycon, targs2)
+        if needsFlexible(tycon) then FlexibleType(appTp2) else appTp2
+      case ptp: PolyType =>
+        derivedLambdaType(ptp)(ptp.paramInfos, this(ptp.resType))
+      case mtp: MethodType =>
+        val oldOutermostNullable = outermostLevelAlreadyNullable
+        outermostLevelAlreadyNullable = false
+        val paramInfos2 = mtp.paramInfos map this /*new JavaNullMap(outermostLevelAlreadyNullable)*/ // FLEX PARAMS
+        outermostLevelAlreadyNullable = oldOutermostNullable
+        derivedLambdaType(mtp)(paramInfos2, this(mtp.resType))
+      case tp: TypeAlias => mapOver(tp)
+      case tp: AndType =>
+        // nullify(A & B) = (nullify(A) & nullify(B)) | Null, but take care not to add
+        // duplicate `Null`s at the outermost level inside `A` and `B`.
+        outermostLevelAlreadyNullable = true
+        FlexibleType(derivedAndType(tp, this(tp.tp1), this(tp.tp2)))
+      case tp: TypeParamRef if needsFlexible(tp) =>
+        FlexibleType(tp)
+      // In all other cases, return the type unchanged.
+      // In particular, if the type is a ConstantType, then we don't nullify it because it is the
+      // type of a final non-nullable field.
+      case _ => tp
+    }
+  }
 }

compiler/src/dotty/tools/dotc/core/NullOpsDecorator.scala

@@ -8,6 +8,12 @@ import Types._
 object NullOpsDecorator:
 
   extension (self: Type)
+    def stripFlexible(using Context): Type = {
+      self match {
+        case FlexibleType(tp) => tp
+        case _ => self
+      }
+    }
     /** Syntactically strips the nullability from this type.
      *  If the type is `T1 | ... | Tn`, and `Ti` references to `Null`,
      *  then return `T1 | ... | Ti-1 | Ti+1 | ... | Tn`.
@@ -33,6 +39,7 @@ object NullOpsDecorator:
             if (tp1s ne tp1) && (tp2s ne tp2) then
               tp.derivedAndType(tp1s, tp2s)
             else tp
+          case tp @ FlexibleType(tp1) => strip(tp1)
           case tp @ TypeBounds(lo, hi) =>
             tp.derivedTypeBounds(strip(lo), strip(hi))
           case tp => tp

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -560,6 +560,9 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
       case CapturingType(parent, refs) =>
         val parent1 = recur(parent)
         if parent1 ne parent then tp.derivedCapturingType(parent1, refs) else tp
+      case tp: FlexibleType =>
+        val underlying = recur(tp.underlying)
+        if underlying ne tp.underlying then tp.derivedFlexibleType(underlying) else tp
       case tp: AnnotatedType =>
         val parent1 = recur(tp.parent)
         if parent1 ne tp.parent then tp.derivedAnnotatedType(parent1, tp.annot) else tp

compiler/src/dotty/tools/dotc/core/PatternTypeConstrainer.scala

@@ -10,6 +10,7 @@ import Contexts.ctx
 import dotty.tools.dotc.reporting.trace
 import config.Feature.migrateTo3
 import config.Printers._
+import dotty.tools.dotc.core.NullOpsDecorator.stripFlexible
 
 trait PatternTypeConstrainer { self: TypeComparer =>
 
@@ -175,7 +176,7 @@ trait PatternTypeConstrainer { self: TypeComparer =>
       case tp => tp
     }
 
-    dealiasDropNonmoduleRefs(scrut) match {
+    dealiasDropNonmoduleRefs(scrut.stripFlexible) match {
       case OrType(scrut1, scrut2) =>
         either(constrainPatternType(pat, scrut1), constrainPatternType(pat, scrut2))
       case AndType(scrut1, scrut2) =>

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -2268,6 +2268,9 @@ object SymDenotations {
           case CapturingType(parent, refs) =>
             tp.derivedCapturingType(recur(parent), refs)
 
+          case tp: FlexibleType =>
+            recur(tp.underlying)
+
           case tp: TypeProxy =>
             def computeTypeProxy = {
               val superTp = tp.superType

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -383,6 +383,10 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
       case OrType(tp21, tp22) =>
         if (tp21.stripTypeVar eq tp22.stripTypeVar) recur(tp1, tp21)
         else secondTry
+      // tp1 <: Flex(T) = T|N..T
+      // iff  tp1 <: T|N
+      case tp2: FlexibleType =>
+        recur(tp1, tp2.lo)
       case TypeErasure.ErasedValueType(tycon1, underlying2) =>
         def compareErasedValueType = tp1 match {
           case TypeErasure.ErasedValueType(tycon2, underlying1) =>
@@ -531,7 +535,10 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
           hardenTypeVars(tp2)
 
         res
-
+      // invariant: tp2 is NOT a FlexibleType
+      // is Flex(T) <: tp2?
+      case tp1: FlexibleType =>
+        recur(tp1.underlying, tp2)
       case CapturingType(parent1, refs1) =>
         if tp2.isAny then true
         else if subCaptures(refs1, tp2.captureSet, frozenConstraint).isOK && sameBoxed(tp1, tp2, refs1)
@@ -2545,53 +2552,73 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
   /** Try to distribute `&` inside type, detect and handle conflicts
    *  @pre !(tp1 <: tp2) && !(tp2 <:< tp1) -- these cases were handled before
    */
-  private def distributeAnd(tp1: Type, tp2: Type): Type = tp1 match {
-    case tp1 @ AppliedType(tycon1, args1) =>
-      tp2 match {
-        case AppliedType(tycon2, args2)
-        if tycon1.typeSymbol == tycon2.typeSymbol && tycon1 =:= tycon2 =>
-          val jointArgs = glbArgs(args1, args2, tycon1.typeParams)
-          if (jointArgs.forall(_.exists)) (tycon1 & tycon2).appliedTo(jointArgs)
-          else NoType
-        case _ =>
-          NoType
-      }
-    case tp1: RefinedType =>
-      // opportunistically merge same-named refinements
-      // this does not change anything semantically (i.e. merging or not merging
-      // gives =:= types), but it keeps the type smaller.
-      tp2 match {
-        case tp2: RefinedType if tp1.refinedName == tp2.refinedName =>
-          val jointInfo = Denotations.infoMeet(tp1.refinedInfo, tp2.refinedInfo, safeIntersection = false)
-          if jointInfo.exists then
-            tp1.derivedRefinedType(tp1.parent & tp2.parent, tp1.refinedName, jointInfo)
-          else
+  private def distributeAnd(tp1: Type, tp2: Type): Type = {
+    var ft1 = false
+    var ft2 = false
+    def recur(tp1: Type, tp2: Type): Type = tp1 match {
+      case tp1 @ FlexibleType(tp) =>
+        // Hack -- doesn't generalise to other intersection/union types
+        // but covers a common special case for pattern matching
+        ft1 = true
+        recur(tp, tp2)
+      case tp1 @ AppliedType(tycon1, args1) =>
+        tp2 match {
+          case AppliedType(tycon2, args2)
+          if tycon1.typeSymbol == tycon2.typeSymbol && tycon1 =:= tycon2 =>
+            val jointArgs = glbArgs(args1, args2, tycon1.typeParams)
+            if (jointArgs.forall(_.exists)) (tycon1 & tycon2).appliedTo(jointArgs)
+            else {
+              NoType
+            }
+          case FlexibleType(tp) =>
+            // Hack from above
+            ft2 = true
+            recur(tp1, tp)
+          case _ =>
             NoType
-        case _ =>
-          NoType
-      }
-    case tp1: RecType =>
-      tp1.rebind(distributeAnd(tp1.parent, tp2))
-    case ExprType(rt1) =>
-      tp2 match {
-        case ExprType(rt2) =>
-          ExprType(rt1 & rt2)
-        case _ =>
-          NoType
-      }
-    case tp1: TypeVar if tp1.isInstantiated =>
-      tp1.underlying & tp2
-    case CapturingType(parent1, refs1) =>
-      if subCaptures(tp2.captureSet, refs1, frozen = true).isOK
-        && tp1.isBoxedCapturing == tp2.isBoxedCapturing
-      then
-        parent1 & tp2
-      else
-        tp1.derivedCapturingType(parent1 & tp2, refs1)
-    case tp1: AnnotatedType if !tp1.isRefining =>
-      tp1.underlying & tp2
-    case _ =>
-      NoType
+        }
+
+      // if result exists and is not notype, maybe wrap result in flex based on whether seen flex on both sides
+      case tp1: RefinedType =>
+        // opportunistically merge same-named refinements
+        // this does not change anything semantically (i.e. merging or not merging
+        // gives =:= types), but it keeps the type smaller.
+        tp2 match {
+          case tp2: RefinedType if tp1.refinedName == tp2.refinedName =>
+            val jointInfo = Denotations.infoMeet(tp1.refinedInfo, tp2.refinedInfo, safeIntersection = false)
+            if jointInfo.exists then
+              tp1.derivedRefinedType(tp1.parent & tp2.parent, tp1.refinedName, jointInfo)
+            else
+              NoType
+          case _ =>
+            NoType
+        }
+      case tp1: RecType =>
+        tp1.rebind(recur(tp1.parent, tp2))
+      case ExprType(rt1) =>
+        tp2 match {
+          case ExprType(rt2) =>
+            ExprType(rt1 & rt2)
+          case _ =>
+            NoType
+        }
+      case tp1: TypeVar if tp1.isInstantiated =>
+        tp1.underlying & tp2
+      case CapturingType(parent1, refs1) =>
+        if subCaptures(tp2.captureSet, refs1, frozen = true).isOK
+          && tp1.isBoxedCapturing == tp2.isBoxedCapturing
+        then
+          parent1 & tp2
+        else
+          tp1.derivedCapturingType(parent1 & tp2, refs1)
+      case tp1: AnnotatedType if !tp1.isRefining =>
+        tp1.underlying & tp2
+      case _ =>
+        NoType
+    }
+    // if flex on both sides, return flex type
+    val ret = recur(tp1, tp2)
+    if (ft1 && ft2) then FlexibleType(ret) else ret
   }
 
   /** Try to distribute `|` inside type, detect and handle conflicts

compiler/src/dotty/tools/dotc/core/TypeErasure.scala

@@ -311,6 +311,8 @@ object TypeErasure {
           repr1.orElse(repr2)
         else
           NoSymbol
+      case tp: FlexibleType =>
+        arrayUpperBound(tp.underlying)
       case _ =>
         NoSymbol
 
@@ -337,6 +339,8 @@ object TypeErasure {
         isGenericArrayElement(tp.tp1, isScala2) && isGenericArrayElement(tp.tp2, isScala2)
       case tp: OrType =>
         isGenericArrayElement(tp.tp1, isScala2) || isGenericArrayElement(tp.tp2, isScala2)
+      case tp: FlexibleType =>
+        isGenericArrayElement(tp.underlying, isScala2)
       case _ => false
     }
   }
@@ -526,6 +530,7 @@ object TypeErasure {
     case tp: TypeProxy => hasStableErasure(tp.translucentSuperType)
     case tp: AndType => hasStableErasure(tp.tp1) && hasStableErasure(tp.tp2)
     case tp: OrType  => hasStableErasure(tp.tp1) && hasStableErasure(tp.tp2)
+    case _: FlexibleType => false
     case _ => false
   }
 
@@ -622,6 +627,7 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
       erasePolyFunctionApply(refinedInfo)
     case RefinedType(parent, nme.apply, refinedInfo: MethodType) if defn.isErasedFunctionType(parent) =>
       eraseErasedFunctionApply(refinedInfo)
+    case FlexibleType(tp) => this(tp)
     case tp: TypeProxy =>
       this(tp.underlying)
     case tp @ AndType(tp1, tp2) =>

compiler/src/dotty/tools/dotc/core/TypeOps.scala

@@ -52,7 +52,6 @@ object TypeOps:
         Stats.record("asSeenFrom skolem prefix required")
       case _ =>
     }
-
     new AsSeenFromMap(pre, cls).apply(tp)
   }
 
@@ -237,6 +236,7 @@ object TypeOps:
         if tp1.isBottomType && (tp1 frozen_<:< tp2) then orBaseClasses(tp2)
         else if tp2.isBottomType && (tp2 frozen_<:< tp1) then orBaseClasses(tp1)
         else intersect(orBaseClasses(tp1), orBaseClasses(tp2))
+      case FlexibleType(tp1) => orBaseClasses(tp1)
       case _ => tp.baseClasses
 
     /** The minimal set of classes in `cs` which derive all other classes in `cs` */