Interpret Java types as implicitly nullable

This commit adds a "translation layer" between Java types (which are
implicitly nullable), and Scala's view of those same Java types (which
most be explicitly nullable to be sound).

e.g. given the Java method

`String foo(String arg) { return arg; }`

Scala should view the method as if had signature

`def foo(arg: String|JavaNull): String|JavaNull`

This transformation should happen both if the Java code is loaded from
bytecode, or if it's loaded from source. The exact rules for how types
are translated can be found in `JavaNullInterop.scala`.

Author: abeln

compiler/src/dotty/tools/dotc/core/Flags.scala

@@ -394,6 +394,15 @@ object Flags {
   /** Symbol is an enum class or enum case (if used with case) */
   final val Enum: FlagSet = commonFlag(40, "<enum>")
 
+  /** A Java enum */
+  final val JavaEnum: FlagConjunction = allOf(JavaDefined, Enum)
+
+  /** A Java enum trait */
+  final val JavaEnumTrait: FlagConjunction = allOf(JavaDefined, Enum)
+
+  /** A Java enum value */
+  final val JavaEnumValue: FlagConjunction = allOf(StableRealizable, JavaStatic, JavaDefined, Enum)
+
   /** Labeled with `erased` modifier (erased value)  */
   final val Erased: FlagSet = termFlag(42, "erased")
 
@@ -482,7 +491,7 @@ object Flags {
   final val FromStartFlags: FlagSet =
     Module | Package | Deferred | Method.toCommonFlags | Case |
     HigherKinded.toCommonFlags | Param | ParamAccessor.toCommonFlags |
-    Scala2ExistentialCommon | MutableOrOpaque | Touched | JavaStatic |
+    Scala2ExistentialCommon | MutableOrOpaque | Touched | JavaStatic | JavaDefined | JavaEnumValue.toCommonFlags |
     CovariantOrOuter | ContravariantOrLabel | CaseAccessor.toCommonFlags |
     Extension.toCommonFlags | NonMember | Implicit | Implied | Permanent | Synthetic |
     SuperAccessorOrScala2x | Inline
@@ -679,15 +688,6 @@ object Flags {
   /** A Java companion object */
   final val JavaProtected: FlagConjunction = allOf(JavaDefined, Protected)
 
-  /** A Java enum */
-  final val JavaEnum: FlagConjunction = allOf(JavaDefined, Enum)
-
-  /** A Java enum trait */
-  final val JavaEnumTrait: FlagConjunction = allOf(JavaDefined, Enum)
-
-  /** A Java enum value */
-  final val JavaEnumValue: FlagConjunction = allOf(StableRealizable, JavaStatic, JavaDefined, Enum)
-
   /** Labeled private[this] */
   final val PrivateLocal: FlagConjunction = allOf(Private, Local)
 

compiler/src/dotty/tools/dotc/core/JavaNullInterop.scala

@@ -0,0 +1,189 @@
+package dotty.tools.dotc.core
+
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.core.Flags.JavaDefined
+import dotty.tools.dotc.core.StdNames.{jnme, nme}
+import dotty.tools.dotc.core.Symbols.{Symbol, defn, _}
+import dotty.tools.dotc.core.Types.{AndType, AppliedType, LambdaType, MethodType, OrType, PolyType, Type, TypeAlias, TypeMap, TypeParamRef, TypeRef}
+
+/** This module defines methods to interpret Java types, which are implicitly nullable,
+ *  as Scala types, which are explicitly nullable.
+ *
+ *  The transformation from Java types to Scala types is (conceptually) a function `n`
+ *  that adheres to the following rules:
+ *    (1) n(T)              = T|JavaNull              if T is a reference type
+ *    (2) n(T)              = T                       if T is a value type
+ *    (3) n(C[T])           = C[T]|JavaNull           if C is Java-defined
+ *    (4) n(C[T])           = C[n(T)]|JavaNull        if C is Scala-defined
+ *    (5) n(A|B)            = n(A)|n(B)|JavaNull
+ *    (6) n(A&B)            = n(A) & n(B)
+ *    (7) n((A1, ..., Am)R) = (n(A1), ..., n(Am))n(R) for a method with arguments (A1, ..., Am) and return type R
+ *    (8) n(T)              = T                       otherwise
+ *
+ *   Notice that since the transformation is only applied to types attached to Java symbols, it doesn't need
+ *   to handle the full spectrum of Scala types. Additionally, some kinds of symbols like constructors and
+ *   enum instances get special treatment.
+ */
+object JavaNullInterop {
+
+  /** Transforms the type `tp` of Java member `sym` to be explicitly nullable.
+   *  `tp` is needed because the type inside `sym` might not be set when this method is called.
+   *
+   *  e.g. given a Java method
+   *  String foo(String arg) { return arg; }
+   *
+   *  After calling `nullifyMember`, Scala will see the method as
+   *
+   *  def foo(arg: String|JavaNull): String|JavaNull
+   *
+   *  This nullability function uses `JavaNull` instead of vanilla `Null`, for usability.
+   *  This means that we can select on the return of `foo`:
+   *
+   *  val len = foo("hello").length
+   *
+   *  But the selection can throw an NPE if the returned value is `null`.
+   */
+  def nullifyMember(sym: Symbol, tp: Type)(implicit ctx: Context): Type = {
+    assert(ctx.settings.YexplicitNulls.value)
+    assert(sym.is(JavaDefined), "can only nullify java-defined members")
+
+    // A list of members that are special-cased.
+    val whitelist: Seq[NullifyPolicy] = Seq(
+      // The `TYPE` field in every class: don't nullify.
+      NoOpP(_.name == nme.TYPE_),
+      // The `toString` method: don't nullify the return type.
+      paramsOnlyP(_.name == nme.toString_),
+      // Constructors: params are nullified, but the result type isn't.
+      paramsOnlyP(_.isConstructor),
+      // Java enum instances: don't nullify.
+      NoOpP(_.is(Flags.JavaEnumValue))
+    )
+
+    val (fromWhitelistTp, handled) = whitelist.foldLeft((tp, false)) {
+      case (res@(_, true), _) => res
+      case ((_, false), pol) =>
+        if (pol.isApplicable(sym)) (pol(tp), true)
+        else (tp, false)
+    }
+
+    if (handled) {
+      fromWhitelistTp
+    } else {
+      // Default case: nullify everything.
+      nullifyType(tp)
+    }
+  }
+
+  /** A policy that special cases the handling of some symbol. */
+  private sealed trait NullifyPolicy {
+    /** Whether the policy applies to `sym`. */
+    def isApplicable(sym: Symbol): Boolean
+    /** Nullifies `tp` according to the policy. Should call `isApplicable` first. */
+    def apply(tp: Type): Type
+  }
+
+  /** A policy that leaves the passed-in type unchanged. */
+  private case class NoOpP(trigger: Symbol => Boolean) extends NullifyPolicy {
+    override def isApplicable(sym: Symbol): Boolean = trigger(sym)
+
+    override def apply(tp: Type): Type = tp
+  }
+
+  /** A policy for handling a method or poly.
+   *  @param trigger determines whether the policy applies to a given symbol.
+   *  @param nnParams the indices of the method parameters that should be considered "non-null" (should not be nullified).
+   *  @param nnRes whether the result type should be nullified.
+   *
+   *  For the purposes of both `nnParams` and `nnRes`, when a parameter or return type is not nullified,
+   *  this applies only at the top level. e.g. suppose we have a Java result type `Array[String]` and `nnRes` is set.
+   *  Scala will see `Array[String|JavaNull]`; the array element type is still nullified.
+   */
+  private case class MethodP(trigger: Symbol => Boolean,
+                             nnParams: Seq[Int],
+                             nnRes: Boolean)(implicit ctx: Context) extends TypeMap with NullifyPolicy {
+    override def isApplicable(sym: Symbol): Boolean = trigger(sym)
+
+    private def spare(tp: Type): Type = {
+      nullifyType(tp).stripNull
+    }
+
+    override def apply(tp: Type): Type = {
+      tp match {
+        case ptp: PolyType =>
+          derivedLambdaType(ptp)(ptp.paramInfos, this(ptp.resType))
+        case mtp: MethodType =>
+          val paramTpes = mtp.paramInfos.zipWithIndex.map {
+            case (paramInfo, index) =>
+              // TODO(abeln): the sequence lookup can be optimized, because the indices
+              // in it appear in increasing order.
+              if (nnParams.contains(index)) spare(paramInfo) else nullifyType(paramInfo)
+          }
+          val resTpe = if (nnRes) spare(mtp.resType) else nullifyType(mtp.resType)
+          derivedLambdaType(mtp)(paramTpes, resTpe)
+      }
+    }
+  }
+
+  /** A policy that nullifies only method parameters (but not result types). */
+  private def paramsOnlyP(trigger: Symbol => Boolean)(implicit ctx: Context): MethodP = {
+    MethodP(trigger, nnParams = Seq.empty, nnRes = true)
+  }
+
+  /** Nullifies a Java type by adding `| JavaNull` in the relevant places. */
+  private def nullifyType(tpe: Type)(implicit ctx: Context): Type = {
+    val nullMap = new JavaNullMap(alreadyNullable = false)
+    nullMap(tpe)
+  }
+
+  /** A type map that adds `| JavaNull`.
+   *  @param alreadyNullable whether the type being mapped is already nullable (at the outermost level).
+   *                         This is needed so that `JavaNullMap(A | B)` gives back `(A | B) | JavaNull`,
+   *                         instead of `(A|JavaNull | B|JavaNull) | JavaNull`.
+   */
+  private class JavaNullMap(alreadyNullable: Boolean)(implicit ctx: Context) extends TypeMap {
+    /** Should we nullify `tp` at the outermost level? */
+    def needsTopLevelNull(tp: Type): Boolean = {
+      !alreadyNullable && (tp match {
+        case tp: TypeRef =>
+          // We don't modify value types because they're non-nullable even in Java.
+          // We don't modify `Any` because it's already nullable.
+          !tp.symbol.isValueClass && !tp.isRef(defn.AnyClass)
+        case _ => true
+      })
+    }
+
+    /** Should we nullify the type arguments to the given generic `tp`?
+     *  We only nullify the inside of Scala-defined generics.
+     *  This is because Java classes are _all_ nullified, so both `java.util.List[String]` and
+     *  `java.util.List[String|Null]` contain nullable elements.
+     */
+    def needsNullArgs(tp: AppliedType): Boolean = {
+      val AppliedType(tycons, _) = tp
+      tycons.widenDealias match {
+        case tp: TypeRef if !tp.symbol.is(JavaDefined) => true
+        case _ => false
+      }
+    }
+
+    override def apply(tp: Type): Type = {
+      tp match {
+        case tp: LambdaType => mapOver(tp)
+        case tp: TypeAlias => mapOver(tp)
+        case tp@AndType(tp1, tp2) =>
+          // nullify(A & B) = (nullify(A) & nullify(B)) | JavaNull, but take care not to add
+          // duplicate `JavaNull`s at the outermost level inside `A` and `B`.
+          val newMap = new JavaNullMap(alreadyNullable = true)
+          derivedAndType(tp, newMap(tp1), newMap(tp2)).toJavaNullableUnion
+        case tp@OrType(tp1, tp2) if !tp.isJavaNullableUnion =>
+          val newMap = new JavaNullMap(alreadyNullable = true)
+          derivedOrType(tp, newMap(tp1), newMap(tp2)).toJavaNullableUnion
+        case tp: TypeRef if needsTopLevelNull(tp) => tp.toJavaNullableUnion
+        case tp: TypeParamRef if needsTopLevelNull(tp) => tp.toJavaNullableUnion
+        case appTp@AppliedType(tycons, targs) =>
+          val targs2 = if (needsNullArgs(appTp)) targs map this else targs
+          derivedAppliedType(appTp, tycons, targs2).toJavaNullableUnion
+        case _ => tp
+      }
+    }
+  }
+}

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -475,6 +475,13 @@ object Types {
       if (diff) rem else this
     }
 
+    /** Injects this type into a union with `JavaNull`. */
+    def toJavaNullableUnion(implicit ctx: Context): Type = {
+      assert(ctx.settings.YexplicitNulls.value)
+      if (this.isJavaNullableUnion) this
+      else OrType(this, defn.JavaNullAliasType)
+    }
+
 // ----- Higher-order combinators -----------------------------------
 
     /** Returns true if there is a part of this type that satisfies predicate `p`.

compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala

@@ -270,6 +270,10 @@ class ClassfileParser(
         if ((denot is Flags.Method) && (jflags & JAVA_ACC_VARARGS) != 0)
           denot.info = arrayToRepeated(denot.info)
 
+        if (ctx.settings.YexplicitNulls.value) {
+          denot.info = JavaNullInterop.nullifyMember(denot.symbol, denot.info)
+        }
+
         // seal java enums
         if (isEnum) {
           val enumClass = sym.owner.linkedClass

compiler/src/dotty/tools/dotc/typer/Namer.scala

@@ -1397,7 +1397,12 @@ class Namer { typer: Typer =>
       case _ =>
         WildcardType
     }
-    paramFn(checkSimpleKinded(typedAheadType(mdef.tpt, tptProto)).tpe)
+    val memTpe = paramFn(checkSimpleKinded(typedAheadType(mdef.tpt, tptProto)).tpe)
+    if (ctx.settings.YexplicitNulls.value && mdef.mods.is(JavaDefined)) {
+      JavaNullInterop.nullifyMember(sym, memTpe)
+    } else {
+      memTpe
+    }
   }
 
   /** The type signature of a DefDef with given symbol */

tests/explicit-nulls/neg/interop-array-src/J.java

@@ -0,0 +1,3 @@
+class J {
+	void foo(String[] ss) {}
+}

tests/explicit-nulls/neg/interop-array-src/S.scala

@@ -0,0 +1,10 @@
+class S {
+
+  val j = new J()
+  val x: Array[String] = ???
+  j.foo(x) // error: expected Array[String|Null] but got Array[String]
+  
+  val x2: Array[String|Null] = ???
+  j.foo(x2) // ok
+  j.foo(null) // ok
+}

tests/explicit-nulls/neg/interop-java-enum-src/Planet.java

@@ -0,0 +1,27 @@
+public enum Planet {
+    MERCURY (3.303e+23, 2.4397e6),
+    VENUS   (4.869e+24, 6.0518e6),
+    EARTH   (5.976e+24, 6.37814e6),
+    MARS    (6.421e+23, 3.3972e6),
+    JUPITER (1.9e+27,   7.1492e7),
+    SATURN  (5.688e+26, 6.0268e7),
+    URANUS  (8.686e+25, 2.5559e7),
+    NEPTUNE (1.024e+26, 2.4746e7);
+
+    private final double mass;   // in kilograms
+    private final double radius; // in meters
+    Planet(double mass, double radius) {
+        this.mass = mass;
+        this.radius = radius;
+    }
+    private double mass() { return mass; }
+    private double radius() { return radius; }
+
+    // This method returns a `Planet`, but since `null` is a valid
+    // return value, the return type should be nullified.
+    // Contrast with accessing the static member corresponding to the enum
+    // _instance_ (e.g. Planet.MERCURY) which shouldn't be nullified.
+    Planet next() {
+	    return null;
+    }
+}

tests/explicit-nulls/neg/interop-java-enum-src/S.scala

@@ -0,0 +1,6 @@
+
+// Verify that enum values aren't nullified.
+class S {
+  val p: Planet = Planet.MARS // ok: accessing static member
+  val p2: Planet = p.next()   // error: expected Planet but got Planet|Null
+}

tests/explicit-nulls/neg/interop-javanull.scala

@@ -0,0 +1,8 @@
+
+// Test that JavaNull can be assigned to Null.
+class Foo {
+  import java.util.ArrayList
+  val l = new ArrayList[String]()
+  val s: String = l.get(0) // error: return type is nullable
+  val s2: String|Null = l.get(0) // ok
+}

tests/explicit-nulls/neg/interop-method-src/J.java

@@ -0,0 +1,5 @@
+
+class J {
+	String foo(String x) { return null; }
+	static String fooStatic(String x) { return null; }
+}

tests/explicit-nulls/neg/interop-method-src/S.scala

@@ -0,0 +1,10 @@
+
+class S {
+  
+  val j = new J()
+  j.foo(null) // ok: argument is nullable
+  val s: String = j.foo("hello") // error: return type is nullable
+  
+  J.fooStatic(null) // ok: argument is nullable
+  val s2: String = J.fooStatic("hello") // error: return type is nullable
+}

tests/explicit-nulls/neg/interop-polytypes.scala

@@ -0,0 +1,7 @@
+class Foo {
+  import java.util.ArrayList
+  // Test that return values in PolyTypes are marked as nullable.
+  val lstring = new ArrayList[String]()
+  val res: String = java.util.Collections.max(lstring) // error: missing |Null
+  val res2: String|Null = java.util.Collections.max(lstring) // ok
+}

tests/explicit-nulls/neg/interop-propagate.scala

@@ -0,0 +1,11 @@
+ class Foo {
+  import java.util.ArrayList
+
+  // Test that as we extract return values, we're missing the |JavaNull in the return type.
+  // i.e. test that the nullability is propagated to nested containers.
+  val ll = new ArrayList[ArrayList[ArrayList[String]]]
+  val level1: ArrayList[ArrayList[String]] = ll.get(0) // error
+  val level2: ArrayList[String] = ll.get(0).get(0) // error
+  val level3: String = ll.get(0).get(0).get(0) // error
+  val ok: String = ll.get(0).get(0).get(0) // error
+}

tests/explicit-nulls/neg/interop-return.scala

@@ -0,0 +1,14 @@
+
+// Test that the return type of Java methods as well as the type of Java fields is marked as nullable.
+class Foo {
+
+  def foo = {
+    import java.util.ArrayList
+    val x = new ArrayList[String]()
+    val r: String = x.get(0) // error: got String|JavaNull instead of String
+
+    val x2 = new ArrayList[Int]()
+    val r2: Int = x2.get(0) // error: even though Int is non-nullable in Scala, its counterpart
+    // (for purposes of generics) in Java (Integer) is. So we're missing |JavaNull
+  }
+}

tests/explicit-nulls/pos/array.scala

@@ -0,0 +1,5 @@
+// Test that array contents are non-nullable.
+class Foo {
+  val x: Array[String] = Array("hello")
+  val s: String = x(0)
+}

tests/explicit-nulls/pos/interop-constructor-src/J.java

@@ -0,0 +1,6 @@
+class J {
+	private String s;
+
+	J(String x) { this.s = x; }
+	J(String x, String y, String z) {}
+}