Implement deep check for variables

Scope extrusion can also happen for nested types, so we need to prevent
{*} capturesets anywhere in the type of a mutable variable.

Author: odersky

compiler/src/dotty/tools/dotc/typer/CheckCaptures.scala

@@ -403,30 +403,46 @@ class CheckCaptures extends Recheck:
           show(unit.tpdTree) // this dows not print tree, but makes its variables visible for dependency printing
       }
 
+    def checkNotGlobal(tree: Tree, tp: Type, allArgs: Tree*)(using Context): Unit =
+      for ref <-tp.captureSet.elems do
+        val isGlobal = ref match
+          case ref: TermRef =>
+            ref.isRootCapability || ref.prefix != NoPrefix && ref.symbol.hasAnnotation(defn.AbilityAnnot)
+          case _ => false
+        if isGlobal then
+          val what = if ref.isRootCapability then "universal" else "global"
+          val notAllowed = i" is not allowed to capture the $what capability $ref"
+          def msg =
+            if allArgs.isEmpty then
+              i"type of mutable variable ${knownType(tree)}$notAllowed"
+            else tree match
+              case tree: InferredTypeTree =>
+                i"""inferred type argument ${knownType(tree)}$notAllowed
+                    |
+                    |The inferred arguments are: [${allArgs.map(knownType)}%, %]"""
+              case _ => s"type argument$notAllowed"
+          report.error(msg, tree.srcPos)
+
     def checkNotGlobal(tree: Tree, allArgs: Tree*)(using Context): Unit =
       if disallowGlobal then
         tree match
           case LambdaTypeTree(_, restpt) =>
             checkNotGlobal(restpt, allArgs*)
           case _ =>
-            for ref <- knownType(tree).captureSet.elems do
-              val isGlobal = ref match
-                case ref: TermRef =>
-                  ref.isRootCapability || ref.prefix != NoPrefix && ref.symbol.hasAnnotation(defn.AbilityAnnot)
-                case _ => false
-              val what = if ref.isRootCapability then "universal" else "global"
-              if isGlobal then
-                val notAllowed = i" is not allowed to capture the $what capability $ref"
-                def msg =
-                  if allArgs.isEmpty then
-                    i"type of mutable variable ${knownType(tree)}$notAllowed"
-                  else tree match
-                    case tree: InferredTypeTree =>
-                      i"""inferred type argument ${knownType(tree)}$notAllowed
-                          |
-                          |The inferred arguments are: [${allArgs.map(knownType)}%, %]"""
-                    case _ => s"type argument$notAllowed"
-                report.error(msg, tree.srcPos)
+            checkNotGlobal(tree, knownType(tree), allArgs*)
+
+    def checkNotGlobalDeep(tree: Tree)(using Context): Unit =
+      val checker = new TypeTraverser:
+        def traverse(tp: Type): Unit = tp match
+          case tp: TypeRef =>
+            tp.info match
+              case TypeBounds(_, hi) => traverse(hi)
+              case _ =>
+          case tp: TermRef =>
+          case _ =>
+            checkNotGlobal(tree, tp)
+            traverseChildren(tp)
+      checker.traverse(knownType(tree))
 
     object PostRefinerCheck extends TreeTraverser:
       def traverse(tree: Tree)(using Context) =
@@ -467,7 +483,7 @@ class CheckCaptures extends Recheck:
                 case _ =>
               inferred.foreachPart(checkPure, StopAt.Static)
           case t: ValDef if t.symbol.is(Mutable) =>
-            checkNotGlobal(t.tpt)
+            checkNotGlobalDeep(t.tpt)
           case _ =>
         traverseChildren(tree)
 

tests/neg-custom-args/captures/vars.check

@@ -9,8 +9,12 @@ longer explanation available when compiling with `-explain`
 13 |  var a: {*} String => String = f // error
    |          ^^^^^^^^^^^^^^^^^^^
    |  type of mutable variable box {*} String => String is not allowed to capture the universal capability (* : Any)
--- Error: tests/neg-custom-args/captures/vars.scala:27:2 ---------------------------------------------------------------
-27 |  local { cap3 => // error
+-- Error: tests/neg-custom-args/captures/vars.scala:14:9 ---------------------------------------------------------------
+14 |  var b: List[{*} String => String] = Nil // error
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |type of mutable variable List[box {*} String => String] is not allowed to capture the universal capability (* : Any)
+-- Error: tests/neg-custom-args/captures/vars.scala:29:2 ---------------------------------------------------------------
+29 |  local { cap3 => // error
    |  ^^^^^
    |inferred type argument {*} (x$0: ? String) => ? String is not allowed to capture the universal capability (* : Any)
    |

tests/neg-custom-args/captures/vars.scala

@@ -11,11 +11,13 @@ def test(cap1: Cap, cap2: Cap) =
   val z2c: () => Unit = z2  // error
 
   var a: {*} String => String = f // error
+  var b: List[{*} String => String] = Nil // error
 
   def scope =
     val cap3: Cap = CC()
     def g(x: String): String = if cap3 == cap3 then "" else "a"
     a = g
+    b = List(g)
     val gc = g
     g