Check outer class prefixes in type projections when pattern matching

A type pattern of the form A#B should be checked if the prefix A
is not already the owner of B or a superclass.

Fixes #16728

Author: odersky

compiler/src/dotty/tools/dotc/transform/PatternMatcher.scala

@@ -11,6 +11,7 @@ import util.Spans._
 import typer.Applications.*
 import SymUtils._
 import TypeUtils.*
+import Annotations.*
 import Flags._, Constants._
 import Decorators._
 import NameKinds.{PatMatStdBinderName, PatMatAltsName, PatMatResultName}
@@ -707,9 +708,9 @@ object PatternMatcher {
     // ----- Generating trees from plans ---------------
 
     /** The condition a test plan rewrites to */
-    private def emitCondition(plan: TestPlan): Tree = {
+    private def emitCondition(plan: TestPlan): Tree =
       val scrutinee = plan.scrutinee
-      (plan.test: @unchecked) match {
+      (plan.test: @unchecked) match
         case NonEmptyTest =>
           constToLiteral(
             scrutinee
@@ -737,41 +738,49 @@ object PatternMatcher {
         case TypeTest(tpt, trusted) =>
           val expectedTp = tpt.tpe
 
-          // An outer test is needed in a situation like  `case x: y.Inner => ...`
-          def outerTestNeeded: Boolean = {
-            def go(expected: Type): Boolean = expected match {
-              case tref @ TypeRef(pre: SingletonType, _) =>
-                tref.symbol.isClass &&
-                ExplicitOuter.needsOuterIfReferenced(tref.symbol.asClass)
-              case AppliedType(tpe, _) => go(tpe)
-              case _ =>
-                false
-            }
-            // See the test for SI-7214 for motivation for dealias. Later `treeCondStrategy#outerTest`
-            // generates an outer test based on `patType.prefix` with automatically dealises.
-            go(expectedTp.dealias)
-          }
+          def typeTest(scrut: Tree, expected: Type): Tree =
+            val ttest = scrut.select(defn.Any_typeTest).appliedToType(expected)
+            if trusted then ttest.pushAttachment(TrustedTypeTestKey, ())
+            ttest
 
-          def outerTest: Tree = thisPhase.transformFollowingDeep {
-            val expectedOuter = singleton(expectedTp.normalizedPrefix)
-            val expectedClass = expectedTp.dealias.classSymbol.asClass
-            ExplicitOuter.ensureOuterAccessors(expectedClass)
-            scrutinee.ensureConforms(expectedTp)
-              .outerSelect(1, expectedClass.owner.typeRef)
-              .select(defn.Object_eq)
-              .appliedTo(expectedOuter)
-          }
+          /** An outer test is needed in a situation like  `case x: y.Inner => ...
+           *  or like  case x: O#Inner  if the owner of Inner is not a subclass of O.
+           *  Outer tests are added here instead of in TypeTestsCasts since they
+           *  might cause outer accessors to be added to inner classes (via ensureOuterAccessors)
+           *  and therefore have to run before ExplicitOuter.
+           */
+          def addOuterTest(tree: Tree, expected: Type): Tree = expected.dealias match
+            case tref @ TypeRef(pre, _) =>
+              tref.symbol match
+                case expectedCls: ClassSymbol if ExplicitOuter.needsOuterIfReferenced(expectedCls) =>
+                  def selectOuter =
+                    ExplicitOuter.ensureOuterAccessors(expectedCls)
+                    scrutinee.ensureConforms(expected).outerSelect(1, expectedCls.owner.typeRef)
+                  if pre.isSingleton then
+                    val expectedOuter = singleton(pre)
+                    tree.and(selectOuter.select(defn.Object_eq).appliedTo(expectedOuter))
+                  else if !expectedCls.isStatic
+                    && expectedCls.owner.isType
+                    && !expectedCls.owner.derivesFrom(pre.classSymbol)
+                  then
+                    val testPre =
+                      if expected.hasAnnotation(defn.UncheckedAnnot) then
+                        AnnotatedType(pre, Annotation(defn.UncheckedAnnot, tree.span))
+                      else pre
+                    tree.and(typeTest(selectOuter, testPre))
+                  else tree
+                case _ => tree
+            case AppliedType(tycon, _) =>
+              addOuterTest(tree, tycon)
+            case _ =>
+              tree
 
-          expectedTp.dealias match {
+          expectedTp.dealias match
             case expectedTp: SingletonType =>
               scrutinee.isInstance(expectedTp)  // will be translated to an equality test
             case _ =>
-              val typeTest = scrutinee.select(defn.Any_typeTest).appliedToType(expectedTp)
-              if (trusted) typeTest.pushAttachment(TrustedTypeTestKey, ())
-              if (outerTestNeeded) typeTest.and(outerTest) else typeTest
-          }
-      }
-    }
+              addOuterTest(typeTest(scrutinee, expectedTp), expectedTp)
+    end emitCondition
 
     @tailrec
     private def canFallThrough(plan: Plan): Boolean = plan match {

tests/neg-custom-args/fatal-warnings/i16728.check

@@ -0,0 +1,4 @@
+-- Error: tests/neg-custom-args/fatal-warnings/i16728.scala:16:11 ------------------------------------------------------
+16 |      case tx : C[Int]#X => // error
+   |           ^
+   |     the type test for C[Int] cannot be checked at runtime because its type arguments can't be determined from A

tests/neg-custom-args/fatal-warnings/i16728.scala

@@ -0,0 +1,32 @@
+class A[T] {
+  class X {
+      def outer : A.this.type = A.this
+  }
+}
+
+class B extends A[Int]
+class C[T] extends A[T]
+
+object Test {
+  def main(args: Array[String]) : Unit = {
+    val b0 = new B
+    val b0x : A[?]#X = new b0.X
+
+    def test = b0x match {
+      case tx : C[Int]#X => // error
+        val c : C[Int] = tx.outer
+        c
+      case _ =>
+        "no match"
+    }
+
+    def test2 = b0x match {
+      case tx : C[Int]#X @unchecked => // ok
+        val c : C[Int] = tx.outer
+        c
+      case _ =>
+        "no match"
+    }
+
+  }
+}

tests/run/i16728.check

@@ -0,0 +1,3 @@
+b1.X
+B#X
+ELSE

tests/run/i16728.scala

@@ -0,0 +1,52 @@
+class A {
+  class X {
+      def outer : A.this.type = A.this
+  }
+}
+
+class B extends A
+class C extends A
+
+object Test {
+  def main(args: Array[String]) : Unit = {
+    val b0 = new B
+    val b1 = b0
+    val b2 = new B
+
+    val c0 = new C
+    val c1 = c0
+    val c2 = new C
+
+    val b0x : A#X = new b0.X
+
+    val pathTypeMatch = b0x match {
+      case _ : c2.X => "c2.X"
+      case _ : c1.X => "c1.x"
+      case _ : c0.X => "c0.X"
+      case _ : b2.X => "b2.X"
+      case _ : b1.X => "b1.X"
+      case _ : b0.X => "b0.X"
+      case _        => "ELSE"
+    }
+
+    println(pathTypeMatch)
+
+    val projectionTypeMatch = b0x match {
+      case _ : C#X => "C#X"
+      case _ : B#X => "B#X"
+      case _ : A#X => "A#X"
+      case _       => "ELSE"
+    }
+
+    println(projectionTypeMatch)
+
+    val failingTypeMatch = b0x match {
+      case cx : C#X =>
+        val c : C = cx.outer
+        c
+      case _ => "ELSE"
+    }
+
+    println(failingTypeMatch)
+  }
+}