Handle GADT bounds in isInstanceOf type test

Author: liufengyun

compiler/src/dotty/tools/dotc/transform/TypeTestsCasts.scala

@@ -105,7 +105,11 @@ object TypeTestsCasts {
       debug.println("P1 : " + P1.show)
       debug.println("X : " + X.show)
 
-      val res = (P1 <:< X) && (P1 <:< P)
+      P1 <:< X
+
+      maximizeType(P1, span, fromScala2x = false)
+
+      val res = P1 <:< P
       debug.println("P1 : " + P1.show)
       debug.println("P1 <:< P = " + res)
 
@@ -133,8 +137,8 @@ object TypeTestsCasts {
           case _ =>
             // first try withou striping type parameters for performance
             X.classSymbol.exists && P.classSymbol.exists && !X.classSymbol.asClass.mayHaveCommonChild(P.classSymbol.asClass) ||
-            isClassDetermined(X, tpe)(ctx.fresh.setNewTyperState()) ||
-            isClassDetermined(stripTypeParam(X), tpe)(ctx.fresh.setNewTyperState())
+            isClassDetermined(X, tpe)(ctx.fresh.setNewTyperState().setFreshGADTBounds) ||
+            isClassDetermined(stripTypeParam(X), tpe)(ctx.fresh.setNewTyperState().setFreshGADTBounds)
         }
       case AndType(tp1, tp2)    => recur(X, tp1) && recur(X, tp2)
       case OrType(tp1, tp2)     => recur(X, tp1) && recur(X, tp2)

tests/neg-custom-args/isInstanceOf/3324g.scala

@@ -12,7 +12,7 @@ class Test {
   }
 
   def quux[T](a: A[T]): Unit = a match {
-    case _: B[T] => // should be an error!!
+    case _: B[T] => // error!!
   }
 
   quux(new C[Int])

tests/neg-custom-args/isInstanceOf/i5495.scala

@@ -0,0 +1,12 @@
+class A
+class B
+
+type AorB = A | B
+
+def foo(any: Any) = any match {
+  case aorb: AorB =>
+  case _ =>
+}
+
+def bar[T](x: List[T]) = x.isInstanceof[List[Int]] // error
+

tests/neg-custom-args/isInstanceOf/patmat-applied.scala

@@ -7,3 +7,5 @@ object Test {
     case _ =>
   }
 }
+
+def bar[T](x: List[T]) = x.isInstanceof[List[Int]] // error