Fix #5876: Don't dereference pending LazyRefs when normalizing

In monitoredIsSubType, don't dereference pending LazyRefs when normalizing a type to check whether
it is in the log of seen types.

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -210,9 +210,9 @@ class TypeComparer(initctx: Context) extends ConstraintHandling[AbsentContext] w
           case t: LazyRef =>
             // Dereference a lazyref to detect underlying matching types, but
             // be careful not to get into an infinite recursion. If recursion count
-            // exceeds `DerefLimit`, approximate with `NoType` instead.
+            // exceeds `DerefLimit`, approximate with `t` instead.
             derefCount += 1
-            if (derefCount >= DerefLimit) NoType
+            if t.pending || derefCount >= DerefLimit then t
             else try mapOver(t.ref) finally derefCount -= 1
           case tp: TypeVar =>
             tp

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -2578,6 +2578,8 @@ object Types {
     private var myRef: Type = null
     private var computed = false
 
+    def pending(using Context): Boolean = computed && myRef == null
+
     def ref(implicit ctx: Context): Type =
       if computed then
         if myRef == null then

tests/pos-deep-subtype/i5876.scala

@@ -0,0 +1,9 @@
+type HasThisB[T] = HasThis { type This <: T }
+trait HasThis {
+  type This >: this.type <: HasThisB[This]
+}
+
+type FB[T] = F { type This <: T }
+class F extends HasThis {
+  type This >: this.type <: FB[This]
+}