Special case comparisons between LazyRefs and bottom/top types

Fixes #11064

When comparing a LazyRef with some other type, we need to force the LazyRef
which can potentially lead to a cycle (as observed in #11064). However, in the situations
```
Nothing <: LazyRef(...)
LazyRef(...) <: Any
```
we can skip that since the comparison is always true. This scheme fixes #11064, but it breaks
down in similar situations if additional bounds are introduced (see neg/i11064.scala).
So it is quite fragile. But it's probably the best we can do.

The example in #11064 looks harmless, but type-theoretically it's actually a pretty explosive
mix of F-bounded types and existential types (in Scala 2). In Scala 2, we need to force more
for F-bounded types and we need to approximate existential types by dependent types, so
not everything works the same way.

My long term advice would be: get rid of F-bounds. Model them with intersections at the
use site.

Author: odersky

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -306,7 +306,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         }
         compareWild
       case tp2: LazyRef =>
-        !tp2.evaluating && recur(tp1, tp2.ref)
+        isBottom(tp1) || !tp2.evaluating && recur(tp1, tp2.ref)
       case tp2: AnnotatedType if !tp2.isRefining =>
         recur(tp1, tp2.parent)
       case tp2: ThisType =>
@@ -373,7 +373,7 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
             if (recur(info1.alias, tp2)) return true
             if (tp1.prefix.isStable) return tryLiftedToThis1
           case _ =>
-            if (tp1 eq NothingType) return true
+            if (tp1 eq NothingType) || isBottom(tp1) then return true
         }
         thirdTry
       case tp1: TypeParamRef =>
@@ -420,7 +420,8 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         // If `tp1` is in train of being evaluated, don't force it
         // because that would cause an assertionError. Return false instead.
         // See i859.scala for an example where we hit this case.
-        !tp1.evaluating && recur(tp1.ref, tp2)
+        tp2.isRef(AnyClass, skipRefined = false)
+        || !tp1.evaluating && recur(tp1.ref, tp2)
       case tp1: AnnotatedType if !tp1.isRefining =>
         recur(tp1.parent, tp2)
       case AndType(tp11, tp12) =>

tests/neg/i11064.scala

@@ -0,0 +1,9 @@
+trait TypedArray[T, Repr]
+
+trait Ops[T <: TypedArray[_, T]] {
+  def typedArray(): T
+}
+
+object Test {
+  def test(ops: Ops[_ <: TypedArray[_ <: AnyRef, _]]) = ops.typedArray()  // error: Recursion limit exceeded.
+}

tests/pos/i11064.scala

@@ -0,0 +1,9 @@
+trait TypedArray[T, Repr]
+
+trait Ops[T <: TypedArray[_, T]] {
+  def typedArray(): T
+}
+
+object Test {
+  def test(ops: Ops[_ <: TypedArray[_, _]]) = ops.typedArray()
+}