Fix replace operation

In OrderingConstraint#replace we moved the actual replacement
of a parameter with a type from the start of replace to its end,
since that was more convenient for dependency adjustments. It turns
out that doing so causes infinite recursion in instantiations in
some cases, specifically if a parameter contains itself as an indirect
lower bound that goes through an alias. Here is a situation that arises
in i16311.scala:
```
type WithTag[T, U] = T & Tagged[U]

  T1 >: WithTag[T2, Int]
  T2 >: T1 & Tagged[Int]
```
The current instantiation for T1 and T2 is Nothing. But we ran into a cycle
instead.

The fix is to move the parameter replacement back to the start of `replace`,
and to account for that in the dependency adjustment logic.

Fixes #16311

Author: odersky

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -297,27 +297,36 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
    */
   private trait ConstraintAwareTraversal[T] extends TypeAccumulator[T]:
 
+    /** Does `param` have bounds in the current constraint? */
+    protected def hasBounds(param: TypeParamRef): Boolean = entry(param).isInstanceOf[TypeBounds]
+
     override def tyconTypeParams(tp: AppliedType)(using Context): List[ParamInfo] =
       def tparams(tycon: Type): List[ParamInfo] = tycon match
         case tycon: TypeVar if !tycon.inst.exists => tparams(tycon.origin)
-        case tycon: TypeParamRef =>
-          entry(tycon) match
-            case _: TypeBounds => tp.tyconTypeParams
-            case tycon1 if tycon1.typeParams.nonEmpty => tycon1.typeParams
-            case _ => tp.tyconTypeParams
+        case tycon: TypeParamRef if !hasBounds(tycon) =>
+          val entryParams = entry(tycon).typeParams
+          if entryParams.nonEmpty then entryParams
+          else tp.tyconTypeParams
         case _ => tp.tyconTypeParams
       tparams(tp.tycon)
 
     override def applyToPrefix(x: T, tp: NamedType): T =
       this(x, tp.prefix)
   end ConstraintAwareTraversal
 
-  private class Adjuster(srcParam: TypeParamRef)(using Context)
+  /** A type traverser that adjust dependencies originating from a given type
+   *  @param ignoreBinding  if not null, a parameter that is assumed to be still uninstantiated.
+   *                        This is necessary to handle replacements.
+   */
+  private class Adjuster(srcParam: TypeParamRef, ignoreBinding: TypeParamRef | Null)(using Context)
   extends TypeTraverser, ConstraintAwareTraversal[Unit]:
 
     var add: Boolean = compiletime.uninitialized
     val seen = util.HashSet[LazyRef]()
 
+    override protected def hasBounds(param: TypeParamRef) =
+      (param eq ignoreBinding) || super.hasBounds(param)
+
     def update(deps: ReverseDeps, referenced: TypeParamRef): ReverseDeps =
       val prev = deps.at(referenced)
       val newSet = if add then prev + srcParam else prev - srcParam
@@ -326,12 +335,11 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
 
     def traverse(t: Type) = t match
       case param: TypeParamRef =>
-        entry(param) match
-          case _: TypeBounds =>
-            if variance >= 0 then coDeps = update(coDeps, param)
-            if variance <= 0 then contraDeps = update(contraDeps, param)
-          case tp =>
-            traverse(tp)
+        if hasBounds(param) then
+          if variance >= 0 then coDeps = update(coDeps, param)
+          if variance <= 0 then contraDeps = update(contraDeps, param)
+        else
+          traverse(entry(param))
       case tp: LazyRef =>
         if !seen.contains(tp) then
           seen += tp
@@ -342,8 +350,8 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
   /** Adjust dependencies to account for the delta of previous entry `prevEntry`
    *  and the new bound `entry` for the type parameter `srcParam`.
    */
-  def adjustDeps(entry: Type | Null, prevEntry: Type | Null, srcParam: TypeParamRef)(using Context): this.type =
-    val adjuster = new Adjuster(srcParam)
+  def adjustDeps(entry: Type | Null, prevEntry: Type | Null, srcParam: TypeParamRef, ignoreBinding: TypeParamRef | Null = null)(using Context): this.type =
+    val adjuster = new Adjuster(srcParam, ignoreBinding)
 
     /** Adjust reverse dependencies of all type parameters referenced by `bound`
      *  @param  isLower `bound` is a lower bound
@@ -676,7 +684,14 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
         override def apply(t: Type): Type =
           if (t eq replacedTypeVar) && t.exists then to else mapOver(t)
 
-      var current = this
+      val coDepsOfParam = coDeps.at(param)
+      val contraDepsOfParam = contraDeps.at(param)
+
+      var current = updateEntry(this, param, replacement)
+        // Need to update param early to avoid infinite recursion on instantiation.
+        // See i16311.scala for a test case. On the other hand, for the purposes of
+        // dependency adjustment, we need to pretend that `param` is still unbound.
+        // We achieve that by passing a `ignoreBinding = param` to `adjustDeps` below.
 
       def removeParamFrom(ps: List[TypeParamRef]) =
         ps.filterConserve(param ne _)
@@ -710,22 +725,15 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
               newDepEntry = mapReplacedTypeVarTo(replacement)(newDepEntry)
           case _ =>
         if oldDepEntry ne newDepEntry then
-          if current eq this then
-            // We can end up here if oldEntry eq newEntry, so posssibly no new constraint
-            // was created, but oldDepEntry ne newDepEntry. In that case we must make
-            // sure we have a new constraint before updating dependencies.
-            current = newConstraint()
-          current.adjustDeps(newDepEntry, oldDepEntry, other)
+          current.adjustDeps(newDepEntry, oldDepEntry, other, ignoreBinding = param)
       end replaceParamIn
 
       if optimizeReplace then
-        val co = current.coDeps.at(param)
-        val contra = current.contraDeps.at(param)
         current.foreachParam { (p, i) =>
           val other = p.paramRefs(i)
           entry(other) match
             case _: TypeBounds =>
-              if co.contains(other) || contra.contains(other) then
+              if coDepsOfParam.contains(other) || contraDepsOfParam.contains(other) then
                 replaceParamIn(other)
             case _ => replaceParamIn(other)
         }
@@ -734,10 +742,7 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
           val other = p.paramRefs(i)
           if other != param then replaceParamIn(other)
         }
-
-      current =
-        if isRemovable(param.binder) then current.remove(param.binder)
-        else updateEntry(current, param, replacement)
+      if isRemovable(param.binder) then current = current.remove(param.binder)
       current.dropDeps(param)
       current.checkWellFormed()
   end replace

tests/pending/pos/i16311.scala tests/pos-deep-subtype/i16311.scalatests/pending/pos/i16311.scala renamed to tests/pos-deep-subtype/i16311.scala

@@ -0,0 +1,17 @@
+trait Tagged[U]
+type WithTag[+T, U] = T & Tagged[U]
+
+trait FromInput[Val]
+implicit def coercedScalaInput[T]: FromInput[T & Tagged[Int]] = ???
+implicit def optionInput[T](implicit ev: FromInput[T]): FromInput[Option[T]] = ???
+
+trait WithoutInputTypeTags[T]
+implicit def coercedOptArgTpe[T]: WithoutInputTypeTags[Option[T & Tagged[Int]]] = ???
+
+trait InputType[+T]
+class OptionInputType[T](ofType: InputType[T]) extends InputType[Option[T]]
+
+type Argument[T]
+def argument[T](argumentType: InputType[T])(implicit fromInput: FromInput[T], res: WithoutInputTypeTags[T]): Argument[Option[T]] = ???
+
+def test = argument(OptionInputType(??? : InputType[Boolean & Tagged[Int]]))