Fix #10769: change synthesized type in def ordinal

As pointed out by @smarter in #10769, for the following code:

    package stm

    trait STMLike[F[_]] {
      import Internals._

      sealed abstract class Txn[+A] {}

      object Txn {
	def abort[A](e: Throwable): Txn[A] = Abort(e)
      }

      object Internals {
	case class Abort(error: Throwable) extends Txn[Nothing]
      }

    }

The compiler synthesized the following code for the object `Txn`:

    object Txn {
      type MirroredMonoType = STMLike.this.Txn[?]
      def ordinal(x: Txn.MirroredMonoType): Int =
	x match {
          case _:stm.STMLike.Internals.Abort => 0
	}

      def abort[A](e: Throwable): Txn[A] = Abort(e)
    }

In the method `ordinal`, the type for `Internals.Abort` is

    TypeRef(ThisType(TypeRef(ThisType(TypeRef(ThisType(TypeRef(NoPrefix,module class stm)),trait STMLike)),module class Internals$)),class Abort)

This type is incorrect, as we are not in the object `Internals`. The
explicit outer can only handle such types if it's static. In this
case, the object is not static, thus it crashes the explicit outer.

Co-authored-by: Guillaume Martres <smarter@ubuntu.com>

Author: liufengyun

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1410,6 +1410,33 @@ object SymDenotations {
     def namedType(using Context): NamedType =
       if (isType) typeRef else termRef
 
+    /** The typeRef where `pre.O$.this` is changed to `pre.O.type` if `O` is a non-static object
+     *
+     *  This is required to avoid owner crash in ExplicitOuter.
+     *  See tests/pos/i10769.scala
+     */
+     def reachableTypeRef(using Context) =
+       TypeRef(owner.reachableThisType, symbol)
+
+    /** The termRef where `pre.O$.this` is changed to `pre.O.type` if `O` is a non-static object
+     *
+     *  This is required to avoid owner crash in ExplicitOuter.
+     *  See tests/pos/i10769.scala
+     */
+    def reachableTermRef(using Context) =
+      TermRef(owner.reachableThisType, symbol)
+
+    /** The thisType where `pre.O$.this` is changed to `pre.O.type` if `O` is a non-static object */
+    def reachableThisType(using Context): Type =
+      if this.is(Package) then
+        symbol.thisType
+      else if this.isTerm then
+        NoPrefix
+      else if this.is(Module) then
+        TermRef(owner.reachableThisType, this.sourceModule)
+      else
+        ThisType.raw(TypeRef(owner.reachableThisType, symbol.asType))
+
     /** The variance of this type parameter or type member as a subset of
      *  {Covariant, Contravariant}
      */

compiler/src/dotty/tools/dotc/transform/SymUtils.scala

@@ -223,9 +223,9 @@ object SymUtils:
       else owner.isLocal
     }
 
-    /** The typeRef with wildcard arguments for each type parameter */
-    def rawTypeRef(using Context) =
-      self.typeRef.appliedTo(self.typeParams.map(_ => TypeBounds.emptyPolyKind))
+    /** The reachable typeRef with wildcard arguments for each type parameter */
+    def reachableRawTypeRef(using Context) =
+      self.reachableTypeRef.appliedTo(self.typeParams.map(_ => TypeBounds.emptyPolyKind))
 
     /** Is symbol a quote operation? */
     def isQuote(using Context): Boolean =

compiler/src/dotty/tools/dotc/transform/SyntheticMembers.scala

@@ -521,7 +521,7 @@ class SyntheticMembers(thisPhase: DenotTransformer) {
     else {
       val cases =
         for ((child, idx) <- cls.children.zipWithIndex) yield {
-          val patType = if (child.isTerm) child.termRef else child.rawTypeRef
+          val patType = if (child.isTerm) child.reachableTermRef else child.reachableRawTypeRef
           val pat = Typed(untpd.Ident(nme.WILDCARD).withType(patType), TypeTree(patType))
           CaseDef(pat, EmptyTree, Literal(Constant(idx)))
         }
@@ -563,7 +563,7 @@ class SyntheticMembers(thisPhase: DenotTransformer) {
       if (existing.exists && !existing.is(Deferred)) existing
       else {
         val monoType =
-          newSymbol(clazz, tpnme.MirroredMonoType, Synthetic, TypeAlias(linked.rawTypeRef), coord = clazz.coord)
+          newSymbol(clazz, tpnme.MirroredMonoType, Synthetic, TypeAlias(linked.reachableRawTypeRef), coord = clazz.coord)
         newBody = newBody :+ TypeDef(monoType).withSpan(ctx.owner.span.focus)
         monoType.entered
       }

tests/pos/i10769.scala

@@ -0,0 +1,21 @@
+package stm
+
+trait STMLike[F[_]] {
+  import Internals._
+
+  sealed abstract class Txn[+A] {}
+
+  object Txn {
+    def abort[A](e: Throwable): Txn[A] = Abort(e)
+  }
+
+  object Internals {
+    case class Abort(error: Throwable) extends Txn[Nothing]
+    case object Noop extends Txn[Nothing]
+  }
+
+  class Foo {
+    case class Abort(error: Throwable) extends Txn[Nothing]
+    case object Noop extends Txn[Nothing]
+  }
+}