Kotlin interop: Find nested class if InnerClass entry is missing

This is a port of scala/scala#5822 which works
around a bug in Kotlin (https://youtrack.jetbrains.com/issue/KT-27936).

Fixes #12086.

Co-Authored-By: Lukas Rytz <lukas.rytz@gmail.com>
Co-Authored-By: Brandon Barker <beb82@cornell.edu>

Author: 3 people

compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala

@@ -111,10 +111,39 @@ class ClassfileParser(
   }
 
   /** Return the class symbol of the given name. */
-  def classNameToSymbol(name: Name)(using Context): Symbol = innerClasses.get(name.toString) match {
-    case Some(entry) => innerClasses.classSymbol(entry)
-    case None => requiredClass(name)
-  }
+  def classNameToSymbol(name: Name)(using Context): Symbol =
+    val nameStr = name.toString
+    innerClasses.get(nameStr) match
+      case Some(entry) => innerClasses.classSymbol(entry)
+      case None =>
+        def lookupTopLevel(): Symbol = requiredClass(name)
+        // For inner classes we usually don't get to this branch: `innerClasses.classSymbol` already returns the symbol
+        // of the inner class based on the InnerClass table. However, if the classfile is missing the
+        // InnerClass entry for `name`, it might still be that there exists an inner symbol (because
+        // some other classfile _does_ have an InnerClass entry for `name`). In this case, we want to
+        // return the actual inner symbol (C.D, with owner C), not the top-level symbol C$D. This is
+        // what the logic below is for (see scala/bug#9937 / lampepfl/dotty#12086).
+        val split = nameStr.lastIndexOf('$')
+        if split < 0 || split >= nameStr.length - 1 then
+          lookupTopLevel()
+        else
+          val outerNameStr = nameStr.substring(0, split)
+          val innerNameStr = nameStr.substring(split + 1, nameStr.length)
+          val outerSym = classNameToSymbol(outerNameStr.toTypeName)
+          outerSym.denot.infoOrCompleter match
+            case _: StubInfo =>
+              // If the outer class C cannot be found, look for a top-level class C$D
+              lookupTopLevel()
+            case _ =>
+              // We have a java-defined class name C$D and look for a member D of C. But we don't know if
+              // D is declared static or not, so we have to search both in class C and its companion.
+              val innerName = innerNameStr.toTypeName
+              val r =
+                if outerSym eq classRoot.symbol then
+                  instanceScope.lookup(innerName).orElse(staticScope.lookup(innerName))
+                else
+                  outerSym.info.member(innerName).orElse(outerSym.asClass.companionModule.info.member(innerName)).symbol
+              r.orElse(lookupTopLevel())
 
   var sawPrivateConstructor: Boolean = false
 

tests/run/i12086/Test_1.java

@@ -0,0 +1,11 @@
+class C$D { public int i() { return 1; } }
+class C$E { public int i() { return 1; } }
+class C$F$G { public int i() { return 1; } }
+
+// Test1 has a reference to C$D, which is a top-level class in this case,
+// so there's no INNERCLASS attribute in Test1
+class Test_1 {
+  static C$D mD(C$D cd) { return cd; }
+  static C$E mE(C$E ce) { return ce; }
+  static C$F$G mG(C$F$G cg ) { return cg; }
+}

tests/run/i12086/Test_2.java

@@ -0,0 +1,12 @@
+class C {
+  class D { public int i() { return 2; } }
+  static class E { public int i() { return 2; } }
+  static class F { static class G { public int i() { return 2; } } }
+}
+
+// Test2 has an INNERCLASS attribute for C$D
+class Test_2 {
+  public static int acceptD(C.D cd) { return cd.i(); }
+  public static int acceptE(C.E ce) { return ce.i(); }
+  public static int acceptG(C.F.G cg ) { return cg.i(); }
+}

tests/run/i12086/Test_3.scala

@@ -0,0 +1,8 @@
+object Test {
+  def main(args: Array[String]): Unit = {
+    val c = new C
+    assert(Test_2.acceptD(Test_1.mD(new c.D)) == 2)
+    assert(Test_2.acceptE(Test_1.mE(new C.E)) == 2)
+    assert(Test_2.acceptG(Test_1.mG(new C.F.G)) == 2)
+  }
+}