Handle HKT bounds in Java generic signatures

compiler/src/dotty/tools/dotc/transform/GenericSignatures.scala

@@ -510,19 +510,45 @@ object GenericSignatures {
   }
 
   private object RefOrAppliedType {
-    def unapply(tp: Type)(using Context): Option[(Symbol, Type, List[Type])] = tp match {
+    private enum ResolvedAppliedType:
+      case Resolved(t: Type)
+      case NotResolved
+      case Bail
+    // In the special case where we see a type parameter applied to type parameters,
+    // such as `K[X, Y]` given `[X, Y, K <: Iterable[(X, Y)]]`, we must find its bound
+    // and instantiate it, otherwise in our example we end up with `Iterable[X, Y]` which is nonsensical.
+    private def resolveAppliedType(a: AppliedType)(using Context): ResolvedAppliedType =
+      a.tycon match
+        case TypeParamRef(binder, paramNum) =>
+          binder.paramInfos(paramNum).hi match
+            case hkt @ HKTypeLambda(_, _) =>
+              val instantiated = hkt.instantiate(a.args).dealias
+              // However, since Java doesn't have a way to refer to HKTs in generic signatures,
+              // we must trade precision for termination by only resolving one level,
+              // otherwise we end up in infinite loops,
+              // e.g., in `X[A] <: Thing[X[A]]` or `X[A] <: X[Thing[A]]` we keep resolving `X`.
+              // In that case we must completely give up on the genericity, i.e.,
+              // in `X[A] <: Y[X[Z[A]]]` it would not be correct to use `Y[A]` as a type signature! 
+              if instantiated.existsPart(_ == a.tycon) then ResolvedAppliedType.Bail
+              else ResolvedAppliedType.Resolved(instantiated)
+            case _ => ResolvedAppliedType.NotResolved
+        case _ => ResolvedAppliedType.NotResolved
+
+    def unapply(tp: Type)(using Context): Option[(Symbol, Type, List[Type])] = tp match
       case TypeParamRef(_, _) =>
         Some((tp.typeSymbol, tp, Nil))
       case TermParamRef(_, _) =>
         Some((tp.termSymbol, tp, Nil))
       case TypeRef(pre, _) if !tp.typeSymbol.isAliasType =>
         val sym = tp.typeSymbol
         Some((sym, pre, Nil))
-      case AppliedType(pre, args) =>
-        Some((pre.typeSymbol, pre, args))
+      case a @ AppliedType(pre, args) =>
+        resolveAppliedType(a) match
+          case ResolvedAppliedType.Resolved(resolved) => unapply(resolved)
+          case ResolvedAppliedType.NotResolved => Some((pre.typeSymbol, pre, args))
+          case ResolvedAppliedType.Bail => None
       case _ =>
         None
-    }
   }
 
   private def needsJavaSig(tp: Type, throwsArgs: List[Type])(using Context): Boolean = !ctx.settings.XnoGenericSig.value && {

tests/run/i15903.check

@@ -0,0 +1 @@
+scala.collection.Iterable<scala.Tuple2<K, A>>

tests/run/i15903.scala

@@ -0,0 +1,10 @@
+// scalajs: --skip
+// (JVM-only, generic signatures)
+
+trait Working:
+  type M[X,Y] = Iterable[(X, Y)]
+  def example[K, A, T[X,Y] <: M[X,Y]](ab: String): T[K,A] = ???
+
+object Test:
+  def main(args: Array[String]): Unit =
+    classOf[Working].getMethods.filter(_.getName == "example").map(_.getGenericReturnType).foreach(println)

tests/run/i15903b.check

@@ -0,0 +1 @@
+scala.collection.Iterable<scala.Tuple2<K, A>>

tests/run/i15903b.scala

@@ -0,0 +1,9 @@
+// scalajs: --skip
+// (JVM-only, generic signatures)
+
+trait NotWorking:
+  def example[K, A, T[X,Y] <: Iterable[(X, Y)]](ab: String): T[K,A] = ???
+
+object Test:
+  def main(args: Array[String]): Unit =
+    classOf[NotWorking].getMethods.filter(_.getName == "example").map(_.getGenericReturnType).foreach(println)

tests/run/i15903c.check

@@ -0,0 +1 @@
+scala.collection.Iterable<scala.Tuple2<A, K>>

tests/run/i15903c.scala

@@ -0,0 +1,10 @@
+// scalajs: --skip
+// (JVM-only, generic signatures)
+
+trait NotWorking:
+  // return type args reversed compared to test case (b)
+  def example[K, A, T[X,Y] <: Iterable[(X, Y)]](ab: String): T[A,K] = ???
+
+object Test:
+  def main(args: Array[String]): Unit =
+    classOf[NotWorking].getMethods.filter(_.getName == "example").map(_.getGenericReturnType).foreach(println)

tests/run/i15903d.check

@@ -0,0 +1 @@
+interface scala.collection.Iterable

tests/run/i15903d.scala

@@ -0,0 +1,9 @@
+// scalajs: --skip
+// (JVM-only, generic signatures)
+
+trait NotWorking:
+  def example[A, X[T] <: Iterable[X[List[T]]]](s: String): X[A] = ???
+
+object Test:
+  def main(args: Array[String]): Unit =
+    classOf[NotWorking].getMethods.filter(_.getName == "example").map(_.getGenericReturnType).foreach(println)