Handle nullability info in by-name arguments

Implements the scheme described in my comment to #7546.

 - We type check an argument without or with flow info,
   depending on whether the function part of the application
   is known to take a call-by-name parameter. If we know nothing
   about the function part, we assume call-by-value.
 - At the end of type checking an application, if the argument is
   known to be by-name, remove all x.$asInstanceOf$[T] casts in the
   argument where x is a mutable variable, and run the type assigner
   (not the type checker!) again on the result. If this succeeds and
   gives a type that is still compatible with the formal parameter type,
   we are done.
 - Otherwise, issue an error saying that the argument cannot be treated
   as call-by-name since it contains flow-assumptions about mutable variables.
   As a remedy, suggest to wrap the argument in a `scala.compiletime.byName(...)`
   call. Here, `byName` is defined as follws:
   ```
   inline def byName[T](x: => T): T
   ```
   Wrapping an argument with byName means that we know statically that
   it is passed to a by-name parameter, so it will be typechecked without
   flow info.

Author: odersky

compiler/src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -861,6 +861,19 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
       case _ => None
     }
   }
+
+  /** Extractor for not-null assertions */
+  object AssertNotNull with
+    def apply(tree: tpd.Tree, tpnn: Type)(given Context): tpd.Tree =
+      tree.select(defn.Any_typeCast).appliedToType(AndType(tree.tpe, tpnn))
+
+    def unapply(tree: tpd.TypeApply)(given Context): Option[tpd.Tree] = tree match
+      case TypeApply(Select(qual: RefTree, nme.asInstanceOfPM), arg :: Nil) =>
+        arg.tpe match
+          case AndType(ref, _) if qual.tpe eq ref => Some(qual)
+          case _ => None
+      case _ => None
+  end AssertNotNull
 }
 
 object TreeInfo {

compiler/src/dotty/tools/dotc/typer/Applications.scala

@@ -794,7 +794,7 @@ trait Applications extends Compatibility {
   /** Subclass of Application for type checking an Apply node with untyped arguments. */
   class ApplyToUntyped(app: untpd.Apply, fun: Tree, methRef: TermRef, proto: FunProto, resultType: Type)(implicit ctx: Context)
   extends TypedApply(app, fun, methRef, proto.args, resultType) {
-    def typedArg(arg: untpd.Tree, formal: Type): TypedArg = proto.typedArg(arg, formal.widenExpr)
+    def typedArg(arg: untpd.Tree, formal: Type): TypedArg = proto.typedArg(arg, formal)
     def treeToArg(arg: Tree): untpd.Tree = untpd.TypedSplice(arg)
     def typeOfArg(arg: untpd.Tree): Type = proto.typeOfArg(arg)
   }
@@ -868,7 +868,8 @@ trait Applications extends Compatibility {
               else
                 new ApplyToUntyped(tree, fun1, funRef, proto, pt)(
                   given fun1.nullableInArgContext(given argCtx(tree)))
-            convertNewGenericArray(app.result).computeNullable()
+            convertNewGenericArray(
+              postProcessByNameArgs(funRef, app.result).computeNullable())
           case _ =>
             handleUnexpectedFunType(tree, fun1)
         }
@@ -1030,7 +1031,7 @@ trait Applications extends Compatibility {
    *  It is performed during typer as creation of generic arrays needs a classTag.
    *  we rely on implicit search to find one.
    */
-  def convertNewGenericArray(tree:  Tree)(implicit ctx: Context):  Tree = tree match {
+  def convertNewGenericArray(tree: Tree)(implicit ctx: Context):  Tree = tree match {
     case Apply(TypeApply(tycon, targs@(targ :: Nil)), args) if tycon.symbol == defn.ArrayConstructor =>
       fullyDefinedType(tree.tpe, "array", tree.span)
 
@@ -1046,6 +1047,56 @@ trait Applications extends Compatibility {
       tree
   }
 
+  /** Post process all arguments to by-name parameters by removing any not-null
+   *  info that was used when typing them. Concretely:
+   *  If an argument corresponds to a call-by-name parameter, drop all
+   *  embedded not-null assertions of the form `x.$asInstanceOf[x.type & T]`
+   *  where `x` is a reference to a mutable variable. If the argument still typechecks
+   *  with the removed assertions and is still compatible with the formal parameter,
+   *  keep it. Otherwise issue an error that the call-by-name argument was typed using
+   *  flow assumptions about mutable variables and suggest that it is enclosed
+   *  in a `byName(...)` call instead.
+   */
+  private def postProcessByNameArgs(fn: TermRef, app: Tree)(given ctx: Context): Tree =
+    fn.widen match
+      case mt: MethodType if mt.paramInfos.exists.isInstanceOf[ExprType] =>
+        app match
+          case Apply(fn, args) =>
+            val dropNotNull = new TreeMap with
+              override def transform(t: Tree)(given Context) = t match
+                case AssertNotNull(t0) if t0.symbol.is(Mutable) => transform(t0)
+                case t: ValDef if !t.symbol.is(Lazy) => super.transform(t)
+                case t: MemberDef => t // stop here since embedded references are out of order anyway
+                case t => super.transform(t)
+
+            def postProcess(formal: Type, arg: Tree): Tree =
+              val arg1 = dropNotNull.transform(arg)
+              if (arg1 ne arg) && !(arg1.tpe <:< formal) then
+                ctx.error(em"""This argument was typed using flow assumptions about mutable variables
+                              |but it is passed to a by-name parameter where such flow assumptions are unsound.
+                              |Wrapping the argument in `byName(...)` fixes the problem by disabling the flow assumptions.
+                              |
+                              |`byName` needs to be imported from the `scala.compiletime` package.""",
+                          arg.sourcePos)
+                arg
+              else
+                arg1
+
+            def recur(formals: List[Type], args: List[Tree]): List[Tree] = (formals, args) match
+              case (formal :: formalsRest, arg :: argsRest) =>
+                val arg1 = postProcess(formal.widenExpr.repeatedToSingle, arg)
+                val argsRest1 = recur(
+                  if formal.isRepeatedParam then formals else formalsRest,
+                  argsRest)
+                if (arg1 eq arg) && (argsRest1 eq argsRest) then args
+                else arg1 :: argsRest1
+              case _ => args
+
+            tpd.cpy.Apply(app)(fn, recur(mt.paramInfos, args))
+          case _ => app
+      case _ => app
+  end postProcessByNameArgs
+
   def typedUnApply(tree: untpd.Apply, selType: Type)(implicit ctx: Context): Tree = {
     record("typedUnApply")
     val Apply(qual, args) = tree

compiler/src/dotty/tools/dotc/typer/Nullables.scala

@@ -182,6 +182,13 @@ object Nullables with
       then infos
       else info :: infos
 
+    /** Retract all references to mutable variables */
+    def retractMutables(given Context) =
+      val mutables = infos.foldLeft(Set[TermRef]())((ms, info) =>
+        ms.union(info.asserted.filter(_.symbol.is(Mutable))))
+      infos.extendWith(NotNullInfo(Set(), mutables))
+  end notNullInfoOps
+
   given refOps: extension (ref: TermRef) with
 
     /** Is the use of a mutable variable out of order

compiler/src/dotty/tools/dotc/typer/ProtoTypes.scala

@@ -322,9 +322,15 @@ object ProtoTypes {
      *  used to avoid repeated typings of trees when backtracking.
      */
     def typedArg(arg: untpd.Tree, formal: Type)(implicit ctx: Context): Tree = {
+      val wideFormal = formal.widenExpr
+      val argCtx =
+        if wideFormal eq formal then ctx
+        else ctx.withNotNullInfos(ctx.notNullInfos.retractMutables)
       val locked = ctx.typerState.ownedVars
-      val targ = cacheTypedArg(arg, typer.typedUnadapted(_, formal, locked), force = true)
-      typer.adapt(targ, formal, locked)
+      val targ = cacheTypedArg(arg,
+        typer.typedUnadapted(_, wideFormal, locked)(given argCtx),
+        force = true)
+      typer.adapt(targ, wideFormal, locked)
     }
 
     /** The type of the argument `arg`, or `NoType` if `arg` has not been typed before

library/src/scala/compiletime/package.scala

@@ -63,4 +63,7 @@ package object compiletime {
    *      }
    */
   type S[N <: Int] <: Int
+
+  /** Assertion that an argument is by-name. Used for nullability checking. */
+  def byName[T](x: => T): T = x
 }