Document RenderObject._relayoutBoundary and its invariant; small refactors (flutter#150527)

Fixes part of flutter#150524.

Recently I was studying RenderObject and related code to understand
how the rendering library tracks which objects need layout, and in
particular just how the concept of "relayout boundary" works.
The [RenderObject._relayoutBoundary] property itself had no docs,
and there were some other areas that I felt could be clearer;
so here's a PR where I add those docs.

In addition to docs, this makes some small pure refactors.  Mostly
those are in order to make clear docs easier to write.  One rename
is to fix a lone straggler from flutter#4425 / flutter#4483.

One key thing I learned, which this documents, is an invariant that
this code maintains on [_relayoutBoundary].  With that invariant in
mind, I found it's possible to simplify that bookkeeping, and to get a
measurable performance improvement as a bonus.  To keep things simple,
though, I'll leave those for follow-up PRs.

For ease of review, the PR is broken into several commits each with
their own commit message.

Author: gnprice

packages/flutter/lib/src/rendering/object.dart

@@ -1853,7 +1853,7 @@ abstract class RenderObject with DiagnosticableTreeMixin implements HitTestTarge
     assert(child._parent == this);
     assert(child.attached == attached);
     assert(child.parentData != null);
-    child._cleanRelayoutBoundary();
+    _cleanChildRelayoutBoundary(child);
     child.parentData!.detach();
     child.parentData = null;
     child._parent = null;
@@ -2185,6 +2185,32 @@ abstract class RenderObject with DiagnosticableTreeMixin implements HitTestTarge
   }
   bool _needsLayout = true;
 
+  /// The nearest relayout boundary enclosing this render object, if known.
+  ///
+  /// When a render object is marked as needing layout, its parent may
+  /// as a result also need to be marked as needing layout.
+  /// For details, see [markNeedsLayout].
+  /// A render object where relayout does not require relayout of the parent
+  /// (because its size cannot change on relayout, or because
+  /// its parent does not use the child's size for its own layout)
+  /// is a "relayout boundary".
+  ///
+  /// This property is set in [layout], and consulted by [markNeedsLayout] in
+  /// deciding whether to recursively mark the parent as also needing layout.
+  ///
+  /// This property is initially null, and becomes null again if this
+  /// render object is removed from the tree (with [dropChild]);
+  /// it remains null until the first layout of this render object
+  /// after it was most recently added to the tree.
+  /// This property can also be null while an ancestor in the tree is
+  /// currently doing layout, until this render object itself does layout.
+  ///
+  /// When not null, the relayout boundary is either this render object itself
+  /// or one of its ancestors, and all the render objects in the ancestry chain
+  /// up through that ancestor have the same [_relayoutBoundary].
+  /// Equivalently: when not null, the relayout boundary is either this render
+  /// object itself or the same as that of its parent.  (So [_relayoutBoundary]
+  /// is one of `null`, `this`, or `parent!._relayoutBoundary!`.)
   RenderObject? _relayoutBoundary;
 
   /// Whether [invokeLayoutCallback] for this render object is currently running.
@@ -2222,7 +2248,7 @@ abstract class RenderObject with DiagnosticableTreeMixin implements HitTestTarge
   /// (where it will always be false).
   static bool debugCheckingIntrinsics = false;
 
-  bool _debugSubtreeRelayoutRootAlreadyMarkedNeedsLayout() {
+  bool _debugRelayoutBoundaryAlreadyMarkedNeedsLayout() {
     if (_relayoutBoundary == null) {
       // We don't know where our relayout boundary is yet.
       return true;
@@ -2281,7 +2307,7 @@ abstract class RenderObject with DiagnosticableTreeMixin implements HitTestTarge
   void markNeedsLayout() {
     assert(_debugCanPerformMutations);
     if (_needsLayout) {
-      assert(_debugSubtreeRelayoutRootAlreadyMarkedNeedsLayout());
+      assert(_debugRelayoutBoundaryAlreadyMarkedNeedsLayout());
       return;
     }
     if (_relayoutBoundary == null) {
@@ -2346,32 +2372,36 @@ abstract class RenderObject with DiagnosticableTreeMixin implements HitTestTarge
     markParentNeedsLayout();
   }
 
-  void _cleanRelayoutBoundary() {
-    if (_relayoutBoundary != this) {
-      _relayoutBoundary = null;
-      visitChildren(_cleanChildRelayoutBoundary);
+  /// Set [_relayoutBoundary] to null throughout this render object's subtree,
+  /// stopping at relayout boundaries.
+  // This is a static method to reduce closure allocation with visitChildren.
+  static void _cleanChildRelayoutBoundary(RenderObject child) {
+    if (child._relayoutBoundary != child) {
+      child.visitChildren(_cleanChildRelayoutBoundary);
+      child._relayoutBoundary = null;
     }
   }
 
-  void _propagateRelayoutBoundary() {
-    if (_relayoutBoundary == this) {
+  // This is a static method to reduce closure allocation with visitChildren.
+  static void _propagateRelayoutBoundaryToChild(RenderObject child) {
+    if (child._relayoutBoundary == child) {
       return;
     }
-    final RenderObject? parentRelayoutBoundary = parent?._relayoutBoundary;
+    final RenderObject? parentRelayoutBoundary = child.parent?._relayoutBoundary;
     assert(parentRelayoutBoundary != null);
-    if (parentRelayoutBoundary != _relayoutBoundary) {
-      _relayoutBoundary = parentRelayoutBoundary;
-      visitChildren(_propagateRelayoutBoundaryToChild);
-    }
+    assert(parentRelayoutBoundary != child._relayoutBoundary);
+    child._setRelayoutBoundary(parentRelayoutBoundary!);
   }
 
-  // Reduces closure allocation for visitChildren use cases.
-  static void _cleanChildRelayoutBoundary(RenderObject child) {
-    child._cleanRelayoutBoundary();
-  }
-
-  static void _propagateRelayoutBoundaryToChild(RenderObject child) {
-    child._propagateRelayoutBoundary();
+  /// Set [_relayoutBoundary] to [value] throughout this render object's
+  /// subtree, including this render object but stopping at relayout boundaries
+  /// thereafter.
+  void _setRelayoutBoundary(RenderObject value) {
+    assert(value != _relayoutBoundary);
+    // This may temporarily break the _relayoutBoundary invariant at children;
+    // the visitChildren restores the invariant.
+    _relayoutBoundary = value;
+    visitChildren(_propagateRelayoutBoundaryToChild);
   }
 
   /// Bootstrap the rendering pipeline by scheduling the very first layout.
@@ -2396,6 +2426,7 @@ abstract class RenderObject with DiagnosticableTreeMixin implements HitTestTarge
 
   @pragma('vm:notify-debugger-on-exception')
   void _layoutWithoutResize() {
+    assert(_needsLayout);
     assert(_relayoutBoundary == this);
     RenderObject? debugPreviousActiveLayout;
     assert(!_debugMutationsLocked);
@@ -2520,8 +2551,7 @@ abstract class RenderObject with DiagnosticableTreeMixin implements HitTestTarge
       }());
 
       if (relayoutBoundary != _relayoutBoundary) {
-        _relayoutBoundary = relayoutBoundary;
-        visitChildren(_propagateRelayoutBoundaryToChild);
+        _setRelayoutBoundary(relayoutBoundary);
       }
 
       if (!kReleaseMode && debugProfileLayoutsEnabled) {
@@ -2530,13 +2560,15 @@ abstract class RenderObject with DiagnosticableTreeMixin implements HitTestTarge
       return;
     }
     _constraints = constraints;
+
     if (_relayoutBoundary != null && relayoutBoundary != _relayoutBoundary) {
       // The local relayout boundary has changed, must notify children in case
       // they also need updating. Otherwise, they will be confused about what
       // their actual relayout boundary is later.
       visitChildren(_cleanChildRelayoutBoundary);
     }
     _relayoutBoundary = relayoutBoundary;
+
     assert(!_debugMutationsLocked);
     assert(!_doingThisLayoutWithCallback);
     assert(() {