ACP2E-3891: Categories in admin are loading very slow

Author: soklymeach

app/code/Magento/Catalog/Model/ResourceModel/Category/Collection.php

@@ -375,35 +375,57 @@ private function getCountFromCategoryTableBulk(
         array $categoryIds,
         int $websiteId
     ) : array {
-        $subSelect = clone $this->_conn->select();
-        $subSelect->from(['ce2' => $this->getTable('catalog_category_entity')], 'ce2.entity_id')
-            ->where("ce2.path LIKE CONCAT(ce.path, '/%') OR ce2.path = ce.path");
-
-        $select = clone $this->_conn->select();
-        $select->from(
+        $connection = $this->_conn;
+        $tempTableName = $connection->getTableName('temp_category_descendants_' . uniqid());
+        $connection->query("CREATE TEMPORARY TABLE {$tempTableName} (
+        category_id INT UNSIGNED NOT NULL,
+        descendant_id INT UNSIGNED NOT NULL,
+        PRIMARY KEY (category_id, descendant_id)
+    )");
+        $selectDescendants = clone $connection->select();
+        $selectDescendants->from(
             ['ce' => $this->getTable('catalog_category_entity')],
-            'ce.entity_id'
-        );
-        $joinCondition =  new \Zend_Db_Expr("cp.category_id IN ({$subSelect})");
-        $select->joinLeft(
-            ['cp' => $this->getProductTable()],
-            $joinCondition,
-            'COUNT(DISTINCT cp.product_id) AS product_count'
+            ['category_id' => 'ce.entity_id', 'descendant_id' => 'ce2.entity_id']
+        )
+            ->joinInner(
+                ['ce2' => $this->getTable('catalog_category_entity')],
+                'ce2.path LIKE CONCAT(ce.path, \'/%\') OR ce2.entity_id = ce.entity_id',
+                []
+            )
+            ->where('ce.entity_id IN (?)', $categoryIds);
+        $connection->query(
+            $connection->insertFromSelect(
+                $selectDescendants,
+                $tempTableName,
+                ['category_id', 'descendant_id']
+            )
         );
+        $select = clone $connection->select();
+        $select->from(
+            ['t' => $tempTableName],
+            ['category_id' => 't.category_id']
+        )
+            ->joinLeft(
+                ['cp' => $this->getTable('catalog_category_product')],
+                'cp.category_id = t.descendant_id',
+                ['product_count' => 'COUNT(DISTINCT cp.product_id)']
+            );
         if ($websiteId) {
             $select->join(
                 ['w' => $this->getProductWebsiteTable()],
                 'cp.product_id = w.product_id',
                 []
-            )->where(
-                'w.website_id = ?',
-                $websiteId
-            );
+            )->where('w.website_id = ?', $websiteId);
+        }
+        $select->group('t.category_id');
+        $result = $connection->fetchPairs($select);
+        $connection->query("DROP TEMPORARY TABLE {$tempTableName}");
+        $counts = array_fill_keys($categoryIds, 0);
+        foreach ($result as $categoryId => $count) {
+            $counts[$categoryId] = (int)$count;
         }
-        $select->where('ce.entity_id IN(?)', $categoryIds);
-        $select->group('ce.entity_id');
 
-        return $this->_conn->fetchPairs($select);
+        return $counts;
     }
 
     /**

app/code/Magento/Catalog/Test/Unit/Model/ResourceModel/Category/CollectionTest.php

@@ -7,7 +7,9 @@
 
 namespace Magento\Catalog\Test\Unit\Model\ResourceModel\Category;
 
+use Magento\Catalog\Model\Category;
 use Magento\Framework\Data\Collection\EntityFactory;
+use Magento\Store\Model\Store;
 use Psr\Log\LoggerInterface;
 use Magento\Framework\Data\Collection\Db\FetchStrategyInterface;
 use Magento\Framework\Event\ManagerInterface;
@@ -215,4 +217,52 @@ public function testLoadProductCount() : void
             ->willReturn([]);
         $this->collection->loadProductCount([]);
     }
+
+    /**
+     * Test that loadProductCount calls getCountFromCategoryTableBulk
+     */
+    public function testLoadProductCountCallsBulkMethodForLargeCategoryCount()
+    {
+        $websiteId = 1;
+        $storeId = 1;
+        $categoryCount = 401;
+        $items = [];
+        $categoryIds = [];
+        for ($i = 1; $i <= $categoryCount; $i++) {
+            $category = $this->getMockBuilder(Category::class)
+                ->addMethods(['getIsAnchor'])
+                ->onlyMethods(['getId', 'setProductCount'])
+                ->disableOriginalConstructor()
+                ->getMock();
+            $category->method('getId')->willReturn($i);
+            $category->method('getIsAnchor')->willReturn(true);
+            $category->expects($this->once())->method('setProductCount')->with(5);
+            $items[$i] = $category;
+            $categoryIds[] = $i;
+        }
+        $storeMock = $this->createMock(Store::class);
+        $storeMock->method('getWebsiteId')->willReturn($websiteId);
+        $this->storeManager->method('getStore')->with($storeId)->willReturn($storeMock);
+        $this->connection->method('select')->willReturn($this->select);
+        $counts = array_fill_keys($categoryIds, 5);
+        $this->select->method('from')->willReturnSelf();
+        $this->select->method('where')->willReturnSelf();
+        $this->select->method('group')->willReturnSelf();
+        $this->connection->method('fetchPairs')
+            ->with($this->isInstanceOf(Select::class))
+            ->willReturn($counts);
+        $tempTableName = 'temp_category_descendants_';
+        $this->connection->method('query')->willReturn($tempTableName);
+        $this->select->method('joinInner')->willReturnSelf();
+        $this->connection->method('insertFromSelect')->willReturn('INSERT QUERY');
+        $bulkSelectMock = $this->createMock(Select::class);
+        $bulkSelectMock->method('from')->willReturnSelf();
+        $bulkSelectMock->method('joinLeft')->willReturnSelf();
+        $bulkSelectMock->method('join')->willReturnSelf();
+        $bulkSelectMock->method('where')->willReturnSelf();
+        $bulkSelectMock->method('group')->willReturnSelf();
+        $this->connection->method('fetchPairs')->willReturn($counts);
+        $this->collection->setProductStoreId($storeId);
+        $this->collection->loadProductCount($items, false, true);
+    }
 }