Skip to content

Commit c8e7c8d

Browse files
magento-engcom-teammagento-engcom-team
committed
ENGCOM-8146: Support batches processing for media gallery synchronization queue messages #29543
- Merge Pull Request #29543 from yolouiese/magento2:1724-support-batches-synchronization - Merged commits: 1. b6a66b3 2. a550e1e 3. 29c3b26 4. 7bacf97 5. bbb99e5 6. 2917c4b 7. d98fcf3 8. 16dc1fa 9. d5276dd 10. d9e9075 11. bf3c03a 12. 56ed88f 13. a5e5c1b 14. b8e89a1 15. 561c849 16. a36a924 17. 6a9f506 18. 7a166c2 19. 2fb949e 20. 000120d 21. 79151d6 22. 70042a6 23. 8a59f10 24. 0030bde 25. b4d73e3 26. d79a043 27. c889371 28. 6bc5a8a 29. f221320 30. ee1d2d2 31. 01a968e
2 parents 1dc62a7 + 01a968e commit c8e7c8d

File tree

18 files changed

+847
-20
lines changed

18 files changed

+847
-20
lines changed

‎app/code/Magento/MediaContentSynchronization/Model/Consume.php‎

Lines changed: 58 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,85 @@
77

88
namespace Magento\MediaContentSynchronization\Model;
99

10+
use Magento\AsynchronousOperations\Api\Data\OperationInterface;
11+
use Magento\Framework\Exception\LocalizedException;
12+
use Magento\Framework\Serialize\SerializerInterface;
13+
use Magento\MediaContentApi\Api\Data\ContentIdentityInterfaceFactory;
14+
use Magento\MediaContentSynchronizationApi\Api\SynchronizeIdentitiesInterface;
1015
use Magento\MediaContentSynchronizationApi\Api\SynchronizeInterface;
1116

1217
/**
1318
* Media content synchronization queue consumer.
1419
*/
1520
class Consume
1621
{
22+
private const ENTITY_TYPE = 'entityType';
23+
private const ENTITY_ID = 'entityId';
24+
private const FIELD = 'field';
25+
26+
/**
27+
* @var SerializerInterface
28+
*/
29+
private $serializer;
30+
31+
/**
32+
* @var ContentIdentityInterfaceFactory
33+
*/
34+
private $contentIdentityFactory;
35+
1736
/**
1837
* @var SynchronizeInterface
1938
*/
2039
private $synchronize;
2140

2241
/**
42+
* @var SynchronizeIdentitiesInterface
43+
*/
44+
private $synchronizeIdentities;
45+
46+
/**
47+
* @param SerializerInterface $serializer
48+
* @param ContentIdentityInterfaceFactory $contentIdentityFactory
2349
* @param SynchronizeInterface $synchronize
50+
* @param SynchronizeIdentitiesInterface $synchronizeIdentities
2451
*/
25-
public function __construct(SynchronizeInterface $synchronize)
26-
{
52+
public function __construct(
53+
SerializerInterface $serializer,
54+
ContentIdentityInterfaceFactory $contentIdentityFactory,
55+
SynchronizeInterface $synchronize,
56+
SynchronizeIdentitiesInterface $synchronizeIdentities
57+
) {
58+
$this->serializer = $serializer;
59+
$this->contentIdentityFactory = $contentIdentityFactory;
2760
$this->synchronize = $synchronize;
61+
$this->synchronizeIdentities = $synchronizeIdentities;
2862
}
2963

3064
/**
3165
* Run media files synchronization.
66+
*
67+
* @param OperationInterface $operation
68+
* @throws LocalizedException
3269
*/
33-
public function execute() : void
70+
public function execute(OperationInterface $operation) : void
3471
{
35-
$this->synchronize->execute();
72+
$identities = $this->serializer->unserialize($operation->getSerializedData());
73+
74+
if (empty($identities)) {
75+
$this->synchronize->execute();
76+
return;
77+
}
78+
79+
$contentIdentities = [];
80+
foreach ($identities as $identity) {
81+
$contentIdentities[] = $this->contentIdentityFactory->create(
82+
[
83+
self::ENTITY_TYPE => $identity[self::ENTITY_TYPE],
84+
self::ENTITY_ID => $identity[self::ENTITY_ID],
85+
self::FIELD => $identity[self::FIELD]
86+
]
87+
);
88+
}
89+
$this->synchronizeIdentities->execute($contentIdentities);
3690
}
3791
}

‎app/code/Magento/MediaContentSynchronization/Model/Publish.php‎

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,11 @@
77

88
namespace Magento\MediaContentSynchronization\Model;
99

10+
use Magento\AsynchronousOperations\Api\Data\OperationInterfaceFactory;
11+
use Magento\Framework\Bulk\OperationInterface;
12+
use Magento\Framework\DataObject\IdentityGeneratorInterface;
1013
use Magento\Framework\MessageQueue\PublisherInterface;
14+
use Magento\Framework\Serialize\SerializerInterface;
1115

1216
/**
1317
* Publish media content synchronization queue.
@@ -19,27 +23,64 @@ class Publish
1923
*/
2024
private const TOPIC_MEDIA_CONTENT_SYNCHRONIZATION = 'media.content.synchronization';
2125

26+
/**
27+
* @var OperationInterfaceFactory
28+
*/
29+
private $operationFactory;
30+
31+
/**
32+
* @var IdentityGeneratorInterface
33+
*/
34+
private $identityService;
35+
2236
/**
2337
* @var PublisherInterface
2438
*/
2539
private $publisher;
2640

2741
/**
42+
* @var SerializerInterface
43+
*/
44+
private $serializer;
45+
46+
/**
47+
* @param OperationInterfaceFactory $operationFactory
48+
* @param IdentityGeneratorInterface $identityService
2849
* @param PublisherInterface $publisher
50+
* @param SerializerInterface $serializer
2951
*/
30-
public function __construct(PublisherInterface $publisher)
31-
{
52+
public function __construct(
53+
OperationInterfaceFactory $operationFactory,
54+
IdentityGeneratorInterface $identityService,
55+
PublisherInterface $publisher,
56+
SerializerInterface $serializer
57+
) {
58+
$this->operationFactory = $operationFactory;
59+
$this->identityService = $identityService;
60+
$this->serializer = $serializer;
3261
$this->publisher = $publisher;
3362
}
3463

3564
/**
36-
* Publish media content synchronization message to the message queue.
65+
* Publish media content synchronization message to the message queue
66+
*
67+
* @param array $contentIdentities
3768
*/
38-
public function execute() : void
69+
public function execute(array $contentIdentities = []) : void
3970
{
71+
$data = [
72+
'data' => [
73+
'bulk_uuid' => $this->identityService->generateId(),
74+
'topic_name' => self::TOPIC_MEDIA_CONTENT_SYNCHRONIZATION,
75+
'serialized_data' => $this->serializer->serialize($contentIdentities),
76+
'status' => OperationInterface::STATUS_TYPE_OPEN,
77+
]
78+
];
79+
$operation = $this->operationFactory->create($data);
80+
4081
$this->publisher->publish(
4182
self::TOPIC_MEDIA_CONTENT_SYNCHRONIZATION,
42-
[self::TOPIC_MEDIA_CONTENT_SYNCHRONIZATION]
83+
$operation
4384
);
4485
}
4586
}

‎app/code/Magento/MediaContentSynchronization/Model/SynchronizeIdentities.php‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
declare(strict_types=1);
7+
8+
namespace Magento\MediaContentSynchronization\Model;
9+
10+
use Magento\Framework\Exception\LocalizedException;
11+
use Magento\Framework\FlagManager;
12+
use Magento\Framework\Stdlib\DateTime\DateTimeFactory;
13+
use Magento\MediaContentSynchronizationApi\Api\SynchronizeIdentitiesInterface;
14+
use Magento\MediaContentSynchronizationApi\Model\SynchronizeIdentitiesPool;
15+
use Psr\Log\LoggerInterface;
16+
17+
/**
18+
* Batch Synchronize content with assets
19+
*/
20+
class SynchronizeIdentities implements SynchronizeIdentitiesInterface
21+
{
22+
/**
23+
* @var LoggerInterface
24+
*/
25+
private $log;
26+
27+
/**
28+
* @var SynchronizeIdentitiesPool
29+
*/
30+
private $synchronizeIdentitiesPool;
31+
32+
/**
33+
* @param LoggerInterface $log
34+
* @param SynchronizeIdentitiesPool $synchronizeIdentitiesPool
35+
*/
36+
public function __construct(
37+
LoggerInterface $log,
38+
SynchronizeIdentitiesPool $synchronizeIdentitiesPool
39+
) {
40+
$this->log = $log;
41+
$this->synchronizeIdentitiesPool = $synchronizeIdentitiesPool;
42+
}
43+
44+
/**
45+
* @inheritdoc
46+
*/
47+
public function execute(array $mediaContentIdentities): void
48+
{
49+
$failed = [];
50+
51+
foreach ($this->synchronizeIdentitiesPool->get() as $name => $synchronizer) {
52+
try {
53+
$synchronizer->execute($mediaContentIdentities);
54+
} catch (\Exception $exception) {
55+
$this->log->critical($exception);
56+
$failed[] = $name;
57+
}
58+
}
59+
60+
if (!empty($failed)) {
61+
throw new LocalizedException(
62+
__(
63+
'Failed to execute the following content synchronizers: %synchronizers',
64+
[
65+
'synchronizers' => implode(', ', $failed)
66+
]
67+
)
68+
);
69+
}
70+
}
71+
}

‎app/code/Magento/MediaContentSynchronization/Test/Integration/Model/PublisherTest.php‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
<?php
2+
/**
3+
* Copyright © Magento, Inc. All rights reserved.
4+
* See COPYING.txt for license details.
5+
*/
6+
declare(strict_types=1);
7+
8+
namespace Magento\MediaContentSynchronization\Test\Integration\Model;
9+
10+
use Magento\Framework\Exception\IntegrationException;
11+
use Magento\Framework\Exception\LocalizedException;
12+
use Magento\Framework\MessageQueue\ConsumerFactory;
13+
use Magento\MediaContentApi\Api\Data\ContentIdentityInterfaceFactory;
14+
use Magento\MediaContentApi\Api\GetAssetIdsByContentIdentityInterface;
15+
use Magento\MediaContentApi\Api\GetContentByAssetIdsInterface;
16+
use Magento\MediaContentSynchronization\Model\Publish;
17+
use Magento\TestFramework\Helper\Bootstrap;
18+
use PHPUnit\Framework\TestCase;
19+
20+
/**
21+
* Test for media content Publisher
22+
*/
23+
class PublisherTest extends TestCase
24+
{
25+
private const TOPIC_MEDIA_CONTENT_SYNCHRONIZATION = 'media.content.synchronization';
26+
27+
/**
28+
* @var ConsumerFactory
29+
*/
30+
private $consumerFactory;
31+
32+
/**
33+
* @var ContentIdentityInterfaceFactory
34+
*/
35+
private $contentIdentityFactory;
36+
37+
/**
38+
* @var GetContentByAssetIdsInterface
39+
*/
40+
private $getContentIdentities;
41+
42+
/**
43+
* @var GetAssetIdsByContentIdentityInterface
44+
*/
45+
private $getAssetIds;
46+
47+
/**
48+
* @var Publish
49+
*/
50+
private $publish;
51+
52+
protected function setUp(): void
53+
{
54+
$this->consumerFactory = Bootstrap::getObjectManager()->get(ConsumerFactory::class);
55+
$this->contentIdentityFactory = Bootstrap::getObjectManager()->get(ContentIdentityInterfaceFactory::class);
56+
$this->getContentIdentities = Bootstrap::getObjectManager()->get(GetContentByAssetIdsInterface::class);
57+
$this->getAssetIds = Bootstrap::getObjectManager()->get(GetAssetIdsByContentIdentityInterface::class);
58+
$this->publish = Bootstrap::getObjectManager()->get(Publish::class);
59+
}
60+
61+
/**
62+
* @dataProvider filesProvider
63+
* @magentoDataFixture Magento/MediaContentCatalog/_files/category_with_asset.php
64+
* @magentoDataFixture Magento/MediaContentCatalog/_files/product_with_asset.php
65+
* @magentoDataFixture Magento/MediaContentCms/_files/page_with_asset.php
66+
* @magentoDataFixture Magento/MediaContentCms/_files/block_with_asset.php
67+
* @magentoDataFixture Magento/MediaGallery/_files/media_asset.php
68+
* @param array $contentIdentities
69+
* @throws IntegrationException
70+
* @throws LocalizedException
71+
*/
72+
public function testExecute(array $contentIdentities): void
73+
{
74+
// publish message to the queue
75+
$this->publish->execute($contentIdentities);
76+
77+
// run and process message
78+
$batchSize = 1;
79+
$maxNumberOfMessages = 1;
80+
$consumer = $this->consumerFactory->get(self::TOPIC_MEDIA_CONTENT_SYNCHRONIZATION, $batchSize);
81+
$consumer->process($maxNumberOfMessages);
82+
83+
// verify synchronized media content
84+
$assetId = 2020;
85+
$entityIds = [];
86+
foreach ($contentIdentities as $contentIdentity) {
87+
$contentIdentityObject = $this->contentIdentityFactory->create($contentIdentity);
88+
$this->assertEquals([$assetId], $this->getAssetIds->execute($contentIdentityObject));
89+
$entityIds[] = $contentIdentityObject->getEntityId();
90+
}
91+
92+
$synchronizedContentIdentities = $this->getContentIdentities->execute([$assetId]);
93+
$this->assertEquals(2, count($synchronizedContentIdentities));
94+
95+
foreach ($synchronizedContentIdentities as $syncedContentIdentity) {
96+
$this->assertContains($syncedContentIdentity->getEntityId(), $entityIds);
97+
}
98+
}
99+
100+
/**
101+
* Data provider
102+
*
103+
* @return array
104+
*/
105+
public function filesProvider(): array
106+
{
107+
return [
108+
[
109+
[
110+
[
111+
'entityType' => 'catalog_category',
112+
'field' => 'description',
113+
'entityId' => 28767
114+
],
115+
[
116+
'entityType' => 'catalog_product',
117+
'field' => 'description',
118+
'entityId' => 1567
119+
]
120+
]
121+
]
122+
];
123+
}
124+
}

‎app/code/Magento/MediaContentSynchronization/composer.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@
44
"require": {
55
"php": "~7.3.0||~7.4.0",
66
"magento/framework": "*",
7+
"magento/framework-bulk": "*",
78
"magento/module-media-content-synchronization-api": "*",
8-
"magento/framework-message-queue": "*",
9-
"magento/module-media-content-api": "*"
9+
"magento/module-media-content-api": "*",
10+
"magento/module-asynchronous-operations": "*"
1011
},
1112
"suggest": {
1213
"magento/module-media-gallery-synchronization": "*"

‎app/code/Magento/MediaContentSynchronization/etc/communication.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
-->
88
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
99
xsi:noNamespaceSchemaLocation="urn:magento:framework:Communication/etc/communication.xsd">
10-
<topic name="media.content.synchronization" is_synchronous="false" request="string[]">
10+
<topic name="media.content.synchronization" is_synchronous="false" request="Magento\AsynchronousOperations\Api\Data\OperationInterface">
1111
<handler name="media.content.synchronization.handler"
1212
type="Magento\MediaContentSynchronization\Model\Consume" method="execute"/>
1313
</topic>

‎app/code/Magento/MediaContentSynchronization/etc/di.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
-->
88
<config xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:framework:ObjectManager/etc/config.xsd">
99
<preference for="Magento\MediaContentSynchronizationApi\Api\SynchronizeInterface" type="Magento\MediaContentSynchronization\Model\Synchronize"/>
10+
<preference for="Magento\MediaContentSynchronizationApi\Api\SynchronizeIdentitiesInterface" type="Magento\MediaContentSynchronization\Model\SynchronizeIdentities"/>
1011
<type name="Magento\Framework\Console\CommandListInterface">
1112
<arguments>
1213
<argument name="commands" xsi:type="array">

0 commit comments

Comments
 (0)