Merge branch 'MC-42330' of https://github.com/magento-l3/magento2ce into PR-2021-08-18

Author: chittima

app/code/Magento/AwsS3/Test/Mftf/Test/AdminAwsS3ImportSimpleProductImagesDuplicationTest.xml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="AdminAwsS3ImportSimpleProductImagesDuplicationTest" extends="AdminImportSimpleProductImagesDuplicationTest">
+        <annotations>
+            <features value="AwsS3"/>
+            <stories value="Import Products"/>
+            <title value="S3 - Duplicated images should not be created if the CSV file is imported more than once"/>
+            <description value="Duplicated images should not be created if the CSV file is imported more than once"/>
+            <severity value="MAJOR"/>
+            <testCaseId value="MC-42986"/>
+            <useCaseId value="MC-42330"/>
+            <group value="catalog_import_export"/>
+            <group value="remote_storage_aws_s3"/>
+            <group value="remote_storage_disabled"/>
+        </annotations>
+
+        <before>
+            <!-- Locally Copy Import Files to Unique Media Import Directory -->
+            <helper class="Magento\Catalog\Test\Mftf\Helper\LocalFileAssertions" method="createDirectory" stepKey="createDirectoryForImportImages">
+                <argument name="path">pub/media/import/test_image_duplication</argument>
+            </helper>
+            <helper class="Magento\Catalog\Test\Mftf\Helper\LocalFileAssertions" method="copy" stepKey="copyProductBaseImage">
+                <argument name="source">dev/tests/acceptance/tests/_data/{{placeholderBaseImage.file}}</argument>
+                <argument name="destination">pub/media/import/test_image_duplication/{{placeholderBaseImage.file}}</argument>
+            </helper>
+            <helper class="Magento\Catalog\Test\Mftf\Helper\LocalFileAssertions" method="copy" stepKey="copyProductSmallImage">
+                <argument name="source">dev/tests/acceptance/tests/_data/{{placeholderSmallImage.file}}</argument>
+                <argument name="destination">pub/media/import/test_image_duplication/{{placeholderSmallImage.file}}</argument>
+            </helper>
+            <helper class="Magento\Catalog\Test\Mftf\Helper\LocalFileAssertions" method="copy" stepKey="copyProductThumbImage">
+                <argument name="source">dev/tests/acceptance/tests/_data/{{placeholderThumbnailImage.file}}</argument>
+                <argument name="destination">pub/media/import/test_image_duplication/{{placeholderThumbnailImage.file}}</argument>
+            </helper>
+
+            <!-- Enable AWS S3 Remote Storage & Sync -->
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.enable_options}}" stepKey="enableRemoteStorage" after="copyProductThumbImage"/>
+            <magentoCLI command="remote-storage:sync" timeout="120" stepKey="syncRemoteStorage" after="enableRemoteStorage"/>
+
+            <!-- Copy to Import Directory in AWS S3 -->
+            <helper class="Magento\AwsS3\Test\Mftf\Helper\S3FileAssertions" method="createDirectory" stepKey="createDirectoryForImportFilesInS3" after="syncRemoteStorage">
+                <argument name="path">var/import/images/test_image_duplication</argument>
+            </helper>
+            <helper class="Magento\AwsS3\Test\Mftf\Helper\S3FileAssertions" method="copy" stepKey="copyProductBaseImageS3" after="createDirectoryForImportFilesInS3">
+                <argument name="source">media/import/test_image_duplication/{{placeholderBaseImage.file}}</argument>
+                <argument name="destination">var/import/images/test_image_duplication/{{placeholderBaseImage.file}}</argument>
+            </helper>
+            <helper class="Magento\AwsS3\Test\Mftf\Helper\S3FileAssertions" method="copy" stepKey="copyProductSmallImageS3" after="copyProductBaseImageS3">
+                <argument name="source">media/import/test_image_duplication/{{placeholderSmallImage.file}}</argument>
+                <argument name="destination">var/import/images/test_image_duplication/{{placeholderSmallImage.file}}</argument>
+            </helper>
+            <helper class="Magento\AwsS3\Test\Mftf\Helper\S3FileAssertions" method="copy" stepKey="copyProductThumbImageS3" after="copyProductSmallImageS3">
+                <argument name="source">media/import/test_image_duplication/{{placeholderThumbnailImage.file}}</argument>
+                <argument name="destination">var/import/images/test_image_duplication/{{placeholderThumbnailImage.file}}</argument>
+            </helper>
+        </before>
+
+        <after>
+            <!-- Delete S3 Data -->
+            <remove keyForRemoval="deleteProductImageDirectory"/>
+            <helper class="Magento\AwsS3\Test\Mftf\Helper\S3FileAssertions" method="deleteDirectory" stepKey="deleteImportFilesDirectoryS3" after="deleteCategory">
+                <argument name="path">media/import/test_image_duplication</argument>
+            </helper>
+            <helper class="Magento\AwsS3\Test\Mftf\Helper\S3FileAssertions" method="deleteDirectory" stepKey="deleteImportImagesFilesDirectoryS3" after="deleteImportFilesDirectoryS3">
+                <argument name="path">var/import/images/test_image_duplication</argument>
+            </helper>
+
+            <!-- Disable AWS S3 Remote Storage & Delete Local Data -->
+            <magentoCLI command="setup:config:set {{RemoteStorageAwsS3ConfigData.disable_options}}" stepKey="disableRemoteStorage" after="logoutFromAdmin"/>
+            <helper class="Magento\Catalog\Test\Mftf\Helper\LocalFileAssertions" method="deleteDirectory" stepKey="deleteImportFilesDirectoryLocal" after="disableRemoteStorage">
+                <argument name="path">pub/media/import/test_image_duplication</argument>
+            </helper>
+        </after>
+    </test>
+</tests>

app/code/Magento/CatalogImportExport/Model/Import/Product.php

@@ -23,6 +23,7 @@
 use Magento\Framework\Exception\NoSuchEntityException;
 use Magento\Framework\Filesystem;
 use Magento\Framework\Filesystem\Driver\File;
+use Magento\Framework\Filesystem\DriverPool;
 use Magento\Framework\Intl\DateTimeFactory;
 use Magento\Framework\Model\ResourceModel\Db\ObjectRelationProcessor;
 use Magento\Framework\Model\ResourceModel\Db\TransactionManagerInterface;
@@ -767,11 +768,6 @@ class Product extends AbstractEntity
      */
     private $linkProcessor;
 
-    /**
-     * @var File
-     */
-    private $fileDriver;
-
     /**
      * @param \Magento\Framework\Json\Helper\Data $jsonHelper
      * @param \Magento\ImportExport\Helper\Data $importExportData
@@ -938,7 +934,6 @@ public function __construct(
         $this->dateTimeFactory = $dateTimeFactory ?? ObjectManager::getInstance()->get(DateTimeFactory::class);
         $this->productRepository = $productRepository ?? ObjectManager::getInstance()
                 ->get(ProductRepositoryInterface::class);
-        $this->fileDriver = $fileDriver ?: ObjectManager::getInstance()->get(File::class);
     }
 
     /**
@@ -1580,7 +1575,6 @@ protected function _saveProducts()
             $previousType = null;
             $prevAttributeSet = null;
 
-            $importDir = $this->_mediaDirectory->getAbsolutePath($this->getUploader()->getTmpDir());
             $existingImages = $this->getExistingImages($bunch);
             $this->addImageHashes($existingImages);
 
@@ -1750,7 +1744,10 @@ protected function _saveProducts()
                 $position = 0;
                 foreach ($rowImages as $column => $columnImages) {
                     foreach ($columnImages as $columnImageKey => $columnImage) {
-                        $uploadedFile = $this->getAlreadyExistedImage($rowExistingImages, $columnImage, $importDir);
+                        $hash = filter_var($columnImage, FILTER_VALIDATE_URL)
+                            ? $this->getRemoteFileHash($columnImage)
+                            : $this->getFileHash($this->joinFilePaths($this->getUploader()->getTmpDir(), $columnImage));
+                        $uploadedFile = $this->findImageByHash($rowExistingImages, $hash);
                         if (!$uploadedFile && !isset($uploadedImages[$columnImage])) {
                             $uploadedFile = $this->uploadMediaFiles($columnImage);
                             $uploadedFile = $uploadedFile ?: $this->getSystemFile($columnImage);
@@ -1955,40 +1952,29 @@ protected function _saveProducts()
      *
      * @param string $path
      * @return string
+     * @throws \Magento\Framework\Exception\FileSystemException
      */
     private function getFileHash(string $path): string
     {
-        return hash_file(self::HASH_ALGORITHM, $path);
+        $content = '';
+        if ($this->_mediaDirectory->isFile($path)
+            && $this->_mediaDirectory->isReadable($path)
+        ) {
+            $content = $this->_mediaDirectory->readFile($path);
+        }
+        return $content ? hash(self::HASH_ALGORITHM, $content) : '';
     }
 
     /**
-     * Returns existed image
+     * Returns hash for remote file
      *
-     * @param array $imageRow
-     * @param string $columnImage
-     * @param string $importDir
+     * @param string $filename
      * @return string
      */
-    private function getAlreadyExistedImage(array $imageRow, string $columnImage, string $importDir): string
+    private function getRemoteFileHash(string $filename): string
     {
-        if (filter_var($columnImage, FILTER_VALIDATE_URL)) {
-            $hash = $this->getFileHash($columnImage);
-        } else {
-            $path = $importDir . DIRECTORY_SEPARATOR . $columnImage;
-            $hash = $this->isFileExists($path) ? $this->getFileHash($path) : '';
-        }
-
-        return array_reduce(
-            $imageRow,
-            function ($exists, $file) use ($hash) {
-                if (!$exists && isset($file['hash']) && $file['hash'] === $hash) {
-                    return $file['value'];
-                }
-
-                return $exists;
-            },
-            ''
-        );
+        $hash = hash_file(self::HASH_ALGORITHM, $filename);
+        return $hash !== false ? $hash : '';
     }
 
     /**
@@ -1999,38 +1985,19 @@ function ($exists, $file) use ($hash) {
      */
     private function addImageHashes(array &$images): void
     {
-        $productMediaPath = $this->filesystem->getDirectoryRead(DirectoryList::MEDIA)
-            ->getAbsolutePath(DIRECTORY_SEPARATOR . 'catalog' . DIRECTORY_SEPARATOR . 'product');
-
+        $productMediaPath = $this->getProductMediaPath();
         foreach ($images as $storeId => $skus) {
             foreach ($skus as $sku => $files) {
                 foreach ($files as $path => $file) {
-                    if ($this->fileDriver->isExists($productMediaPath . $file['value'])) {
-                        $fileName = $productMediaPath . $file['value'];
-                        $images[$storeId][$sku][$path]['hash'] = $this->getFileHash($fileName);
+                    $hash = $this->getFileHash($this->joinFilePaths($productMediaPath, $file['value']));
+                    if ($hash) {
+                        $images[$storeId][$sku][$path]['hash'] = $hash;
                     }
                 }
             }
         }
     }
 
-    /**
-     * Is file exists
-     *
-     * @param string $path
-     * @return bool
-     */
-    private function isFileExists(string $path): bool
-    {
-        try {
-            $fileExists = $this->fileDriver->isExists($path);
-        } catch (\Exception $exception) {
-            $fileExists = false;
-        }
-
-        return $fileExists;
-    }
-
     /**
      * Clears entries from Image Set and Row Data marked as no_selection
      *
@@ -2214,23 +2181,15 @@ protected function _getUploader()
 
             $fileUploader->init();
 
-            $dirConfig = DirectoryList::getDefaultConfig();
-            $dirAddon = $dirConfig[DirectoryList::MEDIA][DirectoryList::PATH];
-
-            // make media folder a primary folder for media in external storages
-            if (!is_a($this->_mediaDirectory->getDriver(), File::class)) {
-                $dirAddon = DirectoryList::MEDIA;
-            }
-
             $tmpPath = $this->getImportDir();
 
             if (!$fileUploader->setTmpDir($tmpPath)) {
                 throw new LocalizedException(
                     __('File directory \'%1\' is not readable.', $tmpPath)
                 );
             }
-            $destinationDir = "catalog/product";
-            $destinationPath = $dirAddon . '/' . $this->_mediaDirectory->getRelativePath($destinationDir);
+
+            $destinationPath = $this->getProductMediaPath();
 
             $this->_mediaDirectory->create($destinationPath);
             if (!$fileUploader->setDestDir($destinationPath)) {
@@ -2284,11 +2243,11 @@ protected function uploadMediaFiles($fileName, $renameFileOff = false)
      */
     private function getSystemFile($fileName)
     {
-        $filePath = 'catalog' . DIRECTORY_SEPARATOR . 'product' . DIRECTORY_SEPARATOR . $fileName;
-        /** @var \Magento\Framework\Filesystem\Directory\ReadInterface $read */
-        $read = $this->filesystem->getDirectoryRead(DirectoryList::MEDIA);
+        $filePath = $this->joinFilePaths($this->getProductMediaPath(), $fileName);
 
-        return $read->isExist($filePath) && $read->isReadable($filePath) ? $fileName : '';
+        return $this->_mediaDirectory->isFile($filePath) && $this->_mediaDirectory->isReadable($filePath)
+            ? $fileName
+            : '';
     }
 
     /**
@@ -3280,4 +3239,68 @@ private function getRowExistingStockItem(array $rowData): StockItemInterface
         $websiteId = $this->stockConfiguration->getDefaultScopeId();
         return $this->stockRegistry->getStockItem($productId, $websiteId);
     }
+
+    /**
+     * Returns image that matches the provided hash
+     *
+     * @param array $images
+     * @param string $hash
+     * @return string
+     */
+    private function findImageByHash(array $images, string $hash): string
+    {
+        $value = '';
+        if ($hash) {
+            foreach ($images as $image) {
+                if (isset($image['hash']) && $image['hash'] === $hash) {
+                    $value = $image['value'];
+                    break;
+                }
+            }
+        }
+        return $value;
+    }
+
+    /**
+     * Returns product media
+     *
+     * @return string relative path to root folder
+     */
+    private function getProductMediaPath(): string
+    {
+        return $this->joinFilePaths($this->getMediaBasePath(), 'catalog','product');
+    }
+
+    /**
+     * Returns media base path
+     *
+     * @return string relative path to root folder
+     */
+    private function getMediaBasePath(): string
+    {
+        $mediaDir = !is_a($this->_mediaDirectory->getDriver(), File::class)
+            // make media folder a primary folder for media in external storages
+            ? $this->filesystem->getDirectoryReadByPath(DirectoryList::MEDIA)
+            : $this->filesystem->getDirectoryRead(DirectoryList::MEDIA);
+
+        return $this->_mediaDirectory->getRelativePath($mediaDir->getAbsolutePath());
+    }
+
+    /**
+     * Joins two paths and remove redundant directory separator
+     *
+     * @param string ...$paths
+     * @return string
+     */
+    private function joinFilePaths(...$paths): string
+    {
+        $result = '';
+        if ($paths) {
+            $result = rtrim(array_shift($paths), DIRECTORY_SEPARATOR);
+            foreach ($paths as $path) {
+                $result .= DIRECTORY_SEPARATOR . ltrim($path, DIRECTORY_SEPARATOR);
+            }
+        }
+        return $result;
+    }
 }