diff --git a/Helper/Data.php b/Helper/Data.php index fb5979c..6df53ad 100755 --- a/Helper/Data.php +++ b/Helper/Data.php @@ -39,7 +39,7 @@ public function getSourceModelByType($sourceType) return $source; } else { throw new \Magento\Framework\Exception\LocalizedException( - __("Import source type class for '%s' is not exist.", $sourceType) + __("Import source type class for '" . $sourceType . "' is not exist.") ); } } diff --git a/Model/Import/Product/Type/Downloadable.php b/Model/Import/Product/Type/Downloadable.php new file mode 100755 index 0000000..a0ef5a8 --- /dev/null +++ b/Model/Import/Product/Type/Downloadable.php @@ -0,0 +1,94 @@ +connection->select(); + $select->from( + ['dl' => $this->_resource->getTableName('downloadable_link')], + [ + 'link_id', + 'product_id', + 'sort_order', + 'number_of_downloads', + 'is_shareable', + 'link_url', + 'link_file', + 'link_type', + 'sample_url', + 'sample_file', + 'sample_type' + ] + ); + $select->joinLeft( + ['dlp' => $this->_resource->getTableName('downloadable_link_price')], + 'dl.link_id = dlp.link_id AND dlp.website_id=' . self::DEFAULT_WEBSITE_ID, + ['price_id'] + ); + $select->where( + 'product_id in (?)', + $this->productIds + ); + $existingOptions = $this->connection->fetchAll($select); + foreach ($options as $option) { + $existOption = $this->downloadableHelper->fillExistOptions( + $this->dataLinkTitle, + $option, + $existingOptions + ); + if (!empty($existOption)) { + $result['title'][] = $existOption; + } + $existOption = $this->downloadableHelper->fillExistOptions( + $this->dataLinkPrice, + $option, + $existingOptions + ); + if (!empty($existOption)) { + $result['price'][] = $existOption; + } + } + return $result; + } + + /** + * Uploading files into the "downloadable/files" media folder. + * Return a new file name if the same file is already exists. + * + * @param string $fileName + * @param string $type + * @param bool $renameFileOff + * @return string + */ + protected function uploadDownloadableFiles($fileName, $type = 'links', $renameFileOff = false) + { + try { + $res = $this->uploaderHelper->getUploader( + $type, + $this->_entityModel->getParameters() + )->move($fileName, $renameFileOff); + return $res['file']; + } catch (\Exception $e) { + $this->_entityModel->addRowError( + $this->_messageTemplates[self::ERROR_MOVE_FILE] . '. ' . $e->getMessage(), + $this->rowNum + ); + return ''; + } + } +} diff --git a/Model/Source/Config.php b/Model/Source/Config.php index 3907323..5ed4e80 100755 --- a/Model/Source/Config.php +++ b/Model/Source/Config.php @@ -6,8 +6,17 @@ namespace Firebear\ImportExport\Model\Source; +/** + * Class Config + * @package Firebear\ImportExport\Model\Source + */ class Config extends \Magento\Framework\Config\Data implements \Firebear\ImportExport\Model\Source\ConfigInterface { + /** + * @param Config\Reader $reader + * @param \Magento\Framework\Config\CacheInterface $cache + * @param string $cacheId + */ public function __construct( \Firebear\ImportExport\Model\Source\Config\Reader $reader, \Magento\Framework\Config\CacheInterface $cache, @@ -16,6 +25,12 @@ public function __construct( parent::__construct($reader, $cache, $cacheId); } + /** + * Get system configuration of source type by name + * + * @param string $name + * @return array|mixed|null + */ public function getType($name) { return $this->get('type/' . $name); } diff --git a/Model/Source/Config/Converter.php b/Model/Source/Config/Converter.php index 5847933..bd5bc51 100755 --- a/Model/Source/Config/Converter.php +++ b/Model/Source/Config/Converter.php @@ -40,12 +40,13 @@ public function convert($source) continue; } - $result[$typeName]['fields'][$childNode->attributes->getNamedItem('name')->nodeValue] = array( + $result[$typeName]['fields'][$childNode->attributes->getNamedItem('name')->nodeValue] = [ 'id' => $childNode->attributes->getNamedItem('id')->nodeValue, 'label' => $childNode->attributes->getNamedItem('label')->nodeValue, 'type' => $childNode->attributes->getNamedItem('type')->nodeValue, - 'required' => ($childNode->attributes->getNamedItem('required')) ? $childNode->attributes->getNamedItem('required')->nodeValue : false, - ); + 'required' => ($childNode->attributes->getNamedItem('required')) + ? $childNode->attributes->getNamedItem('required')->nodeValue : false, + ]; } } diff --git a/Model/Source/Type/AbstractType.php b/Model/Source/Type/AbstractType.php index eae0422..64192ac 100755 --- a/Model/Source/Type/AbstractType.php +++ b/Model/Source/Type/AbstractType.php @@ -8,47 +8,106 @@ use Magento\Framework\App\Filesystem\DirectoryList; +/** + * Abstract class for import source types + * @package Firebear\ImportExport\Model\Source\Type + */ abstract class AbstractType extends \Magento\Framework\DataObject { - const IMPORT_DIR = 'var/import'; + /** + * Temp directory for downloaded files + */ + const IMPORT_DIR = 'import'; + /** + * Temp directory for downloaded images + */ const MEDIA_IMPORT_DIR = 'pub/media/import'; + /** + * Source type code + * @var string + */ protected $_code; + /** + * @var \Magento\Framework\App\Config\ScopeConfigInterface + */ protected $_scopeConfig; + /** + * @var \Magento\Framework\Filesystem\Directory\WriteInterface + */ protected $_directory; - protected $_client; - + /** + * @var \Magento\Framework\Filesystem + */ protected $_filesystem; + /** + * @var \Magento\Framework\Filesystem\File\ReadFactory + */ protected $_readFactory; + /** + * @var array + */ + protected $_metadata = []; + + protected $_client; + + /** + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + * @param \Magento\Framework\Filesystem $filesystem + * @param \Magento\Framework\Filesystem\File\ReadFactory $readFactory + */ public function __construct( \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig, \Magento\Framework\Filesystem $filesystem, \Magento\Framework\Filesystem\File\ReadFactory $readFactory - ){ + ) { $this->_scopeConfig = $scopeConfig; $this->_filesystem = $filesystem; $this->_directory = $filesystem->getDirectoryWrite(DirectoryList::ROOT); $this->_readFactory = $readFactory; } + /** + * Prepare temp dir for import files + * + * @return string + */ protected function getImportPath() { return self::IMPORT_DIR . '/' . $this->_code; } + /** + * @return string + */ + protected function getImportVarPath() + { + return DirectoryList::VAR_DIR . '/' . $this->getImportPath(); + } + + /** + * Prepare temp dir for import images + * + * @return string + */ protected function getMediaImportPath() { return self::MEDIA_IMPORT_DIR . '/' . $this->_code; } + /** + * Get file path + * + * @return bool|string + */ public function getImportFilePath() { - if($sourceType = $this->getImportSource()) { + if ($sourceType = $this->getImportSource()) { $filePath = $this->getData($sourceType . '_file_path'); return $filePath; } @@ -56,14 +115,47 @@ public function getImportFilePath() { return false; } + /** + * Get source type code + * + * @return string + */ public function getCode() { return $this->_code; } + /** + * @return mixed + */ + public function getClient() + { + return $this->_client; + } + + /** + * @param $client + */ + public function setClient($client) + { + $this->_client = $client; + } + + /** + * @return mixed + */ abstract function uploadSource(); + /** + * @param $importImage + * @param $imageSting + * + * @return mixed + */ abstract function importImage($importImage, $imageSting); + /** + * @return mixed + */ abstract protected function _getSourceClient(); } \ No newline at end of file diff --git a/Model/Source/Type/Dropbox.php b/Model/Source/Type/Dropbox.php index 7bb89a4..46b14b8 100755 --- a/Model/Source/Type/Dropbox.php +++ b/Model/Source/Type/Dropbox.php @@ -5,31 +5,54 @@ */ namespace Firebear\ImportExport\Model\Source\Type; +use Magento\Store\Model\ScopeInterface; + class Dropbox extends AbstractType { + /** + * @var string + */ protected $_code = 'dropbox'; + /** + * @var null + */ + protected $_accessToken = null; + + /** + * Download remote source file to temporary directory + * + * @return string + * @throws \Dropbox\Exception_BadResponseCode + * @throws \Dropbox\Exception_OverQuota + * @throws \Dropbox\Exception_RetryLater + * @throws \Dropbox\Exception_ServerError + * @throws \Magento\Framework\Exception\LocalizedException + */ public function uploadSource() { - if($client = $this->_getSourceClient()) { - //$filePath = '/var/www/local-magento2.com/magento2/var/import/dropbox/test-dropbox.csv'; + if ($client = $this->_getSourceClient()) { $sourceFilePath = $this->getData($this->_code . '_file_path'); $fileName = basename($sourceFilePath); - $filePath = $this->_directory->getAbsolutePath($this->getImportPath() . '/' . $fileName); + $filePath = $this->_directory->getAbsolutePath($this->getImportVarPath() . '/' . $fileName); try { $dirname = dirname($filePath); - if (!is_dir($dirname)) - { + if (!is_dir($dirname)) { mkdir($dirname, 0775, true); } - $f = fopen($filePath, 'w+b'); + $file = fopen($filePath, 'w+b'); } catch(\Exception $e) { - throw new \Magento\Framework\Exception\LocalizedException(__("Can't create local file /var/import/dropbox'. Please check files permissions.")); + throw new \Magento\Framework\Exception\LocalizedException( + __( + "Can't create local file /var/import/dropbox'. Please check files permissions. " + . $e->getMessage() + ) + ); } - $fileMetadata = $client->getFile($sourceFilePath, $f); - fclose($f); - if($fileMetadata) { + $fileMetadata = $client->getFile($sourceFilePath, $file); + fclose($file); + if ($fileMetadata) { return $this->_directory->getRelativePath($this->getImportPath() . '/' . $fileName); } else { throw new \Magento\Framework\Exception\LocalizedException(__("File not found on Dropbox")); @@ -39,33 +62,88 @@ public function uploadSource() } } + /** + * Download remote images to temporary media directory + * + * @param $importImage + * @param $imageSting + * + * @throws \Dropbox\Exception_BadResponseCode + * @throws \Dropbox\Exception_OverQuota + * @throws \Dropbox\Exception_RetryLater + * @throws \Dropbox\Exception_ServerError + */ public function importImage($importImage, $imageSting) { - if($client = $this->_getSourceClient()) { + if ($client = $this->_getSourceClient()) { $filePath = $this->_directory->getAbsolutePath($this->getMediaImportPath() . $imageSting); + $sourceFilePath = $this->getData($this->_code . '_file_path'); + $sourceDir = dirname($sourceFilePath); $dirname = dirname($filePath); if (!is_dir($dirname)) { mkdir($dirname, 0775, true); } - $f = fopen($filePath, 'w+b'); - $filePath = $this->getImportFilePath(); + $file = fopen($filePath, 'w+b'); - if($filePath) { - $dir = dirname($filePath); - $fileMetadata = $client->getFile($dir . '/' . $importImage, $f); + if ($filePath) { + $client->getFile($sourceDir . '/' . $importImage, $file); } - fclose($f); + fclose($file); } } + /** + * Get access token + * + * @return string|null + */ + public function getAccessToken() + { + if (!$this->_accessToken) { + + /** + * Data sent by cron job + * @see \Firebear\ImportExport\Plugin\Model\Import::uploadSource() + * + * else get token from admin config if import processed directly via admin panel + */ + if ($token = $this->getData('access_token')) { + $this->_accessToken = $token; + } else { + $this->_accessToken = $this->_scopeConfig->getValue( + 'firebear_importexport/dropbox/token', + ScopeInterface::SCOPE_STORE + ); + } + } + + return $this->_accessToken; + } + + /** + * Set access token + * + * @param $token + * + * @return Dropbox + */ + public function setAccessToken($token) + { + $this->_accessToken = $token; + + return $this; + } + + /** + * Prepare and return API client + * + * @return \Dropbox\Client + */ protected function _getSourceClient() { - if(!$this->_client) { - $accessToken = $this->_scopeConfig->getValue( - 'firebear_importexport/dropbox/token', - \Magento\Store\Model\ScopeInterface::SCOPE_STORE - ); - if($accessToken) { + if (!$this->_client) { + $accessToken = $this->getAccessToken(); + if ($accessToken) { $this->_client = new \Dropbox\Client($accessToken, "PHP-Example/1.0"); } } diff --git a/Model/Source/Type/Ftp.php b/Model/Source/Type/Ftp.php index 1c3d169..cd0c66c 100755 --- a/Model/Source/Type/Ftp.php +++ b/Model/Source/Type/Ftp.php @@ -8,23 +8,32 @@ class Ftp extends AbstractType { + /** + * @var string + */ protected $_code = 'ftp'; + /** + * Download remote source file to temporary directory + * + * @return string + * @throws \Magento\Framework\Exception\LocalizedException + */ public function uploadSource() { - if($client = $this->_getSourceClient()) { + if ($client = $this->_getSourceClient()) { $sourceFilePath = $this->getData($this->_code . '_file_path'); $fileName = basename($sourceFilePath); - $filePath = $this->_directory->getAbsolutePath($this->getImportPath() . '/' . $fileName); + $filePath = $this->_directory->getAbsolutePath($this->getImportVarPath() . '/' . $fileName); $filesystem = new \Magento\Framework\Filesystem\Io\File(); $filesystem->setAllowCreateFolders(true); - $filesystem->checkAndCreateFolder($this->_directory->getAbsolutePath($this->getImportPath())); + $filesystem->checkAndCreateFolder($this->_directory->getAbsolutePath($this->getImportVarPath())); $result = $client->read($sourceFilePath, $filePath); - if($result) { - return $this->_directory->getRelativePath($this->getImportPath() . '/' . $fileName); + if ($result) { + return $this->_directory->getAbsolutePath($this->getImportPath() . '/' . $fileName); } else { throw new \Magento\Framework\Exception\LocalizedException(__("File not found")); } @@ -33,9 +42,17 @@ public function uploadSource() } } + /** + * Download remote images to temporary media directory + * + * @param $importImage + * @param $imageSting + * + * @throws \Magento\Framework\Exception\LocalizedException + */ public function importImage($importImage, $imageSting) { - if($client = $this->_getSourceClient()) { + if ($client = $this->_getSourceClient()) { $sourceFilePath = $this->getData($this->_code . '_file_path'); $sourceDirName = dirname($sourceFilePath); $filePath = $this->_directory->getAbsolutePath($this->getMediaImportPath() . $imageSting); @@ -43,7 +60,7 @@ public function importImage($importImage, $imageSting) if (!is_dir($dirname)) { mkdir($dirname, 0775, true); } - if($filePath) { + if ($filePath) { $result = $client->read($sourceDirName . '/' . $importImage, $filePath); } } @@ -51,14 +68,24 @@ public function importImage($importImage, $imageSting) protected function _getSourceClient() { - if(!$this->_client) { - $settings = $this->_scopeConfig->getValue( - 'firebear_importexport/ftp', - \Magento\Store\Model\ScopeInterface::SCOPE_STORE - ); + if (!$this->getClient()) { + if ( + $this->getData('host') + && $this->getData('port') + && $this->getData('user') + && $this->getData('password') + ) { + $settings = $this->getData(); + } else { + $settings = $this->_scopeConfig->getValue( + 'firebear_importexport/ftp', + \Magento\Store\Model\ScopeInterface::SCOPE_STORE + ); + } + $settings['passive'] = true; try { - $connection = new \Magento\Framework\Filesystem\Io\Ftp(); + $connection = new \Firebear\ImportExport\Model\Filesystem\Io\Ftp(); $connection->open( $settings ); @@ -69,6 +96,6 @@ protected function _getSourceClient() } - return $this->_client; + return $this->getClient(); } } \ No newline at end of file diff --git a/Model/Source/Type/Url.php b/Model/Source/Type/Url.php index f32d532..fc029ea 100755 --- a/Model/Source/Type/Url.php +++ b/Model/Source/Type/Url.php @@ -10,33 +10,116 @@ class Url extends AbstractType { + /** + * @var string + */ protected $_code = 'url'; + /** + * @var string + */ + protected $_fileName; + + /** + * Download remote source file to temporary directory + * + * @return bool|string + */ public function uploadSource() { - $fileName = $this->getData($this->_code . '_file_path'); - if (preg_match('/\bhttps?:\/\//i', $fileName, $matches)) { - $url = str_replace($matches[0], '', $fileName); - $read = $this->_readFactory->create($url, DriverPool::HTTP); - $fileName = preg_replace('/[^a-z0-9\._-]+/i', '', $fileName); + if ($read = $this->_getSourceClient()) { + $fileName = preg_replace('/[^a-z0-9\._-]+/i', '', $this->_fileName); $this->_directory->writeFile( - $this->_directory->getRelativePath($this->getImportPath() . '/' . $fileName), + $this->_directory->getRelativePath($this->getImportVarPath() . '/' . $fileName), $read->readAll() ); return $this->_directory->getRelativePath($this->getImportPath() . '/' . $fileName); - } else { - throw new \Magento\Framework\Exception\LocalizedException(__("Please, provide correct URL")); } + + return false; } + /** + * Download remote images to temporary media directory + * + * @param $importImage + * @param $imageSting + * @return bool + */ public function importImage($importImage, $imageSting) { + $filePath = $this->_directory->getAbsolutePath($this->getMediaImportPath() . $imageSting); + $dirname = dirname($filePath); + if (!is_dir($dirname)) { + mkdir($dirname, 0775, true); + } + + if (preg_match('/\bhttps?:\/\//i', $importImage, $matches)) { + $url = str_replace($matches[0], '', $importImage); + } else { + $sourceFilePath = $this->getData($this->_code . '_file_path'); + $sourceDir = dirname($sourceFilePath); + $url = $sourceDir . '/' . $importImage; + if (preg_match('/\bhttps?:\/\//i', $url, $matches)) { + $url = str_replace($matches[0], '', $url); + } + } + + if ($url) { + $read = $this->_readFactory->create($url, DriverPool::HTTP); + $this->_directory->writeFile( + $this->_directory->getRelativePath($filePath), + $read->readAll() + ); + } + + return true; + } + + /** + * Check if remote file was modified since the last import + * + * @param int $timestamp + * @return bool|int + */ + public function checkModified($timestamp) + { + $fileName = $this->getData($this->_code . '_file_path'); + if (preg_match('/\bhttps?:\/\//i', $fileName, $matches)) { + $url = str_replace($matches[0], '', $fileName); + $read = $this->_readFactory->create($url, DriverPool::HTTP); + + if (!$this->_metadata) { + $this->_metadata = $read->stat(); + } + + $modified = strtotime($this->_metadata['mtime']); + + return ($timestamp != $modified) ? $modified : false; + } + return false; } + /** + * Prepare and return Driver client + * + * @return \Magento\Framework\Filesystem\File\ReadInterface + */ protected function _getSourceClient() { + if (!$this->_fileName) { + $this->_fileName = $this->getData($this->_code . '_file_path'); + } + + if (!$this->_client) { + if (preg_match('/\bhttps?:\/\//i', $this->_fileName, $matches)) { + $url = str_replace($matches[0], '', $this->_fileName); + $this->_client = $this->_readFactory->create($url, DriverPool::HTTP); + } + } + return $this->_client; } } \ No newline at end of file diff --git a/Plugin/Block/Import/Form.php b/Plugin/Block/Import/Form.php index 54b0b2f..542575d 100755 --- a/Plugin/Block/Import/Form.php +++ b/Plugin/Block/Import/Form.php @@ -6,21 +6,41 @@ namespace Firebear\ImportExport\Plugin\Block\Import; +/** + * Class Form + * @package Firebear\ImportExport\Plugin\Block\Import + */ class Form { + /** + * @var \Firebear\ImportExport\Model\Source\ConfigInterface|null + */ protected $_config = null; + /** + * @var \Magento\Framework\App\Config\ScopeConfigInterface + */ protected $_scopeConfig; + /** + * @param \Firebear\ImportExport\Model\Source\ConfigInterface $config + * @param \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig + */ public function __construct( \Firebear\ImportExport\Model\Source\ConfigInterface $config, \Magento\Framework\App\Config\ScopeConfigInterface $scopeConfig - ) - { + ) { $this->_config = $config; $this->_scopeConfig = $scopeConfig; } + /** + * Add import source fieldset to default import form + * + * @param \Magento\ImportExport\Block\Adminhtml\Import\Edit\Form $subject + * @param $form + * @return array + */ public function beforeSetForm(\Magento\ImportExport\Block\Adminhtml\Import\Edit\Form $subject, $form) { $fileFieldset = $form->getElement('upload_file_fieldset'); @@ -65,6 +85,10 @@ public function beforeSetForm(\Magento\ImportExport\Block\Adminhtml\Import\Edit\ foreach ($type['fields'] as $fieldName => $field) { + if ($fieldName != 'file_path') { + continue; + } + $fieldsets[$typeName]->addField( $typeName . '_' . $fieldName, $field['type'], diff --git a/Plugin/Model/Import.php b/Plugin/Model/Import.php index 8d61443..017ae0c 100755 --- a/Plugin/Model/Import.php +++ b/Plugin/Model/Import.php @@ -6,8 +6,20 @@ namespace Firebear\ImportExport\Plugin\Model; +use Magento\ImportExport\Model\Import\AbstractSource; +use Magento\ImportExport\Model\Import\Entity\AbstractEntity; +use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingError; +use Magento\Framework\App\Filesystem\DirectoryList; + class Import extends \Magento\ImportExport\Model\Import { + /** + * Limit displayed errors on Import History page. + */ + const LIMIT_VISIBLE_ERRORS = 5; + + const CREATE_ATTRIBUTES_CONF_PATH = 'firebear_importexport/general/create_attributes'; + /** * @var \Firebear\ImportExport\Model\Source\ConfigInterface */ @@ -19,17 +31,34 @@ class Import extends \Magento\ImportExport\Model\Import { protected $_helper; /** - * @param \Firebear\ImportExport\Model\Source\ConfigInterface $config, - * @param \Firebear\ImportExport\Helper\Data $helper, - * @param \Psr\Log\LoggerInterface $logger - * @param \Magento\Framework\Filesystem $filesystem - * @param \Magento\ImportExport\Helper\Data $importExportData - * @param \Magento\Framework\App\Config\ScopeConfigInterface $coreConfig - * @param \Magento\ImportExport\Model\Import\ConfigInterface $importConfig - * @param \Magento\ImportExport\Model\Import\Entity\Factory $entityFactory - * @param \Magento\ImportExport\Model\Export\Adapter\CsvFactory $csvFactory - * @param \Magento\Framework\HTTP\Adapter\FileTransferFactory $httpFactory - * @param \Magento\MediaStorage\Model\File\UploaderFactory $uploaderFactory + * @var \Magento\Framework\Stdlib\DateTime\Timezone + */ + protected $_timezone; + + /** + * @var \Firebear\ImportExport\Model\Source\Type\AbstractType + */ + protected $_source; + + /** + * @var \Magento\Framework\Filesystem\Directory\WriteInterface + */ + protected $directory; + + /** + * @param \Firebear\ImportExport\Model\Source\ConfigInterface $config + * @param \Firebear\ImportExport\Helper\Data $helper + * @param \Magento\Framework\Stdlib\DateTime\TimezoneInterface $timezone + * @param \Psr\Log\LoggerInterface $logger + * @param \Magento\Framework\Filesystem $filesystem + * @param \Magento\ImportExport\Helper\Data $importExportData + * @param \Magento\Framework\App\Config\ScopeConfigInterface $coreConfig + * @param \Magento\ImportExport\Model\Import\ConfigInterface $importConfig + * @param \Magento\ImportExport\Model\Import\Entity\Factory $entityFactory + * @param \Magento\ImportExport\Model\ResourceModel\Import\Data $importData + * @param \Magento\ImportExport\Model\Export\Adapter\CsvFactory $csvFactory + * @param \Magento\Framework\HTTP\Adapter\FileTransferFactory $httpFactory + * @param \Magento\MediaStorage\Model\File\UploaderFactory $uploaderFactory * @param \Magento\ImportExport\Model\Source\Import\Behavior\Factory $behaviorFactory * @param \Magento\Indexer\Model\IndexerRegistry $indexerRegistry * @param \Magento\ImportExport\Model\History $importHistoryModel @@ -40,6 +69,7 @@ class Import extends \Magento\ImportExport\Model\Import { public function __construct( \Firebear\ImportExport\Model\Source\ConfigInterface $config, \Firebear\ImportExport\Helper\Data $helper, + \Magento\Framework\Stdlib\DateTime\TimezoneInterface $timezone, \Psr\Log\LoggerInterface $logger, \Magento\Framework\Filesystem $filesystem, \Magento\ImportExport\Helper\Data $importExportData, @@ -58,6 +88,8 @@ public function __construct( ) { $this->_config = $config; $this->_helper = $helper; + $this->_timezone = $timezone; + $this->directory = $filesystem->getDirectoryWrite(DirectoryList::ROOT); parent::__construct( $logger, @@ -78,19 +110,27 @@ public function __construct( ); } + /** + * Prepare source type class name + * + * @param $sourceType + * @return string + */ protected function _prepareSourceClassName($sourceType) { return 'Firebear\ImportExport\Model\Source\Type\\' . ucfirst(strtolower($sourceType)); } + /** + * @return mixed|null|string + * @throws \Magento\Framework\Exception\LocalizedException + */ public function uploadSource() { $result = null; - if($this->getImportSource() && $this->getImportSource() != 'file') { - $sourceType = $this->getImportSource(); - $source = $this->_helper->getSourceModelByType($sourceType); - $source->setData($this->getData()); + if ($this->getImportSource() && $this->getImportSource() != 'file') { + $source = $this->getSource(); try { $result = $source->uploadSource(); @@ -99,10 +139,106 @@ public function uploadSource() } } - if($result) { - return $result; + if ($result) { + $sourceFileRelative = $this->directory->getRelativePath($result); + $entity = $this->getEntity(); + $this->createHistoryReport($sourceFileRelative, $entity); + return DirectoryList::VAR_DIR . '/' . $result; } return parent::uploadSource(); } + + /** + * Validates source file and returns validation result. + * + * @param AbstractSource $source + * @return bool + */ + public function validateSource(AbstractSource $source) + { + $this->addLogComment(__('Begin data validation')); + try { + $adapter = $this->_getEntityAdapter()->setSource($source); + $errorAggregator = $adapter->validateData(); + } catch (\Exception $e) { + $errorAggregator = $this->getErrorAggregator(); + $errorAggregator->addError( + AbstractEntity::ERROR_CODE_SYSTEM_EXCEPTION + . '. ' . $e->getMessage(), + ProcessingError::ERROR_LEVEL_CRITICAL, + null, + null, + null, + $e->getMessage() + ); + } + + $messages = $this->getOperationResultMessages($errorAggregator); + $this->addLogComment($messages); + + $result = !$errorAggregator->getErrorsCount(); + if ($result) { + $this->addLogComment(__('Import data validation is complete.')); + } else { + if ($this->isReportEntityType()) { + $this->importHistoryModel->load($this->importHistoryModel->getLastItemId()); + $summary = ''; + if ($errorAggregator->getErrorsCount() > self::LIMIT_VISIBLE_ERRORS) { + $summary = __('Too many errors. Please check your debug log file.') . '
'; + } else { + if ($this->getJobId()) { + $summary = __('Import job #' . $this->getJobId() . ' failed.') . '
'; + } + + foreach ($errorAggregator->getRowsGroupedByErrorCode() as $errorMessage => $rows) { + $error = $errorMessage . ' ' . __('in rows') . ': ' . implode(', ', $rows); + $summary .= $error . '
'; + } + } + $date = $this->_timezone->formatDateTime( + new \DateTime(), + \IntlDateFormatter::MEDIUM, + \IntlDateFormatter::MEDIUM, + null, + null + ); + $summary .= '' . $date . ''; + $this->importHistoryModel->setSummary($summary); + $this->importHistoryModel->setExecutionTime(\Magento\ImportExport\Model\History::IMPORT_FAILED); + $this->importHistoryModel->save(); + } + } + return $result; + } + + /** + * Get import source by type. + * + * @return \Firebear\ImportExport\Model\Source\Type\AbstractType + */ + public function getSource() + { + if (!$this->_source) { + $sourceType = $this->getImportSource(); + try { + $this->_source = $this->_helper->getSourceModelByType($sourceType); + $this->_source->setData($this->getData()); + } catch(\Exception $e) { + + } + } + + return $this->_source; + } + + /** + * Get import history model + * + * @return mixed + */ + public function getImportHistoryModel() + { + return $this->importHistoryModel; + } } \ No newline at end of file diff --git a/Plugin/Model/Import/Product.php b/Plugin/Model/Import/Product.php index 36f31f9..d276149 100755 --- a/Plugin/Model/Import/Product.php +++ b/Plugin/Model/Import/Product.php @@ -1,6 +1,7 @@ _request = $request; $this->_helper = $helper; + $this->attributeFactory = $attributeFactory; + $this->eavEntityFactory = $eavEntityFactory; + $this->groupCollectionFactory = $groupCollectionFactory; + $this->productHelper = $productHelper; parent::__construct( $jsonHelper, @@ -103,9 +207,15 @@ public function __construct( ); } + /** + * Initialize source type model + * + * @param $type + * @throws \Magento\Framework\Exception\LocalizedException + */ protected function _initSourceType($type) { - if(!$this->_sourceType) { + if (!$this->_sourceType) { $this->_sourceType = $this->_helper->getSourceModelByType($type); $this->_sourceType->setData($this->_parameters); } @@ -136,21 +246,23 @@ protected function _saveProducts() $this->websitesCache = []; $this->categoriesCache = []; $tierPrices = []; - $groupPrices = []; $mediaGallery = []; + $uploadedImages = []; $previousType = null; $prevAttributeSet = null; if ($this->_sourceType) { $bunch = $this->_prepareImagesFromSource($bunch); } - $bunchImages = $this->getBunchImages($bunch); - $existingImages = $this->getExistingImages($bunchImages); foreach ($bunch as $rowNum => $rowData) { if (!$this->validateRow($rowData, $rowNum)) { continue; } + if ($this->getErrorAggregator()->hasToBeTerminated()) { + $this->getErrorAggregator()->addRowToSkip($rowNum); + continue; + } $rowScope = $this->getRowScope($rowData); $rowSku = $rowData[self::COL_SKU]; @@ -209,10 +321,12 @@ protected function _saveProducts() if (!array_key_exists($rowSku, $this->categoriesCache)) { $this->categoriesCache[$rowSku] = []; } + $rowData['rowNum'] = $rowNum; $categoryIds = $this->processRowCategories($rowData); foreach ($categoryIds as $id) { $this->categoriesCache[$rowSku][$id] = true; } + unset($rowData['rowNum']); // 4.1. Tier prices phase if (!empty($rowData['_tier_price_website'])) { @@ -231,22 +345,6 @@ protected function _saveProducts() continue; } - // 4.2. Group prices phase - if (!empty($rowData['_group_price_website'])) { - $groupPrices[$rowSku][] = [ - 'all_groups' => $rowData['_group_price_customer_group'] == self::VALUE_ALL, - 'customer_group_id' => $rowData['_group_price_customer_group'] == - self::VALUE_ALL ? 0 : $rowData['_group_price_customer_group'], - 'value' => $rowData['_group_price_price'], - 'website_id' => self::VALUE_ALL == $rowData['_group_price_website'] || - $priceIsGlobal ? 0 : $this->storeResolver->getWebsiteCodeToId($rowData['_group_price_website']), - ]; - } - - if (!$this->validateRow($rowData, $rowNum)) { - continue; - } - // 5. Media gallery phase $disabledImages = []; list($rowImages, $rowLabels) = $this->getImagesFromRow($rowData); @@ -279,8 +377,7 @@ protected function _saveProducts() $rowData[$column] = $uploadedFile; } - $imageNotAssigned = !isset($existingImages[$uploadedFile]) - || !in_array($rowSku, $existingImages[$uploadedFile]); + $imageNotAssigned = !isset($existingImages[$rowSku][$uploadedFile]); if ($uploadedFile && $imageNotAssigned) { if ($column == self::COL_MEDIA_IMAGE) { @@ -293,7 +390,7 @@ protected function _saveProducts() 'disabled' => isset($disabledImages[$columnImage]) ? '1' : '0', 'value' => $uploadedFile, ]; - $existingImages[$uploadedFile][] = $rowSku; + $existingImages[$rowSku][$uploadedFile] = true; } } } @@ -303,7 +400,7 @@ protected function _saveProducts() ? $this->storeResolver->getStoreCodeToId($rowData[self::COL_STORE]) : 0; $productType = isset($rowData[self::COL_TYPE]) ? $rowData[self::COL_TYPE] : null; - if (!is_null($productType)) { + if ($productType !== null) { $previousType = $productType; } if (isset($rowData[self::COL_ATTR_SET])) { @@ -311,13 +408,13 @@ protected function _saveProducts() } if (self::SCOPE_NULL == $rowScope) { // for multiselect attributes only - if (!is_null($prevAttributeSet)) { + if ($prevAttributeSet !== null) { $rowData[self::COL_ATTR_SET] = $prevAttributeSet; } - if (is_null($productType) && !is_null($previousType)) { + if ($productType === null && $previousType !== null) { $productType = $previousType; } - if (is_null($productType)) { + if ($productType === null) { continue; } } @@ -384,10 +481,18 @@ protected function _saveProducts() } } - $this->_saveProductEntity( - $entityRowsIn, - $entityRowsUp - )->_saveProductWebsites( + if (method_exists($this, '_saveProductEntity')) { + $this->_saveProductEntity( + $entityRowsIn, + $entityRowsUp + ); + } else { + $this->saveProductEntity( + $entityRowsIn, + $entityRowsUp + ); + } + $this->_saveProductWebsites( $this->websitesCache )->_saveProductCategories( $this->categoriesCache @@ -407,6 +512,82 @@ protected function _saveProducts() return $this; } + /** + * Stock item saving. + * + * @return $this + */ + protected function _saveStockItem() + { + $indexer = $this->indexerRegistry->get('catalog_product_category'); + /** @var $stockResource \Magento\CatalogInventory\Model\ResourceModel\Stock\Item */ + $stockResource = $this->_stockResItemFac->create(); + $entityTable = $stockResource->getMainTable(); + while ($bunch = $this->_dataSourceModel->getNextBunch()) { + $stockData = []; + $productIdsToReindex = []; + // Format bunch to stock data rows + foreach ($bunch as $rowNum => $rowData) { + if (!$this->isRowAllowedToImport($rowData, $rowNum)) { + continue; + } + + $row = []; + $row['product_id'] = $this->skuProcessor->getNewSku($rowData[self::COL_SKU])['entity_id']; + $productIdsToReindex[] = $row['product_id']; + + $row['website_id'] = $this->stockConfiguration->getDefaultScopeId(); + $row['stock_id'] = $this->stockRegistry->getStock($row['website_id'])->getStockId(); + + $stockItemDo = $this->stockRegistry->getStockItem($row['product_id'], $row['website_id']); + $existStockData = $stockItemDo->getData(); + + $row = array_merge( + $this->defaultStockData, + array_intersect_key($existStockData, $this->defaultStockData), + array_intersect_key($rowData, $this->defaultStockData), + $row + ); + + if ($this->stockConfiguration->isQty( + $this->skuProcessor->getNewSku($rowData[self::COL_SKU])['type_id'] + )) { + $stockItemDo->setData($row); + $row['is_in_stock'] = $this->stockStateProvider->verifyStock($stockItemDo); + if ($this->stockStateProvider->verifyNotification($stockItemDo)) { + $row['low_stock_date'] = $this->dateTime->gmDate( + 'Y-m-d H:i:s', + (new \DateTime())->getTimestamp() + ); + } + $row['stock_status_changed_auto'] = + (int) !$this->stockStateProvider->verifyStock($stockItemDo); + } else { + $row['qty'] = 0; + } + if (!isset($stockData[$rowData[self::COL_SKU]])) { + $stockData[$rowData[self::COL_SKU]] = $row; + } + } + + // Insert rows + if (!empty($stockData)) { + $this->_connection->insertOnDuplicate($entityTable, array_values($stockData)); + } + + if ($productIdsToReindex) { + $indexer->reindexList($productIdsToReindex); + } + } + return $this; + } + + /** + * Import images via initialized source type + * + * @param $bunch + * @return mixed + */ protected function _prepareImagesFromSource($bunch) { foreach ($bunch as &$rowData) { @@ -423,7 +604,7 @@ protected function _prepareImagesFromSource($bunch) $dispersionPath . '/' . preg_replace('/[^a-z0-9\._-]+/i', '', $importImage) ); - if($this->_sourceType) { + if ($this->_sourceType) { $this->_sourceType->importImage($importImage, $imageSting); } $rowData[$image] = $this->_sourceType->getCode() . $imageSting; @@ -476,7 +657,7 @@ protected function getBunchImages($bunch) private function _customFieldsMapping($rowData) { foreach ($this->_fieldsMap as $systemFieldName => $fileFieldName) { - if (isset($rowData[$fileFieldName])) { + if (array_key_exists($fileFieldName, $rowData)) { $rowData[$systemFieldName] = $rowData[$fileFieldName]; } } @@ -484,10 +665,12 @@ private function _customFieldsMapping($rowData) $rowData = $this->_parseAdditionalAttributes($rowData); $rowData = $this->_setStockUseConfigFieldsValues($rowData); - if (isset($rowData['status'])) { - if (($rowData['status'] == \Magento\Catalog\Model\Product\Attribute\Source\Status::STATUS_ENABLED) || $rowData['status'] == 'yes') { + if (array_key_exists('status', $rowData) + && $rowData['status'] != \Magento\Catalog\Model\Product\Attribute\Source\Status::STATUS_ENABLED + ) { + if ($rowData['status'] == 'yes') { $rowData['status'] = \Magento\Catalog\Model\Product\Attribute\Source\Status::STATUS_ENABLED; - } else { + } elseif (!empty($rowData['status']) || $this->getRowScope($rowData) == self::SCOPE_DEFAULT) { $rowData['status'] = \Magento\Catalog\Model\Product\Attribute\Source\Status::STATUS_DISABLED; } } @@ -507,11 +690,16 @@ private function _parseAdditionalAttributes($rowData) return $rowData; } - $attributeNameValuePairs = explode($this->getMultipleValueSeparator(), $rowData['additional_attributes']); - foreach ($attributeNameValuePairs as $attributeNameValuePair) { - $nameAndValue = explode(self::PAIR_NAME_VALUE_SEPARATOR, $attributeNameValuePair); - if (!empty($nameAndValue)) { - $rowData[$nameAndValue[0]] = isset($nameAndValue[1]) ? $nameAndValue[1] : ''; + $valuePairs = explode($this->getMultipleValueSeparator(), $rowData['additional_attributes']); + foreach ($valuePairs as $valuePair) { + $separatorPosition = strpos($valuePair, self::PAIR_NAME_VALUE_SEPARATOR); + if ($separatorPosition !== false) { + $key = substr($valuePair, 0, $separatorPosition); + $value = substr( + $valuePair, + $separatorPosition + strlen(self::PAIR_NAME_VALUE_SEPARATOR) + ); + $rowData[$key] = $value === false ? '' : $value; } } return $rowData; @@ -526,10 +714,15 @@ private function _parseAdditionalAttributes($rowData) */ private function _setStockUseConfigFieldsValues($rowData) { - $useConfigFields = array(); + $useConfigFields = []; foreach ($rowData as $key => $value) { - if (isset($this->defaultStockData[$key]) && isset($this->defaultStockData[self::INVENTORY_USE_CONFIG_PREFIX . $key]) && !empty($value)) { - $useConfigFields[self::INVENTORY_USE_CONFIG_PREFIX . $key] = ($value == self::INVENTORY_USE_CONFIG) ? 1 : 0; + if ( + isset($this->defaultStockData[$key]) + && isset($this->defaultStockData[self::INVENTORY_USE_CONFIG_PREFIX . $key]) + && !empty($value) + ) { + $fullKey = self::INVENTORY_USE_CONFIG_PREFIX . $key; + $useConfigFields[$fullKey] = ($value == self::INVENTORY_USE_CONFIG) ? 1 : 0; } } $rowData = array_merge($rowData, $useConfigFields); diff --git a/Plugin/Model/Import/Product/Validator.php b/Plugin/Model/Import/Product/Validator.php new file mode 100755 index 0000000..770827c --- /dev/null +++ b/Plugin/Model/Import/Product/Validator.php @@ -0,0 +1,139 @@ +scopeConfig = $scopeConfig; + $this->prodAttrFac = $prodAttrFac; + } + + /** + * Rewrite method which allow create attributes & values on the fly + * + * @param string $attrCode + * @param array $attrParams + * @param array $rowData + * @return bool + * @SuppressWarnings(PHPMD.CyclomaticComplexity) + * @SuppressWarnings(PHPMD.NPathComplexity) + */ + public function isAttributeValid($attrCode, array $attrParams, array $rowData) + { + $this->_rowData = $rowData; + if (isset($rowData['product_type']) && !empty($attrParams['apply_to']) + && !in_array($rowData['product_type'], $attrParams['apply_to']) + ) { + return true; + } + + if (!$this->isRequiredAttributeValid($attrCode, $attrParams, $rowData)) { + $valid = false; + $this->_addMessages( + [ + sprintf( + $this->context->retrieveMessageTemplate( + RowValidatorInterface::ERROR_VALUE_IS_REQUIRED + ), + $attrCode + ) + ] + ); + return $valid; + } + + if (!strlen(trim($rowData[$attrCode]))) { + return true; + } + switch ($attrParams['type']) { + case 'varchar': + case 'text': + $valid = $this->textValidation($attrCode, $attrParams['type']); + break; + case 'decimal': + case 'int': + $valid = $this->numericValidation($attrCode, $attrParams['type']); + break; + case 'select': + case 'boolean': + case 'multiselect': + $createValuesAllowed = (bool) $this->scopeConfig->getValue( + \Firebear\ImportExport\Plugin\Model\Import::CREATE_ATTRIBUTES_CONF_PATH, + \Magento\Store\Model\ScopeInterface::SCOPE_STORE + ); + $attribute = $this->prodAttrFac->create(); + $attribute->load($attrParams['id']); + $values = explode(Product::PSEUDO_MULTI_LINE_SEPARATOR, $rowData[$attrCode]); + $valid = true; + foreach ($values as $value) { + if ($createValuesAllowed && $attribute->getIsUserDefined()) { + $valid = $valid && ($this->string->strlen($value) < Product::DB_MAX_VARCHAR_LENGTH); + } else { + $valid = $valid && isset($attrParams['options'][strtolower($value)]); + } + } + if (!$valid) { + $this->_addMessages( + [ + sprintf( + $this->context->retrieveMessageTemplate( + RowValidatorInterface::ERROR_INVALID_ATTRIBUTE_OPTION + ), + $attrCode + ) + ] + ); + } + break; + case 'datetime': + $val = trim($rowData[$attrCode]); + $valid = strtotime($val) !== false; + if (!$valid) { + $this->_addMessages([RowValidatorInterface::ERROR_INVALID_ATTRIBUTE_TYPE]); + } + break; + default: + $valid = true; + break; + } + + if ($valid && !empty($attrParams['is_unique'])) { + if (isset($this->_uniqueAttributes[$attrCode][$rowData[$attrCode]]) + && ($this->_uniqueAttributes[$attrCode][$rowData[$attrCode]] != $rowData[Product::COL_SKU])) { + $this->_addMessages([RowValidatorInterface::ERROR_DUPLICATE_UNIQUE_ATTRIBUTE]); + return false; + } + $this->_uniqueAttributes[$attrCode][$rowData[$attrCode]] = $rowData[Product::COL_SKU]; + } + return (bool)$valid; + + } +} diff --git a/composer.json b/composer.json index 86813c5..8b4bec6 100755 --- a/composer.json +++ b/composer.json @@ -7,7 +7,7 @@ "dropbox/dropbox-sdk": "1.1.*" }, "type": "magento2-module", - "version": "1.0.2", + "version": "1.0.3", "license": [ "GPL-2.0" ], @@ -19,4 +19,4 @@ "Firebear\\ImportExport\\": "" } } -} +} \ No newline at end of file diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml index c1a8d41..964760a 100755 --- a/etc/adminhtml/system.xml +++ b/etc/adminhtml/system.xml @@ -17,16 +17,9 @@ Firebear_ImportExport::config_importexport - - - - - - App Console]]> - - + App Console. Don’t share your access token with anyone!]]> diff --git a/etc/di.xml b/etc/di.xml index 1989db7..2f622b1 100755 --- a/etc/di.xml +++ b/etc/di.xml @@ -6,4 +6,12 @@ --> + + + + + + + + \ No newline at end of file diff --git a/etc/module.xml b/etc/module.xml index d13f7d3..3b32384 100755 --- a/etc/module.xml +++ b/etc/module.xml @@ -5,5 +5,5 @@ */ --> - + diff --git a/view/base/web/js/grid/columns/status.js b/view/base/web/js/grid/columns/status.js new file mode 100755 index 0000000..a20bdd1 --- /dev/null +++ b/view/base/web/js/grid/columns/status.js @@ -0,0 +1,54 @@ +/** + * Copyright © 2015 Magento. All rights reserved. + * See COPYING.txt for license details. + */ +define([ + 'underscore', + 'Magento_Ui/js/grid/columns/column' +], function (_, Column) { + 'use strict'; + + return Column.extend({ + + defaults: { + bodyTmpl: 'Firebear_ImportExport/grid/cells/status', + fieldClass: { + 'data-grid-thumbnail-cell': true + } + }, + + getIsActive: function (record) { + return (record[this.index] == 1); + }, + + /** + * Retrieves label associated with a provided value. + * + * @returns {String} + */ + getLabel: function () { + + var options = this.options || [], + values = this._super(), + label = []; + + if (!Array.isArray(values)) { + values = [values]; + } + + values = values.map(function (value) { + return value + ''; + }); + + options.forEach(function (item) { + if (_.contains(values, item.value + '')) { + label.push(item.label); + } + }); + + return label.join(', '); + } + + /*eslint-enable eqeqeq*/ + }); +}); diff --git a/view/base/web/template/grid/cells/status.html b/view/base/web/template/grid/cells/status.html new file mode 100755 index 0000000..2763869 --- /dev/null +++ b/view/base/web/template/grid/cells/status.html @@ -0,0 +1,18 @@ + +
+ + + + + + + + + + +
\ No newline at end of file