feat: Code Refactoring

Use GX Services where possible Use dynamic limitation for export Batches Refactored Entity Batching Improved Logging
MakairaIO · Nov 25, 2024 · 763b7b5 · 763b7b5
1 parent 48ddd70
commit 763b7b5
Show file tree

Hide file tree

Showing 16 changed files with 317 additions and 479 deletions.
diff --git a/.idea/gambio-connect.iml b/.idea/gambio-connect.iml
diff --git a/.idea/php.xml b/.idea/php.xml
diff --git a/src/GXModules/MakairaIO/MakairaConnect/Admin/CronJobs/MakairaConnectCronjobTask.php b/src/GXModules/MakairaIO/MakairaConnect/Admin/CronJobs/MakairaConnectCronjobTask.php
@@ -19,6 +19,12 @@ class MakairaConnectCronjobTask extends AbstractCronjobTask
 
     protected Connection $connection;
 
+    private const CHANGE_TYPES = [
+        'manufacturer',
+        'category',
+        'product'
+    ];
+
     public function getCallback($cronjobStartAsMicrotime): \Closure
     {
         $dependencies = $this->dependencies->getDependencies();
@@ -41,6 +47,8 @@ public function getCallback($cronjobStartAsMicrotime): \Closure
             $this->gambioConnectPublicFieldsService = $gambioConnectService->getGambioConnectPublicFieldsService();
 
             return function () use ($gambioConnectService): void {
+                $this->logInfo('MakairaConnect Cronjob Started');
+
                 if (! $this->checkImporterSetup()) {
                     $this->logInfo('Importer was not created yet - creating it now');
                     $this->gambioConnectImporterConfigService->setUpImporter();
@@ -58,62 +66,66 @@ public function getCallback($cronjobStartAsMicrotime): \Closure
                     ->execute()
                     ->fetchAll(FetchMode::ASSOCIATIVE);
 
-                $changes = $this->connection->createQueryBuilder()
-                    ->select('id')
-                    ->from(\GXModules\MakairaIO\MakairaConnect\App\ChangesService::TABLE_NAME)
-                    ->execute()
-                    ->fetchAll(FetchMode::ASSOCIATIVE);
+                $limit = $this->moduleConfigService->getBatchSize();
 
-                $limit = 500;
+                $query = $this->connection->createQueryBuilder()
+                    ->select('gambio_id')
+                    ->from(\GXModules\MakairaIO\MakairaConnect\App\ChangesService::TABLE_NAME)
+                    ->setMaxResults($limit);
 
-                $currentPosition = 0;
+                $changes = [];
 
-                $this->logInfo('MakairaConnect Cronjob Started');
+                foreach(self::CHANGE_TYPES as $changeType) {
+                    $changes[$changeType] = $query->where('type = :type')->setParameter('type', $changeType)->execute()->fetchAll(FetchMode::ASSOCIATIVE);
+                    $this->logInfo("Loaded " . $changeType . " from changes table: " . count($changes[$changeType]) . " changes.");
+                }
 
-                $this->logInfo('Check count of Changes for Makaira');
+                $deleteIds = [];
 
-                if(count($changes) > 1000) {
-                    foreach($languages as $index => $language) {
-                        $lastLanguage = $index === (count($languages) - 1);
-                        $this->logInfo('Begin Export of '.$limit.' Datasets for Language ' . $language['code']);
+                foreach(self::CHANGE_TYPES as $changeType) {
+                    foreach($languages as $language) {
+                        $this->logInfo('Begin Export of '.$changeType . ': ' . $limit.' Datasets for Language ' . $language['code']);
                         $client = new \GuzzleHttp\Client([
                             'base_uri' => $host . $language['code'],
                         ]);
                         try {
-                            $client->get(
-                                'shop.php?do=MakairaCronService/doExport&language=' . $language['code'] . '&start=' . $currentPosition . '&limit=' . $limit . '&complete=' . $lastLanguage
+                            $client->post(
+                                'shop.php?do=MakairaCronService/doExport&language=' . $language['code'],
+                                [
+                                    'json' => [
+                                        'type' => $changeType,
+                                        'changes' => $changes[$changeType],
+                                    ]
+                                ]
                             );
+                            $deleteIds = array_merge($deleteIds, $changes[$changeType]);
                         } catch (Exception $exception) {
-                            $this->logInfo('Error in Export for Language ' . $language['code']);
+                            $this->logInfo('Error in Export of ' . $changeType . ' for Language ' . $language['code']);
                             $this->logError($exception->getMessage());
+                            $errors = true;
+                            $newBatchSize = $limit / 2;
+                            $this->logInfo("Set Batch Limit to $newBatchSize");
+                            $this->moduleConfigService->setBatchSize($newBatchSize);
                         }
-                        $this->logInfo('End Export of '.$limit.' Datasets for Language ' . $language['code']);
+                        $this->logInfo('End Export of '. $changeType . ': ' . $limit.' Datasets for Language ' . $language['code']);
                     }
-                } else {
-                    do {
-                        foreach ($languages as $index => $language) {
-                            $lastLanguage = $index === (count($languages) - 1);
-                            $this->logInfo('Begin Export for Language ' . $language['code']);
-                            $client = new \GuzzleHttp\Client([
-                                'base_uri' => $host . $language['code'],
-                            ]);
-                            try {
-                                $client->get(
-                                    'shop.php?do=MakairaCronService/doExport&language=' . $language['code'] . '&start=' . $currentPosition . '&limit=' . $limit . '&complete=' . $lastLanguage
-                                );
-                            } catch (Exception $exception) {
-                                $this->logInfo('Error in Export for Language ' . $language['code']);
-                                $this->logError($exception->getMessage());
-                            }
-                            $this->logInfo('End Export for Language ' . $language['code']);
-
-                            if ($lastLanguage) {
-                                $currentPosition += $limit;
-                            }
-                        }
-                    } while ($currentPosition >= $changes);
                 }
 
+                if($errors === false) {
+                    $newBatchSize = $limit + 1;
+                    $this->logInfo("Set Batch Limit to $newBatchSize");
+                    $this->moduleConfigService->setBatchSize($newBatchSize);
+                }
+
+                $this->logInfo("Delete " . count($deleteIds) . " changes.");
+
+                $this->connection->createQueryBuilder()
+                    ->delete(\GXModules\MakairaIO\MakairaConnect\App\ChangesService::TABLE_NAME)
+                    ->add('where', $this->connection->createQueryBuilder()->expr()->in('id', $deleteIds))
+                ->execute();
+
+                $this->logInfo("Deleted " . count($deleteIds) . " changes.");
+
                 if (! $this->checkPublicFieldsSetup()) {
                     try {
                         $this->logInfo('Makaira Public Fields Setup Has Started');

diff --git a/src/GXModules/MakairaIO/MakairaConnect/Admin/Services/ModuleConfigService.php b/src/GXModules/MakairaIO/MakairaConnect/Admin/Services/ModuleConfigService.php
@@ -18,6 +18,8 @@ class ModuleConfigService
 
     public const CONFIG_MAKAIRA_STATUS = 'status';
 
+    public const CONFIG_MAKAIRA_BATCH_SIZE = 'batchSize';
+
     public const CONFIG_MAKAIRA_PUBLICFIELDS_SETUP_DONE = 'publicFieldsSetupDone';
 
     public const CONFIG_MAKAIRA_IMPORTER_SETUP_DONE = 'makairaImporterSetupDone';
@@ -42,6 +44,19 @@ class ModuleConfigService
 
     public function __construct(private ConfigurationService $configurationService) {}
 
+    public function setBatchSize(int $batchSize): self
+    {
+        $this->setConfigValue(self::CONFIG_PREFIX . self::CONFIG_MAKAIRA_BATCH_SIZE, $batchSize);
+
+        return $this;
+    }
+
+    public function getBatchSize(): int
+    {
+        return empty($value = $this->getConfigValue(self::CONFIG_PREFIX . self::CONFIG_MAKAIRA_BATCH_SIZE))
+            ? 500 : (int)$value;
+    }
+
     public function getMakairaUrl(): string
     {
         return $this->getConfigValue(self::CONFIG_PREFIX.self::CONFIG_MAKAIRA_URL);

diff --git a/src/GXModules/MakairaIO/MakairaConnect/App/Core/MakairaRequest.php b/src/GXModules/MakairaIO/MakairaConnect/App/Core/MakairaRequest.php
@@ -23,6 +23,7 @@ public function __construct(string $makairaUrl, string $makairaInstance, string
     {
         $this->makairaInstance = $makairaInstance;
         $this->makairaUrl = $makairaUrl;
+        $this->makairaSecret = $makairaSecret;
         $this->language = $language;
         $this->nonce = bin2hex(random_bytes(8));
 

diff --git a/src/GXModules/MakairaIO/MakairaConnect/App/Documents/MakairaCategory.php b/src/GXModules/MakairaIO/MakairaConnect/App/Documents/MakairaCategory.php
@@ -96,6 +96,14 @@ class MakairaCategory extends MakairaEntity
 
     public function toArray(): array
     {
+        if ($this->isDelete()) {
+            return [
+                'id' => $this->getId(),
+                'categories_id' => $this->getCategoriesId(),
+                'delete' => $this->isDelete()
+            ];
+        }
+
         return array_merge(
             /* Makaira fields */
             parent::toArray(),

diff --git a/src/GXModules/MakairaIO/MakairaConnect/App/Documents/MakairaEntity.php b/src/GXModules/MakairaIO/MakairaConnect/App/Documents/MakairaEntity.php
@@ -136,4 +136,11 @@ public function setDelete(bool $delete): static
 
         return $this;
     }
+
+    public function delete(): MakairaEntity
+    {
+        $this->delete = true;
+
+        return $this;
+    }
 }
diff --git a/src/GXModules/MakairaIO/MakairaConnect/App/Documents/MakairaManufacturer.php b/src/GXModules/MakairaIO/MakairaConnect/App/Documents/MakairaManufacturer.php
@@ -26,6 +26,12 @@ class MakairaManufacturer extends MakairaEntity
 
     public function toArray(): array
     {
+        if($this->delete) {
+            return [
+                'delete' => $this->delete
+            ];
+        }
+
         return [
             /* MAKAIRA fields */
             ...parent::toArray(),
@@ -43,6 +49,17 @@ public function toArray(): array
         ];
     }
 
+    public function isDelete(): bool
+    {
+        return $this->delete;
+    }
+
+    public function setDelete(bool $delete): static
+    {
+        $this->delete = $delete;
+        return $this;
+    }
+
     public function setTitle(string $title): static
     {
         $this->title = $title;

diff --git a/src/GXModules/MakairaIO/MakairaConnect/App/Documents/MakairaProduct.php b/src/GXModules/MakairaIO/MakairaConnect/App/Documents/MakairaProduct.php
@@ -266,58 +266,6 @@ private function mapCooProduct(): array
         return $coo_product->buildDataArray(self::toGambio($this));
     }
 
-    private function mapProduct(): array
-    {
-        if (! $this->product) {
-            return false;
-        }
-
-        $document = [
-            'data' => [
-                'type' => self::DOC_TYPE,
-                'id' => $this->product['products_id'],
-                'parent' => '',
-                'shop' => 1,
-                'ean' => $this->product['products_ean'],
-                'activeto' => '',
-                'activefrom' => '',
-                'isVariant' => false,
-                'active' => $this->getActive(),
-                'sort' => 0,
-                'stock' => $this->getStock(),
-                'onstock' => $this->getStock() > 0,
-                'picture_url_main' => $this->product['products_image'],
-                'title' => $this->product['products_name'],
-                'shortdesc' => $this->product['products_short_description'],
-                'longdesc' => $this->product['products_description'],
-                'price' => $this->product['products_price'],
-                'soldamount' => '',
-                'searchable' => true,
-                'searchkeys' => $this->product['products_keywords'] ?? '',
-                'url' => $this->getUrl(),
-                'maincategory' => $this->product['main_category_id'],
-                'maincategoryurl' => '',
-                'category' => [],
-                'attributes' => [],
-                'mak_boost_norm_insert' => 0.0,
-                'mak_boost_norm_sold' => 0.0,
-                'mak_boost_norm_rating' => 0.0,
-                'mak_boost_norm_revenue' => 0.0,
-                'mak_boost_norm_profit_margin' => 0.0,
-                'timestamp' => $this->now,
-                'manufacturerid' => $this->product['manufacturers_id'],
-                'manufacturer_title' => '',
-            ],
-            'source_revision' => 1,
-            'language_id' => $this->getLanguage(),
-        ];
-        if ($this->delete) {
-            $document['delete'] = true;
-        }
-
-        return $document;
-    }
-
     public static function toGambio(object $product)
     {
         return [

diff --git a/src/GXModules/MakairaIO/MakairaConnect/App/GambioConnectService.php b/src/GXModules/MakairaIO/MakairaConnect/App/GambioConnectService.php
@@ -71,6 +71,11 @@ protected function getLanguages(): Languages
         return $this->languageService->getAvailableLanguages();
     }
 
+    public function callStoredProcedure(int $id, string $type): void
+    {
+        $this->connection->executeQuery('CALL makairaChange('.$id.','.$type.')');
+    }
+
     public function exportIsDoneForType(string $type)
     {
         $this->connection->delete(ChangesService::TABLE_NAME, [