From 895cdba8e7f4a78008a6f228eefcb7fbe96f30e0 Mon Sep 17 00:00:00 2001 From: Lorenz Ulrich Date: Fri, 7 Jan 2022 23:43:12 +0100 Subject: [PATCH] TASK: Improve export when form definition changes, allow setting nodeTypesIgnoredInExport This package only stores the FormElement identifier at the time of form submission in its entry data. This has some drawbacks: * If a field identifier is added or removed later, it breaks the export * The field identifier could be a uuid which makes it hard to read This change improves this as follows: * A unique list of each field identifier of each submission is compiled * The ContentRepository is used to look up the actual field labels if the form still exists. * There is a fallback to the field identifier if the form or field don't exist anymore. Furthermore, the export contains too much information, such as Sections, Captchas (if configured) and possibly sensible information such as Passwords. A new configuration nodeTypesIgnoredInExport is added to prevent the output of fields configured. The following types are excluded by default: - Neos.Form.Builder:Section - Neos.Form.Builder:StaticText - Neos.Form.Builder:Password - Neos.Form.Builder:PasswordWithConfirmation --- .../Controller/DatabaseStorageController.php | 180 +++++------ Classes/Service/DatabaseStorageService.php | 306 ++++++++++++++++++ Configuration/Settings.yaml | 5 + Readme.md | 9 +- 4 files changed, 394 insertions(+), 106 deletions(-) create mode 100644 Classes/Service/DatabaseStorageService.php diff --git a/Classes/Controller/DatabaseStorageController.php b/Classes/Controller/DatabaseStorageController.php index 63488c9..8b86a75 100644 --- a/Classes/Controller/DatabaseStorageController.php +++ b/Classes/Controller/DatabaseStorageController.php @@ -5,8 +5,6 @@ * * This file is part of the Flow Framework Package "Wegmeister.DatabaseStorage". * - * PHP version 7 - * * @category Controller * @package Wegmeister\DatabaseStorage * @author Benjamin Klix @@ -19,8 +17,6 @@ use Neos\Flow\Annotations as Flow; use Neos\Flow\I18n\Translator; use Neos\Flow\Mvc\Controller\ActionController; -use Neos\Flow\ResourceManagement\ResourceManager; -use Neos\Flow\ResourceManagement\PersistentResource; use Wegmeister\DatabaseStorage\Domain\Model\DatabaseStorage; use Wegmeister\DatabaseStorage\Domain\Repository\DatabaseStorageRepository; @@ -29,6 +25,7 @@ use PhpOffice\PhpSpreadsheet\Style\Alignment; use PhpOffice\PhpSpreadsheet\IOFactory; use PhpOffice\PhpSpreadsheet\Writer\Exception as WriterException; +use Wegmeister\DatabaseStorage\Service\DatabaseStorageService; /** * The Database Storage controller @@ -45,23 +42,23 @@ class DatabaseStorageController extends ActionController protected static $types = [ 'Xls' => [ 'extension' => 'xls', - 'mimeType' => 'application/vnd.ms-excel', + 'mimeType' => 'application/vnd.ms-excel', ], 'Xlsx' => [ 'extension' => 'xlsx', - 'mimeType' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'mimeType' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', ], 'Ods' => [ 'extension' => 'ods', - 'mimeType' => 'application/vnd.oasis.opendocument.spreadsheet', + 'mimeType' => 'application/vnd.oasis.opendocument.spreadsheet', ], 'Csv' => [ 'extension' => 'csv', - 'mimeType' => 'text/csv', + 'mimeType' => 'text/csv', ], 'Html' => [ 'extension' => 'html', - 'mimeType' => 'text/html', + 'mimeType' => 'text/html', ], ]; @@ -74,12 +71,9 @@ class DatabaseStorageController extends ActionController protected $databaseStorageRepository; /** - * Instance of the resource manager. - * - * @Flow\Inject - * @var ResourceManager + * @var DatabaseStorageService */ - protected $resourceManager; + protected $databaseStorageService; /** * Instance of the translator interface. @@ -96,7 +90,6 @@ class DatabaseStorageController extends ActionController */ protected $settings; - /** * Inject the settings * @@ -119,7 +112,6 @@ public function indexAction() $this->view->assign('identifiers', $this->databaseStorageRepository->findStorageidentifiers()); } - /** * List entries of a given storage identifier. * @@ -130,30 +122,34 @@ public function indexAction() */ public function showAction(string $identifier) { + $this->databaseStorageService = new DatabaseStorageService($identifier); + $entries = $this->databaseStorageRepository->findByStorageidentifier($identifier); - $titles = []; - if (isset($entries[0])) { - foreach ($entries[0]->getProperties() as $title => $value) { - $titles[] = $title; - } - foreach ($entries as $entry) { - $properties = $entry->getProperties(); + $formElementLabels = $this->databaseStorageService->getFormElementLabels( + $entries + ); - foreach ($properties as &$value) { - $value = $this->getStringValue($value); - } + if (empty($entries)) { + $this->redirect('index'); + } - $entry->setProperties($properties); + /** @var DatabaseStorage $entry */ + foreach ($entries as $entry) { + $values = []; + foreach ($formElementLabels as $formElementLabel) { + $values[$formElementLabel] = $this->databaseStorageService->getValueFromEntryProperty( + $entry, + $formElementLabel + ); } - $this->view->assign('identifier', $identifier); - $this->view->assign('titles', $titles); - $this->view->assign('entries', $entries); - $this->view->assign('datetimeFormat', $this->settings['datetimeFormat']); - } else { - $this->redirect('index'); + $entry->setProperties($values); } - } + $this->view->assign('identifier', $identifier); + $this->view->assign('titles', $formElementLabels); + $this->view->assign('entries', $entries); + $this->view->assign('datetimeFormat', $this->settings['datetimeFormat']); + } /** * Delete an entry from the list of identifiers. @@ -166,16 +162,24 @@ public function deleteAction(DatabaseStorage $entry) { $identifier = $entry->getStorageidentifier(); $this->databaseStorageRepository->remove($entry); - $this->addFlashMessage($this->translator->translateById('storage.flashmessage.entryRemoved', [], null, null, 'Main', 'Wegmeister.DatabaseStorage')); + $this->addFlashMessage( + $this->translator->translateById( + 'storage.flashmessage.entryRemoved', + [], + null, + null, + 'Main', + 'Wegmeister.DatabaseStorage' + ) + ); $this->redirect('show', null, null, ['identifier' => $identifier]); } - /** * Delete all entries for the given identifier. * * @param string $identifier The storage identifier for the entries to be removed. - * @param bool $redirect Redirect to index? + * @param bool $redirect Redirect to index? * * @return void */ @@ -192,18 +196,26 @@ public function deleteAllAction(string $identifier, bool $redirect = false) if ($redirect) { // TODO: Translate flash message. - $this->addFlashMessage($this->translator->translateById('storage.flashmessage.entriesRemoved', [], null, null, 'Main', 'Wegmeister.DatabaseStorage')); + $this->addFlashMessage( + $this->translator->translateById( + 'storage.flashmessage.entriesRemoved', + [], + null, + null, + 'Main', + 'Wegmeister.DatabaseStorage' + ) + ); $this->redirect('index'); } } - /** * Export all entries for a specific identifier as xls. * - * @param string $identifier The storage identifier that should be exported. - * @param string $writerType The writer type/export format to be used. - * @param bool $exportDateTime Should the datetime be exported? + * @param string $identifier The storage identifier that should be exported. + * @param string $writerType The writer type/export format to be used. + * @param bool $exportDateTime Should the datetime be exported? * * @return void */ @@ -213,46 +225,46 @@ public function exportAction(string $identifier, string $writerType = 'Xlsx', bo throw new WriterException('No writer available for type ' . $writerType . '.', 1521787983); } - $entries = $this->databaseStorageRepository->findByStorageidentifier($identifier)->toArray(); + $this->databaseStorageService = new DatabaseStorageService($identifier); + + $entries = $this->databaseStorageRepository->findByStorageidentifier($identifier); $dataArray = []; $spreadsheet = new Spreadsheet(); - $spreadsheet->getProperties() - ->setCreator($this->settings['creator']) - ->setTitle($this->settings['title']) - ->setSubject($this->settings['subject']); + $spreadsheet->getProperties()->setCreator($this->settings['creator'])->setTitle( + $this->settings['title'] + )->setSubject($this->settings['subject']); $spreadsheet->setActiveSheetIndex(0); $spreadsheet->getActiveSheet()->setTitle($this->settings['title']); - $titles = []; - $columns = 0; - foreach ($entries[0]->getProperties() as $title => $value) { - $titles[] = $title; - $columns++; - } + $formElementLabels = $this->databaseStorageService->getFormElementLabels( + $entries + ); + $columns = count($formElementLabels); + if ($exportDateTime) { // TODO: Translate title for datetime - $titles[] = 'DateTime'; + $formElementLabels['DateTime'] = 'DateTime'; $columns++; } - $dataArray[] = $titles; - + $dataArray[] = $formElementLabels; + /** @var DatabaseStorage $entry */ foreach ($entries as $entry) { $values = []; - - foreach ($entry->getProperties() as $value) { - $values[] = $this->getStringValue($value); + foreach ($formElementLabels as $formElementLabel) { + $values[$formElementLabel] = $this->databaseStorageService->getValueFromEntryProperty( + $entry, + $formElementLabel + ); } - if ($exportDateTime) { - $values[] = $entry->getDateTime()->format($this->settings['datetimeFormat']); + $values['DateTime'] = $entry->getDateTime()->format($this->settings['datetimeFormat']); } - $dataArray[] = $values; } @@ -265,9 +277,9 @@ public function exportAction(string $identifier, string $writerType = 'Xlsx', bo $index = $i % 26; $columnStyle = $spreadsheet->getActiveSheet()->getStyle($prefixKey . chr(65 + $index) . '1'); $columnStyle->getFont()->setBold(true); - $columnStyle->getAlignment() - ->setHorizontal(Alignment::HORIZONTAL_CENTER) - ->setVertical(Alignment::VERTICAL_CENTER); + $columnStyle->getAlignment()->setHorizontal(Alignment::HORIZONTAL_CENTER)->setVertical( + Alignment::VERTICAL_CENTER + ); if ($index + 1 > 25) { $prefixIndex++; @@ -275,7 +287,6 @@ public function exportAction(string $identifier, string $writerType = 'Xlsx', bo } } - if (ini_get('zlib.output_compression')) { ini_set('zlib.output_compression', 'Off'); } @@ -300,41 +311,4 @@ public function exportAction(string $identifier, string $writerType = 'Xlsx', bo exit; } - /** - * Internal function to replace value with a string for export / listing. - * - * @param mixed $value The database column value. - * @param int $indent The level of indentation (for array values). - * - * @return string - */ - protected function getStringValue($value, int $indent = 0): string - { - if ($value instanceof PersistentResource) { - return $this->resourceManager->getPublicPersistentResourceUri($value) ?: '-'; - } elseif (is_string($value)) { - return $value; - } elseif (is_object($value) && method_exists($value, '__toString')) { - return (string)$value; - } elseif (isset($value['dateFormat'], $value['date'])) { - $timezone = null; - if (isset($value['timezone'])) { - $timezone = new \DateTimeZone($value['timezone']); - } - $dateTime = \DateTime::createFromFormat($value['dateFormat'], $value['date'], $timezone); - return $dateTime->format($this->settings['datetimeFormat']); - } elseif (is_array($value)) { - foreach ($value as &$innerValue) { - $innerValue = $this->getStringValue($innerValue, $indent + 1); - } - $prefix = str_repeat(' ', $indent * 2) . '- '; - return sprintf( - '%s%s', - $prefix, - implode("\r\n" . $prefix, $value) - ); - } - - return '-'; - } } diff --git a/Classes/Service/DatabaseStorageService.php b/Classes/Service/DatabaseStorageService.php new file mode 100644 index 0000000..7b6349e --- /dev/null +++ b/Classes/Service/DatabaseStorageService.php @@ -0,0 +1,306 @@ + + * @license https://github.com/die-wegmeister/Wegmeister.DatabaseStorage/blob/master/LICENSE GPL-3.0-or-later + * @link https://github.com/die-wegmeister/Wegmeister.DatabaseStorage + */ + +namespace Wegmeister\DatabaseStorage\Service; + +use Doctrine\ORM\EntityNotFoundException; +use Neos\ContentRepository\Domain\Model\NodeInterface; +use Neos\ContentRepository\Domain\Service\ContextFactory; +use Neos\Eel\FlowQuery\FlowQuery; +use Neos\Flow\Annotations as Flow; +use Neos\Flow\Persistence\QueryResultInterface; +use Neos\Flow\ResourceManagement\PersistentResource; +use Neos\Flow\ResourceManagement\ResourceManager; +use Neos\Neos\Domain\Service\SiteService; +use Wegmeister\DatabaseStorage\Domain\Model\DatabaseStorage; + +/** + * @Flow\Scope("singleton") + */ +class DatabaseStorageService +{ + + /** + * @var array + */ + protected $formElementsNodeData; + + /** + * @var string + */ + protected $formStorageIdentifier; + + /** + * @var array + * @Flow\InjectConfiguration(path="nodeTypesIgnoredInExport", package="Wegmeister.DatabaseStorage") + */ + protected $nodeTypesIgnoredInExport; + + /** + * @var string + * @Flow\InjectConfiguration(path="datetimeFormat", package="Wegmeister.DatabaseStorage") + */ + protected $datetimeFormat; + + /** + * @Flow\Inject + * @var ContextFactory + */ + protected $contextFactory; + + /** + * @Flow\Inject + * @var ResourceManager + */ + protected $resourceManager; + + public function __construct(string $formStorageIdentifier = '') + { + $this->formStorageIdentifier = $formStorageIdentifier; + } + + public function nodeTypeMustBeIgnoredInExport($nodeTypeName): bool + { + return in_array($nodeTypeName, $this->nodeTypesIgnoredInExport); + } + + /** + * Return the form element mapping for a Node-based form + * Three possible values are looked up: + * - The Node identifier + * - The speaking identifier of the FormElement + * - The label of the FormElement + * + * @param string $identifier + * @return array|null + */ + public function getFormElementDataByIdentifier(string $identifier): ?array + { + $formElementsNodeData = $this->getFormElementsNodeData(); + if (!$formElementsNodeData) { + return null; + } + foreach ($formElementsNodeData as $formElementNodeData) { + // Given identifier can be either the nodeIdentifier or the speakingIdentifier, so we must search for both + if ($formElementNodeData['nodeIdentifier'] === $identifier) { + return $formElementNodeData; + } + if ($formElementNodeData['speakingIdentifier'] === $identifier) { + return $formElementNodeData; + } + if ($formElementNodeData['displayLabel'] === $identifier) { + return $formElementNodeData; + } + } + return null; + } + + /** + * If the Node-based form is still available, node data such as the "speaking" identifier, the label + * are looked up to provide the best possible label and value matching for the export. + * + * @return array|null + * @throws \Neos\ContentRepository\Exception\NodeException + * @throws \Neos\Eel\Exception + */ + protected function getFormElementsNodeData(): ?array + { + if (!empty($this->formElementsNodeData)) { + // First-level cache + return $this->formElementsNodeData; + } + $context = $this->contextFactory->create( + [ + 'workspaceName' => 'live', + 'invisibleContentShown' => true, + 'removedContentShown' => true, + 'inaccessibleContentShown' => false + ] + ); + + // Find the finisher belonging to the formStorageIdentifier + $q = new FlowQuery([$context->getNode(SiteService::SITES_ROOT_PATH)]); + $finisherNodes = $q->find( + "[instanceof Wegmeister.DatabaseStorage:DatabaseStorageFinisher][identifier='" . $this->formStorageIdentifier . "']" + )->get(); + if (count($finisherNodes) !== 1) { + // None or more than one Finisher with the same identifier --> could be a Fusion or YAML form or ambiguous --> return + return null; + } + + // Find the NodeBasedForm owning the Finisher + $q = new FlowQuery([$finisherNodes[0]]); + $formNode = $q->parents('[instanceof Neos.Form.Builder:NodeBasedForm]')->get(0); + if (!$formNode instanceof NodeInterface) { + // No NodeBasedForm found, return + return null; + } + + // Find all FormElements belonging to the Form + $q = new FlowQuery([$formNode]); + $formElements = $q->find('[instanceof Neos.Form.Builder:FormElement]')->get(); + + if (empty($formElements)) { + // No FormElements found, return + return null; + } + + $mapping = []; + + /** @var NodeInterface $formElement */ + foreach ($formElements as $formElement) { + // UUID of the FormElement node + $nodeIdentifier = (string)$formElement->getNodeAggregateIdentifier(); + // Given identifier of the FormElement + $speakingIdentifier = $formElement->getProperty('identifier'); + // Label of the FormElement + $label = $formElement->getProperty('label'); + // "Best available" label + $displayLabel = $label ?: $speakingIdentifier ?: $nodeIdentifier; + + $mapping[] = [ + 'nodeTypeName' => $formElement->getNodeType()->getName(), + 'nodeIdentifier' => $nodeIdentifier, + 'speakingIdentifier' => $speakingIdentifier, + 'label' => $label, + 'displayLabel' => $displayLabel, + ]; + } + + $this->formElementsNodeData = $mapping; + return $mapping; + } + + /** + * Get field labels of all entries to allow exporting all fields added/removed/changed over time + * + * @param QueryResultInterface $entries + * @return array + */ + public function getFormElementLabels(QueryResultInterface $entries): array + { + $mapping = []; + + /** @var DatabaseStorage $entry */ + foreach ($entries as $entry) { + foreach ($entry->getProperties() as $key => $value) { + $formElementMapping = $this->getFormElementDataByIdentifier($key); + if (!$formElementMapping) { + /* + * There is no mapping for one of the following reasons: + * - It is a Fusion-based form + * - It is a YAML-based form + * - The form could have been removed + * - The field could have been removed + * - The field identifier could have been renamed + * In this case, we use the key as fallback, meaning the field + * is labelled as stored in the entry, which is usually "speaking + * enough" at least for Fusion-based and YAML-based forms. + * + */ + $mapping[$key] = $key; + continue; + } + if ($this->nodeTypeMustBeIgnoredInExport($formElementMapping['nodeTypeName'])) { + continue; + } + $mapping[$formElementMapping['displayLabel']] = $formElementMapping['displayLabel']; + } + } + return $mapping; + } + + /** + * We check the given entry if there is a value for the given display label + * The check is performed against the key, the nodeIdentifier and the speakingIdentifier + * + * @param DatabaseStorage $entry + * @param string $formElementLabel + * @return string + */ + public function getValueFromEntryProperty(DatabaseStorage $entry, string $formElementLabel): string + { + + // For Fusion- or YAML-based forms + if (array_key_exists($formElementLabel, $entry->getProperties())) { + return $this->getStringValue($entry->getProperties()[$formElementLabel]); + } + + $formElementData = $this->getFormElementDataByIdentifier($formElementLabel); + + // No data for this field in this entry, it was probably removed + if (!$formElementData) { + return ''; + } + + // Key is node identifier + if (array_key_exists($formElementData['nodeIdentifier'], $entry->getProperties())) { + return $this->getStringValue($entry->getProperties()[$formElementData['nodeIdentifier']]); + } + + // Key is speaking identifier + if (array_key_exists($formElementData['speakingIdentifier'], $entry->getProperties())) { + return $this->getStringValue($entry->getProperties()[$formElementData['speakingIdentifier']]); + } + + // Really no data for this field + return ''; + } + + /** + * Internal function to replace value with a string for export / listing. + * + * @param mixed $value The database column value. + * @param int $indent The level of indentation (for array values). + * + * @return string + */ + protected function getStringValue($value, int $indent = 0): string + { + if ($value instanceof PersistentResource) { + try { + $resourceUri = $this->resourceManager->getPublicPersistentResourceUri($value); + } catch (EntityNotFoundException $e) { + return ''; + } + return $resourceUri; + } + if (is_string($value)) { + return $value; + } + if (is_object($value) && method_exists($value, '__toString')) { + return (string)$value; + } + if (isset($value['dateFormat'], $value['date'])) { + $timezone = null; + if (isset($value['timezone'])) { + $timezone = new \DateTimeZone($value['timezone']); + } + $dateTime = \DateTime::createFromFormat($value['dateFormat'], $value['date'], $timezone); + return $dateTime->format($this->datetimeFormat); + } + if (is_array($value)) { + foreach ($value as &$innerValue) { + $innerValue = $this->getStringValue($innerValue, $indent + 1); + } + $prefix = str_repeat(' ', $indent * 2) . '- '; + return sprintf( + '%s%s', + $prefix, + implode("\r\n" . $prefix, $value) + ); + } + + return ''; + } + +} diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 2c45156..28a84e2 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -30,3 +30,8 @@ Wegmeister: title: 'Database Export' subject: 'Database Export' datetimeFormat: 'Y-m-d H:i:s' + nodeTypesIgnoredInExport: + - 'Neos.Form.Builder:Section' + - 'Neos.Form.Builder:StaticText' + - 'Neos.Form.Builder:Password' + - 'Neos.Form.Builder:PasswordWithConfirmation' diff --git a/Readme.md b/Readme.md index 4c787b5..38e5489 100644 --- a/Readme.md +++ b/Readme.md @@ -12,11 +12,8 @@ composer require wegmeister/databasestorage ## Usage -> :exclamation: The DatabaseStorage stores your data as JSON. Therefore only the labels of the first entry can be used for the headline/export. Keep that in mind and try to avoid changing your forms later on. Whenever you add a field **after** someone already entered some data, the new field would not exist in the headline row of the exported table :exclamation: - You can add the DatabaseStorage Finisher in the following ways: - ### Add DatabaseStorage using YAML definitions Add the DatabaseStorage a finisher in your form definition/yaml file: @@ -74,4 +71,10 @@ Wegmeister: subject: 'Database Export' # DateTime format if the datetime is included in the export datetimeFormat: 'Y-m-d H:i:s' + # Form element types that should not be part of the export + nodeTypesIgnoredInExport: + - 'Neos.Form.Builder:Section' + - 'Neos.Form.Builder:StaticText' + - 'Neos.Form.Builder:Password' + - 'Neos.Form.Builder:PasswordWithConfirmation' ```