From 602ec2d52c75826178ef6197770ae5c1b76fe3d3 Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 15 Mar 2024 18:06:54 +0000 Subject: [PATCH 01/22] Feat: Long term storage. Long term storage of Webform submission data in an off-site database. Work in progress. --- .../localgov_forms_lts.info.yml | 9 ++ .../localgov_forms_lts.install | 117 ++++++++++++++++++ .../localgov_forms_lts.module | 6 + 3 files changed, 132 insertions(+) create mode 100644 modules/localgov_forms_lts/localgov_forms_lts.info.yml create mode 100644 modules/localgov_forms_lts/localgov_forms_lts.install create mode 100644 modules/localgov_forms_lts/localgov_forms_lts.module diff --git a/modules/localgov_forms_lts/localgov_forms_lts.info.yml b/modules/localgov_forms_lts/localgov_forms_lts.info.yml new file mode 100644 index 0000000..68f94cc --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.info.yml @@ -0,0 +1,9 @@ +name: LocalGov Forms long term storage +type: module +description: Long term storage for Webform submissions. +core_version_requirement: ^10 +php: 8.0 +package: LocalGov Drupal + +dependencies: +- webform:webform diff --git a/modules/localgov_forms_lts/localgov_forms_lts.install b/modules/localgov_forms_lts/localgov_forms_lts.install new file mode 100644 index 0000000..fee3c54 --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.install @@ -0,0 +1,117 @@ +set(key: 'localgov_forms_lts_db', value: $localgov_forms_lts_db); + + $webform_submission_schema = _localgov_forms_lts_get_webform_submission_storage_schema(); + + foreach ($webform_submission_schema as $table_name => $table_schema) { + _localgov_forms_lts_copy_table($table_name, $table_schema, db: $localgov_forms_lts_db); + } +} + +/** + * Implements hook_requirements(). + * + * Checks for the presence of the localgov_forms_lts database. + */ +function localgov_forms_lts_requirements($phase) { + + $requirements = [ + 'localgov_forms_lts' => [ + 'title' => t('LocalGov Forms LTS'), + 'value' => t('Available'), + 'description' => t('LocalGov Forms LTS database available.'), + 'severity' => REQUIREMENT_OK, + ], + ]; + + if (!$localgov_forms_lts_db = _localgov_forms_lts_has_db()) { + $requirements['localgov_forms_lts']['value'] = t('Unavailable'); + $requirements['localgov_forms_lts']['description'] = t('The LocalGov Forms LTS database must exist for this module to function.'); + $requirements['localgov_forms_lts']['severity'] = REQUIREMENT_ERROR; + } + + return $requirements; +} + +/** + * Tests presence of LTS database. + */ +function _localgov_forms_lts_has_db(): bool|string { + + $localgov_forms_lts_db = Drupal::service('state')->get(key: 'localgov_forms_lts_db', default: 'localgov_forms_lts'); + + try { + Database::getConnection(key: $localgov_forms_lts_db); + } + catch (Exception $e) { + return FALSE; + } + + return $localgov_forms_lts_db; +} + +/** + * Extracts entity storage schema. + * + * Returns the entity storage schema for the webform_submission content entity. + */ +function _localgov_forms_lts_get_webform_submission_storage_schema(): array { + + $entity_type_manager = Drupal::service('entity_type.manager'); + $entity_storage = $entity_type_manager->getStorage('webform_submission'); + $entity_type = $entity_storage->getEntityType(); + $entity_field_manager = Drupal::service('entity_field.manager'); + $db_service = Drupal::service('database'); + + $entity_schema = (new class($entity_type_manager, $entity_type, $entity_storage, $db_service, $entity_field_manager) extends WebformSubmissionStorageSchema { + + /** + * Public wrapper over protected method. + */ + public function getEntitySchemaWrapper(ContentEntityTypeInterface $entity_type) { + + return parent::getEntitySchema($entity_type); + } + + })->getEntitySchemaWrapper($entity_type); + + return $entity_schema; +} + +/** + * Creates database tables. + */ +function _localgov_forms_lts_copy_table(string $table_name, array $table_schema, string $db): void { + + $db_connection = Database::getConnection(key: $db); + $tx = $db_connection->startTransaction(); + + try { + $db_connection->schema()->createTable($table_name, $table_schema); + } + catch (Exception $e) { + $tx->rollBack(); + } +} diff --git a/modules/localgov_forms_lts/localgov_forms_lts.module b/modules/localgov_forms_lts/localgov_forms_lts.module new file mode 100644 index 0000000..60e45ec --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.module @@ -0,0 +1,6 @@ + Date: Fri, 22 Mar 2024 12:31:01 +0000 Subject: [PATCH 02/22] Feat: Long term storage. Alternate Webform submission entity storage class for storing and retrieving entities from the LTS database. --- .../localgov_forms_lts.install | 32 ++++------- .../localgov_forms_lts.module | 18 ++++++ .../localgov_forms_lts.services.yml | 12 ++++ modules/localgov_forms_lts/src/Constants.php | 22 ++++++++ .../src/LtsStorageForWebformSubmission.php | 55 +++++++++++++++++++ 5 files changed, 117 insertions(+), 22 deletions(-) create mode 100644 modules/localgov_forms_lts/localgov_forms_lts.services.yml create mode 100644 modules/localgov_forms_lts/src/Constants.php create mode 100644 modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php diff --git a/modules/localgov_forms_lts/localgov_forms_lts.install b/modules/localgov_forms_lts/localgov_forms_lts.install index fee3c54..4dc0f1b 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.install +++ b/modules/localgov_forms_lts/localgov_forms_lts.install @@ -7,6 +7,7 @@ use Drupal\Core\Database\Database; use Drupal\Core\Entity\ContentEntityTypeInterface; +use Drupal\localgov_forms_lts\Constants; use Drupal\webform\WebformSubmissionStorageSchema; /** @@ -17,16 +18,14 @@ use Drupal\webform\WebformSubmissionStorageSchema; */ function localgov_forms_lts_install($is_syncing): void { - if (!$localgov_forms_lts_db = _localgov_forms_lts_has_db()) { + if (!localgov_forms_lts_has_db()) { return; } - Drupal::service('state')->set(key: 'localgov_forms_lts_db', value: $localgov_forms_lts_db); - $webform_submission_schema = _localgov_forms_lts_get_webform_submission_storage_schema(); foreach ($webform_submission_schema as $table_name => $table_schema) { - _localgov_forms_lts_copy_table($table_name, $table_schema, db: $localgov_forms_lts_db); + _localgov_forms_lts_copy_table($table_name, $table_schema, db: Constants::LTS_DB_KEY); } } @@ -46,7 +45,13 @@ function localgov_forms_lts_requirements($phase) { ], ]; - if (!$localgov_forms_lts_db = _localgov_forms_lts_has_db()) { + if (!function_exists('localgov_forms_lts_has_db')) { + // Some necessary files have not been loaded yet during the "install" phase. + require_once __DIR__ . '/localgov_forms_lts.module'; + require_once __DIR__ . '/src/Constants.php'; + } + + if (!localgov_forms_lts_has_db()) { $requirements['localgov_forms_lts']['value'] = t('Unavailable'); $requirements['localgov_forms_lts']['description'] = t('The LocalGov Forms LTS database must exist for this module to function.'); $requirements['localgov_forms_lts']['severity'] = REQUIREMENT_ERROR; @@ -55,23 +60,6 @@ function localgov_forms_lts_requirements($phase) { return $requirements; } -/** - * Tests presence of LTS database. - */ -function _localgov_forms_lts_has_db(): bool|string { - - $localgov_forms_lts_db = Drupal::service('state')->get(key: 'localgov_forms_lts_db', default: 'localgov_forms_lts'); - - try { - Database::getConnection(key: $localgov_forms_lts_db); - } - catch (Exception $e) { - return FALSE; - } - - return $localgov_forms_lts_db; -} - /** * Extracts entity storage schema. * diff --git a/modules/localgov_forms_lts/localgov_forms_lts.module b/modules/localgov_forms_lts/localgov_forms_lts.module index 60e45ec..3779fa9 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.module +++ b/modules/localgov_forms_lts/localgov_forms_lts.module @@ -4,3 +4,21 @@ * @file * Hook implementations. */ + +use Drupal\Core\Database\Database; +use Drupal\localgov_forms_lts\Constants; + +/** + * Tests presence of LTS database. + */ +function localgov_forms_lts_has_db(): bool { + + try { + Database::getConnection(key: Constants::LTS_DB_KEY); + } + catch (Exception $e) { + return FALSE; + } + + return TRUE; +} diff --git a/modules/localgov_forms_lts/localgov_forms_lts.services.yml b/modules/localgov_forms_lts/localgov_forms_lts.services.yml new file mode 100644 index 0000000..1ddce4c --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.services.yml @@ -0,0 +1,12 @@ +services: + localgov_forms_lts_db: + class: Drupal\Core\Database\Connection + factory: Drupal\Core\Database\Database::getConnection + arguments: + $key: localgov_forms_lts + + localgov_forms_lts.query.sql: + class: Drupal\Core\Entity\Query\Sql\QueryFactory + arguments: ['@localgov_forms_lts_db'] + tags: + - { name: backend_overridable } diff --git a/modules/localgov_forms_lts/src/Constants.php b/modules/localgov_forms_lts/src/Constants.php new file mode 100644 index 0000000..1e233de --- /dev/null +++ b/modules/localgov_forms_lts/src/Constants.php @@ -0,0 +1,22 @@ +setLtsDatabaseConnection($lts_db_connection); + * $a_webform_submission = $lts_storage->load($a_webform_submission_id); + * @endcode + */ +class LtsStorageForWebformSubmission extends WebformSubmissionStorage { + + /** + * Constructor wrapper. + * + * Switches to the LTS database. + */ + public function __construct(...$args) { + + parent::__construct(...$args); + + $this->database = Database::getConnection(key: Constants::LTS_DB_KEY); + } + + /** + * Setter for database connection. + */ + public function setDatabaseConnection(DbConnection $db_connection): void { + + $this->database = $db_connection; + } + + /** + * {@inheritdoc} + * + * Names our custom entity query service that speaks to the LTS database. + */ + protected function getQueryServiceName() { + + return Constants::LTS_ENTITY_QUERY_SERVICE; + } + +} From 349949fc55d7233c815bde5c45af61d24fecb740 Mon Sep 17 00:00:00 2001 From: Adnan Date: Thu, 23 May 2024 19:23:31 +0100 Subject: [PATCH 03/22] Feat: Long term storage. - PHP class to copy Webform submissions to Long term storage. - Cron job to periodically copy Webform submissions to Long term storage. - Deploy hook to copy Webform submissions to Long term storage. - Personally Identifiable Information redaction before copying Webform submissions to Long term storage. - A requirement hook implementation to warn against misconfiguration. --- modules/localgov_forms_lts/README.md | 29 +++ .../localgov_forms_lts.deploy.php | 41 ++++ .../localgov_forms_lts.module | 55 +++++ .../localgov_forms_lts.services.yml | 4 +- modules/localgov_forms_lts/src/Constants.php | 22 ++ modules/localgov_forms_lts/src/LtsCopy.php | 200 ++++++++++++++++++ .../src/LtsStorageForWebformSubmission.php | 8 + .../localgov_forms_lts/src/PIIRedactor.php | 117 ++++++++++ 8 files changed, 474 insertions(+), 2 deletions(-) create mode 100644 modules/localgov_forms_lts/README.md create mode 100644 modules/localgov_forms_lts/localgov_forms_lts.deploy.php create mode 100644 modules/localgov_forms_lts/src/LtsCopy.php create mode 100644 modules/localgov_forms_lts/src/PIIRedactor.php diff --git a/modules/localgov_forms_lts/README.md b/modules/localgov_forms_lts/README.md new file mode 100644 index 0000000..e921a91 --- /dev/null +++ b/modules/localgov_forms_lts/README.md @@ -0,0 +1,29 @@ +## Long term storage for Webform submission + +### Setup process +- Create a database which will serve as the Long term storage. +- Declare it in Drupal's settings.php using the **localgov_forms_lts** key. Example: + ``` + $databases['localgov_forms_lts']['default'] = [ + 'database' => 'our_longer_term_storage_database', + 'username' => 'database_username_goes_here' + 'password' => 'database-password-goes-here', + 'host' => 'database-hostname-goes-here', + 'port' => '3306', + 'driver' => 'mysql', + 'prefix' => '', + ]; + ``` +- Install the localgov_forms_lts submodule. +- Check the module requirement report from Drupal's status page at `admin/reports/status`. This should be under the **LocalGov Forms LTS** key. +- If all looks good in the previous step, run `drush deploy:hook` which will copy existing Webform submissions into the Long term storage. If you are using `drush deploy`, this will be taken care of as part of it and there would be no need for `drush deploy:hook`. +- Ensure cron is running periodically. This will copy any new Webform submissions or changes to existing Webform submissions since deployment or the last cron run. +- [Optional] Tell individual Webforms to purge submissions older than a chosen period. This is configured for each Webform from its `Settings > Submissions > Submission purge settings` configuration section. + +### Good to know +- Each cron run copies 50 Webform submissions. If your site is getting more than that many Webform submissions between subsequent cron runs, not all Webform submissions will get copied to Long term storage during a certain period. If that happens, adjust cron run frequency. +- Elements with Personally Identifiable Information (PII) are redacted. At the moment, this includes all email, telephone, and number type elements. Additionally, any text or radio or checkbox element whose machine name contains the following also gets redacted: name, mail, phone, date_of_birth, personal, title, gender, sex, ethnicity. + +### Todo +- Machine names which are indicative of PII are hardcoded within the Drupal\localgov_forms_lts\PIIRedactor class at the moment. This should have a configuration UI. +- Automated tests. diff --git a/modules/localgov_forms_lts/localgov_forms_lts.deploy.php b/modules/localgov_forms_lts/localgov_forms_lts.deploy.php new file mode 100644 index 0000000..1159dc5 --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.deploy.php @@ -0,0 +1,41 @@ +copy(start_offset: $sandbox['webform_sub_id_offset']); + + if (count($copy_results) < Constants::COPY_LIMIT) { + $sandbox['#finished'] = 1; + } + else { + $sandbox['webform_sub_id_offset'] += Constants::COPY_LIMIT; + } + + $feedback = _localgov_forms_lts_prepare_feedback_msg($copy_results); + return $feedback; +} diff --git a/modules/localgov_forms_lts/localgov_forms_lts.module b/modules/localgov_forms_lts/localgov_forms_lts.module index 3779fa9..31706e8 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.module +++ b/modules/localgov_forms_lts/localgov_forms_lts.module @@ -5,8 +5,12 @@ * Hook implementations. */ +declare(strict_types=1); + +use Drupal\Component\Render\MarkupInterface; use Drupal\Core\Database\Database; use Drupal\localgov_forms_lts\Constants; +use Drupal\localgov_forms_lts\LtsCopy; /** * Tests presence of LTS database. @@ -22,3 +26,54 @@ function localgov_forms_lts_has_db(): bool { return TRUE; } + +/** + * Implements hook_cron(). + */ +function localgov_forms_lts_cron() { + + localgov_forms_lts_copy_recently_added_n_updated_subs(); +} + +/** + * Copies Webform submissions. + * + * Copies Webform submissions added or updated since last run to the lts + * storage. At most 50 Webform submissions are copied. + */ +function localgov_forms_lts_copy_recently_added_n_updated_subs() :void { + + $lts_copy_obj = LtsCopy::create(Drupal::getContainer()); + $copy_results = $lts_copy_obj->copy(); + + $feedback_msg = _localgov_forms_lts_prepare_feedback_msg($copy_results); + Drupal::service('logger.factory') + ->get('localgov_forms_lts') + ->info($feedback_msg); +} + +/** + * Prepares feedback message for copying. + * + * The feedback message is prepared from the outcome of Webform submission copy + * operations. + */ +function _localgov_forms_lts_prepare_feedback_msg(array $copy_results): MarkupInterface { + + $copy_successes = array_filter($copy_results); + $copy_failures = array_diff_key($copy_results, $copy_successes); + + $successfully_copied_sid_list = array_keys($copy_successes); + $unsuccessfully_copied_sid_list = array_keys($copy_failures); + + $successfully_copied_sid_list_msg = $successfully_copied_sid_list ? implode(', ', $successfully_copied_sid_list) : 'None'; + $unsuccessfully_copied_sid_list_msg = $unsuccessfully_copied_sid_list ? implode(', ', $unsuccessfully_copied_sid_list) : 'None'; + + $feedback_msg = t('Successfully copied Webform submission ids: %successes. :newline Failed copies: %failures.', [ + '%successes' => $successfully_copied_sid_list_msg, + '%failures' => $unsuccessfully_copied_sid_list_msg, + ':newline' => PHP_EOL, + ]); + + return $feedback_msg; +} diff --git a/modules/localgov_forms_lts/localgov_forms_lts.services.yml b/modules/localgov_forms_lts/localgov_forms_lts.services.yml index 1ddce4c..6440676 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.services.yml +++ b/modules/localgov_forms_lts/localgov_forms_lts.services.yml @@ -2,9 +2,9 @@ services: localgov_forms_lts_db: class: Drupal\Core\Database\Connection factory: Drupal\Core\Database\Database::getConnection - arguments: + arguments: $key: localgov_forms_lts - + localgov_forms_lts.query.sql: class: Drupal\Core\Entity\Query\Sql\QueryFactory arguments: ['@localgov_forms_lts_db'] diff --git a/modules/localgov_forms_lts/src/Constants.php b/modules/localgov_forms_lts/src/Constants.php index 1e233de..9c041aa 100644 --- a/modules/localgov_forms_lts/src/Constants.php +++ b/modules/localgov_forms_lts/src/Constants.php @@ -14,9 +14,31 @@ class Constants { */ const LTS_DB_KEY = 'localgov_forms_lts'; + /** + * Relevant key value store name. + */ + const LTS_KEYVALUE_STORE_ID = 'localgov_forms_lts'; + + /** + * Relevant Logger channel. + */ + const LTS_LOGGER_CHANNEL_ID = 'localgov_forms_lts'; + + /** + * When did the last copied Webform submission change? + */ + const LAST_CHANGE_TIMESTAMP = 'last_copied_webform_sub_changed_ts'; + /** * LTS database-based Webform submission entity query service. */ const LTS_ENTITY_QUERY_SERVICE = 'localgov_forms_lts.query.sql'; + /** + * How many Webform submissions to copy at a time. + * + * Useful in batch jobs. + */ + const COPY_LIMIT = 50; + } diff --git a/modules/localgov_forms_lts/src/LtsCopy.php b/modules/localgov_forms_lts/src/LtsCopy.php new file mode 100644 index 0000000..afffc65 --- /dev/null +++ b/modules/localgov_forms_lts/src/LtsCopy.php @@ -0,0 +1,200 @@ +findLastCopiedSubId(); + $webform_subs_to_copy = $this->findCopyTargets($start_offset, $count); + + $copy_results = []; + $has_copied = FALSE; + foreach ($webform_subs_to_copy as $webform_sub_id) { + $is_new_webform_sub = $webform_sub_id > $last_copied_webform_sub_id; + $copy_results[$webform_sub_id] = $this->copySub((int) $webform_sub_id, $is_new_webform_sub); + $has_copied = TRUE; + } + + if ($has_copied) { + $this->setLatestUpdateTimestamp($copy_results); + } + + return $copy_results; + } + + /** + * Saves a single webform submission in LTS database. + */ + public function copySub(int $webform_sub_id, bool $is_new_webform_sub) :bool { + + $webform_sub = WebformSubmission::load($webform_sub_id); + PIIRedactor::redact($webform_sub); + + $db_connection = $this->ltsStorage->getDatabaseConnection(); + $tx = $db_connection->startTransaction(); + + try { + if ($is_new_webform_sub) { + $this->ltsStorage->save($webform_sub->enforceIsNew()); + } + else { + $this->ltsStorage->save($webform_sub); + } + } + catch (Exception $e) { + $tx->rollBack(); + + $this->ltsLogger->error('Failed to add/edit Webform submission: %sub-id', [ + '%sub-id' => $webform_sub_id, + ]); + + return FALSE; + } + + return TRUE; + } + + /** + * Finds last copied Webform submission's id. + */ + public function findLastCopiedSubId() :int { + + $last_copied_webform_sub_id_raw = $this->ltsStorage->getAggregateQuery() + ->accessCheck(FALSE) + ->aggregate('sid', 'MAX') + ->execute(); + $last_copied_webform_sub_id = $last_copied_webform_sub_id_raw[0]['sid_max'] ?? 0; + return (int) $last_copied_webform_sub_id; + } + + /** + * Finds Webform submissions to copy. + * + * These are the Webform submissions that have been added or edited since the + * last copy operation. + */ + public function findCopyTargets(int $start_offset = 0, int $count = -1) :array { + + $last_copied_webform_sub_changed_ts = $this->findLatestUpdateTimestamp(); + + $webform_subs_to_copy = $this->entityTypeManager + ->getStorage('webform_submission') + ->getQuery() + ->accessCheck(FALSE) + ->condition('changed', $last_copied_webform_sub_changed_ts, '>') + ->condition('in_draft', 0); + + if ($count < 0) { + $webform_subs_to_copy->range(start: $start_offset, length: $count); + } + else { + $webform_subs_to_copy->range(start: $start_offset); + } + + $webform_subs_to_copy + ->sort('changed') + ->execute(); + + return $webform_subs_to_copy; + } + + /** + * When did the last copied Webform submission change? + */ + public function findLatestUpdateTimestamp() :int { + + $ts = (int) $this->ltsKeyValueStore->get(Constants::LAST_CHANGE_TIMESTAMP, default: 0); + return $ts; + } + + /** + * Records time of latest copy operation. + */ + public function setLatestUpdateTimestamp(array $copy_results) :void { + + $last_copied_webform_sub_id = array_key_last($copy_results); + $last_copied_webform_sub = WebformSubmission::load($last_copied_webform_sub_id); + $this->ltsKeyValueStore->set(Constants::LAST_CHANGE_TIMESTAMP, $last_copied_webform_sub->getChangedTime()); + } + + /** + * Constructor. + * + * Keeps track of dependencies. + */ + public function __construct(EntityTypeManagerInterface $entity_type_manager, KeyValueFactoryInterface $key_value_factory, LoggerChannelFactoryInterface $logger_factory, WebformSubmissionStorageInterface $lts_storage) { + + $this->entityTypeManager = $entity_type_manager; + $this->ltsStorage = $lts_storage; + $this->ltsKeyValueStore = $key_value_factory->get(Constants::LTS_KEYVALUE_STORE_ID); + $this->ltsLogger = $logger_factory->get(Constants::LTS_LOGGER_CHANNEL_ID); + } + + /** + * Factory. + */ + public static function create(ContainerInterface $container) :LtsCopy { + + $webform_sub_def = $container->get('entity_type.manager')->getDefinition('webform_submission'); + + return new LtsCopy( + $container->get('entity_type.manager'), + $container->get('keyvalue'), + $container->get('logger.factory'), + LtsStorageForWebformSubmission::createInstance($container, $webform_sub_def) + ); + } + + /** + * Key value store for LTS related state. + * + * @var Drupal\Core\KeyValueStore\KeyValueStoreInterface + */ + protected $ltsKeyValueStore; + + /** + * Entity type manager service. + * + * @var Drupal\Core\Entity\EntityTypeManagerInterface + */ + protected $entityTypeManager; + + /** + * Database service for the Long term storage database. + * + * @var Drupal\webform\WebformSubmissionStorageInterface + */ + protected $ltsStorage; + + /** + * Logger channel. + * + * @var Drupal\Core\Logger\LoggerChannelInterface + */ + protected $ltsLogger; + +} diff --git a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php index 2c6b049..27d8e39 100644 --- a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php +++ b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php @@ -42,6 +42,14 @@ public function setDatabaseConnection(DbConnection $db_connection): void { $this->database = $db_connection; } + /** + * Getter for database connection. + */ + public function getDatabaseConnection(): DbConnection { + + return $this->database; + } + /** * {@inheritdoc} * diff --git a/modules/localgov_forms_lts/src/PIIRedactor.php b/modules/localgov_forms_lts/src/PIIRedactor.php new file mode 100644 index 0000000..32385e5 --- /dev/null +++ b/modules/localgov_forms_lts/src/PIIRedactor.php @@ -0,0 +1,117 @@ +getElementData($elem)) { + $webform_sub->setElementData($elem, NULL); + + return $elem; + } + }, $elems_to_redact); + + $redacted_elems = array_filter($redaction_result); + static::addRedactionNote($webform_sub, $redacted_elems); + + return $redacted_elems; + } + + /** + * Finds the Webform element names to redact. + */ + public static function findElemsToRedact(WebformSubmissionInterface $webform_sub) :array { + + $elem_type_mapping = static::listElemsAndTypes($webform_sub); + $pii_mapping = array_intersect($elem_type_mapping, static::PII_ELEMENT_TYPES); + $pii_elems = array_keys($pii_mapping); + + $potential_mapping = array_intersect($elem_type_mapping, static::POTENTIAL_PII_ELEMENT_TYPES); + $guessed_pii_elems = preg_grep(static::GUESSED_PII_ELEM_PATTERN, array_keys($potential_mapping)); + + $elems_to_redact = $pii_elems + $guessed_pii_elems; + return $elems_to_redact; + } + + /** + * Prepares mapping of element ids and types. + */ + public static function listElemsAndTypes(WebformSubmissionInterface $webform_sub) :array { + + $elems = $webform_sub->getWebform()->getElementsDecodedAndFlattened(); + return array_map(fn($elem_def) => $elem_def['#type'], $elems); + } + + /** + * Adds redaction note. + * + * Adds a note to the Webform submission to highlight the redacted elements. + */ + public static function addRedactionNote(WebformSubmissionInterface $webform_sub, array $redacted_elems) :void { + + if (empty($redacted_elems)) { + return; + } + + $redaction_note = 'Redacted elements: ' . implode(', ', $redacted_elems) . '.'; + + $existing_note = $webform_sub->getNotes(); + $updated_note = $existing_note . PHP_EOL . $redaction_note; + + $webform_sub->setNotes($updated_note); + } + + /** + * Element types carrying PII for certain. + */ + const PII_ELEMENT_TYPES = [ + 'email', + 'tel', + 'number', + ]; + + /** + * Element types that *may* carry PII. + */ + const POTENTIAL_PII_ELEMENT_TYPES = [ + 'textfield', + 'processed_text', + 'checkboxes', + 'radios', + ]; + + /** + * Preg pattern. + * + * Element type naming pattern indicating possible link with PII. + */ + const GUESSED_PII_ELEM_PATTERN = '#name|mail|phone|date_of_birth|personal|title|gender|sex|ethnicity#i'; + +} From 094618f7f472bfef043dfc802590ffe28e0b3730 Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 24 May 2024 11:15:07 +0100 Subject: [PATCH 04/22] Fix: Long term storage. Fixed return value of LtsCopy::findCopyTargets(). Also, ensured that no Webform handlers run while Webform submissions are copied to Long term storage. --- .../localgov_forms_lts.deploy.php | 18 +++++------ modules/localgov_forms_lts/src/LtsCopy.php | 30 +++++++++---------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/modules/localgov_forms_lts/localgov_forms_lts.deploy.php b/modules/localgov_forms_lts/localgov_forms_lts.deploy.php index 1159dc5..2ae2e35 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.deploy.php +++ b/modules/localgov_forms_lts/localgov_forms_lts.deploy.php @@ -9,7 +9,7 @@ use Drupal\Component\Render\MarkupInterface; use Drupal\localgov_forms_lts\Constants; -use Drupal\localgov_forms_lts\LtsStorageForWebformSubmission; +use Drupal\localgov_forms_lts\LtsCopy; /** * Implements hook_deploy_NAME(). @@ -23,19 +23,15 @@ function localgov_forms_lts_deploy_copy_webform_subs(array &$sandbox): MarkupInt return t('The LocalGov Forms LTS database must exist for this module to function.'); } - $sandbox['webform_sub_id_offset'] = $sandbox['webform_sub_id_offset'] ?? 0; - $sandbox['#finished'] = 0; - $lts_copy_obj = LtsCopy::create(Drupal::getContainer()); - $copy_results = $lts_copy_obj->copy(start_offset: $sandbox['webform_sub_id_offset']); + $copy_results = $lts_copy_obj->copy(); - if (count($copy_results) < Constants::COPY_LIMIT) { - $sandbox['#finished'] = 1; - } - else { - $sandbox['webform_sub_id_offset'] += Constants::COPY_LIMIT; - } + $sandbox['#finished'] = (count($copy_results) < Constants::COPY_LIMIT) ? 1 : 0; $feedback = _localgov_forms_lts_prepare_feedback_msg($copy_results); + Drupal::service('logger.factory') + ->get('localgov_forms_lts') + ->info($feedback); + return $feedback; } diff --git a/modules/localgov_forms_lts/src/LtsCopy.php b/modules/localgov_forms_lts/src/LtsCopy.php index afffc65..3c39a97 100644 --- a/modules/localgov_forms_lts/src/LtsCopy.php +++ b/modules/localgov_forms_lts/src/LtsCopy.php @@ -26,10 +26,10 @@ class LtsCopy implements ContainerInjectionInterface { * * By default, 50 Webform submissions are copied. */ - public function copy(int $start_offset = 0, int $count = Constants::COPY_LIMIT) :array { + public function copy(int $count = Constants::COPY_LIMIT) :array { $last_copied_webform_sub_id = $this->findLastCopiedSubId(); - $webform_subs_to_copy = $this->findCopyTargets($start_offset, $count); + $webform_subs_to_copy = $this->findCopyTargets($count); $copy_results = []; $has_copied = FALSE; @@ -59,10 +59,10 @@ public function copySub(int $webform_sub_id, bool $is_new_webform_sub) :bool { try { if ($is_new_webform_sub) { - $this->ltsStorage->save($webform_sub->enforceIsNew()); + $this->ltsStorage->resave($webform_sub->enforceIsNew()); } else { - $this->ltsStorage->save($webform_sub); + $this->ltsStorage->resave($webform_sub); } } catch (Exception $e) { @@ -96,8 +96,11 @@ public function findLastCopiedSubId() :int { * * These are the Webform submissions that have been added or edited since the * last copy operation. + * + * For offset to work, *both* parameters must be provided with nonnegative + * values. */ - public function findCopyTargets(int $start_offset = 0, int $count = -1) :array { + public function findCopyTargets(int $count = -1) :array { $last_copied_webform_sub_changed_ts = $this->findLatestUpdateTimestamp(); @@ -106,20 +109,15 @@ public function findCopyTargets(int $start_offset = 0, int $count = -1) :array { ->getQuery() ->accessCheck(FALSE) ->condition('changed', $last_copied_webform_sub_changed_ts, '>') - ->condition('in_draft', 0); + ->condition('in_draft', 0) + ->sort('changed'); - if ($count < 0) { - $webform_subs_to_copy->range(start: $start_offset, length: $count); - } - else { - $webform_subs_to_copy->range(start: $start_offset); + if ($count > -1) { + $webform_subs_to_copy->range(start: 0, length: $count); } - $webform_subs_to_copy - ->sort('changed') - ->execute(); - - return $webform_subs_to_copy; + $copy_targets = $webform_subs_to_copy->execute(); + return $copy_targets; } /** From 8939994c3f330f13691f47b9fa759409cf39b6e4 Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 24 May 2024 19:18:59 +0100 Subject: [PATCH 05/22] Feat: Long term storage. Updated scope of Personally Identifiable Information. --- modules/localgov_forms_lts/README.md | 2 +- modules/localgov_forms_lts/src/PIIRedactor.php | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/localgov_forms_lts/README.md b/modules/localgov_forms_lts/README.md index e921a91..b91c62a 100644 --- a/modules/localgov_forms_lts/README.md +++ b/modules/localgov_forms_lts/README.md @@ -22,7 +22,7 @@ ### Good to know - Each cron run copies 50 Webform submissions. If your site is getting more than that many Webform submissions between subsequent cron runs, not all Webform submissions will get copied to Long term storage during a certain period. If that happens, adjust cron run frequency. -- Elements with Personally Identifiable Information (PII) are redacted. At the moment, this includes all email, telephone, and number type elements. Additionally, any text or radio or checkbox element whose machine name contains the following also gets redacted: name, mail, phone, date_of_birth, personal, title, gender, sex, ethnicity. +- Elements with Personally Identifiable Information (PII) are redacted. At the moment, this includes all name, email, telephone, number, and various address type elements. Additionally, any text or radio or checkbox element whose machine name (AKA Key) contains the following also gets redacted: name, mail, phone, contact_number, date_of_birth, dob_, personal_, title, nino, passport, postcode, address, serial_number, reg_number, pcn_, and driver_. ### Todo - Machine names which are indicative of PII are hardcoded within the Drupal\localgov_forms_lts\PIIRedactor class at the moment. This should have a configuration UI. diff --git a/modules/localgov_forms_lts/src/PIIRedactor.php b/modules/localgov_forms_lts/src/PIIRedactor.php index 32385e5..49f1193 100644 --- a/modules/localgov_forms_lts/src/PIIRedactor.php +++ b/modules/localgov_forms_lts/src/PIIRedactor.php @@ -92,9 +92,15 @@ public static function addRedactionNote(WebformSubmissionInterface $webform_sub, * Element types carrying PII for certain. */ const PII_ELEMENT_TYPES = [ + 'address', 'email', - 'tel', + 'localgov_webform_uk_address', 'number', + 'tel', + 'webform_name', + 'webform_address', + 'webform_contact', + 'webform_telephone', ]; /** @@ -112,6 +118,6 @@ public static function addRedactionNote(WebformSubmissionInterface $webform_sub, * * Element type naming pattern indicating possible link with PII. */ - const GUESSED_PII_ELEM_PATTERN = '#name|mail|phone|date_of_birth|personal|title|gender|sex|ethnicity#i'; + const GUESSED_PII_ELEM_PATTERN = '#name|mail|phone|contact_number|date_of_birth|dob_|nino|address|postcode|post_code|personal_|title|passport|serial_number|reg_number|pcn_|driver_#i'; } From 53149642f5e09e162cc1d23858ac705113f7547f Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 31 May 2024 13:06:40 +0100 Subject: [PATCH 06/22] Feat: Long term storage. Listing and viewing Webform submissions in Long term storage. --- modules/localgov_forms_lts/README.md | 7 +- .../localgov_forms_lts.links.task.yml | 20 +++++ .../localgov_forms_lts.routing.yml | 29 +++++++ modules/localgov_forms_lts/src/Constants.php | 10 +++ .../WebformSubmissionLtsViewController.php | 71 ++++++++++++++++ .../src/LtsStorageForWebformSubmission.php | 36 ++++++-- .../localgov_forms_lts/src/PIIRedactor.php | 2 +- .../src/WebformSubmissionLtsListBuilder.php | 82 +++++++++++++++++++ 8 files changed, 250 insertions(+), 7 deletions(-) create mode 100644 modules/localgov_forms_lts/localgov_forms_lts.links.task.yml create mode 100644 modules/localgov_forms_lts/localgov_forms_lts.routing.yml create mode 100644 modules/localgov_forms_lts/src/Controller/WebformSubmissionLtsViewController.php create mode 100644 modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php diff --git a/modules/localgov_forms_lts/README.md b/modules/localgov_forms_lts/README.md index b91c62a..5270824 100644 --- a/modules/localgov_forms_lts/README.md +++ b/modules/localgov_forms_lts/README.md @@ -20,10 +20,15 @@ - Ensure cron is running periodically. This will copy any new Webform submissions or changes to existing Webform submissions since deployment or the last cron run. - [Optional] Tell individual Webforms to purge submissions older than a chosen period. This is configured for each Webform from its `Settings > Submissions > Submission purge settings` configuration section. +### Inspection +To inspect Webform submissions kept in Long term storage, look for the "LTS" tab in the Webform submissions listing page. This is usually at /admin/structure/webform/submissions/manage. + ### Good to know - Each cron run copies 50 Webform submissions. If your site is getting more than that many Webform submissions between subsequent cron runs, not all Webform submissions will get copied to Long term storage during a certain period. If that happens, adjust cron run frequency. +- Files attached to Webform submissions are *not* moved to Long term storage. - Elements with Personally Identifiable Information (PII) are redacted. At the moment, this includes all name, email, telephone, number, and various address type elements. Additionally, any text or radio or checkbox element whose machine name (AKA Key) contains the following also gets redacted: name, mail, phone, contact_number, date_of_birth, dob_, personal_, title, nino, passport, postcode, address, serial_number, reg_number, pcn_, and driver_. ### Todo -- Machine names which are indicative of PII are hardcoded within the Drupal\localgov_forms_lts\PIIRedactor class at the moment. This should have a configuration UI. +- Removal of Webform submissions from Long term storage after a predefined period e.g. 5 years. +- Machine names which are indicative of PII are hardcoded within the Drupal\localgov_forms_lts\PIIRedactor class at the moment. This needs a configuration UI. - Automated tests. diff --git a/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml b/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml new file mode 100644 index 0000000..1f005ca --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml @@ -0,0 +1,20 @@ +# Submissions > LTS +entity.webform_submission.lts_collection: + title: 'LTS' + route_name: entity.webform_submission.lts_collection + parent_id: entity.webform_submission.collection + weight: 21 + +# View +entity.webform_submission.lts_view: + title: 'View' + route_name: entity.webform_submission.lts_view + base_route: entity.webform_submission.lts_view + weight: 1 + +# Notes +entity.webform_submission.lts_notes: + title: 'Notes' + route_name: entity.webform_submission.lts_notes + base_route: entity.webform_submission.lts_view + weight: 3 diff --git a/modules/localgov_forms_lts/localgov_forms_lts.routing.yml b/modules/localgov_forms_lts/localgov_forms_lts.routing.yml new file mode 100644 index 0000000..a06b574 --- /dev/null +++ b/modules/localgov_forms_lts/localgov_forms_lts.routing.yml @@ -0,0 +1,29 @@ +# List of Webform submissions from Long term storage. +entity.webform_submission.lts_collection: + path: '/admin/structure/webform/submissions/lts/{submission_view}' + defaults: + _controller: '\Drupal\localgov_forms_lts\WebformSubmissionLtsListBuilder::render' + _title: 'Webforms: Submissions' + submission_view: '' + requirements: + _custom_access: '\Drupal\webform\Access\WebformAccountAccess:checkSubmissionAccess' + +# Individual Webform submission view from Long term storage. +entity.webform_submission.lts_view: + path: '/admin/structure/webform/manage/{webform}/submission/{webform_sid}/lts' + defaults: + _controller: '\Drupal\localgov_forms_lts\Controller\WebformSubmissionLtsViewController::viewFromLts' + _title_callback: '\Drupal\localgov_forms_lts\Controller\WebformSubmissionLtsViewController::titleFromLts' + view_mode: 'html' + requirements: + _custom_access: '\Drupal\webform\Access\WebformAccountAccess:checkSubmissionAccess' + +# Individual Webform submission notes from Long term storage. +entity.webform_submission.lts_notes: + path: '/admin/structure/webform/manage/{webform}/submission/{webform_sid}/notes/lts' + defaults: + _controller: '\Drupal\localgov_forms_lts\Controller\WebformSubmissionLtsViewController::noteViewFromLts' + _title_callback: '\Drupal\localgov_forms_lts\Controller\WebformSubmissionLtsViewController::titleFromLts' + view_mode: 'html' + requirements: + _custom_access: '\Drupal\webform\Access\WebformAccountAccess:checkSubmissionAccess' diff --git a/modules/localgov_forms_lts/src/Constants.php b/modules/localgov_forms_lts/src/Constants.php index 9c041aa..b27badb 100644 --- a/modules/localgov_forms_lts/src/Constants.php +++ b/modules/localgov_forms_lts/src/Constants.php @@ -41,4 +41,14 @@ class Constants { */ const COPY_LIMIT = 50; + /** + * Cache Id prefix. + * + * For default storage, the prefix is "values". We need to differentiate this + * for LTS. + * + * @see EntityStorageBase::buildCacheId() + */ + const LTS_CACHE_ID_PREFIX = 'lts_values'; + } diff --git a/modules/localgov_forms_lts/src/Controller/WebformSubmissionLtsViewController.php b/modules/localgov_forms_lts/src/Controller/WebformSubmissionLtsViewController.php new file mode 100644 index 0000000..8f2bca0 --- /dev/null +++ b/modules/localgov_forms_lts/src/Controller/WebformSubmissionLtsViewController.php @@ -0,0 +1,71 @@ +ltsStorage->load($webform_sid); + return parent::view($webform_sub, $view_mode, $langcode); + } + + /** + * Webform submission notes callback. + * + * Loads the Webform submission from Long term storage. + */ + public function noteViewFromLts(int $webform_sid, $view_mode = 'default', $langcode = NULL) { + + $webform_sub = $this->ltsStorage->load($webform_sid); + return [ + '#markup' => '
' . $webform_sub->getNotes() . '
', + ]; + } + + /** + * Entity title callback. + * + * Loads the Webform submission from Long term storage. + */ + public function titleFromLts(int $webform_sid, $duplicate = FALSE) { + + $webform_sub = $this->ltsStorage->load($webform_sid); + return parent::title($webform_sub, $duplicate); + } + + /** + * Factory. + */ + public static function create(ContainerInterface $container) { + + $instance = parent::create($container); + + $webform_sub_entity_type = $container->get('entity_type.manager')->getDefinition('webform_submission'); + $instance->ltsStorage = LtsStorageForWebformSubmission::createInstance($container, $webform_sub_entity_type); + + return $instance; + } + + /** + * Database service for the Long term storage database. + * + * @var Drupal\webform\WebformSubmissionStorageInterface + */ + protected $ltsStorage; + +} diff --git a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php index 27d8e39..96232c3 100644 --- a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php +++ b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php @@ -5,18 +5,18 @@ use Drupal\webform\WebformSubmissionStorage; use Drupal\Core\Database\Connection as DbConnection; use Drupal\Core\Database\Database; -use Drupal\localgov_forms_lts\Constants; /** * Alternate storage class for Webform submission. * - * Saves copies of Webform submission entities in the given database instead of - * the default one. + * - Saves copies of Webform submission entities in the given database instead + * of the default one. + * - Disables persistent entity cache as LTS does not provide any. * * Usage: * @code * $lts_storage = LtsStorageForWebformSubmission::createInstance($container, $entity_type_definition); - * $lts_storage->setLtsDatabaseConnection($lts_db_connection); + * $lts_storage->setLtsDatabaseConnection($lts_db_connection); // Optional. * $a_webform_submission = $lts_storage->load($a_webform_submission_id); * @endcode */ @@ -25,7 +25,7 @@ class LtsStorageForWebformSubmission extends WebformSubmissionStorage { /** * Constructor wrapper. * - * Switches to the LTS database. + * - Switches to the LTS database. */ public function __construct(...$args) { @@ -50,6 +50,32 @@ public function getDatabaseConnection(): DbConnection { return $this->database; } + /** + * Disables persistent cache. + * + * Because we have not got any in LTS. + */ + protected function getFromPersistentCache(array &$ids = NULL) { + + return []; + } + + /** + * See above. + */ + protected function setPersistentCache($entities) {} + + /** + * Customizes cache Ids for LTS. + * + * Although we have disabled persistent cache above, cache ids are still used + * in static cache. + */ + protected function buildCacheId($id) { + + return Constants::LTS_CACHE_ID_PREFIX . ":{$this->entityTypeId}:$id"; + } + /** * {@inheritdoc} * diff --git a/modules/localgov_forms_lts/src/PIIRedactor.php b/modules/localgov_forms_lts/src/PIIRedactor.php index 49f1193..2c7f183 100644 --- a/modules/localgov_forms_lts/src/PIIRedactor.php +++ b/modules/localgov_forms_lts/src/PIIRedactor.php @@ -56,7 +56,7 @@ public static function findElemsToRedact(WebformSubmissionInterface $webform_sub $potential_mapping = array_intersect($elem_type_mapping, static::POTENTIAL_PII_ELEMENT_TYPES); $guessed_pii_elems = preg_grep(static::GUESSED_PII_ELEM_PATTERN, array_keys($potential_mapping)); - $elems_to_redact = $pii_elems + $guessed_pii_elems; + $elems_to_redact = [...$pii_elems, ...$guessed_pii_elems]; return $elems_to_redact; } diff --git a/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php b/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php new file mode 100644 index 0000000..e0096b9 --- /dev/null +++ b/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php @@ -0,0 +1,82 @@ +id(); + $webform_id = $webform_submission->getWebform()->id(); + + $ops = parent::getDefaultOperations($entity); + + $lts_ops = [ + 'view' => $ops['view'] ?? [], + 'notes' => $ops['notes'] ?? [], + ]; + $lts_ops['view']['url'] = Url::fromRoute('entity.webform_submission.lts_view', [ + 'webform' => $webform_id, + 'webform_sid' => $webform_submission_id, + ]); + $lts_ops['notes']['url'] = Url::fromRoute('entity.webform_submission.lts_notes', [ + 'webform' => $webform_id, + 'webform_sid' => $webform_submission_id, + ]); + + return $lts_ops; + } + + /** + * {@inheritdoc} + * + * Tells the list builder to use our Webform submissions LTS storage. + */ + public static function createInstance(ContainerInterface $container, EntityTypeInterface $entity_type) { + /** @var \Drupal\webform\WebformSubmissionLtsListBuilder $instance */ + $instance = parent::createInstance($container, $entity_type); + + $lts_storage = LtsStorageForWebformSubmission::createInstance($container, $entity_type); + $instance->storage = $lts_storage; + $instance->initialize(); + $instance->columns = $instance->storage->getSubmissionsColumns(); + + return $instance; + } + + /** + * {@inheritdoc} + */ + public static function create(ContainerInterface $container) { + + $webform_sub_def = $container->get('entity_type.manager')->getDefinition('webform_submission'); + return self::createInstance($container, $webform_sub_def); + } + +} From f25924adcb0568ad08fdfbbf66f1d7a3e0109f47 Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 31 May 2024 16:36:09 +0100 Subject: [PATCH 07/22] Fix: Long term storage. Fix for Personally Identifiable Information redaction. Date elements from this module were unaccounted for. --- modules/localgov_forms_lts/src/PIIRedactor.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/localgov_forms_lts/src/PIIRedactor.php b/modules/localgov_forms_lts/src/PIIRedactor.php index 2c7f183..42971d9 100644 --- a/modules/localgov_forms_lts/src/PIIRedactor.php +++ b/modules/localgov_forms_lts/src/PIIRedactor.php @@ -94,6 +94,7 @@ public static function addRedactionNote(WebformSubmissionInterface $webform_sub, const PII_ELEMENT_TYPES = [ 'address', 'email', + 'localgov_forms_dob', 'localgov_webform_uk_address', 'number', 'tel', @@ -107,10 +108,11 @@ public static function addRedactionNote(WebformSubmissionInterface $webform_sub, * Element types that *may* carry PII. */ const POTENTIAL_PII_ELEMENT_TYPES = [ - 'textfield', - 'processed_text', + 'localgov_forms_date', 'checkboxes', + 'processed_text', 'radios', + 'textfield', ]; /** From 1c18230dc1391972f915235b49f9bcd80d8b38d5 Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 16 Aug 2024 18:23:59 +0100 Subject: [PATCH 08/22] Feat: Textarea PII redaction. Spots and redacts any email address, postcode, and number within the text value of a textarea field. --- .../localgov_forms_lts/src/PIIRedactor.php | 49 ++++++++-- .../src/PIIRedactorForText.php | 92 +++++++++++++++++++ .../tests/src/Unit/PIIRedactorForTextTest.php | 33 +++++++ 3 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 modules/localgov_forms_lts/src/PIIRedactorForText.php create mode 100644 modules/localgov_forms_lts/tests/src/Unit/PIIRedactorForTextTest.php diff --git a/modules/localgov_forms_lts/src/PIIRedactor.php b/modules/localgov_forms_lts/src/PIIRedactor.php index 42971d9..b689954 100644 --- a/modules/localgov_forms_lts/src/PIIRedactor.php +++ b/modules/localgov_forms_lts/src/PIIRedactor.php @@ -15,6 +15,7 @@ * - All Webform elements of type email, telephone, number. * - Any element with the following happening in its machine id: name, mail, * phone, date_of_birth, personal, title, gender, sex, ethnicity. + * - All textareas are cleaned of email, postcode, and any number. */ class PIIRedactor { @@ -36,16 +37,35 @@ public static function redact(WebformSubmissionInterface $webform_sub) :array { return $elem; } - }, $elems_to_redact); + }, $elems_to_redact['full']); + + $partial_redaction_result = array_map(function ($elem) use ($webform_sub) { + if ($text = $webform_sub->getElementData($elem)) { + [$redacted_text, $redaction_count] = PIIRedactorForText::redact($text); + + if ($redaction_count) { + $webform_sub->setElementData($elem, $redacted_text); + return $elem; + } + } + }, $elems_to_redact['part']); $redacted_elems = array_filter($redaction_result); + $partly_redacted_elems = array_filter($partial_redaction_result); static::addRedactionNote($webform_sub, $redacted_elems); + static::addRedactionNote($webform_sub, $partly_redacted_elems, note_prefix: 'Partly redacted elements: '); - return $redacted_elems; + $all_redacted_elems = [...$redacted_elems, ...$partly_redacted_elems]; + return $all_redacted_elems; } /** * Finds the Webform element names to redact. + * + * The result array contains two keys: + * - full: These elements are to be fully redacted. + * - part: The values of these elements contain PII among other text and are + * to be partly redacted. */ public static function findElemsToRedact(WebformSubmissionInterface $webform_sub) :array { @@ -56,7 +76,12 @@ public static function findElemsToRedact(WebformSubmissionInterface $webform_sub $potential_mapping = array_intersect($elem_type_mapping, static::POTENTIAL_PII_ELEMENT_TYPES); $guessed_pii_elems = preg_grep(static::GUESSED_PII_ELEM_PATTERN, array_keys($potential_mapping)); - $elems_to_redact = [...$pii_elems, ...$guessed_pii_elems]; + $elems_w_some_pii = array_keys(array_intersect($elem_type_mapping, static::PII_ELEMENT_TYPES_TO_REDUCT_IN_PART)); + + $elems_to_redact = [ + 'full' => [...$pii_elems, ...$guessed_pii_elems], + 'part' => $elems_w_some_pii, + ]; return $elems_to_redact; } @@ -74,18 +99,17 @@ public static function listElemsAndTypes(WebformSubmissionInterface $webform_sub * * Adds a note to the Webform submission to highlight the redacted elements. */ - public static function addRedactionNote(WebformSubmissionInterface $webform_sub, array $redacted_elems) :void { + public static function addRedactionNote(WebformSubmissionInterface $webform_sub, array $redacted_elems, string $note_prefix = 'Redacted elements: '): void { if (empty($redacted_elems)) { return; } - $redaction_note = 'Redacted elements: ' . implode(', ', $redacted_elems) . '.'; + $redaction_note = $note_prefix . implode(', ', $redacted_elems) . '.'; + $existing_note = $webform_sub->getNotes(); - $existing_note = $webform_sub->getNotes(); - $updated_note = $existing_note . PHP_EOL . $redaction_note; - - $webform_sub->setNotes($updated_note); + $note_list = array_filter([$existing_note, $redaction_note]); + $webform_sub->setNotes(implode(PHP_EOL, $note_list)); } /** @@ -115,6 +139,13 @@ public static function addRedactionNote(WebformSubmissionInterface $webform_sub, 'textfield', ]; + /** + * Element types with PII mixed with other text. + */ + const PII_ELEMENT_TYPES_TO_REDUCT_IN_PART = [ + 'textarea', + ]; + /** * Preg pattern. * diff --git a/modules/localgov_forms_lts/src/PIIRedactorForText.php b/modules/localgov_forms_lts/src/PIIRedactorForText.php new file mode 100644 index 0000000..3c214fa --- /dev/null +++ b/modules/localgov_forms_lts/src/PIIRedactorForText.php @@ -0,0 +1,92 @@ +assertEquals($redaction_count, 8); + + $nonredactable_text = 'preg_replace() performs a regex search and replace.'; + [, $redaction_count] = PIIRedactorForText::redact($nonredactable_text); + + $this->assertEquals($redaction_count, 0); + } + +} From e5071327c314e6f3acc20a1c66173d491baab568 Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 20 Sep 2024 18:20:27 +0100 Subject: [PATCH 09/22] Test: Automated tests. Tests following functionalities: - LtsCopy's ability to copy both new and existing Webform submissions. - LtsStorageForWebformSubmission's ability to copy into the LTS database. - PIIRedactor's ability to correctly identify which Webform fields should be redacted. --- modules/localgov_forms_lts/README.md | 3 +- .../localgov_forms_lts.info.yml | 3 +- modules/localgov_forms_lts/src/LtsCopy.php | 17 +- .../src/LtsStorageForWebformSubmission.php | 2 +- .../localgov_forms_lts/src/PIIRedactor.php | 4 +- .../LtsStorageForWebformSubmissionTest.php | 119 +++++++++++++ .../tests/src/Unit/LtsCopyTest.php | 161 ++++++++++++++++++ .../tests/src/Unit/PIIRedactorTest.php | 57 +++++++ .../Driver/Database/FakeLts/Connection.php | 29 ++++ 9 files changed, 381 insertions(+), 14 deletions(-) create mode 100644 modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php create mode 100644 modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php create mode 100644 modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php create mode 100644 tests/modules/localgov_forms_test/src/Driver/Database/FakeLts/Connection.php diff --git a/modules/localgov_forms_lts/README.md b/modules/localgov_forms_lts/README.md index 5270824..9eb2544 100644 --- a/modules/localgov_forms_lts/README.md +++ b/modules/localgov_forms_lts/README.md @@ -27,8 +27,9 @@ To inspect Webform submissions kept in Long term storage, look for the "LTS" tab - Each cron run copies 50 Webform submissions. If your site is getting more than that many Webform submissions between subsequent cron runs, not all Webform submissions will get copied to Long term storage during a certain period. If that happens, adjust cron run frequency. - Files attached to Webform submissions are *not* moved to Long term storage. - Elements with Personally Identifiable Information (PII) are redacted. At the moment, this includes all name, email, telephone, number, and various address type elements. Additionally, any text or radio or checkbox element whose machine name (AKA Key) contains the following also gets redacted: name, mail, phone, contact_number, date_of_birth, dob_, personal_, title, nino, passport, postcode, address, serial_number, reg_number, pcn_, and driver_. +- This module is currently in experimental stage. +- If you are using this module in multiple instances of the same site (e.g. dev/stage/live), ensure that the database settings array points to *different* databases. ### Todo - Removal of Webform submissions from Long term storage after a predefined period e.g. 5 years. - Machine names which are indicative of PII are hardcoded within the Drupal\localgov_forms_lts\PIIRedactor class at the moment. This needs a configuration UI. -- Automated tests. diff --git a/modules/localgov_forms_lts/localgov_forms_lts.info.yml b/modules/localgov_forms_lts/localgov_forms_lts.info.yml index 68f94cc..efcf787 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.info.yml +++ b/modules/localgov_forms_lts/localgov_forms_lts.info.yml @@ -1,9 +1,10 @@ name: LocalGov Forms long term storage type: module description: Long term storage for Webform submissions. -core_version_requirement: ^10 +core_version_requirement: ^10 || ^11 php: 8.0 package: LocalGov Drupal +lifecycle: experimental dependencies: - webform:webform diff --git a/modules/localgov_forms_lts/src/LtsCopy.php b/modules/localgov_forms_lts/src/LtsCopy.php index 3c39a97..af777d7 100644 --- a/modules/localgov_forms_lts/src/LtsCopy.php +++ b/modules/localgov_forms_lts/src/LtsCopy.php @@ -51,7 +51,7 @@ public function copy(int $count = Constants::COPY_LIMIT) :array { */ public function copySub(int $webform_sub_id, bool $is_new_webform_sub) :bool { - $webform_sub = WebformSubmission::load($webform_sub_id); + $webform_sub = $this->webformSubStorage->load($webform_sub_id); PIIRedactor::redact($webform_sub); $db_connection = $this->ltsStorage->getDatabaseConnection(); @@ -104,8 +104,7 @@ public function findCopyTargets(int $count = -1) :array { $last_copied_webform_sub_changed_ts = $this->findLatestUpdateTimestamp(); - $webform_subs_to_copy = $this->entityTypeManager - ->getStorage('webform_submission') + $webform_subs_to_copy_query = $this->webformSubStorage ->getQuery() ->accessCheck(FALSE) ->condition('changed', $last_copied_webform_sub_changed_ts, '>') @@ -113,10 +112,10 @@ public function findCopyTargets(int $count = -1) :array { ->sort('changed'); if ($count > -1) { - $webform_subs_to_copy->range(start: 0, length: $count); + $webform_subs_to_copy_query->range(start: 0, length: $count); } - $copy_targets = $webform_subs_to_copy->execute(); + $copy_targets = $webform_subs_to_copy_query->execute(); return $copy_targets; } @@ -135,7 +134,7 @@ public function findLatestUpdateTimestamp() :int { public function setLatestUpdateTimestamp(array $copy_results) :void { $last_copied_webform_sub_id = array_key_last($copy_results); - $last_copied_webform_sub = WebformSubmission::load($last_copied_webform_sub_id); + $last_copied_webform_sub = $this->webformSubStorage->load($last_copied_webform_sub_id); $this->ltsKeyValueStore->set(Constants::LAST_CHANGE_TIMESTAMP, $last_copied_webform_sub->getChangedTime()); } @@ -146,7 +145,7 @@ public function setLatestUpdateTimestamp(array $copy_results) :void { */ public function __construct(EntityTypeManagerInterface $entity_type_manager, KeyValueFactoryInterface $key_value_factory, LoggerChannelFactoryInterface $logger_factory, WebformSubmissionStorageInterface $lts_storage) { - $this->entityTypeManager = $entity_type_manager; + $this->webformSubStorage = $entity_type_manager->getStorage('webform_submission'); $this->ltsStorage = $lts_storage; $this->ltsKeyValueStore = $key_value_factory->get(Constants::LTS_KEYVALUE_STORE_ID); $this->ltsLogger = $logger_factory->get(Constants::LTS_LOGGER_CHANNEL_ID); @@ -177,9 +176,9 @@ public static function create(ContainerInterface $container) :LtsCopy { /** * Entity type manager service. * - * @var Drupal\Core\Entity\EntityTypeManagerInterface + * @var Drupal\webform\WebformSubmissionStorageInterface */ - protected $entityTypeManager; + protected $webformSubStorage; /** * Database service for the Long term storage database. diff --git a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php index 96232c3..107810d 100644 --- a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php +++ b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php @@ -53,7 +53,7 @@ public function getDatabaseConnection(): DbConnection { /** * Disables persistent cache. * - * Because we have not got any in LTS. + * Because we do not have any in LTS. */ protected function getFromPersistentCache(array &$ids = NULL) { diff --git a/modules/localgov_forms_lts/src/PIIRedactor.php b/modules/localgov_forms_lts/src/PIIRedactor.php index b689954..8e8aa16 100644 --- a/modules/localgov_forms_lts/src/PIIRedactor.php +++ b/modules/localgov_forms_lts/src/PIIRedactor.php @@ -79,7 +79,7 @@ public static function findElemsToRedact(WebformSubmissionInterface $webform_sub $elems_w_some_pii = array_keys(array_intersect($elem_type_mapping, static::PII_ELEMENT_TYPES_TO_REDUCT_IN_PART)); $elems_to_redact = [ - 'full' => [...$pii_elems, ...$guessed_pii_elems], + 'full' => array_unique([...$pii_elems, ...$guessed_pii_elems]), 'part' => $elems_w_some_pii, ]; return $elems_to_redact; @@ -151,6 +151,6 @@ public static function addRedactionNote(WebformSubmissionInterface $webform_sub, * * Element type naming pattern indicating possible link with PII. */ - const GUESSED_PII_ELEM_PATTERN = '#name|mail|phone|contact_number|date_of_birth|dob_|nino|address|postcode|post_code|personal_|title|passport|serial_number|reg_number|pcn_|driver_#i'; + const GUESSED_PII_ELEM_PATTERN = '#name|mail|phone|contact_number|date_of_birth|dob_|nino|address|postcode|post_code|personal_|title|gender|sex|ethnicity|passport|serial_number|reg_number|pcn_|driver_#i'; } diff --git a/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php b/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php new file mode 100644 index 0000000..5c321c1 --- /dev/null +++ b/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php @@ -0,0 +1,119 @@ +container->get('entity_type.manager')->getStorage('webform')->load(self::TEST_WEBFORM_ID); + $this->assertNotNull($contact2_webform); + + $a_webform_submission = WebformSubmission::create([ + 'webform_id' => self::TEST_WEBFORM_ID, + 'data' => [ + 'name' => 'Foo Bar', + 'email' => 'foo@example.net', + ], + ]); + + // Temporary measure to satisfy + // LtsStorageForWebformSubmission::__construct(). + // Gets overwritten by the call to $test_obj->setDatabaseConnection() below. + Database::addConnectionInfo(Constants::LTS_DB_KEY, 'default', [ + 'driver' => 'fake_lts', + 'namespace' => 'Drupal\\localgov_forms_test\\Driver\\Database\\FakeLts', + ]); + $webform_sub_def = $this->container->get('entity_type.manager')->getDefinition('webform_submission'); + $test_obj = LtsStorageForWebformSubmission::createInstance($this->container, $webform_sub_def); + $test_obj->setDatabaseConnection($this->mockLtsDbConnection); + + $test_obj->resave($a_webform_submission); + } + + /** + * Prepares the mock LTS database connection. + */ + protected function setUp(): void { + + parent::setUp(); + + $this->installSchema('webform', ['webform']); + $this->installConfig(['webform']); + $this->installEntitySchema('user'); + + $mock_insert_query = $this->createConfiguredMock(Insert::class, [ + 'execute' => random_int(1, 10), + ]); + $mock_insert_query->method('fields')->willReturnSelf(); + + $this->mockLtsDbConnection = $this->createMock(DbConnection::class); + $this->mockLtsDbConnection->expects($this->exactly(self::WEBFORM_SUB_LTS_INSERT_COUNT)) + ->method('insert') + ->willReturnMap([ + ['webform_submission', $mock_insert_query], + ['webform_submission', [ + 'return' => Database::RETURN_INSERT_ID, + ], $mock_insert_query, + ], + ['webform_submission_data', $mock_insert_query], + ['webform_submission_data', [], $mock_insert_query], + ]); + } + + /** + * A new Webform submission is inserted into the LTS database twice. + * + * Once in the webform_submission table and then in the + * webform_submission_data table. + */ + const WEBFORM_SUB_LTS_INSERT_COUNT = 2; + + /** + * A Webform from the localgov_forms_test module. + */ + const TEST_WEBFORM_ID = 'contact'; + + /** + * Modules to enable. + * + * @var array + */ + protected static $modules = [ + 'localgov_forms_test', + 'system', + 'user', + 'webform', + ]; + + /** + * Mock LTS database connection. + * + * Is it being used at all? That's what this test is about. + * + * @var Drupal\Core\Database\Connection + */ + protected $mockLtsDbConnection; + +} diff --git a/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php b/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php new file mode 100644 index 0000000..66971ad --- /dev/null +++ b/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php @@ -0,0 +1,161 @@ +mockEntityTypeManager, $this->mockLtsKeyValueFactory, $this->mockLtsLoggerFactory, $this->mockLtsStorage); + + $copy_results = $test_obj->copy(); + + $this->assertCount(expectedCount: self::WEBFORM_SUB_EXPECTED_COPY_COUNT, haystack: $copy_results); + } + + /** + * Creates mock dependencies. + * + * Initializes all objects needed to create an LtsCopy object. + */ + public function setup(): void { + + parent::setup(); + + $mock_webform_sub_storage = static::setupMockWebformSubmissionStorage(); + $this->mockEntityTypeManager = $this->createConfiguredMock(EntityTypeManagerInterface::class, [ + 'getStorage' => $mock_webform_sub_storage, + ]); + + $mock_lts_storage_query = $this->createMock(QueryAggregateInterface::class); + $mock_lts_storage_query->method('execute') + ->willReturn([['sid_max' => self::LAST_COPIED_WEBFORM_SUB_ID]]); + $mock_lts_storage_query->method('accessCheck')->willReturnSelf(); + $mock_lts_storage_query->method('aggregate')->willReturnSelf(); + $this->mockLtsStorage = $this->createConfiguredMock(LtsStorageForWebformSubmission::class, [ + 'getAggregateQuery' => $mock_lts_storage_query, + 'getDatabaseConnection' => $this->createMock(DbConnection::class), + ]); + $this->mockLtsStorage->expects($this->exactly(self::WEBFORM_SUB_EXPECTED_COPY_COUNT))->method('resave'); + + $this->mockLtsKeyValueFactory = $this->createConfiguredMock(KeyValueFactoryInterface::class, [ + 'get' => $this->createMock(KeyValueStoreInterface::class), + ]); + + $this->mockLtsLoggerFactory = $this->createMock(LoggerChannelFactoryInterface::class); + } + + /** + * Prepares a mock Webform submission storage object. + * + * When queried, returns three mock Webform submission entities. + */ + public function setupMockWebformSubmissionStorage(): WebformSubmissionStorageInterface { + $mock_webform_sub_query = $this->createMock(QueryInterface::class); + $mock_webform_sub_query->expects($this->any()) + ->method('execute') + ->willReturn([ + self::LAST_COPIED_WEBFORM_SUB_ID => (string) self::LAST_COPIED_WEBFORM_SUB_ID, + self::NEW_WEBFORM_SUB_ID0 => (string) self::NEW_WEBFORM_SUB_ID0 , + self::NEW_WEBFORM_SUB_ID1 => (string) self::NEW_WEBFORM_SUB_ID1 , + ]); + $mock_webform_sub_query->method('accessCheck')->willReturnSelf(); + $mock_webform_sub_query->method('condition')->willReturnSelf(); + $mock_webform_sub_query->method('sort')->willReturnSelf(); + $mock_webform_sub_storage = $this->createConfiguredMock(WebformSubmissionStorageInterface::class, [ + 'getQuery' => $mock_webform_sub_query, + ]); + + $mock_webform = $this->createConfiguredMock(WebformInterface::class, [ + 'getElementsDecodedAndFlattened' => [], + ]); + $mock_existing_webform_sub = $this->createConfiguredMock(WebformSubmissionInterface::class, [ + 'id' => self::LAST_COPIED_WEBFORM_SUB_ID, + 'getWebform' => $mock_webform, + ]); + $mock_new_webform_sub0 = $this->createConfiguredMock(WebformSubmissionInterface::class, [ + 'id' => self::NEW_WEBFORM_SUB_ID0, + 'getWebform' => $mock_webform, + ]); + $mock_new_webform_sub0->expects($this->once())->method('enforceIsNew')->willReturnSelf(); + $mock_new_webform_sub1 = $this->createConfiguredMock(WebformSubmissionInterface::class, [ + 'id' => self::NEW_WEBFORM_SUB_ID1, + 'getWebform' => $mock_webform, + ]); + $mock_new_webform_sub1->expects($this->once())->method('enforceIsNew')->willReturnSelf(); + $mock_webform_sub_storage->expects($this->exactly(self::WEBFORM_SUB_EXPECTED_LOAD_COUNT)) + ->method('load') + ->willReturnMap([ + [self::LAST_COPIED_WEBFORM_SUB_ID, $mock_existing_webform_sub], + [self::NEW_WEBFORM_SUB_ID0, $mock_new_webform_sub0], + [self::NEW_WEBFORM_SUB_ID1, $mock_new_webform_sub1], + ]); + + return $mock_webform_sub_storage; + } + + const LAST_COPIED_WEBFORM_SUB_ID = 99; + + const NEW_WEBFORM_SUB_ID0 = 100; + + const NEW_WEBFORM_SUB_ID1 = 101; + + const WEBFORM_SUB_EXPECTED_COPY_COUNT = 3; + + const WEBFORM_SUB_EXPECTED_LOAD_COUNT = 4; + + /** + * Mock KeyValue factory. + * + * @var Drupal\Core\KeyValueStore\KeyValueFactoryInterface + */ + protected $mockLtsKeyValueFactory; + + /** + * Mock EntityTypeManager. + * + * @var Drupal\Core\Entity\EntityTypeManagerInterface + */ + protected $mockEntityTypeManager; + + /** + * Mock Long term webform submission storage. + * + * @var Drupal\webform\WebformSubmissionStorageInterface + */ + protected $mockLtsStorage; + + /** + * Mock logger. + * + * @var Drupal\Core\Logger\LoggerChannelFactoryInterface + */ + protected $mockLtsLoggerFactory; + +} diff --git a/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php b/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php new file mode 100644 index 0000000..1ff522e --- /dev/null +++ b/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php @@ -0,0 +1,57 @@ +createConfiguredMock(WebformInterface::class, [ + 'getElementsDecodedAndFlattened' => [ + 'name' => ['#type' => 'textfield'], + 'email' => ['#type' => 'email'], + 'subject' => ['#type' => 'textfield'], + 'message' => ['#type' => 'textarea'], + 'work_number' => ['#type' => 'tel'], + 'nino' => ['#type' => 'textfield'], + 'location' => ['#type' => 'address'], + 'cars' => ['#type' => 'number'], + 'gender' => ['#type' => 'radios'], + 'ethnicity' => ['#type' => 'checkboxes'], + 'date_of_birth' => ['#type' => 'localgov_forms_date'], + ], + ]); + $mock_webform_sub = $this->createConfiguredMock(WebformSubmissionInterface::class, [ + 'getWebform' => $mock_webform, + ]); + + $elems_to_redact = PIIRedactor::findElemsToRedact($mock_webform_sub); + + $this->assertSame([ + 'email', + 'work_number', + 'location', + 'cars', + 'name', + 'nino', + 'gender', + 'ethnicity', + 'date_of_birth', + ], $elems_to_redact['full']); + $this->assertSame(['message'], $elems_to_redact['part']); + } + +} diff --git a/tests/modules/localgov_forms_test/src/Driver/Database/FakeLts/Connection.php b/tests/modules/localgov_forms_test/src/Driver/Database/FakeLts/Connection.php new file mode 100644 index 0000000..0f4cff0 --- /dev/null +++ b/tests/modules/localgov_forms_test/src/Driver/Database/FakeLts/Connection.php @@ -0,0 +1,29 @@ + Date: Fri, 20 Sep 2024 19:15:02 +0100 Subject: [PATCH 10/22] Chore: Coding standard fixes. --- modules/localgov_forms_lts/src/LtsCopy.php | 5 ++--- .../src/LtsStorageForWebformSubmission.php | 2 +- .../src/WebformSubmissionLtsListBuilder.php | 2 +- .../Kernel/LtsStorageForWebformSubmissionTest.php | 2 +- .../tests/src/Unit/LtsCopyTest.php | 14 +++++++------- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/modules/localgov_forms_lts/src/LtsCopy.php b/modules/localgov_forms_lts/src/LtsCopy.php index af777d7..373ef1f 100644 --- a/modules/localgov_forms_lts/src/LtsCopy.php +++ b/modules/localgov_forms_lts/src/LtsCopy.php @@ -4,12 +4,11 @@ namespace Drupal\localgov_forms_lts; -use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\DependencyInjection\ContainerInjectionInterface; +use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\KeyValueStore\KeyValueFactoryInterface; use Drupal\Core\Logger\LoggerChannelFactoryInterface; use Drupal\webform\WebformSubmissionStorageInterface; -use Drupal\webform\Entity\WebformSubmission; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -65,7 +64,7 @@ public function copySub(int $webform_sub_id, bool $is_new_webform_sub) :bool { $this->ltsStorage->resave($webform_sub); } } - catch (Exception $e) { + catch (\Exception $e) { $tx->rollBack(); $this->ltsLogger->error('Failed to add/edit Webform submission: %sub-id', [ diff --git a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php index 107810d..66000b6 100644 --- a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php +++ b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php @@ -2,9 +2,9 @@ namespace Drupal\localgov_forms_lts; -use Drupal\webform\WebformSubmissionStorage; use Drupal\Core\Database\Connection as DbConnection; use Drupal\Core\Database\Database; +use Drupal\webform\WebformSubmissionStorage; /** * Alternate storage class for Webform submission. diff --git a/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php b/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php index e0096b9..9466217 100644 --- a/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php +++ b/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php @@ -5,8 +5,8 @@ namespace Drupal\localgov_forms_lts; use Drupal\Core\DependencyInjection\ContainerInjectionInterface; -use Drupal\Core\Entity\EntityTypeInterface; use Drupal\Core\Entity\EntityInterface; +use Drupal\Core\Entity\EntityTypeInterface; use Drupal\Core\Url; use Drupal\webform\WebformSubmissionListBuilder; use Drupal\webform\WebformSubmissionInterface; diff --git a/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php b/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php index 5c321c1..532ae19 100644 --- a/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php +++ b/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php @@ -2,8 +2,8 @@ namespace Drupal\Tests\localgov_forms_lts\Kernel; -use Drupal\Core\Database\Database; use Drupal\Core\Database\Connection as DbConnection; +use Drupal\Core\Database\Database; use Drupal\Core\Database\Query\Insert; use Drupal\KernelTests\KernelTestBase; use Drupal\localgov_forms_lts\LtsStorageForWebformSubmission; diff --git a/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php b/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php index 66971ad..ff958ac 100644 --- a/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php +++ b/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php @@ -36,7 +36,7 @@ public function testCopySub() { $copy_results = $test_obj->copy(); - $this->assertCount(expectedCount: self::WEBFORM_SUB_EXPECTED_COPY_COUNT, haystack: $copy_results); + $this->assertCount(self::WEBFORM_SUB_EXPECTED_COPY_COUNT, $copy_results); } /** @@ -44,9 +44,9 @@ public function testCopySub() { * * Initializes all objects needed to create an LtsCopy object. */ - public function setup(): void { + public function setUp(): void { - parent::setup(); + parent::setUp(); $mock_webform_sub_storage = static::setupMockWebformSubmissionStorage(); $this->mockEntityTypeManager = $this->createConfiguredMock(EntityTypeManagerInterface::class, [ @@ -133,28 +133,28 @@ public function setupMockWebformSubmissionStorage(): WebformSubmissionStorageInt /** * Mock KeyValue factory. * - * @var Drupal\Core\KeyValueStore\KeyValueFactoryInterface + * @var \Drupal\Core\KeyValueStore\KeyValueFactoryInterface */ protected $mockLtsKeyValueFactory; /** * Mock EntityTypeManager. * - * @var Drupal\Core\Entity\EntityTypeManagerInterface + * @var \Drupal\Core\Entity\EntityTypeManagerInterface */ protected $mockEntityTypeManager; /** * Mock Long term webform submission storage. * - * @var Drupal\webform\WebformSubmissionStorageInterface + * @var \Drupal\webform\WebformSubmissionStorageInterface */ protected $mockLtsStorage; /** * Mock logger. * - * @var Drupal\Core\Logger\LoggerChannelFactoryInterface + * @var \Drupal\Core\Logger\LoggerChannelFactoryInterface */ protected $mockLtsLoggerFactory; From ddebdbc3544d454ea6e6e3e190605a6d6139afbb Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 20 Sep 2024 20:09:06 +0100 Subject: [PATCH 11/22] Chore: More coding standard fixes. --- modules/localgov_forms_lts/localgov_forms_lts.install | 5 +++-- .../src/WebformSubmissionLtsListBuilder.php | 2 +- .../src/Kernel/LtsStorageForWebformSubmissionTest.php | 9 +++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/localgov_forms_lts/localgov_forms_lts.install b/modules/localgov_forms_lts/localgov_forms_lts.install index 4dc0f1b..5c4c621 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.install +++ b/modules/localgov_forms_lts/localgov_forms_lts.install @@ -67,8 +67,9 @@ function localgov_forms_lts_requirements($phase) { */ function _localgov_forms_lts_get_webform_submission_storage_schema(): array { - $entity_type_manager = Drupal::service('entity_type.manager'); - $entity_storage = $entity_type_manager->getStorage('webform_submission'); + $entity_type_manager = Drupal::service('entity_type.manager'); + $entity_storage = $entity_type_manager->getStorage('webform_submission'); + // @phpstan-ignore-next-line $entity_type = $entity_storage->getEntityType(); $entity_field_manager = Drupal::service('entity_field.manager'); $db_service = Drupal::service('database'); diff --git a/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php b/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php index 9466217..4192694 100644 --- a/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php +++ b/modules/localgov_forms_lts/src/WebformSubmissionLtsListBuilder.php @@ -8,8 +8,8 @@ use Drupal\Core\Entity\EntityInterface; use Drupal\Core\Entity\EntityTypeInterface; use Drupal\Core\Url; -use Drupal\webform\WebformSubmissionListBuilder; use Drupal\webform\WebformSubmissionInterface; +use Drupal\webform\WebformSubmissionListBuilder; use Symfony\Component\DependencyInjection\ContainerInterface; /** diff --git a/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php b/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php index 532ae19..ee106a3 100644 --- a/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php +++ b/modules/localgov_forms_lts/tests/src/Kernel/LtsStorageForWebformSubmissionTest.php @@ -6,8 +6,8 @@ use Drupal\Core\Database\Database; use Drupal\Core\Database\Query\Insert; use Drupal\KernelTests\KernelTestBase; -use Drupal\localgov_forms_lts\LtsStorageForWebformSubmission; use Drupal\localgov_forms_lts\Constants; +use Drupal\localgov_forms_lts\LtsStorageForWebformSubmission; use Drupal\webform\Entity\WebformSubmission; /** @@ -74,7 +74,7 @@ protected function setUp(): void { ->willReturnMap([ ['webform_submission', $mock_insert_query], ['webform_submission', [ - 'return' => Database::RETURN_INSERT_ID, + 'return' => self::DEPRECATED_D10_RETURN_INSERT_ID, ], $mock_insert_query, ], ['webform_submission_data', $mock_insert_query], @@ -95,6 +95,11 @@ protected function setUp(): void { */ const TEST_WEBFORM_ID = 'contact'; + /** + * Mirrors the deprecated Database::RETURN_INSERT_ID. + */ + const DEPRECATED_D10_RETURN_INSERT_ID = 3; + /** * Modules to enable. * From aedf44089a7771d28683c59e1ee057a25f099600 Mon Sep 17 00:00:00 2001 From: Adnan Date: Sat, 21 Sep 2024 10:30:45 +0100 Subject: [PATCH 12/22] Chore: More coding standard fixes. PHPStan is incorrectly inferring the type of the $entity_type object used within _localgov_forms_lts_get_webform_submission_storage_schema(). Inferred type is Drupal\Core\Entity\EntityTypeInterface. Expected type is Drupal\Core\Entity\ContentEntityTypeInterface. Actual type is Drupal\Core\Entity\ContentEntityType which is an instance of Drupal\Core\Entity\ContentEntityTypeInterface. --- modules/localgov_forms_lts/localgov_forms_lts.install | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/localgov_forms_lts/localgov_forms_lts.install b/modules/localgov_forms_lts/localgov_forms_lts.install index 5c4c621..6994c61 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.install +++ b/modules/localgov_forms_lts/localgov_forms_lts.install @@ -67,13 +67,13 @@ function localgov_forms_lts_requirements($phase) { */ function _localgov_forms_lts_get_webform_submission_storage_schema(): array { - $entity_type_manager = Drupal::service('entity_type.manager'); - $entity_storage = $entity_type_manager->getStorage('webform_submission'); - // @phpstan-ignore-next-line + $entity_type_manager = Drupal::service('entity_type.manager'); + $entity_storage = $entity_type_manager->getStorage('webform_submission'); $entity_type = $entity_storage->getEntityType(); $entity_field_manager = Drupal::service('entity_field.manager'); $db_service = Drupal::service('database'); + // @phpstan-ignore-next-line Avoid incorrect type inference of $entity_type. $entity_schema = (new class($entity_type_manager, $entity_type, $entity_storage, $db_service, $entity_field_manager) extends WebformSubmissionStorageSchema { /** From a431c1c0df294f16b15df1c0c9c888732edbbcbe Mon Sep 17 00:00:00 2001 From: Adnan Date: Sat, 21 Sep 2024 10:55:19 +0100 Subject: [PATCH 13/22] Doc: Better docblock comments for mock database driver. --- .../src/Driver/Database/FakeLts/Connection.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/modules/localgov_forms_test/src/Driver/Database/FakeLts/Connection.php b/tests/modules/localgov_forms_test/src/Driver/Database/FakeLts/Connection.php index 0f4cff0..724d325 100644 --- a/tests/modules/localgov_forms_test/src/Driver/Database/FakeLts/Connection.php +++ b/tests/modules/localgov_forms_test/src/Driver/Database/FakeLts/Connection.php @@ -8,9 +8,12 @@ use Drupal\Tests\Core\Database\Stub\StubPDO; /** - * A stub of the abstract Connection class for testing purposes. + * A mock Drupal database driver class. * - * Includes minimal implementations of Connection's abstract methods. + * Useful during testing. + * + * Good enough to serve as a database connection object but cannot actually + * perform any query operation yet. */ class Connection extends StubConnection { From 5301517aa1cee8c35fe28f72e86cb3f66d2e822d Mon Sep 17 00:00:00 2001 From: Finn Lewis Date: Thu, 17 Oct 2024 10:35:16 +0100 Subject: [PATCH 14/22] Add missing comma. --- modules/localgov_forms_lts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/localgov_forms_lts/README.md b/modules/localgov_forms_lts/README.md index 9eb2544..a3b8d91 100644 --- a/modules/localgov_forms_lts/README.md +++ b/modules/localgov_forms_lts/README.md @@ -6,7 +6,7 @@ ``` $databases['localgov_forms_lts']['default'] = [ 'database' => 'our_longer_term_storage_database', - 'username' => 'database_username_goes_here' + 'username' => 'database_username_goes_here', 'password' => 'database-password-goes-here', 'host' => 'database-hostname-goes-here', 'port' => '3306', From fefbb76f336b818348221fa723b7521c007cf9e0 Mon Sep 17 00:00:00 2001 From: Finn Date: Thu, 17 Oct 2024 11:12:05 +0100 Subject: [PATCH 15/22] Add specific notes on setting up and testing in DDEV. --- modules/localgov_forms_lts/README.md | 72 ++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/modules/localgov_forms_lts/README.md b/modules/localgov_forms_lts/README.md index a3b8d91..f2d06e8 100644 --- a/modules/localgov_forms_lts/README.md +++ b/modules/localgov_forms_lts/README.md @@ -33,3 +33,75 @@ To inspect Webform submissions kept in Long term storage, look for the "LTS" tab ### Todo - Removal of Webform submissions from Long term storage after a predefined period e.g. 5 years. - Machine names which are indicative of PII are hardcoded within the Drupal\localgov_forms_lts\PIIRedactor class at the moment. This needs a configuration UI. + +### Testing in DDEV + +To set up testing in ddev, we'll need to set up a second database. + +There are a few ways to do this, but the following seems to work. + +#### 1. Add a post-start hook to your .ddev/config.yml + +Edit `.ddev/config.yml` and add the following to create a new database on start. + +``` +hooks: + post-start: + - exec: mysql -uroot -proot -e "CREATE DATABASE IF NOT EXISTS localgov_forms_lts; GRANT ALL ON localgov_forms_lts.* to 'db'@'%';" + service: db +``` + +#### 2. Add the database connection string to settings.php: + +Edit sites/default/settings.php and add a new database connection string at the +end of the file. + +``` +// Database connection for localgov_forms_lts. +$databases['localgov_forms_lts']['default'] = [ + 'database' => 'localgov_forms_lts', + 'username' => 'db', + 'password' => 'db', + 'host' => 'db', + 'port' => '3306', + 'driver' => 'mysql', + 'prefix' => '', +]; +``` + +#### 3. Install Adminer + +Adminer is useful if you want to inspect databases and tables. + +``` +ddev get ddev/ddev-adminer +``` + +#### 4. Restart ddev + +``` +ddev restart +``` + +#### 5. Require and install the module. + +``` +ddev composer require localgovdrupal/localgov_forms +ddev drush si localgov_forms_lts -y +``` + +#### 5. Make some submissions. + +For example, in LocalGov Drupal we tend to have a contact form at /form/contact. + +Make a couple of submissions there. + +#### 6. Run cron. + +ddev drush cron + +#### 7. Inspect the LTS tab + +Go to /admin/structure/webform/submissions/lts + +Here you should see your submissions with redacted name and email address. From c5e953cd2d4def94e9e574d912b2e726de9f25f6 Mon Sep 17 00:00:00 2001 From: Finn Date: Thu, 17 Oct 2024 11:14:48 +0100 Subject: [PATCH 16/22] Fix simple coding standards issues. --- .../localgov_forms_lts/src/LtsStorageForWebformSubmission.php | 2 +- modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php | 2 +- .../tests/src/Unit/PIIRedactorForTextTest.php | 2 +- modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php index 66000b6..241f009 100644 --- a/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php +++ b/modules/localgov_forms_lts/src/LtsStorageForWebformSubmission.php @@ -55,7 +55,7 @@ public function getDatabaseConnection(): DbConnection { * * Because we do not have any in LTS. */ - protected function getFromPersistentCache(array &$ids = NULL) { + protected function getFromPersistentCache(?array &$ids = NULL) { return []; } diff --git a/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php b/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php index ff958ac..9ea869d 100644 --- a/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php +++ b/modules/localgov_forms_lts/tests/src/Unit/LtsCopyTest.php @@ -11,9 +11,9 @@ use Drupal\Core\KeyValueStore\KeyValueFactoryInterface; use Drupal\Core\KeyValueStore\KeyValueStoreInterface; use Drupal\Core\Logger\LoggerChannelFactoryInterface; +use Drupal\Tests\UnitTestCase; use Drupal\localgov_forms_lts\LtsCopy; use Drupal\localgov_forms_lts\LtsStorageForWebformSubmission; -use Drupal\Tests\UnitTestCase; use Drupal\webform\WebformInterface; use Drupal\webform\WebformSubmissionInterface; use Drupal\webform\WebformSubmissionStorageInterface; diff --git a/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorForTextTest.php b/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorForTextTest.php index b38caee..7fd6219 100644 --- a/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorForTextTest.php +++ b/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorForTextTest.php @@ -4,8 +4,8 @@ namespace Drupal\Tests\localgov_forms_lts\Unit; -use Drupal\localgov_forms_lts\PIIRedactorForText; use Drupal\Tests\UnitTestCase; +use Drupal\localgov_forms_lts\PIIRedactorForText; /** * Unit tests for PIIRedactorForText. diff --git a/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php b/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php index 1ff522e..9690ad6 100644 --- a/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php +++ b/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php @@ -4,8 +4,8 @@ namespace Drupal\Tests\localgov_forms_lts\Unit; -use Drupal\localgov_forms_lts\PIIRedactor; use Drupal\Tests\UnitTestCase; +use Drupal\localgov_forms_lts\PIIRedactor; use Drupal\webform\WebformInterface; use Drupal\webform\WebformSubmissionInterface; From 2b38ffb89b893515351dbbe5186744840b2a7b86 Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 25 Oct 2024 18:58:26 +0100 Subject: [PATCH 17/22] Feat: PII redaction plugin. We should have a choice of PII redaction plugins before saving Webform submissions in the long term storage database. WIP. --- localgov_forms.services.yml | 5 +++ .../PIIRedactor/BestEffortPIIRedactor.php | 35 +++++++++++++++++ src/Annotations/PIIRedactor.php | 38 +++++++++++++++++++ src/Plugin/PIIRedactorManager.php | 30 +++++++++++++++ src/Plugin/PIIRedactorPluginInterface.php | 20 ++++++++++ 5 files changed, 128 insertions(+) create mode 100644 modules/localgov_forms_lts/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php create mode 100644 src/Annotations/PIIRedactor.php create mode 100644 src/Plugin/PIIRedactorManager.php create mode 100644 src/Plugin/PIIRedactorPluginInterface.php diff --git a/localgov_forms.services.yml b/localgov_forms.services.yml index 233229d..f8d4fe3 100644 --- a/localgov_forms.services.yml +++ b/localgov_forms.services.yml @@ -6,3 +6,8 @@ services: localgov_forms.address_lookup: class: Drupal\localgov_forms\AddressLookup arguments: ['@geocoder', '@localgov_forms.geocoder_selection'] + + # Plugin manager service for PII redaction from Webform submissions. + localgov_forms.pii_redactor_manager: + class: Drupal\localgov_forms\Plugin\PIIRedactorManager + parent: default_plugin_manager diff --git a/modules/localgov_forms_lts/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php b/modules/localgov_forms_lts/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php new file mode 100644 index 0000000..15b7048 --- /dev/null +++ b/modules/localgov_forms_lts/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php @@ -0,0 +1,35 @@ +alterInfo('pii_redactor_info'); + + $this->setCacheBackend($cache_backend, 'pii_redactor_plugins'); + } + +} diff --git a/src/Plugin/PIIRedactorPluginInterface.php b/src/Plugin/PIIRedactorPluginInterface.php new file mode 100644 index 0000000..f04dc59 --- /dev/null +++ b/src/Plugin/PIIRedactorPluginInterface.php @@ -0,0 +1,20 @@ + Date: Thu, 31 Oct 2024 12:01:15 +0000 Subject: [PATCH 18/22] Feat: PII redaction plugin. Moved PII redaction functionality from the localgov_forms_lts submodule into the localgov_forms module in the form of a plugin. --- localgov_forms.services.yml | 4 +-- .../localgov_forms_lts.module | 4 ++- modules/localgov_forms_lts/src/LtsCopy.php | 29 ++++++++++----- .../PIIRedactor/BestEffortPIIRedactor.php | 35 ------------------- src/Attribute/PIIRedactor.php | 27 ++++++++++++++ .../BestEffortPIIRedactor.php | 6 ++-- .../BestEffortPIIRedactorForText.php | 4 +-- .../PIIRedactor/BestEffortPIIRedactor.php | 33 +++++++++++++++++ src/Plugin/PIIRedactorPluginBase.php | 15 ++++++++ src/Plugin/PIIRedactorPluginInterface.php | 5 ++- ...nager.php => PIIRedactorPluginManager.php} | 7 ++-- .../PIIRedactorPluginManagerInterface.php | 10 ++++++ .../Unit/BestEffortPIIRedactorForTextTest.php | 12 +++---- .../src/Unit/BestEffortPIIRedactorTest.php | 8 ++--- 14 files changed, 134 insertions(+), 65 deletions(-) delete mode 100644 modules/localgov_forms_lts/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php create mode 100644 src/Attribute/PIIRedactor.php rename modules/localgov_forms_lts/src/PIIRedactor.php => src/BestEffortPIIRedactor.php (96%) rename modules/localgov_forms_lts/src/PIIRedactorForText.php => src/BestEffortPIIRedactorForText.php (97%) create mode 100644 src/Plugin/PIIRedactor/BestEffortPIIRedactor.php create mode 100644 src/Plugin/PIIRedactorPluginBase.php rename src/Plugin/{PIIRedactorManager.php => PIIRedactorPluginManager.php} (78%) create mode 100644 src/Plugin/PIIRedactorPluginManagerInterface.php rename modules/localgov_forms_lts/tests/src/Unit/PIIRedactorForTextTest.php => tests/src/Unit/BestEffortPIIRedactorForTextTest.php (61%) rename modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php => tests/src/Unit/BestEffortPIIRedactorTest.php (85%) diff --git a/localgov_forms.services.yml b/localgov_forms.services.yml index f8d4fe3..959c86b 100644 --- a/localgov_forms.services.yml +++ b/localgov_forms.services.yml @@ -8,6 +8,6 @@ services: arguments: ['@geocoder', '@localgov_forms.geocoder_selection'] # Plugin manager service for PII redaction from Webform submissions. - localgov_forms.pii_redactor_manager: - class: Drupal\localgov_forms\Plugin\PIIRedactorManager + plugin.manager.pii_redactor: + class: Drupal\localgov_forms\Plugin\PIIRedactorPluginManager parent: default_plugin_manager diff --git a/modules/localgov_forms_lts/localgov_forms_lts.module b/modules/localgov_forms_lts/localgov_forms_lts.module index 31706e8..99ec3ef 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.module +++ b/modules/localgov_forms_lts/localgov_forms_lts.module @@ -43,7 +43,9 @@ function localgov_forms_lts_cron() { */ function localgov_forms_lts_copy_recently_added_n_updated_subs() :void { - $lts_copy_obj = LtsCopy::create(Drupal::getContainer()); + $service_container = Drupal::service('service_container'); + $pii_redaction_plugin = $service_container->has('plugin.manager.pii_redactor') ? $service_container->get('plugin.manager.pii_redactor')->createInstance('best_effort_pii_redactor') : NULL; + $lts_copy_obj = LtsCopy::create(Drupal::getContainer(), $pii_redaction_plugin); $copy_results = $lts_copy_obj->copy(); $feedback_msg = _localgov_forms_lts_prepare_feedback_msg($copy_results); diff --git a/modules/localgov_forms_lts/src/LtsCopy.php b/modules/localgov_forms_lts/src/LtsCopy.php index 373ef1f..187c46c 100644 --- a/modules/localgov_forms_lts/src/LtsCopy.php +++ b/modules/localgov_forms_lts/src/LtsCopy.php @@ -4,6 +4,7 @@ namespace Drupal\localgov_forms_lts; +use Drupal\Component\Plugin\PluginInspectionInterface; use Drupal\Core\DependencyInjection\ContainerInjectionInterface; use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\KeyValueStore\KeyValueFactoryInterface; @@ -51,7 +52,10 @@ public function copy(int $count = Constants::COPY_LIMIT) :array { public function copySub(int $webform_sub_id, bool $is_new_webform_sub) :bool { $webform_sub = $this->webformSubStorage->load($webform_sub_id); - PIIRedactor::redact($webform_sub); + + if ($this->optionalPIIRedactorPlugin) { + $this->optionalPIIRedactorPlugin->redact($webform_sub); + } $db_connection = $this->ltsStorage->getDatabaseConnection(); $tx = $db_connection->startTransaction(); @@ -142,18 +146,19 @@ public function setLatestUpdateTimestamp(array $copy_results) :void { * * Keeps track of dependencies. */ - public function __construct(EntityTypeManagerInterface $entity_type_manager, KeyValueFactoryInterface $key_value_factory, LoggerChannelFactoryInterface $logger_factory, WebformSubmissionStorageInterface $lts_storage) { + public function __construct(EntityTypeManagerInterface $entity_type_manager, KeyValueFactoryInterface $key_value_factory, LoggerChannelFactoryInterface $logger_factory, WebformSubmissionStorageInterface $lts_storage, ?PluginInspectionInterface $pii_redactor_plugin = NULL) { - $this->webformSubStorage = $entity_type_manager->getStorage('webform_submission'); - $this->ltsStorage = $lts_storage; - $this->ltsKeyValueStore = $key_value_factory->get(Constants::LTS_KEYVALUE_STORE_ID); - $this->ltsLogger = $logger_factory->get(Constants::LTS_LOGGER_CHANNEL_ID); + $this->webformSubStorage = $entity_type_manager->getStorage('webform_submission'); + $this->ltsStorage = $lts_storage; + $this->ltsKeyValueStore = $key_value_factory->get(Constants::LTS_KEYVALUE_STORE_ID); + $this->ltsLogger = $logger_factory->get(Constants::LTS_LOGGER_CHANNEL_ID); + $this->optionalPIIRedactorPlugin = $pii_redactor_plugin; } /** * Factory. */ - public static function create(ContainerInterface $container) :LtsCopy { + public static function create(ContainerInterface $container, ?PluginInspectionInterface $pii_redactor_plugin = NULL) :LtsCopy { $webform_sub_def = $container->get('entity_type.manager')->getDefinition('webform_submission'); @@ -161,7 +166,8 @@ public static function create(ContainerInterface $container) :LtsCopy { $container->get('entity_type.manager'), $container->get('keyvalue'), $container->get('logger.factory'), - LtsStorageForWebformSubmission::createInstance($container, $webform_sub_def) + LtsStorageForWebformSubmission::createInstance($container, $webform_sub_def), + $pii_redactor_plugin, ); } @@ -193,4 +199,11 @@ public static function create(ContainerInterface $container) :LtsCopy { */ protected $ltsLogger; + /** + * Optional PII redactor plugin manager. + * + * @var Drupal\Component\Plugin\PluginInspectionInterface + */ + protected $optionalPIIRedactorPlugin = NULL; + } diff --git a/modules/localgov_forms_lts/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php b/modules/localgov_forms_lts/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php deleted file mode 100644 index 15b7048..0000000 --- a/modules/localgov_forms_lts/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php +++ /dev/null @@ -1,35 +0,0 @@ -getElementData($elem)) { - [$redacted_text, $redaction_count] = PIIRedactorForText::redact($text); + [$redacted_text, $redaction_count] = BestEffortPIIRedactorForText::redact($text); if ($redaction_count) { $webform_sub->setElementData($elem, $redacted_text); diff --git a/modules/localgov_forms_lts/src/PIIRedactorForText.php b/src/BestEffortPIIRedactorForText.php similarity index 97% rename from modules/localgov_forms_lts/src/PIIRedactorForText.php rename to src/BestEffortPIIRedactorForText.php index 3c214fa..882def7 100644 --- a/modules/localgov_forms_lts/src/PIIRedactorForText.php +++ b/src/BestEffortPIIRedactorForText.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Drupal\localgov_forms_lts; +namespace Drupal\localgov_forms; /** * Redacts from given text. @@ -17,7 +17,7 @@ * foo@example.net will be fully redacted, but foo\@bar@example will only be * partially redacted. */ -class PIIRedactorForText { +class BestEffortPIIRedactorForText { /** * Redacts email, postcode, number. diff --git a/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php b/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php new file mode 100644 index 0000000..201f9db --- /dev/null +++ b/src/Plugin/PIIRedactor/BestEffortPIIRedactor.php @@ -0,0 +1,33 @@ +alterInfo('pii_redactor_info'); diff --git a/src/Plugin/PIIRedactorPluginManagerInterface.php b/src/Plugin/PIIRedactorPluginManagerInterface.php new file mode 100644 index 0000000..a92e660 --- /dev/null +++ b/src/Plugin/PIIRedactorPluginManagerInterface.php @@ -0,0 +1,10 @@ +assertEquals($redaction_count, 8); $nonredactable_text = 'preg_replace() performs a regex search and replace.'; - [, $redaction_count] = PIIRedactorForText::redact($nonredactable_text); + [, $redaction_count] = BestEffortPIIRedactorForText::redact($nonredactable_text); $this->assertEquals($redaction_count, 0); } diff --git a/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php b/tests/src/Unit/BestEffortPIIRedactorTest.php similarity index 85% rename from modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php rename to tests/src/Unit/BestEffortPIIRedactorTest.php index 9690ad6..4b04734 100644 --- a/modules/localgov_forms_lts/tests/src/Unit/PIIRedactorTest.php +++ b/tests/src/Unit/BestEffortPIIRedactorTest.php @@ -2,17 +2,17 @@ declare(strict_types=1); -namespace Drupal\Tests\localgov_forms_lts\Unit; +namespace Drupal\Tests\localgov_forms\Unit; use Drupal\Tests\UnitTestCase; -use Drupal\localgov_forms_lts\PIIRedactor; +use Drupal\localgov_forms\BestEffortPIIRedactor; use Drupal\webform\WebformInterface; use Drupal\webform\WebformSubmissionInterface; /** * Unit tests for PIIRedactor. */ -class PIIRedactorTest extends UnitTestCase { +class BestEffortPIIRedactorTest extends UnitTestCase { /** * Tests PIIRedactorTest::findElemsToRedact(). @@ -38,7 +38,7 @@ public function testFindElemsToRedact() { 'getWebform' => $mock_webform, ]); - $elems_to_redact = PIIRedactor::findElemsToRedact($mock_webform_sub); + $elems_to_redact = BestEffortPIIRedactor::findElemsToRedact($mock_webform_sub); $this->assertSame([ 'email', From 16dc035d80925f1798d36e815426613a374fd5fc Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 1 Nov 2024 15:56:35 +0000 Subject: [PATCH 19/22] Feat: Drush command. Adding the `localgov_forms_lts:copy` Drush command. This command copies all existing Webform submissions into the Long Term Storage (LTS). This is particularly useful soon after the localgov_forms_lts module has been installed. --- .../localgov_forms_lts.deploy.php | 37 ------- .../Commands/LocalgovFormsLtsCommands.php | 97 +++++++++++++++++++ modules/localgov_forms_lts/src/LtsCopy.php | 22 ++--- 3 files changed, 108 insertions(+), 48 deletions(-) delete mode 100644 modules/localgov_forms_lts/localgov_forms_lts.deploy.php create mode 100644 modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php diff --git a/modules/localgov_forms_lts/localgov_forms_lts.deploy.php b/modules/localgov_forms_lts/localgov_forms_lts.deploy.php deleted file mode 100644 index 2ae2e35..0000000 --- a/modules/localgov_forms_lts/localgov_forms_lts.deploy.php +++ /dev/null @@ -1,37 +0,0 @@ -copy(); - - $sandbox['#finished'] = (count($copy_results) < Constants::COPY_LIMIT) ? 1 : 0; - - $feedback = _localgov_forms_lts_prepare_feedback_msg($copy_results); - Drupal::service('logger.factory') - ->get('localgov_forms_lts') - ->info($feedback); - - return $feedback; -} diff --git a/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php b/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php new file mode 100644 index 0000000..7dbbade --- /dev/null +++ b/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php @@ -0,0 +1,97 @@ + FALSE]) { + + if (!localgov_forms_lts_has_db()) { + $this->logger->error(dt('The LocalGov Forms LTS database must exist for this Drush command to function.')); + return; + } + + $is_proceed = $options['force'] ?: $this->configFactory->get('localgov_forms_lts.settings')?->get('is_copy_active'); + if (!$is_proceed) { + $this->logger->warning(dt('Copying is disabled in localgov_forms_lts module configuration. Use --force to override.')); + return; + } + + $pii_redactor_plugin_id = $this->configFactory->get('localgov_forms_lts.settings')?->get('pii_redactor_plugin_id'); + $pii_redactor_plugin = ($pii_redactor_plugin_id && $this->serviceContainer->has('plugin.manager.pii_redactor')) ? $this->serviceContainer->get('plugin.manager.pii_redactor')->createInstance($pii_redactor_plugin_id) : NULL; + + $lts_copy_obj = LtsCopy::create(\Drupal::getContainer(), $pii_redactor_plugin); + $webform_sub_ids_to_copy = $lts_copy_obj->findCopyTargets(); + $batch_count = ceil(count($webform_sub_ids_to_copy) / Constants::COPY_LIMIT); + + $batch_builder = new BatchBuilder(); + $drupal_logger = $this->drupalLoggerFactory->get('localgov_forms_lts'); + for ($i = 0; $i < $batch_count; $i++) { + $batch_builder->addOperation([self::class, 'copyInBatch'], [ + $pii_redactor_plugin, + $drupal_logger, + ]); + } + + batch_set($batch_builder->toArray()); + drush_backend_batch_process(); + + // Info messages are not appearing in the console, so settling for Notices. + $this->logger->notice('Batch operation for copying Webform submissions to Long term storage ends.'); + } + + /** + * Batch operation callback. + * + * Copies a fixed number of Webform submissions to LTS. + */ + public static function copyInBatch(?PluginInspectionInterface $pii_redactor_plugin, LoggerInterface $drupal_logger, &$context) { + + $lts_copy_obj = LtsCopy::create(\Drupal::getContainer(), $pii_redactor_plugin); + $copy_results = $lts_copy_obj->copy(); + + $feedback = _localgov_forms_lts_prepare_feedback_msg($copy_results); + $drupal_logger->info($feedback); + + $context['results'][] = $copy_results; + $context['message'] = $feedback; + } + + /** + * Constructs a LocalgovFormsLtsCommands object. + */ + public function __construct( + private readonly ConfigFactoryInterface $configFactory, + private readonly LoggerChannelFactoryInterface $drupalLoggerFactory, + private readonly ContainerInterface $serviceContainer, + ) { + parent::__construct(); + } + +} diff --git a/modules/localgov_forms_lts/src/LtsCopy.php b/modules/localgov_forms_lts/src/LtsCopy.php index 187c46c..afdfa2f 100644 --- a/modules/localgov_forms_lts/src/LtsCopy.php +++ b/modules/localgov_forms_lts/src/LtsCopy.php @@ -53,8 +53,8 @@ public function copySub(int $webform_sub_id, bool $is_new_webform_sub) :bool { $webform_sub = $this->webformSubStorage->load($webform_sub_id); - if ($this->optionalPIIRedactorPlugin) { - $this->optionalPIIRedactorPlugin->redact($webform_sub); + if ($this->optionalPIIRedactionPlugin) { + $this->optionalPIIRedactionPlugin->redact($webform_sub); } $db_connection = $this->ltsStorage->getDatabaseConnection(); @@ -146,19 +146,19 @@ public function setLatestUpdateTimestamp(array $copy_results) :void { * * Keeps track of dependencies. */ - public function __construct(EntityTypeManagerInterface $entity_type_manager, KeyValueFactoryInterface $key_value_factory, LoggerChannelFactoryInterface $logger_factory, WebformSubmissionStorageInterface $lts_storage, ?PluginInspectionInterface $pii_redactor_plugin = NULL) { + public function __construct(EntityTypeManagerInterface $entity_type_manager, KeyValueFactoryInterface $key_value_factory, LoggerChannelFactoryInterface $logger_factory, WebformSubmissionStorageInterface $lts_storage, ?PluginInspectionInterface $pii_redaction_plugin = NULL) { - $this->webformSubStorage = $entity_type_manager->getStorage('webform_submission'); - $this->ltsStorage = $lts_storage; - $this->ltsKeyValueStore = $key_value_factory->get(Constants::LTS_KEYVALUE_STORE_ID); - $this->ltsLogger = $logger_factory->get(Constants::LTS_LOGGER_CHANNEL_ID); - $this->optionalPIIRedactorPlugin = $pii_redactor_plugin; + $this->webformSubStorage = $entity_type_manager->getStorage('webform_submission'); + $this->ltsStorage = $lts_storage; + $this->ltsKeyValueStore = $key_value_factory->get(Constants::LTS_KEYVALUE_STORE_ID); + $this->ltsLogger = $logger_factory->get(Constants::LTS_LOGGER_CHANNEL_ID); + $this->optionalPIIRedactionPlugin = $pii_redaction_plugin; } /** * Factory. */ - public static function create(ContainerInterface $container, ?PluginInspectionInterface $pii_redactor_plugin = NULL) :LtsCopy { + public static function create(ContainerInterface $container, ?PluginInspectionInterface $pii_redaction_plugin = NULL) :LtsCopy { $webform_sub_def = $container->get('entity_type.manager')->getDefinition('webform_submission'); @@ -167,7 +167,7 @@ public static function create(ContainerInterface $container, ?PluginInspectionIn $container->get('keyvalue'), $container->get('logger.factory'), LtsStorageForWebformSubmission::createInstance($container, $webform_sub_def), - $pii_redactor_plugin, + $pii_redaction_plugin, ); } @@ -204,6 +204,6 @@ public static function create(ContainerInterface $container, ?PluginInspectionIn * * @var Drupal\Component\Plugin\PluginInspectionInterface */ - protected $optionalPIIRedactorPlugin = NULL; + protected $optionalPIIRedactionPlugin = NULL; } From 30a8a36265ba3b8fd94ae8acf06aedb325c61194 Mon Sep 17 00:00:00 2001 From: Adnan Date: Fri, 1 Nov 2024 20:08:49 +0000 Subject: [PATCH 20/22] Feat: LTS config form. A config form for: - Activating or deactivating copying to LTS database. - Selecting a PII redactor plugin if PII redaction is necessary. --- .../install/localgov_forms_lts.settings.yml | 2 + .../schema/localgov_forms_lts.schema.yml | 12 +++ .../localgov_forms_lts.links.task.yml | 7 ++ .../localgov_forms_lts.module | 13 ++- .../localgov_forms_lts.routing.yml | 9 ++ .../Commands/LocalgovFormsLtsCommands.php | 2 +- .../src/Form/LTSSettingsForm.php | 82 +++++++++++++++++++ 7 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 modules/localgov_forms_lts/config/install/localgov_forms_lts.settings.yml create mode 100644 modules/localgov_forms_lts/config/schema/localgov_forms_lts.schema.yml create mode 100644 modules/localgov_forms_lts/src/Form/LTSSettingsForm.php diff --git a/modules/localgov_forms_lts/config/install/localgov_forms_lts.settings.yml b/modules/localgov_forms_lts/config/install/localgov_forms_lts.settings.yml new file mode 100644 index 0000000..09fcfc7 --- /dev/null +++ b/modules/localgov_forms_lts/config/install/localgov_forms_lts.settings.yml @@ -0,0 +1,2 @@ +is_copying_enabled: false +pii_redactor_plugin_id: '' diff --git a/modules/localgov_forms_lts/config/schema/localgov_forms_lts.schema.yml b/modules/localgov_forms_lts/config/schema/localgov_forms_lts.schema.yml new file mode 100644 index 0000000..b4af9d6 --- /dev/null +++ b/modules/localgov_forms_lts/config/schema/localgov_forms_lts.schema.yml @@ -0,0 +1,12 @@ +# Schema for the configuration files of the localgov_forms_lts submodule. + +localgov_forms_lts.settings: + type: config_object + label: 'Webform submissions LTS config' + mapping: + is_copying_enabled: + type: boolean + label: 'Is copying to LTS database enabled?' + pii_redactor_plugin_id: + type: machine_name + label: 'PII redactor plugin id' diff --git a/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml b/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml index 1f005ca..f3ad60b 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml +++ b/modules/localgov_forms_lts/localgov_forms_lts.links.task.yml @@ -18,3 +18,10 @@ entity.webform_submission.lts_notes: route_name: entity.webform_submission.lts_notes base_route: entity.webform_submission.lts_view weight: 3 + +# Config tab +localgov_forms_lts.lts_config: + title: 'LTS' + route_name: localgov_forms_lts.lts_config + parent_id: webform.config + weight: 43 diff --git a/modules/localgov_forms_lts/localgov_forms_lts.module b/modules/localgov_forms_lts/localgov_forms_lts.module index 99ec3ef..4fe2484 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.module +++ b/modules/localgov_forms_lts/localgov_forms_lts.module @@ -44,8 +44,17 @@ function localgov_forms_lts_cron() { function localgov_forms_lts_copy_recently_added_n_updated_subs() :void { $service_container = Drupal::service('service_container'); - $pii_redaction_plugin = $service_container->has('plugin.manager.pii_redactor') ? $service_container->get('plugin.manager.pii_redactor')->createInstance('best_effort_pii_redactor') : NULL; - $lts_copy_obj = LtsCopy::create(Drupal::getContainer(), $pii_redaction_plugin); + $lts_config = $service_container->get('config.factory')->get('localgov_forms_lts.settings'); + + $is_copying_enabled = $lts_config->get('is_copying_enabled'); + if (!$is_copying_enabled) { + return; + } + + $pii_redactor_plugin_id = $lts_config->get('pii_redactor_plugin_id'); + $pii_redactor_plugin = ($pii_redactor_plugin_id && $service_container->has('plugin.manager.pii_redactor')) ? $service_container->get('plugin.manager.pii_redactor')->createInstance($pii_redactor_plugin_id) : NULL; + + $lts_copy_obj = LtsCopy::create(Drupal::getContainer(), $pii_redactor_plugin); $copy_results = $lts_copy_obj->copy(); $feedback_msg = _localgov_forms_lts_prepare_feedback_msg($copy_results); diff --git a/modules/localgov_forms_lts/localgov_forms_lts.routing.yml b/modules/localgov_forms_lts/localgov_forms_lts.routing.yml index a06b574..29d41de 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.routing.yml +++ b/modules/localgov_forms_lts/localgov_forms_lts.routing.yml @@ -27,3 +27,12 @@ entity.webform_submission.lts_notes: view_mode: 'html' requirements: _custom_access: '\Drupal\webform\Access\WebformAccountAccess:checkSubmissionAccess' + +# Config form. +localgov_forms_lts.lts_config: + path: '/admin/structure/webform/config/submissions-lts' + defaults: + _form: '\Drupal\localgov_forms_lts\Form\LTSSettingsForm' + _title: 'Webform submissions LTS' + requirements: + _permission: 'administer site configuration' diff --git a/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php b/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php index 7dbbade..5120276 100644 --- a/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php +++ b/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php @@ -37,7 +37,7 @@ public function copy($options = ['force' => FALSE]) { return; } - $is_proceed = $options['force'] ?: $this->configFactory->get('localgov_forms_lts.settings')?->get('is_copy_active'); + $is_proceed = $options['force'] ?: $this->configFactory->get('localgov_forms_lts.settings')?->get('is_copying_enabled'); if (!$is_proceed) { $this->logger->warning(dt('Copying is disabled in localgov_forms_lts module configuration. Use --force to override.')); return; diff --git a/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php b/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php new file mode 100644 index 0000000..fcc84a5 --- /dev/null +++ b/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php @@ -0,0 +1,82 @@ + 'radios', + '#title' => $this->t('Activate?'), + '#description' => $this->t('Activates copying Webform submissions to the Long Term Storage (LTS) database.'), + '#config_target' => self::CONFIG_ID . ':is_copying_enabled', + '#options' => [ + TRUE => $this->t('Yes'), + FALSE => $this->t('No'), + ], + ]; + + $pii_redactor_plugin_id_list = $this->optionalPIIRedactorPluginManager ? array_map(fn(array $def): string => $def['label'], $this->optionalPIIRedactorPluginManager->getDefinitions()) : []; + $form['pii_redactor_plugin_id'] = [ + '#type' => 'select', + '#title' => $this->t('PII redactor plugin'), + '#description' => $this->t('Select a plugin to redact Personally Identifiable Information (PII) while copying to LTS database.'), + '#config_target' => self::CONFIG_ID . ':pii_redactor_plugin_id', + '#options' => $pii_redactor_plugin_id_list, + '#empty_value' => '', + ]; + + return parent::buildForm($form, $form_state); + } + + /** + * {@inheritdoc} + */ + public function getFormId() { + + return 'localgov_forms_lts_settings'; + } + + /** + * Keeps track of the optional PII redactor plugin manager. + */ + public function __construct(protected ?PluginManagerInterface $optionalPIIRedactorPluginManager) {} + + /** + * Factory. + * + * If the PII redactor plugin manager is available, passes it to the + * constructor. + */ + public static function create(ContainerInterface $container) { + + $pii_redactor_plugin_manager = $container->has(self::PII_REDACTION_PLUGIN_MANAGER_ID) ? $container->get(self::PII_REDACTION_PLUGIN_MANAGER_ID) : NULL; + + return new static($pii_redactor_plugin_manager); + } + + /** + * Config settings. + * + * @var string + */ + const CONFIG_ID = 'localgov_forms_lts.settings'; + + const PII_REDACTION_PLUGIN_MANAGER_ID = 'plugin.manager.pii_redactor'; + +} From 0ef2a2619c6ad1e3310a9002139b029313d244dc Mon Sep 17 00:00:00 2001 From: Adnan Date: Mon, 4 Nov 2024 12:54:14 +0000 Subject: [PATCH 21/22] Chore: Replaced a few reused strings with constants. These constants represent the newly added localgov_forms_lts module configuration. --- .../localgov_forms_lts.module | 10 +++++----- modules/localgov_forms_lts/src/Constants.php | 16 +++++++++++++++ .../Commands/LocalgovFormsLtsCommands.php | 12 +++++------ .../src/Form/LTSSettingsForm.php | 20 ++++++------------- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/modules/localgov_forms_lts/localgov_forms_lts.module b/modules/localgov_forms_lts/localgov_forms_lts.module index 4fe2484..18ed91c 100644 --- a/modules/localgov_forms_lts/localgov_forms_lts.module +++ b/modules/localgov_forms_lts/localgov_forms_lts.module @@ -44,22 +44,22 @@ function localgov_forms_lts_cron() { function localgov_forms_lts_copy_recently_added_n_updated_subs() :void { $service_container = Drupal::service('service_container'); - $lts_config = $service_container->get('config.factory')->get('localgov_forms_lts.settings'); + $lts_config = $service_container->get('config.factory')->get(Constants::LTS_CONFIG_ID); - $is_copying_enabled = $lts_config->get('is_copying_enabled'); + $is_copying_enabled = $lts_config->get(Constants::LTS_CONFIG_COPY_STATE); if (!$is_copying_enabled) { return; } - $pii_redactor_plugin_id = $lts_config->get('pii_redactor_plugin_id'); - $pii_redactor_plugin = ($pii_redactor_plugin_id && $service_container->has('plugin.manager.pii_redactor')) ? $service_container->get('plugin.manager.pii_redactor')->createInstance($pii_redactor_plugin_id) : NULL; + $pii_redactor_plugin_id = $lts_config->get(Constants::LTS_CONFIG_PII_REDACTOR_PLUGIN_ID); + $pii_redactor_plugin = ($pii_redactor_plugin_id && $service_container->has(Constants::PII_REDACTOR_PLUGIN_MANAGER)) ? $service_container->get(Constants::PII_REDACTOR_PLUGIN_MANAGER)->createInstance($pii_redactor_plugin_id) : NULL; $lts_copy_obj = LtsCopy::create(Drupal::getContainer(), $pii_redactor_plugin); $copy_results = $lts_copy_obj->copy(); $feedback_msg = _localgov_forms_lts_prepare_feedback_msg($copy_results); Drupal::service('logger.factory') - ->get('localgov_forms_lts') + ->get(Constants::LTS_LOGGER_CHANNEL_ID) ->info($feedback_msg); } diff --git a/modules/localgov_forms_lts/src/Constants.php b/modules/localgov_forms_lts/src/Constants.php index b27badb..dc41f3c 100644 --- a/modules/localgov_forms_lts/src/Constants.php +++ b/modules/localgov_forms_lts/src/Constants.php @@ -51,4 +51,20 @@ class Constants { */ const LTS_CACHE_ID_PREFIX = 'lts_values'; + /** + * Drupal config id for this module. + */ + const LTS_CONFIG_ID = 'localgov_forms_lts.settings'; + + const LTS_CONFIG_COPY_STATE = 'is_copying_enabled'; + + const LTS_CONFIG_PII_REDACTOR_PLUGIN_ID = 'pii_redactor_plugin_id'; + + /** + * Service name for the PII redactor plugin manager. + * + * @see localgov_forms.services.yml + */ + const PII_REDACTOR_PLUGIN_MANAGER = 'plugin.manager.pii_redactor'; + } diff --git a/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php b/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php index 5120276..667609f 100644 --- a/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php +++ b/modules/localgov_forms_lts/src/Drush/Commands/LocalgovFormsLtsCommands.php @@ -27,9 +27,9 @@ final class LocalgovFormsLtsCommands extends DrushCommands { /** * Drush command to copy all Webform submissions to LTS. */ - #[CLI\Command(name: 'localgov_forms_lts:copy', aliases: ['forms-lts-copy'])] + #[CLI\Command(name: 'localgov-forms-lts:copy', aliases: ['forms-lts-copy'])] #[CLI\Option(name: 'force', description: 'Ignore copy disablement config and copy anyway. Useful immediately after module installation.')] - #[CLI\Usage(name: 'localgov_forms_lts:copy', description: 'Copies all existing Webform submissions.')] + #[CLI\Usage(name: 'localgov-forms-lts:copy', description: 'Copies all existing Webform submissions.')] public function copy($options = ['force' => FALSE]) { if (!localgov_forms_lts_has_db()) { @@ -37,21 +37,21 @@ public function copy($options = ['force' => FALSE]) { return; } - $is_proceed = $options['force'] ?: $this->configFactory->get('localgov_forms_lts.settings')?->get('is_copying_enabled'); + $is_proceed = $options['force'] ?: $this->configFactory->get(Constants::LTS_CONFIG_ID)?->get(Constants::LTS_CONFIG_COPY_STATE); if (!$is_proceed) { $this->logger->warning(dt('Copying is disabled in localgov_forms_lts module configuration. Use --force to override.')); return; } - $pii_redactor_plugin_id = $this->configFactory->get('localgov_forms_lts.settings')?->get('pii_redactor_plugin_id'); - $pii_redactor_plugin = ($pii_redactor_plugin_id && $this->serviceContainer->has('plugin.manager.pii_redactor')) ? $this->serviceContainer->get('plugin.manager.pii_redactor')->createInstance($pii_redactor_plugin_id) : NULL; + $pii_redactor_plugin_id = $this->configFactory->get(Constants::LTS_CONFIG_ID)?->get(Constants::LTS_CONFIG_PII_REDACTOR_PLUGIN_ID); + $pii_redactor_plugin = ($pii_redactor_plugin_id && $this->serviceContainer->has(Constants::PII_REDACTOR_PLUGIN_MANAGER)) ? $this->serviceContainer->get(Constants::PII_REDACTOR_PLUGIN_MANAGER)->createInstance($pii_redactor_plugin_id) : NULL; $lts_copy_obj = LtsCopy::create(\Drupal::getContainer(), $pii_redactor_plugin); $webform_sub_ids_to_copy = $lts_copy_obj->findCopyTargets(); $batch_count = ceil(count($webform_sub_ids_to_copy) / Constants::COPY_LIMIT); $batch_builder = new BatchBuilder(); - $drupal_logger = $this->drupalLoggerFactory->get('localgov_forms_lts'); + $drupal_logger = $this->drupalLoggerFactory->get(Constants::LTS_LOGGER_CHANNEL_ID); for ($i = 0; $i < $batch_count; $i++) { $batch_builder->addOperation([self::class, 'copyInBatch'], [ $pii_redactor_plugin, diff --git a/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php b/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php index fcc84a5..09bbfd4 100644 --- a/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php +++ b/modules/localgov_forms_lts/src/Form/LTSSettingsForm.php @@ -6,6 +6,7 @@ use Drupal\Core\Form\ConfigFormBase; use Drupal\Core\Form\FormStateInterface; use Drupal\Core\Form\RedundantEditableConfigNamesTrait; +use Drupal\localgov_forms_lts\Constants; use Symfony\Component\DependencyInjection\ContainerInterface; /** @@ -20,11 +21,11 @@ class LTSSettingsForm extends ConfigFormBase { */ public function buildForm(array $form, FormStateInterface $form_state) { - $form['is_copying_enabled'] = [ + $form[Constants::LTS_CONFIG_COPY_STATE] = [ '#type' => 'radios', '#title' => $this->t('Activate?'), '#description' => $this->t('Activates copying Webform submissions to the Long Term Storage (LTS) database.'), - '#config_target' => self::CONFIG_ID . ':is_copying_enabled', + '#config_target' => Constants::LTS_CONFIG_ID . ':' . Constants::LTS_CONFIG_COPY_STATE, '#options' => [ TRUE => $this->t('Yes'), FALSE => $this->t('No'), @@ -32,11 +33,11 @@ public function buildForm(array $form, FormStateInterface $form_state) { ]; $pii_redactor_plugin_id_list = $this->optionalPIIRedactorPluginManager ? array_map(fn(array $def): string => $def['label'], $this->optionalPIIRedactorPluginManager->getDefinitions()) : []; - $form['pii_redactor_plugin_id'] = [ + $form[Constants::LTS_CONFIG_PII_REDACTOR_PLUGIN_ID] = [ '#type' => 'select', '#title' => $this->t('PII redactor plugin'), '#description' => $this->t('Select a plugin to redact Personally Identifiable Information (PII) while copying to LTS database.'), - '#config_target' => self::CONFIG_ID . ':pii_redactor_plugin_id', + '#config_target' => Constants::LTS_CONFIG_ID . ':' . Constants::LTS_CONFIG_PII_REDACTOR_PLUGIN_ID, '#options' => $pii_redactor_plugin_id_list, '#empty_value' => '', ]; @@ -65,18 +66,9 @@ public function __construct(protected ?PluginManagerInterface $optionalPIIRedact */ public static function create(ContainerInterface $container) { - $pii_redactor_plugin_manager = $container->has(self::PII_REDACTION_PLUGIN_MANAGER_ID) ? $container->get(self::PII_REDACTION_PLUGIN_MANAGER_ID) : NULL; + $pii_redactor_plugin_manager = $container->has(Constants::PII_REDACTOR_PLUGIN_MANAGER) ? $container->get(Constants::PII_REDACTOR_PLUGIN_MANAGER) : NULL; return new static($pii_redactor_plugin_manager); } - /** - * Config settings. - * - * @var string - */ - const CONFIG_ID = 'localgov_forms_lts.settings'; - - const PII_REDACTION_PLUGIN_MANAGER_ID = 'plugin.manager.pii_redactor'; - } From 1ecc4663f83be559a0188b584cb3613a814fd1c9 Mon Sep 17 00:00:00 2001 From: Adnan Date: Mon, 4 Nov 2024 13:41:02 +0000 Subject: [PATCH 22/22] Doc: Describing optional PII redaction functionality. --- README.md | 4 +++- modules/localgov_forms_lts/README.md | 27 ++++++++++++++------------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5f8a320..cc98965 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ Provides additional configuration, styling and components for the Drupal Webform * LocalGov Forms Date - A date input field based on the [GDS Date Input pattern](https://design-system.service.gov.uk/components/date-input/) * LocalGov address lookup - Webform element with a configurable address lookup backend. Geocoder plugins act as backends. +## Plugins +- Personally Identifiable Information (PII) redactor from Webform submissions: At the moment, a plugin manager `plugin.manager.pii_redactor` and a sample plugin are provided. + ## Dependencies The geocoder-php/nominatim-provider package is necessary to run automated tests: ``` @@ -23,4 +26,3 @@ To avoid the configuration being removed by deployments, install the [Config ign webform.webform.* webform.webform_options.* ``` - diff --git a/modules/localgov_forms_lts/README.md b/modules/localgov_forms_lts/README.md index f2d06e8..b0ae34b 100644 --- a/modules/localgov_forms_lts/README.md +++ b/modules/localgov_forms_lts/README.md @@ -1,12 +1,13 @@ -## Long term storage for Webform submission +## Long term storage for Webform submissions +This module copies Webform submissions to a separate database for Long Term Storage (LTS). The LTS database can then be used for data warehousing needs such as reporting and analysis. Optionally, Personally Identifiable Information (PII) can be redacted from Webform submissions while they are copied to the LTS database. ### Setup process -- Create a database which will serve as the Long term storage. +- Create a database which will serve as the LTS database. - Declare it in Drupal's settings.php using the **localgov_forms_lts** key. Example: ``` $databases['localgov_forms_lts']['default'] = [ - 'database' => 'our_longer_term_storage_database', - 'username' => 'database_username_goes_here', + 'database' => 'our-long-term-storage-database', + 'username' => 'database-username-goes-here', 'password' => 'database-password-goes-here', 'host' => 'database-hostname-goes-here', 'port' => '3306', @@ -16,23 +17,23 @@ ``` - Install the localgov_forms_lts submodule. - Check the module requirement report from Drupal's status page at `admin/reports/status`. This should be under the **LocalGov Forms LTS** key. -- If all looks good in the previous step, run `drush deploy:hook` which will copy existing Webform submissions into the Long term storage. If you are using `drush deploy`, this will be taken care of as part of it and there would be no need for `drush deploy:hook`. -- Ensure cron is running periodically. This will copy any new Webform submissions or changes to existing Webform submissions since deployment or the last cron run. +- [Optional] If all looks good in the previous step, run `drush localgov-forms-lts:copy --force` which will copy existing Webform submissions into the LTS database. +- By default, periodic Webform submissions copying to the LTS database is disabled. Activate it from `/admin/structure/webform/config/submissions-lts`. +- Ensure cron is running periodically. This will copy any new Webform submissions or changes to existing Webform submissions since the last cron run or the last `drush localgov-forms-lts:copy` run. - [Optional] Tell individual Webforms to purge submissions older than a chosen period. This is configured for each Webform from its `Settings > Submissions > Submission purge settings` configuration section. ### Inspection -To inspect Webform submissions kept in Long term storage, look for the "LTS" tab in the Webform submissions listing page. This is usually at /admin/structure/webform/submissions/manage. +To inspect Webform submissions kept in Long term storage, look for the **LTS** tab in the Webform submissions listing page. This is usually at `/admin/structure/webform/submissions/manage`. ### Good to know -- Each cron run copies 50 Webform submissions. If your site is getting more than that many Webform submissions between subsequent cron runs, not all Webform submissions will get copied to Long term storage during a certain period. If that happens, adjust cron run frequency. -- Files attached to Webform submissions are *not* moved to Long term storage. -- Elements with Personally Identifiable Information (PII) are redacted. At the moment, this includes all name, email, telephone, number, and various address type elements. Additionally, any text or radio or checkbox element whose machine name (AKA Key) contains the following also gets redacted: name, mail, phone, contact_number, date_of_birth, dob_, personal_, title, nino, passport, postcode, address, serial_number, reg_number, pcn_, and driver_. +- Each cron run copies 50 Webform submissions. If your site is getting more than that many Webform submissions between subsequent cron runs, not all Webform submissions will get copied to LTS during a certain period. If that happens, adjust cron run frequency. +- Files attached to Webform submissions are *not* moved to LTS. +- You can choose to redact elements with Personally Identifiable Information (PII) while they are copied to the LTS database. For that, select *Best effort PII redactor* (or another redactor) from the `PII redactor plugin` dropdown in the LTS config page at `/admin/structure/webform/config/submissions-lts`. At the moment, this plugin redacts all name, email, telephone, number, and various address type elements. Additionally, any text or radio or checkbox element whose machine name (AKA Key) contains the following also gets redacted: name, mail, phone, contact_number, date_of_birth, dob_, personal_, title, nino, passport, postcode, address, serial_number, reg_number, pcn_, and driver_. +- If you are using this module in multiple instances of the same site (e.g. dev/stage/live), ensure that the database settings array points to *different* databases. Alternatively, disable copying for the non-live environments from `/admin/structure/webform/config/submissions-lts`. The relevant settings `localgov_forms_lts.settings:is_copying_enabled` can be [overridden](https://www.drupal.org/docs/drupal-apis/configuration-api/configuration-override-system#s-global-overrides) from settings.php. The [config_split](https://www.drupal.org/project/config_split) module can be handy as well. - This module is currently in experimental stage. -- If you are using this module in multiple instances of the same site (e.g. dev/stage/live), ensure that the database settings array points to *different* databases. ### Todo -- Removal of Webform submissions from Long term storage after a predefined period e.g. 5 years. -- Machine names which are indicative of PII are hardcoded within the Drupal\localgov_forms_lts\PIIRedactor class at the moment. This needs a configuration UI. +- Removal of Webform submissions from LTS after a predefined period e.g. 5 years. ### Testing in DDEV