YDA-5319: implement delay times for async jobs

Implement optional delays after last modification before a data object is processed by the asynchronous replication and revision job. Setting a delay reduces the risk of consistency issues due to concurrent access bugs.
UtrechtUniversity · Aug 8, 2023 · 71cd12c · 71cd12c
1 parent 49ac7ae
commit 71cd12c
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 0 deletions.
diff --git a/docs/administration/configuring-yoda.md b/docs/administration/configuring-yoda.md
@@ -222,6 +222,8 @@ enable_async_replication       | Enable asynchronous replication: yes (1) or no
 async_replication_jobs         | Number of asynchronous replication jobs, when decreasing the number of jobs, manually remove jobs from the crontab (default: 1)
 async_replication_batch_size   | Asynchronous replication jobs batch size (default: 1000)
 async_replication_verbose_mode | Run asynchronous replication job in verbose mode (default: true)
+async_replication_delay_time   | Delay after last modification to data object before replication job can process it (in seconds, default: 0)
+
 
 ### Revision configuration
 
@@ -232,6 +234,7 @@ revision_strategy              | Revision strategy: A, B, J or Simple (default:
 async_revision_jobs            | Number of asynchronous replication jobs, when decreasing the number of jobs, manually remove jobs from the crontab (default: 1)
 async_revision_batch_size      | Asynchronous revision jobs batch size (default: 1000)
 async_revision_verbose_mode    | Run asynchronous revision job in verbose mode (default: true)
+async_revision_delay_time      | Delay after last modification to data object before revision job can process it (in seconds, default: 0)
 enable_revision_cleanup: true  | Enable revision cleanup job (true/false)
 
 ### Deposit module configuration

diff --git a/roles/yoda_rulesets/defaults/main.yml b/roles/yoda_rulesets/defaults/main.yml
@@ -24,13 +24,15 @@ enable_async_replication: 1            # Enable asynchronous replication: yes (1
 async_replication_jobs: 1              # Number of asynchronous replication jobs, when decreasing the number of jobs, manually remove jobs from the crontab
 async_replication_batch_size: 1000     # Asynchronous replication jobs batch size
 async_replication_verbose_mode: true   # Run asynchronous replication job in verbose mode
+async_replication_delay_time: 0        # Delay after last modification to data object before replication job can process it (sec)
 
 # Revision configuration
 enable_revisions: 1                # Enable asynchronous revisions: yes (1) or no (0)
 revision_strategy: B               # Revision strategy: A, B, J or Simple
 async_revision_jobs: 1             # Number of asynchronous replication jobs, when decreasing the number of jobs, manually remove jobs from the crontab
 async_revision_batch_size: 1000    # Asynchronous revision jobs batch size
 async_revision_verbose_mode: true  # Run asynchronous revision job in verbose mode
+async_revision_delay_time: 0       # Delay after last modification to data object before revision job can process it (sec)
 enable_revision_cleanup: true      # Enable revision cleanup job (true/false)
 
 # Research module specific configuration.

diff --git a/roles/yoda_rulesets/templates/rules_uu.cfg.j2 b/roles/yoda_rulesets/templates/rules_uu.cfg.j2
@@ -64,6 +64,9 @@ token_lifetime                 = '{{ token_lifetime }}'
 token_expiration_notification  = '{{ token_expiration_notification }}'
 {% endif %}
 
+async_replication_delay_time   = '{{ async_replication_delay_time }}'
+async_revision_delay_time      = '{{ async_revision_delay_time }}'
+
 enable_tape_archive            = '{{ ["false", "true"][enable_tape_archive|int] }}'
 
 enable_data_package_archive    = '{{ ["false", "true"][enable_data_package_archive|int] }}'