Merge pull request #33963 from dimagi/mjr/remove-dedupe-toggle

Remove general purpose dedupe toggle

corehq/apps/data_interfaces/migrations/0037_add_dedupe_update_toggle.py

@@ -0,0 +1,27 @@
+from django.db import migrations
+from corehq.util.django_migrations import skip_on_fresh_install
+
+from corehq.toggles import CASE_DEDUPE, CASE_DEDUPE_UPDATES, NAMESPACE_DOMAIN
+from corehq.toggles.models import Toggle
+
+
+@skip_on_fresh_install
+def add_dedupe_update_toggle(apps, schema_editor):
+    for domain in CASE_DEDUPE.get_enabled_domains():
+        CASE_DEDUPE_UPDATES.set(domain, enabled=True, namespace=NAMESPACE_DOMAIN)
+
+
+def reverse(apps, schema_editor):
+    toggle = Toggle.cached_get(CASE_DEDUPE_UPDATES.slug)
+    toggle.delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_interfaces', '0036_backfill_dedupe_match_values'),
+    ]
+
+    operations = [
+        migrations.RunPython(add_dedupe_update_toggle, reverse_code=reverse),
+    ]

corehq/apps/data_interfaces/models.py

@@ -69,6 +69,7 @@
     paginate_query,
     paginate_query_across_partitioned_databases, create_unique_index_name,
 )
+from corehq import toggles
 from corehq.util.log import with_progress_bar
 from corehq.util.quickcache import quickcache
 from corehq.util.test_utils import unit_testing_only
@@ -1176,7 +1177,10 @@ def _handle_case_duplicate_new(self, case, rule):
                 existing_duplicate.delete()
             duplicate_ids = self._create_duplicates(case, rule, current_hash)
 
-        num_updates = self._update_duplicates(duplicate_ids, case, rule)
+        if toggles.CASE_DEDUPE_UPDATES.enabled(rule.domain):
+            num_updates = self._update_duplicates(duplicate_ids, case, rule)
+        else:
+            num_updates = 0
         return CaseRuleActionResult(num_updates=num_updates)
 
     def _create_duplicates(self, case, rule, current_hash):

corehq/apps/data_interfaces/pillow.py

@@ -10,7 +10,6 @@
 from corehq.form_processor.models import CommCareCase
 from corehq.form_processor.models.forms import XFormInstance
 from corehq.apps.hqcase.constants import UPDATE_REASON_RESAVE
-from corehq.toggles import CASE_DEDUPE
 from corehq.util.soft_assert import soft_assert
 
 
@@ -28,8 +27,6 @@ class CaseDeduplicationProcessor(PillowProcessor):
 
     def process_change(self, change):
         domain = change.metadata.domain
-        if not CASE_DEDUPE.enabled(domain):
-            return
 
         if change.deleted:
             # TODO: If a case gets deleted, we don't remove the duplicates for this case?

corehq/apps/data_interfaces/tasks.py

@@ -27,7 +27,7 @@
 from corehq.apps.case_importer.do_import import SubmitCaseBlockHandler, RowAndCase
 from corehq.motech.repeaters.models import SQLRepeatRecord
 from corehq.sql_db.util import get_db_aliases_for_partitioned_query
-from corehq.toggles import CASE_DEDUPE, DISABLE_CASE_UPDATE_RULE_SCHEDULED_TASK
+from corehq.toggles import DISABLE_CASE_UPDATE_RULE_SCHEDULED_TASK
 from corehq.util.celery_utils import no_result_task
 from corehq.util.decorators import serial_task
 from corehq.util.log import send_HTML_email
@@ -66,9 +66,6 @@ def _progress_tracker(current, total):
 @no_result_task(queue='case_rule_queue', acks_late=True,
                 soft_time_limit=15 * settings.CELERY_TASK_SOFT_TIME_LIMIT)
 def reset_and_backfill_deduplicate_rule_task(domain, rule_id):
-    if not CASE_DEDUPE.enabled(domain):
-        return
-
     try:
         rule = AutomaticUpdateRule.objects.get(
             id=rule_id,

corehq/apps/data_interfaces/templates/data_interfaces/edit_deduplication_rule.html

@@ -87,6 +87,7 @@
           {% crispy criteria_form %}
         </div>
       </div>
+      {% if update_actions_enabled %}
       <fieldset {% if readonly %} disabled="disabled" {% endif %}>
         <legend>{% trans "Actions" %}</legend>
         <div data-bind="template: {name: 'update-case-property-action', foreach: propertiesToUpdate, as: 'propertyToUpdate'} "></div>
@@ -101,6 +102,7 @@
           <input type="text" hidden name="properties_to_update" data-bind="value: serializedPropertiesToUpdate" />
         </div>
       </fieldset>
+      {%  endif %}
       {% if not readonly %}
         <div class="form-actions">
           <div class="col-xs-1">

corehq/apps/data_interfaces/templates/data_interfaces/list_deduplication_rules.html

@@ -13,7 +13,7 @@
     <div class="col-sm-8">
       <p>
         {% blocktrans %}
-          Create rules for finding and updating duplicate cases.
+          Create rules for finding duplicate cases.
           Rules are run on each form submission.
           When rules are created or edited the rule will be locked for further changes and a backfill process will be run.
         {% endblocktrans %}

corehq/apps/data_interfaces/tests/test_case_deduplication.py

@@ -483,6 +483,7 @@ def test_find_duplicates_exclude_copied_cases(self):
         )
 
 
+@flag_enabled('CASE_DEDUPE_UPDATES')
 class CaseDeduplicationActionTest(TestCase):
     def setUp(self):
         super().setUp()
@@ -751,7 +752,6 @@ def test_rule_deletion(self):
         self.rule.soft_delete()
         self.assertEqual(CaseDuplicateNew.objects.filter(action=self.action).count(), 0)
 
-    @flag_enabled('CASE_DEDUPE')
     def test_case_deletion(self):
         """Test that deleting cases also deletes Duplicate Relationships
         """
@@ -815,7 +815,7 @@ def test_update_parent(self):
 
 @override_settings(RUN_UNKNOWN_USER_PILLOW=False)
 @override_settings(RUN_FORM_META_PILLOW=False)
-@flag_enabled('CASE_DEDUPE')
+@flag_enabled('CASE_DEDUPE_UPDATES')
 class DeduplicationPillowTest(TestCase):
 
     @classmethod
@@ -941,6 +941,7 @@ def _configure_properties_to_update(self, rule, prop_map):
         action.save()
 
 
+@flag_enabled('CASE_DEDUPE_UPDATES')
 @es_test(requires=[case_search_adapter, user_adapter])
 class TestDeduplicationRuleRuns(TestCase):
     def setUp(self):
@@ -1233,7 +1234,7 @@ def test_rule_with_user_as_owner(self):
         self.assertEqual(refreshed_fake_cases[1].get_case_property('age'), '14')
 
 
-@flag_enabled('CASE_DEDUPE')
+@flag_enabled('CASE_DEDUPE_UPDATES')
 @es_test(requires=[case_search_adapter], setup_class=True)
 class DeduplicationBackfillTest(TestCase):
     @classmethod

corehq/apps/data_interfaces/views.py

@@ -991,7 +991,7 @@ def read_only_mode(self):
         return True
 
 
-@method_decorator(toggles.CASE_DEDUPE.required_decorator(), name='dispatch')
+@method_decorator(requires_privilege_with_fallback(privileges.CASE_DEDUPE), name='dispatch')
 class DeduplicationRuleListView(DataInterfaceSection, CRUDPaginatedViewMixin):
     template_name = 'data_interfaces/list_deduplication_rules.html'
     urlname = 'deduplication_rules'
@@ -1148,7 +1148,7 @@ def _get_duplicates_count(self, rule):
         return CaseDuplicateNew.objects.filter(action=action).count()
 
 
-@method_decorator(toggles.CASE_DEDUPE.required_decorator(), name='dispatch')
+@method_decorator(requires_privilege_with_fallback(privileges.CASE_DEDUPE), name='dispatch')
 class DeduplicationRuleCreateView(DataInterfaceSection):
     template_name = "data_interfaces/edit_deduplication_rule.html"
     urlname = 'add_deduplication_rule'
@@ -1161,6 +1161,7 @@ def page_context(self):
             'all_case_properties': self.get_augmented_data_dict_props_by_case_type(self.domain),
             'case_types': sorted(list(get_case_types_for_domain(self.domain))),
             'criteria_form': self.case_filter_form,
+            'update_actions_enabled': toggles.CASE_DEDUPE_UPDATES.enabled(self.domain),
         })
         return context
 
@@ -1250,11 +1251,17 @@ def parse_params(self, request):
                 if prop
             ],
             "include_closed": request.POST.get("include_closed") == "on",
-            "properties_to_update": [
+        }
+        if toggles.CASE_DEDUPE_UPDATES.enabled(self.domain):
+            properties_to_update = [
                 {"name": prop["name"], "value_type": prop["valueType"], "value": prop["value"]}
                 for prop in json.loads(request.POST.get("properties_to_update"))
-            ],
-        }
+            ]
+        else:
+            properties_to_update = []
+
+        action_params["properties_to_update"] = properties_to_update
+
         return rule_params, action_params
 
     def validate_rule_params(self, domain, rule_params, rule=None):
@@ -1307,7 +1314,7 @@ def _create_rule(self, domain, name, case_type):
         )
 
 
-@method_decorator(toggles.CASE_DEDUPE.required_decorator(), name='dispatch')
+@method_decorator(requires_privilege_with_fallback(privileges.CASE_DEDUPE), name='dispatch')
 class DeduplicationRuleEditView(DeduplicationRuleCreateView):
     urlname = 'edit_deduplication_rule'
     page_title = gettext_lazy("Edit Deduplication Rule")

corehq/apps/users/tasks.py

@@ -107,8 +107,7 @@ def tag_cases_as_deleted_and_remove_indices(domain, case_ids, deletion_id, delet
     _remove_indices_from_deleted_cases_task.delay(domain, case_ids)
     delete_phone_numbers_for_owners.delay(case_ids)
     delete_schedule_instances_for_cases.delay(domain, case_ids)
-    if toggles.CASE_DEDUPE.enabled(domain):
-        delete_duplicates_for_cases.delay(case_ids)
+    delete_duplicates_for_cases.delay(case_ids)
 
 
 @task(serializer='pickle', rate_limit=2, queue='background_queue', ignore_result=True, acks_late=True)

corehq/reports.py

@@ -108,7 +108,7 @@ def REPORTS(project):
     if toggles.CASE_LIST_EXPLORER.enabled(project.name) or domain_can_access_case_list_explorer:
         inspect_reports.append(CaseListExplorer)
 
-    if toggles.CASE_DEDUPE.enabled(project.name):
+    if domain_has_privilege(project.name, privileges.CASE_DEDUPE):
         inspect_reports.append(DuplicateCasesExplorer)
 
     deployments_reports = (

corehq/tabs/tabclasses.py

@@ -482,6 +482,11 @@ def can_edit_commcare_data(self):
     def can_use_data_cleanup(self):
         return domain_has_privilege(self.domain, privileges.DATA_CLEANUP)
 
+    @property
+    @memoized
+    def can_use_dedupe(self):
+        return domain_has_privilege(self.domain, privileges.CASE_DEDUPE)
+
     @property
     @memoized
     def can_export_data(self):
@@ -574,11 +579,6 @@ def can_view_ecd_preview(self):
         return (EXPLORE_CASE_DATA_PREVIEW.enabled_for_request(self._request)
                 and is_eligible_for_ecd_preview(self._request))
 
-    @property
-    @memoized
-    def can_deduplicate_cases(self):
-        return toggles.CASE_DEDUPE.enabled_for_request(self._request)
-
     @property
     def _can_view_geospatial(self):
         return toggles.GEOSPATIAL.enabled(self.domain)
@@ -943,7 +943,7 @@ def _get_edit_section(self):
             else:
                 edit_section = [(gettext_lazy('Edit Data'), [automatic_update_rule_list_view])]
 
-        if self.can_deduplicate_cases:
+        if self.can_use_dedupe:
             from corehq.apps.data_interfaces.views import (
                 DeduplicationRuleListView,
             )
@@ -952,6 +952,7 @@ def _get_edit_section(self):
                 'url': reverse(DeduplicationRuleListView.urlname, args=[self.domain]),
             }
             edit_section[0][1].append(deduplication_list_view)
+
         return edit_section
 
     def _get_explore_data_views(self):

corehq/toggles/__init__.py

@@ -1490,9 +1490,9 @@ def _commtrackify(domain_name, toggle_is_enabled):
     [NAMESPACE_DOMAIN],
 )
 
-CASE_DEDUPE = StaticToggle(
-    'case_dedupe',
-    'Case deduplication feature',
+CASE_DEDUPE_UPDATES = StaticToggle(
+    'case_dedupe_updates',
+    'Allow Case deduplication update actions',
     TAG_SOLUTIONS_LIMITED,
     [NAMESPACE_DOMAIN],
     help_link='https://confluence.dimagi.com/display/saas/Surfacing+Case+Duplicates+in+CommCare',
@@ -2713,6 +2713,14 @@ def domain_has_privilege_from_toggle(privilege_slug, domain):
     help_link='https://confluence.dimagi.com/display/commcarepublic/Data+Dictionary'
 )
 
+CASE_DEDUPE = FrozenPrivilegeToggle(
+    privileges.CASE_DEDUPE,
+    'case_dedupe',
+    label='Case deduplication feature',
+    tag=TAG_SOLUTIONS_LIMITED,
+    namespaces=[NAMESPACE_DOMAIN],
+    help_link='https://confluence.dimagi.com/display/saas/Surfacing+Case+Duplicates+in+CommCare',
+)
 
 USE_LOGO_IN_SYSTEM_EMAILS = StaticToggle(
     slug='use_logo_in_system_emails',

migrations.lock

@@ -342,6 +342,7 @@ data_interfaces
  0034_case_name_actions
  0035_add_case_duplicate_new
  0036_backfill_dedupe_match_values
+ 0037_add_dedupe_update_toggle
 dhis2
  0001_initial
  0002_auto_20170322_1323