Improve RDM to CLM migration #23

arches_references/management/commands/controlled_lists.py

@@ -1,5 +1,6 @@
-from arches.app.models.models import Value
-from django.core.management.base import BaseCommand
+from arches.app.models.models import Value, Language
+from arches_references.models import List
+from django.core.management.base import BaseCommand, CommandError
 
 
 class Command(BaseCommand):
@@ -58,11 +59,28 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         if options["operation"] == "migrate_collections_to_controlled_lists":
+            psl = options["preferred_sort_language"]
+            try:
+                Language.objects.get(code=psl)
+            except Language.DoesNotExist:
+                raise CommandError(
+                    "The preferred sort language, {0}, does not exist in the database.".format(
+                        psl
+                    )
+                )
+
+            if not options["overwrite"]:
+                for collection_name in options["collections_to_migrate"]:
+                    if List.objects.filter(name=collection_name).exists():
+                        raise CommandError(
+                            f"The collection '{collection_name}' already exists."
+                        )
+
             self.migrate_collections_to_controlled_lists(
                 collections_to_migrate=options["collections_to_migrate"],
                 host=options["host"],
                 overwrite=options["overwrite"],
-                preferred_sort_language=options["preferred_sort_language"],
+                preferred_sort_language=psl,
             )
 
     def migrate_collections_to_controlled_lists(
@@ -82,6 +100,9 @@ def migrate_collections_to_controlled_lists(
                 -ho 'http://localhost:8000/plugins/controlled-list-manager/item/'
                 -psl 'fr'
                 -ow
+
+            for collections that contain an apostrophe, wrap the concept in double quotes, e.g. "John''s list"
+
         """
 
         collections_in_db = list(

arches_references/settings.py

@@ -422,6 +422,9 @@
 # TODO: remove when finalizing release
 SILENCED_SYSTEM_CHECKS += ["arches.E002"]
 
+# Location for test data fixtures
+FIXTURE_DIRS = [os.path.join(APP_ROOT, "..", "tests", "fixtures", "data")]
+
 try:
     from .package_settings import *
 except ImportError:

tests/cli_tests.py

@@ -6,8 +6,9 @@
 from django.test import TestCase
 from django.test.client import Client
 from django.test.utils import captured_stdout
+from django.core.management.base import CommandError
 
-from arches_references.models import List
+from arches_references.models import List, ListItem, ListItemValue
 from arches.app.utils.skos import SKOSReader
 
 from .test_settings import PROJECT_TEST_ROOT
@@ -45,7 +46,9 @@ def test_export_controlled_list(self):
 class ListImportPackageTests(TestCase):
 
     def test_import_controlled_list(self):
-        input_file = os.path.join(PROJECT_TEST_ROOT, "data", "controlled_lists.xlsx")
+        input_file = os.path.join(
+            PROJECT_TEST_ROOT, "fixtures", "data", "controlled_lists.xlsx"
+        )
         output = io.StringIO()
         # packages command does not yet fully avoid print()
         with captured_stdout():
@@ -62,41 +65,66 @@ def test_import_controlled_list(self):
 
 
 class RDMToControlledListsETLTests(TestCase):
-
-    @classmethod
-    def setUpTestData(cls):
-
-        skos = SKOSReader()
-        rdf = skos.read_file(
-            os.path.join(PROJECT_TEST_ROOT, "data", "concept_label_test_collection.xml")
-        )
-        ret = skos.save_concepts_from_skos(rdf)
-
-        client = Client()
-        client.login(username="admin", password="admin")
-        response = client.get(
-            reverse(
-                "make_collection",
-                kwargs={"conceptid": "7c90899a-dbe9-4574-9175-e69481a80b3c"},
-            )
-        )
+    fixtures = ["polyhierarchical_collections"]
 
     def test_migrate_collections_to_controlled_lists(self):
         output = io.StringIO()
         management.call_command(
             "controlled_lists",
             operation="migrate_collections_to_controlled_lists",
-            collections_to_migrate=["Concept Label Import Test"],
+            collections_to_migrate=[
+                "Polyhierarchical Collection Test",
+                "Polyhierarchy Collection 2",
+            ],
             host="http://localhost:8000/plugins/controlled-list-manager/item/",
             preferred_sort_language="en",
             overwrite=False,
             stdout=output,
         )
 
-        imported_list = List.objects.get(name="Concept Label Import Test")
+        imported_list = List.objects.get(name="Polyhierarchical Collection Test")
         imported_items = imported_list.list_items.all()
         self.assertEqual(len(imported_items), 3)
 
+        imported_item_values = ListItemValue.objects.filter(
+            list_item__in=imported_items
+        )
+        self.assertQuerySetEqual(
+            imported_item_values.values_list("value", flat=True).order_by("value"),
+            [
+                "French Test Concept 1",
+                "French Test Concept 2",
+                "French Test Concept 3",
+                "Test Concept 1",
+                "Test Concept 2",
+                "Test Concept 3",
+            ],
+        )
+
+        imported_list_2 = List.objects.get(name="Polyhierarchy Collection 2")
+        imported_items_2 = imported_list_2.list_items.all()
+        imported_item_values_2 = ListItemValue.objects.filter(
+            list_item__in=imported_items_2
+        )
+
+        # Check that new uuids were generated for polyhierarchical concepts
+        self.assertNotEqual(
+            imported_item_values.filter(value="Test Concept 1"),
+            imported_item_values_2.filter(value="Test Concept 1"),
+        )
+
+        # Check that items with multiple prefLabels in different languages have same listitemid
+        self.assertEqual(
+            imported_item_values.get(value="Test Concept 1").list_item_id,
+            imported_item_values.get(value="French Test Concept 1").list_item_id,
+        )
+
+        # But that items with prefLabels in different languages have different listitemvalue ids
+        self.assertNotEqual(
+            imported_item_values.get(value="Test Concept 1").pk,
+            imported_item_values.get(value="French Test Concept 1").pk,
+        )
+
     def test_no_matching_collection_error(self):
         expected_output = "Failed to find the following collections in the database: Collection That Doesn't Exist"
         output = io.StringIO()
@@ -110,3 +138,20 @@ def test_no_matching_collection_error(self):
             stderr=output,
         )
         self.assertIn(expected_output, output.getvalue().strip())
+
+    def test_no_matching_language_error(self):
+        expected_output = (
+            "The preferred sort language, nonexistent, does not exist in the database."
+        )
+        output = io.StringIO()
+        with self.assertRaises(CommandError) as e:
+            management.call_command(
+                "controlled_lists",
+                operation="migrate_collections_to_controlled_lists",
+                collections_to_migrate=["Polyhierarchical Collection Test"],
+                host="http://localhost:8000/plugins/controlled-list-manager/item/",
+                preferred_sort_language="nonexistent",
+                overwrite=False,
+                stderr=output,
+            )
+        self.assertEqual(expected_output, str(e.exception))