Merge branch 'master' of https://github.com/City-of-Helsinki/kerrokan…

…tasi

democracy/importing/json_importer.py

@@ -31,7 +31,10 @@ def parse_aware_datetime(value):
     return make_aware(dt, timezone=source_timezone)
 
 
-def import_comments(target, comments_data):
+def import_comments(target, comments_data, patch=False):
+    if patch:
+        for comment in target.comments.all():
+            comment.soft_delete()
     CommentModel = BaseComment.find_subclass(target)
     assert issubclass(CommentModel, BaseComment)
     for datum in sorted(comments_data, key=itemgetter("id")):
@@ -60,7 +63,10 @@ def import_comment(CommentModel, datum, target):
     return CommentModel.objects.create(**c_args)
 
 
-def import_images(target, datum):
+def import_images(target, datum, patch=False):
+    if patch:
+        for image in target.images.all():
+            image.soft_delete()
     main_image = datum.pop("main_image", None)
     main_image_id = datum.pop("main_image_id", None)
     if main_image:
@@ -112,7 +118,7 @@ def import_section(hearing, section_datum, section_type, force=False):
         if len(pk) > 32:
             log.warning("Truncating section pk %s to %s", pk, pk[:32])
             pk = pk[:32]
-        old_section = Section.objects.filter(pk=pk).first()
+        old_section = Section.objects.everything().filter(pk=pk).first()
         if old_section:
             if settings.DEBUG or force:
                 log.info("Section %s already exists, importing new entry with mutated pk", pk)
@@ -126,13 +132,27 @@ def import_section(hearing, section_datum, section_type, force=False):
     import_images(section, section_datum)
 
 
-def import_hearing(hearing_datum, force=False):
+def import_sections(hearing, hearing_datum, force=False, patch=False):
+    if patch:
+        for section in hearing.sections.all():
+            section.soft_delete()
+    for section_datum in sorted(hearing_datum.pop("sections", ()), key=itemgetter("position")):
+        import_section(hearing, section_datum, InitialSectionType.PART, force)
+    for alt_datum in sorted(hearing_datum.pop("alternatives", ()), key=itemgetter("position")):
+        import_section(hearing, alt_datum, InitialSectionType.SCENARIO, force)
+
+
+def import_hearing(hearing_datum, force=False, patch=False):
     hearing_datum = deepcopy(hearing_datum)  # We'll be mutating the data as we go, so it's courteous to take a copy.
     hearing_datum.pop("id")
     slug = hearing_datum.pop("slug")
     old_hearing = Hearing.objects.filter(id=slug).first()
     if old_hearing:  # pragma: no cover
-        if settings.DEBUG or force:
+        if patch:
+            log.info("Hearing %s already exists, patching existing hearing", slug)
+            # force is needed to import all the components with new ids if need be
+            force = True
+        elif settings.DEBUG or force:
             log.info("Hearing %s already exists, importing new entry with mutated slug", slug)
             slug += "_%s" % get_random_string(5)
         else:
@@ -161,13 +181,9 @@ def import_hearing(hearing_datum, force=False):
         abstract=(hearing_datum.pop("lead") or ""),
         content=(hearing_datum.pop("body") or ""),
     )
-    import_comments(hearing, hearing_datum.pop("comments", ()))
-    import_images(hearing, hearing_datum)
-
-    for section_datum in sorted(hearing_datum.pop("sections", ()), key=itemgetter("position")):
-        import_section(hearing, section_datum, InitialSectionType.PART, force)
-    for alt_datum in sorted(hearing_datum.pop("alternatives", ()), key=itemgetter("position")):
-        import_section(hearing, alt_datum, InitialSectionType.SCENARIO, force)
+    import_comments(hearing, hearing_datum.pop("comments", ()), patch)
+    import_images(hearing, hearing_datum, patch)
+    import_sections(hearing, hearing_datum, force, patch)
 
     # Compact section ordering...
     for index, section in enumerate(hearing.sections.order_by("ordering"), 1):
@@ -179,17 +195,21 @@ def import_hearing(hearing_datum, force=False):
     return hearing
 
 
-def import_from_data(data):
+def import_from_data(data, force=False, patch=False):
     """
     Import data from a data blob parsed from JSON
 
     :param data: A parsed data blob
     :type data: dict
+    :param force: Force creation of a new object even if a hearing or section exists in the db
+    :type force: bool
+    :param patch: Overwrite hearings with the same slug, instead of creating new ones
+    :type patch: bool
     :return: The created hearings in a dict, keyed by original hearing key
     :rtype: dict[object, Hearing]
     """
     hearings = {}
     for hearing_id, hearing_data in sorted(data.get("hearings", {}).items()):
         log.info("Beginning import of hearing %s", hearing_id)
-        hearings[hearing_id] = import_hearing(hearing_data)
+        hearings[hearing_id] = import_hearing(hearing_data, force=force, patch=patch)
     return hearings

democracy/management/commands/democracy_import_json.py

@@ -13,6 +13,9 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument("input_file", type=argparse.FileType("r", encoding="utf8"), nargs=1)
+        parser.add_argument("--hearing", nargs=1)
+        parser.add_argument("--force", action="store_true")
+        parser.add_argument("--patch", action="store_true")
         parser.add_argument("--nuke", dest="nuke", action="store_true")
 
     def handle(self, input_file, **options):
@@ -25,9 +28,16 @@ def handle(self, input_file, **options):
             3: logging.DEBUG,
         }
         logging.basicConfig(level=verbosity_to_level[options["verbosity"]])
-        self.do_import(input_file[0])
+        self.do_import(input_file[0],
+                       hearing=options.pop("hearing")[0],
+                       force=options.pop("force", False),
+                       patch=options.pop("patch", False))
 
     @atomic
-    def do_import(self, filename):
+    def do_import(self, filename, hearing=False, force=False, patch=False):
         json_data = json.load(filename)
-        import_from_data(json_data)
+        if hearing:
+            # picks the hearing corresponding to given slug
+            hearing_data = next(value for key, value in json_data['hearings'].items() if value['slug'] == hearing)
+            json_data = {'hearings': {'1': hearing_data}}
+        import_from_data(json_data, force=force, patch=patch)

democracy/models/base.py

@@ -3,7 +3,7 @@
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db import models
-from django.db.models import ManyToOneRel
+from django.db.models import ManyToOneRel, Sum
 from django.utils import timezone
 from django.utils.crypto import get_random_string
 from django.utils.translation import ugettext_lazy as _
@@ -75,6 +75,10 @@ def soft_delete(self, using=None):
         self.deleted = True
         self.save(update_fields=("deleted",), using=using, force_update=True)
 
+    def undelete(self, using=None):
+        self.deleted = False
+        self.save(update_fields=("deleted",), using=using, force_update=True)
+
     def delete(self, using=None):
         raise NotImplementedError("This model does not support hard deletion")
 
@@ -121,9 +125,16 @@ class Commentable(models.Model):
 
     def recache_n_comments(self):
         new_n_comments = self.comments.count()
+        # if commentable has sections, include them in the total comment count
+        if hasattr(self, 'sections'):
+            if self.sections.all():
+                new_n_comments += self.sections.all().aggregate(Sum('n_comments'))['n_comments__sum']
         if new_n_comments != self.n_comments:
             self.n_comments = new_n_comments
             self.save(update_fields=("n_comments",))
+        # if commentable has a parent hearing, recache the hearing comment count
+        if hasattr(self, 'hearing'):
+            self.hearing.recache_n_comments()
 
     def check_commenting(self, request):
         """

democracy/tests/test_comment.py

@@ -109,7 +109,7 @@ def test_54_get_hearing_with_comments_check_amount_of_comments(api_client, defau
     data = get_data_from_response(response)
     assert 'comments' in data
     assert len(data['comments']) == 3
-    assert data['n_comments'] == 3
+    assert data['n_comments'] == 12
 
 
 @pytest.mark.django_db
@@ -208,11 +208,11 @@ def test_56_get_hearing_with_section_check_n_comments_property(api_client):
 
 @pytest.mark.django_db
 def test_n_comments_updates(admin_user, default_hearing):
-    assert Hearing.objects.get(pk=default_hearing.pk).n_comments == 3
+    assert Hearing.objects.get(pk=default_hearing.pk).n_comments == 12
     comment = default_hearing.comments.create(created_by=admin_user, content="Hello")
-    assert Hearing.objects.get(pk=default_hearing.pk).n_comments == 4
+    assert Hearing.objects.get(pk=default_hearing.pk).n_comments == 13
     comment.soft_delete()
-    assert Hearing.objects.get(pk=default_hearing.pk).n_comments == 3
+    assert Hearing.objects.get(pk=default_hearing.pk).n_comments == 12
 
 
 @pytest.mark.django_db

democracy/tests/test_factories.py

@@ -1,4 +1,5 @@
 import pytest
+from django.db.models import Sum
 from django.utils.timezone import now
 
 from democracy.models import Hearing, Label
@@ -9,6 +10,7 @@ def test_hearing_factory(random_label, random_hearing):
     assert isinstance(random_label, Label)
     assert isinstance(random_hearing, Hearing)
     assert random_hearing.close_at > now()
-    assert random_hearing.n_comments == random_hearing.comments.count()
+    assert random_hearing.n_comments == (random_hearing.comments.count() +
+                                         random_hearing.sections.all().aggregate(Sum('n_comments'))['n_comments__sum'])
     assert random_hearing.sections.count()
     assert all(comment.content for comment in random_hearing.comments.all())