Move show populated to backend (#4866)

* Move show populated to backend * Include derived columns * Update seed/static/seed/js/controllers/show_populated_columns_modal_controller.js Co-authored-by: Ross Perry <[email protected]> * Fix for detail page --------- Co-authored-by: Ross Perry <[email protected]> Co-authored-by: Katherine Fleming <[email protected]>
SEED-platform · Nov 5, 2024 · b8ec8cb · b8ec8cb
1 parent 0264539
commit b8ec8cb
Show file tree

Hide file tree

Showing 8 changed files with 211 additions and 172 deletions.
diff --git a/seed/models/columns.py b/seed/models/columns.py
@@ -14,8 +14,8 @@
 
 from django.apps import apps
 from django.core.exceptions import ValidationError
-from django.db import IntegrityError, models, transaction
-from django.db.models import Q
+from django.db import IntegrityError, connection, models, transaction
+from django.db.models import Count, Q
 from django.db.models.signals import pre_save
 from django.utils.translation import gettext_lazy as _
 
@@ -1665,6 +1665,48 @@ def retrieve_all_by_tuple(org_id):
 
         return result
 
+    @staticmethod
+    def get_num_of_nonnulls_by_column_name(state_ids, inventory_class, columns):
+        states = inventory_class.objects.filter(id__in=state_ids)
+
+        # init dicts
+        num_of_nonnulls_by_column_name = {c.column_name: 0 for c in columns}
+        canonical_columns = [c.column_name for c in columns if not c.is_extra_data]
+
+        # add non-null counts for extra_data columns
+        with connection.cursor() as cursor:
+            table_name = "seed_propertystate" if inventory_class.__name__ == "PropertyState" else "seed_taxlotstate"
+            non_null_extra_data_counts_query = (
+                f'SELECT key, COUNT(*)\n'
+                f'FROM {table_name}, LATERAL JSONB_EACH_TEXT(extra_data) AS each_entry(key, value)\n'
+                f'WHERE id IN ({", ".join(map(str, state_ids))})\n'
+                f'  AND value IS NOT NULL\n'
+                f'GROUP BY key;'
+            )
+            cursor.execute(non_null_extra_data_counts_query)
+            extra_data_counts = dict(cursor.fetchall())
+            num_of_nonnulls_by_column_name.update(extra_data_counts)
+
+        # add non-null counts for derived_data columns
+        with connection.cursor() as cursor:
+            table_name = "seed_propertystate" if inventory_class.__name__ == "PropertyState" else "seed_taxlotstate"
+            non_null_derived_data_counts_query = (
+                f'SELECT key, COUNT(*)\n'
+                f'FROM {table_name}, LATERAL JSONB_EACH_TEXT(derived_data) AS each_entry(key, value)\n'
+                f'WHERE id IN ({", ".join(map(str, state_ids))})\n'
+                f'  AND value IS NOT NULL\n'
+                f'GROUP BY key;'
+            )
+            cursor.execute(non_null_derived_data_counts_query)
+            derived_data_counts = dict(cursor.fetchall())
+            num_of_nonnulls_by_column_name.update(derived_data_counts)
+
+        # add non-null counts for canonical columns
+        canonical_counts = states.aggregate(**{col: Count(col) for col in canonical_columns})
+        num_of_nonnulls_by_column_name.update(canonical_counts)
+
+        return num_of_nonnulls_by_column_name
+
 
 def validate_model(sender, **kwargs):
     instance = kwargs["instance"]

diff --git a/seed/static/seed/js/controllers/inventory_detail_controller.js b/seed/static/seed/js/controllers/inventory_detail_controller.js
@@ -259,7 +259,7 @@ angular.module('SEED.controller.inventory_detail', []).controller('inventory_det
         resolve: {
           columns: () => columns,
           currentProfile: () => $scope.currentProfile,
-          cycle: () => null,
+          cycle: () => $scope.cycle,
           inventory_type: () => $stateParams.inventory_type,
           provided_inventory() {
             const provided_inventory = [];

diff --git a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js
@@ -17,118 +17,22 @@ angular.module('SEED.controller.show_populated_columns_modal', []).controller('s
   'inventory_type',
   // eslint-disable-next-line func-names
   function ($scope, $window, $uibModalInstance, Notification, inventory_service, modified_service, spinner_utility, columns, currentProfile, cycle, provided_inventory, inventory_type) {
-    $scope.columns = columns;
-    $scope.currentProfile = currentProfile;
-    $scope.cycle = cycle;
-    $scope.inventory_type = inventory_type;
-
-    _.forEach($scope.columns, (col) => {
-      col.pinnedLeft = false;
-      col.visible = true;
-    });
-
-    const notEmpty = (value) => !_.isNil(value) && value !== '';
-
-    const fetch = (page, chunk) => {
-      let fn;
-      if ($scope.inventory_type === 'properties') {
-        fn = inventory_service.get_properties;
-      } else if ($scope.inventory_type === 'taxlots') {
-        fn = inventory_service.get_taxlots;
-      }
-      return fn(page, chunk, $scope.cycle, -1).then((data) => {
-        $scope.progress = Math.round((data.pagination.end / data.pagination.total) * 100);
-        if (data.pagination.has_next) {
-          return fetch(page + 1, chunk).then((data2) => data.results.concat(data2));
-        }
-        return data.results;
-      });
-    };
-
-    const update_profile_with_populated_columns = (inventory) => {
-      $scope.status = `Processing ${$scope.columns.length} columns in ${inventory.length} records`;
-
-      const cols = _.reject($scope.columns, 'related');
-      // console.log('cols', cols);
-
-      const relatedCols = _.filter($scope.columns, 'related');
-      // console.log('relatedCols', relatedCols);
-
-      const col_key = provided_inventory ? 'column_name' : 'name';
-
-      _.forEach(inventory, (record, index) => {
-        // console.log(cols.length + ' remaining cols to check');
-        _.forEachRight(cols, (col, colIndex) => {
-          if (notEmpty(record[col[col_key]])) {
-            // console.log('Removing ' + col[col_key] + ' from cols');
-            cols.splice(colIndex, 1);
-          }
-        });
-
-        _.forEach(record.related, (relatedRecord) => {
-          // console.log(relatedCols.length + ' remaining related cols to check');
-          _.forEachRight(relatedCols, (col, colIndex) => {
-            if (notEmpty(relatedRecord[col[col_key]])) {
-              // console.log('Removing ' + col[col_key] + ' from relatedCols');
-              relatedCols.splice(colIndex, 1);
-            }
-          });
-        });
-
-        $scope.progress = (index / inventory.length) * 50 + 50;
-      });
-
-      // determine hidden columns
-      const visible = _.reject($scope.columns, (col) => {
-        if (!col.related) {
-          return _.find(cols, { id: col.id });
-        }
-        return _.find(relatedCols, { id: col.id });
-      });
-
-      const hidden = _.reject($scope.columns, (col) => _.find(visible, { id: col.id }));
-
-      _.forEach(hidden, (col) => {
-        col.visible = false;
-      });
-
-      const columns = [];
-      _.forEach(visible, (col) => {
-        columns.push({
-          column_name: col.column_name,
-          id: col.id,
-          order: columns.length + 1,
-          pinned: col.pinnedLeft,
-          table_name: col.table_name
-        });
-      });
+    $scope.start = () => {
+      $scope.state = 'running';
+      $scope.status = 'Processing...';
+      $scope.inventory_type = inventory_type === 'properties' ? 'Property' : 'Tax lot';
 
-      const { id } = $scope.currentProfile;
-      const profile = _.omit($scope.currentProfile, 'id');
-      profile.columns = columns;
-      inventory_service.update_column_list_profile(id, profile).then((/* updatedProfile */) => {
+      inventory_service.update_column_list_profile_to_show_populated(currentProfile.id, cycle.id, $scope.inventory_type).then((/* updatedProfile */) => {
         modified_service.resetModified();
         $scope.progress = 100;
         $scope.state = 'done';
-        $scope.status = `Found ${visible.length} populated columns`;
+        $scope.refresh();
       });
     };
 
-    $scope.start = () => {
-      $scope.state = 'running';
-      $scope.status = 'Fetching Inventory';
-
-      if (provided_inventory) {
-        update_profile_with_populated_columns(provided_inventory);
-      } else {
-        const page = 1;
-        const chunk = 5000;
-        fetch(page, chunk).then(update_profile_with_populated_columns);
-      }
-    };
-
     $scope.refresh = () => {
       spinner_utility.show();
+      $uibModalInstance.close();
       $window.location.reload();
     };
 

diff --git a/seed/static/seed/js/services/inventory_service.js b/seed/static/seed/js/services/inventory_service.js
@@ -1200,6 +1200,26 @@ angular.module('SEED.service.inventory', []).factory('inventory_service', [
         .then((response) => response.data.data);
     };
 
+    inventory_service.update_column_list_profile_to_show_populated = (id, cycle_id, inventory_type) => {
+      if (id === null) {
+        Notification.error('This settings profile is protected from modifications');
+        return $q.reject();
+      }
+      return $http
+        .put(
+          `/api/v3/column_list_profiles/${id}/show_populated/`,
+          {
+            cycle_id,
+            inventory_type
+          },
+          {
+            params: {
+              organization_id: user_service.get_organization().id
+            }
+          }
+        ).then((response) => response.data.data);
+    };
+
     inventory_service.remove_column_list_profile = (id) => {
       if (id === null) {
         Notification.error('This settings profile is protected from modifications');

diff --git a/seed/static/seed/partials/show_populated_columns_modal.html b/seed/static/seed/partials/show_populated_columns_modal.html
@@ -4,7 +4,7 @@ <h4 class="modal-title" ng-switch-when="running" translate>Processing records...
 </div>
 <div class="modal-body row" ng-switch="state">
   <div class="col-sm-12">
-    <div ng-switch-default translate>This will reset your visible columns and column order to only non-derived columns that contain data. Are you sure you want to continue?</div>
+    <div ng-switch-default translate>This will reset your visible columns and column order to only columns that contain data. Are you sure you want to continue?</div>
     <div class="progress_bar_container" ng-switch-when="running">
       <div class="progress_bar_copy_top">{$ status $}</div>
       <uib-progressbar class="progress-striped active" value="progress" type="success"></uib-progressbar>

diff --git a/seed/tests/test_column_list_profiles_views.py b/seed/tests/test_column_list_profiles_views.py
@@ -9,9 +9,16 @@
 from rest_framework import status
 
 from seed.landing.models import SEEDUser as User
-from seed.models import Column
+from seed.models import VIEW_LIST_TAXLOT, Column
 from seed.models.derived_columns import DerivedColumn
-from seed.test_helpers.fake import FakeColumnListProfileFactory
+from seed.test_helpers.fake import (
+    FakeColumnListProfileFactory,
+    FakeCycleFactory,
+    FakePropertyStateFactory,
+    FakePropertyViewFactory,
+    FakeTaxLotStateFactory,
+    FakeTaxLotViewFactory,
+)
 from seed.tests.util import AccessLevelBaseTestCase, DeleteModelsTestCase
 from seed.utils.organizations import create_organization
 
@@ -26,6 +33,15 @@ def setUp(self):
         self.user = User.objects.create_superuser(**user_details)
         self.org, _, _ = create_organization(self.user, "test-organization-a")
 
+        self.cycle_factory = FakeCycleFactory(organization=self.org, user=self.user)
+        self.cycle = self.cycle_factory.get_cycle()
+
+        self.column_list_factory = FakeColumnListProfileFactory(organization=self.org)
+        self.property_view_factory = FakePropertyViewFactory(organization=self.org, cycle=self.cycle)
+        self.property_state_factory = FakePropertyStateFactory(organization=self.org)
+        self.taxlot_view_factory = FakeTaxLotViewFactory(organization=self.org, cycle=self.cycle)
+        self.taxlot_state_factory = FakeTaxLotStateFactory(organization=self.org)
+
         self.column_1 = Column.objects.get(organization=self.org, table_name="PropertyState", column_name="address_line_1")
         self.column_2 = Column.objects.get(organization=self.org, table_name="PropertyState", column_name="city")
         self.column_3 = Column.objects.create(
@@ -193,6 +209,84 @@ def test_update_column_profile(self):
         self.assertEqual(result["data"]["columns"][0]["order"], 999)
         self.assertEqual(result["data"]["columns"][0]["pinned"], True)
 
+    def test_column_profile_show_populated(self):
+        # Set Up
+        columnlistprofile = self.column_list_factory.get_columnlistprofile(columns=["address_line_1", "city"])
+        state = self.property_state_factory.get_property_state(no_default_data=True, city="Denver")
+        self.property_view_factory.get_property_view(state=state)
+
+        # Action
+        response = self.client.put(
+            reverse("api:v3:column_list_profiles-show-populated", args=[columnlistprofile.id]) + f"?organization_id={self.org.id}",
+            data=json.dumps({"cycle_id": self.cycle.id, "inventory_type": "Property"}),
+            content_type="application/json",
+        )
+        result = json.loads(response.content)
+
+        # Assertion
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        columns = {c["column_name"] for c in result["data"]["columns"]}
+        self.assertSetEqual(columns, {"city", "updated", "created"})
+
+    def test_column_profile_show_populated_taxlots(self):
+        columnlistprofile = self.column_list_factory.get_columnlistprofile(
+            columns=["address_line_1", "city"], inventory_type=VIEW_LIST_TAXLOT
+        )
+        state = self.taxlot_state_factory.get_taxlot_state(no_default_data=True, longitude=12345)
+        self.taxlot_view_factory.get_taxlot_view(state=state)
+
+        response = self.client.put(
+            reverse("api:v3:column_list_profiles-show-populated", args=[columnlistprofile.id]) + f"?organization_id={self.org.id}",
+            data=json.dumps({"cycle_id": self.cycle.id, "inventory_type": "Tax Lot"}),
+            content_type="application/json",
+        )
+        result = json.loads(response.content)
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        columns = {c["column_name"] for c in result["data"]["columns"]}
+        self.assertSetEqual(columns, {"updated", "longitude", "created"})
+
+    def test_column_profile_show_populated_extra_data(self):
+        # Set Up
+        columnlistprofile = self.column_list_factory.get_columnlistprofile(columns=["address_line_1", "city"])
+        state = self.property_state_factory.get_property_state(no_default_data=True, extra_data={self.column_3.column_name: "Denver"})
+        self.property_view_factory.get_property_view(state=state)
+
+        # Action
+        response = self.client.put(
+            reverse("api:v3:column_list_profiles-show-populated", args=[columnlistprofile.id]) + f"?organization_id={self.org.id}",
+            data=json.dumps({"cycle_id": self.cycle.id, "inventory_type": "Property"}),
+            content_type="application/json",
+        )
+        result = json.loads(response.content)
+
+        # Assertion
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        columns = {c["column_name"] for c in result["data"]["columns"]}
+        self.assertSetEqual(columns, {self.column_3.column_name, "updated", "created"})
+
+    def test_column_profile_show_populated_derived_data(self):
+        # Set Up
+        self.derived_column = DerivedColumn.objects.create(name="dc", expression="$a + 10", organization=self.org, inventory_type=0)
+        columnlistprofile = self.column_list_factory.get_columnlistprofile(columns=["address_line_1", "city"])
+        state = self.property_state_factory.get_property_state(
+            no_default_data=True, derived_data={self.derived_column.column.column_name: "20"}
+        )
+        self.property_view_factory.get_property_view(state=state)
+
+        # Action
+        response = self.client.put(
+            reverse("api:v3:column_list_profiles-show-populated", args=[columnlistprofile.id]) + f"?organization_id={self.org.id}",
+            data=json.dumps({"cycle_id": self.cycle.id, "inventory_type": "Property"}),
+            content_type="application/json",
+        )
+        result = json.loads(response.content)
+
+        # Assertion
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        columns = {c["column_name"] for c in result["data"]["columns"]}
+        self.assertSetEqual(columns, {self.derived_column.column.column_name, "updated", "created"})
+
 
 class ColumnsListProfileViewPermissionsTests(AccessLevelBaseTestCase, DeleteModelsTestCase):
     def setUp(self):