From b8ec8cb156fa3caed49a40cf7d27f57953e728d0 Mon Sep 17 00:00:00 2001 From: Hannah Eslinger Date: Mon, 4 Nov 2024 17:04:01 -0700 Subject: [PATCH] Move show populated to backend (#4866) * Move show populated to backend * Include derived columns * Update seed/static/seed/js/controllers/show_populated_columns_modal_controller.js Co-authored-by: Ross Perry * Fix for detail page --------- Co-authored-by: Ross Perry Co-authored-by: Katherine Fleming <2205659+kflemin@users.noreply.github.com> --- seed/models/columns.py | 46 +++++++- .../inventory_detail_controller.js | 2 +- ...show_populated_columns_modal_controller.js | 110 ++---------------- .../seed/js/services/inventory_service.js | 20 ++++ .../show_populated_columns_modal.html | 2 +- seed/tests/test_column_list_profiles_views.py | 98 +++++++++++++++- seed/views/v3/analyses.py | 68 +---------- seed/views/v3/column_list_profiles.py | 37 ++++++ 8 files changed, 211 insertions(+), 172 deletions(-) diff --git a/seed/models/columns.py b/seed/models/columns.py index e2b10d043b..c701d10205 100644 --- a/seed/models/columns.py +++ b/seed/models/columns.py @@ -14,8 +14,8 @@ from django.apps import apps from django.core.exceptions import ValidationError -from django.db import IntegrityError, models, transaction -from django.db.models import Q +from django.db import IntegrityError, connection, models, transaction +from django.db.models import Count, Q from django.db.models.signals import pre_save from django.utils.translation import gettext_lazy as _ @@ -1665,6 +1665,48 @@ def retrieve_all_by_tuple(org_id): return result + @staticmethod + def get_num_of_nonnulls_by_column_name(state_ids, inventory_class, columns): + states = inventory_class.objects.filter(id__in=state_ids) + + # init dicts + num_of_nonnulls_by_column_name = {c.column_name: 0 for c in columns} + canonical_columns = [c.column_name for c in columns if not c.is_extra_data] + + # add non-null counts for extra_data columns + with connection.cursor() as cursor: + table_name = "seed_propertystate" if inventory_class.__name__ == "PropertyState" else "seed_taxlotstate" + non_null_extra_data_counts_query = ( + f'SELECT key, COUNT(*)\n' + f'FROM {table_name}, LATERAL JSONB_EACH_TEXT(extra_data) AS each_entry(key, value)\n' + f'WHERE id IN ({", ".join(map(str, state_ids))})\n' + f' AND value IS NOT NULL\n' + f'GROUP BY key;' + ) + cursor.execute(non_null_extra_data_counts_query) + extra_data_counts = dict(cursor.fetchall()) + num_of_nonnulls_by_column_name.update(extra_data_counts) + + # add non-null counts for derived_data columns + with connection.cursor() as cursor: + table_name = "seed_propertystate" if inventory_class.__name__ == "PropertyState" else "seed_taxlotstate" + non_null_derived_data_counts_query = ( + f'SELECT key, COUNT(*)\n' + f'FROM {table_name}, LATERAL JSONB_EACH_TEXT(derived_data) AS each_entry(key, value)\n' + f'WHERE id IN ({", ".join(map(str, state_ids))})\n' + f' AND value IS NOT NULL\n' + f'GROUP BY key;' + ) + cursor.execute(non_null_derived_data_counts_query) + derived_data_counts = dict(cursor.fetchall()) + num_of_nonnulls_by_column_name.update(derived_data_counts) + + # add non-null counts for canonical columns + canonical_counts = states.aggregate(**{col: Count(col) for col in canonical_columns}) + num_of_nonnulls_by_column_name.update(canonical_counts) + + return num_of_nonnulls_by_column_name + def validate_model(sender, **kwargs): instance = kwargs["instance"] diff --git a/seed/static/seed/js/controllers/inventory_detail_controller.js b/seed/static/seed/js/controllers/inventory_detail_controller.js index 3ca350b575..4dd9ad85d1 100644 --- a/seed/static/seed/js/controllers/inventory_detail_controller.js +++ b/seed/static/seed/js/controllers/inventory_detail_controller.js @@ -259,7 +259,7 @@ angular.module('SEED.controller.inventory_detail', []).controller('inventory_det resolve: { columns: () => columns, currentProfile: () => $scope.currentProfile, - cycle: () => null, + cycle: () => $scope.cycle, inventory_type: () => $stateParams.inventory_type, provided_inventory() { const provided_inventory = []; diff --git a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js index aa5bd47e86..54e980a262 100644 --- a/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js +++ b/seed/static/seed/js/controllers/show_populated_columns_modal_controller.js @@ -17,118 +17,22 @@ angular.module('SEED.controller.show_populated_columns_modal', []).controller('s 'inventory_type', // eslint-disable-next-line func-names function ($scope, $window, $uibModalInstance, Notification, inventory_service, modified_service, spinner_utility, columns, currentProfile, cycle, provided_inventory, inventory_type) { - $scope.columns = columns; - $scope.currentProfile = currentProfile; - $scope.cycle = cycle; - $scope.inventory_type = inventory_type; - - _.forEach($scope.columns, (col) => { - col.pinnedLeft = false; - col.visible = true; - }); - - const notEmpty = (value) => !_.isNil(value) && value !== ''; - - const fetch = (page, chunk) => { - let fn; - if ($scope.inventory_type === 'properties') { - fn = inventory_service.get_properties; - } else if ($scope.inventory_type === 'taxlots') { - fn = inventory_service.get_taxlots; - } - return fn(page, chunk, $scope.cycle, -1).then((data) => { - $scope.progress = Math.round((data.pagination.end / data.pagination.total) * 100); - if (data.pagination.has_next) { - return fetch(page + 1, chunk).then((data2) => data.results.concat(data2)); - } - return data.results; - }); - }; - - const update_profile_with_populated_columns = (inventory) => { - $scope.status = `Processing ${$scope.columns.length} columns in ${inventory.length} records`; - - const cols = _.reject($scope.columns, 'related'); - // console.log('cols', cols); - - const relatedCols = _.filter($scope.columns, 'related'); - // console.log('relatedCols', relatedCols); - - const col_key = provided_inventory ? 'column_name' : 'name'; - - _.forEach(inventory, (record, index) => { - // console.log(cols.length + ' remaining cols to check'); - _.forEachRight(cols, (col, colIndex) => { - if (notEmpty(record[col[col_key]])) { - // console.log('Removing ' + col[col_key] + ' from cols'); - cols.splice(colIndex, 1); - } - }); - - _.forEach(record.related, (relatedRecord) => { - // console.log(relatedCols.length + ' remaining related cols to check'); - _.forEachRight(relatedCols, (col, colIndex) => { - if (notEmpty(relatedRecord[col[col_key]])) { - // console.log('Removing ' + col[col_key] + ' from relatedCols'); - relatedCols.splice(colIndex, 1); - } - }); - }); - - $scope.progress = (index / inventory.length) * 50 + 50; - }); - - // determine hidden columns - const visible = _.reject($scope.columns, (col) => { - if (!col.related) { - return _.find(cols, { id: col.id }); - } - return _.find(relatedCols, { id: col.id }); - }); - - const hidden = _.reject($scope.columns, (col) => _.find(visible, { id: col.id })); - - _.forEach(hidden, (col) => { - col.visible = false; - }); - - const columns = []; - _.forEach(visible, (col) => { - columns.push({ - column_name: col.column_name, - id: col.id, - order: columns.length + 1, - pinned: col.pinnedLeft, - table_name: col.table_name - }); - }); + $scope.start = () => { + $scope.state = 'running'; + $scope.status = 'Processing...'; + $scope.inventory_type = inventory_type === 'properties' ? 'Property' : 'Tax lot'; - const { id } = $scope.currentProfile; - const profile = _.omit($scope.currentProfile, 'id'); - profile.columns = columns; - inventory_service.update_column_list_profile(id, profile).then((/* updatedProfile */) => { + inventory_service.update_column_list_profile_to_show_populated(currentProfile.id, cycle.id, $scope.inventory_type).then((/* updatedProfile */) => { modified_service.resetModified(); $scope.progress = 100; $scope.state = 'done'; - $scope.status = `Found ${visible.length} populated columns`; + $scope.refresh(); }); }; - $scope.start = () => { - $scope.state = 'running'; - $scope.status = 'Fetching Inventory'; - - if (provided_inventory) { - update_profile_with_populated_columns(provided_inventory); - } else { - const page = 1; - const chunk = 5000; - fetch(page, chunk).then(update_profile_with_populated_columns); - } - }; - $scope.refresh = () => { spinner_utility.show(); + $uibModalInstance.close(); $window.location.reload(); }; diff --git a/seed/static/seed/js/services/inventory_service.js b/seed/static/seed/js/services/inventory_service.js index a7d4905091..8e48e89b83 100644 --- a/seed/static/seed/js/services/inventory_service.js +++ b/seed/static/seed/js/services/inventory_service.js @@ -1200,6 +1200,26 @@ angular.module('SEED.service.inventory', []).factory('inventory_service', [ .then((response) => response.data.data); }; + inventory_service.update_column_list_profile_to_show_populated = (id, cycle_id, inventory_type) => { + if (id === null) { + Notification.error('This settings profile is protected from modifications'); + return $q.reject(); + } + return $http + .put( + `/api/v3/column_list_profiles/${id}/show_populated/`, + { + cycle_id, + inventory_type + }, + { + params: { + organization_id: user_service.get_organization().id + } + } + ).then((response) => response.data.data); + }; + inventory_service.remove_column_list_profile = (id) => { if (id === null) { Notification.error('This settings profile is protected from modifications'); diff --git a/seed/static/seed/partials/show_populated_columns_modal.html b/seed/static/seed/partials/show_populated_columns_modal.html index 7a0a2b44c9..edc415f29d 100644 --- a/seed/static/seed/partials/show_populated_columns_modal.html +++ b/seed/static/seed/partials/show_populated_columns_modal.html @@ -4,7 +4,7 @@

Processing records...
-
This will reset your visible columns and column order to only non-derived columns that contain data. Are you sure you want to continue?
+
This will reset your visible columns and column order to only columns that contain data. Are you sure you want to continue?
{$ status $}
diff --git a/seed/tests/test_column_list_profiles_views.py b/seed/tests/test_column_list_profiles_views.py index e4843f948d..5939ec29e3 100644 --- a/seed/tests/test_column_list_profiles_views.py +++ b/seed/tests/test_column_list_profiles_views.py @@ -9,9 +9,16 @@ from rest_framework import status from seed.landing.models import SEEDUser as User -from seed.models import Column +from seed.models import VIEW_LIST_TAXLOT, Column from seed.models.derived_columns import DerivedColumn -from seed.test_helpers.fake import FakeColumnListProfileFactory +from seed.test_helpers.fake import ( + FakeColumnListProfileFactory, + FakeCycleFactory, + FakePropertyStateFactory, + FakePropertyViewFactory, + FakeTaxLotStateFactory, + FakeTaxLotViewFactory, +) from seed.tests.util import AccessLevelBaseTestCase, DeleteModelsTestCase from seed.utils.organizations import create_organization @@ -26,6 +33,15 @@ def setUp(self): self.user = User.objects.create_superuser(**user_details) self.org, _, _ = create_organization(self.user, "test-organization-a") + self.cycle_factory = FakeCycleFactory(organization=self.org, user=self.user) + self.cycle = self.cycle_factory.get_cycle() + + self.column_list_factory = FakeColumnListProfileFactory(organization=self.org) + self.property_view_factory = FakePropertyViewFactory(organization=self.org, cycle=self.cycle) + self.property_state_factory = FakePropertyStateFactory(organization=self.org) + self.taxlot_view_factory = FakeTaxLotViewFactory(organization=self.org, cycle=self.cycle) + self.taxlot_state_factory = FakeTaxLotStateFactory(organization=self.org) + self.column_1 = Column.objects.get(organization=self.org, table_name="PropertyState", column_name="address_line_1") self.column_2 = Column.objects.get(organization=self.org, table_name="PropertyState", column_name="city") self.column_3 = Column.objects.create( @@ -193,6 +209,84 @@ def test_update_column_profile(self): self.assertEqual(result["data"]["columns"][0]["order"], 999) self.assertEqual(result["data"]["columns"][0]["pinned"], True) + def test_column_profile_show_populated(self): + # Set Up + columnlistprofile = self.column_list_factory.get_columnlistprofile(columns=["address_line_1", "city"]) + state = self.property_state_factory.get_property_state(no_default_data=True, city="Denver") + self.property_view_factory.get_property_view(state=state) + + # Action + response = self.client.put( + reverse("api:v3:column_list_profiles-show-populated", args=[columnlistprofile.id]) + f"?organization_id={self.org.id}", + data=json.dumps({"cycle_id": self.cycle.id, "inventory_type": "Property"}), + content_type="application/json", + ) + result = json.loads(response.content) + + # Assertion + self.assertEqual(response.status_code, status.HTTP_200_OK) + columns = {c["column_name"] for c in result["data"]["columns"]} + self.assertSetEqual(columns, {"city", "updated", "created"}) + + def test_column_profile_show_populated_taxlots(self): + columnlistprofile = self.column_list_factory.get_columnlistprofile( + columns=["address_line_1", "city"], inventory_type=VIEW_LIST_TAXLOT + ) + state = self.taxlot_state_factory.get_taxlot_state(no_default_data=True, longitude=12345) + self.taxlot_view_factory.get_taxlot_view(state=state) + + response = self.client.put( + reverse("api:v3:column_list_profiles-show-populated", args=[columnlistprofile.id]) + f"?organization_id={self.org.id}", + data=json.dumps({"cycle_id": self.cycle.id, "inventory_type": "Tax Lot"}), + content_type="application/json", + ) + result = json.loads(response.content) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + columns = {c["column_name"] for c in result["data"]["columns"]} + self.assertSetEqual(columns, {"updated", "longitude", "created"}) + + def test_column_profile_show_populated_extra_data(self): + # Set Up + columnlistprofile = self.column_list_factory.get_columnlistprofile(columns=["address_line_1", "city"]) + state = self.property_state_factory.get_property_state(no_default_data=True, extra_data={self.column_3.column_name: "Denver"}) + self.property_view_factory.get_property_view(state=state) + + # Action + response = self.client.put( + reverse("api:v3:column_list_profiles-show-populated", args=[columnlistprofile.id]) + f"?organization_id={self.org.id}", + data=json.dumps({"cycle_id": self.cycle.id, "inventory_type": "Property"}), + content_type="application/json", + ) + result = json.loads(response.content) + + # Assertion + self.assertEqual(response.status_code, status.HTTP_200_OK) + columns = {c["column_name"] for c in result["data"]["columns"]} + self.assertSetEqual(columns, {self.column_3.column_name, "updated", "created"}) + + def test_column_profile_show_populated_derived_data(self): + # Set Up + self.derived_column = DerivedColumn.objects.create(name="dc", expression="$a + 10", organization=self.org, inventory_type=0) + columnlistprofile = self.column_list_factory.get_columnlistprofile(columns=["address_line_1", "city"]) + state = self.property_state_factory.get_property_state( + no_default_data=True, derived_data={self.derived_column.column.column_name: "20"} + ) + self.property_view_factory.get_property_view(state=state) + + # Action + response = self.client.put( + reverse("api:v3:column_list_profiles-show-populated", args=[columnlistprofile.id]) + f"?organization_id={self.org.id}", + data=json.dumps({"cycle_id": self.cycle.id, "inventory_type": "Property"}), + content_type="application/json", + ) + result = json.loads(response.content) + + # Assertion + self.assertEqual(response.status_code, status.HTTP_200_OK) + columns = {c["column_name"] for c in result["data"]["columns"]} + self.assertSetEqual(columns, {self.derived_column.column.column_name, "updated", "created"}) + class ColumnsListProfileViewPermissionsTests(AccessLevelBaseTestCase, DeleteModelsTestCase): def setUp(self): diff --git a/seed/views/v3/analyses.py b/seed/views/v3/analyses.py index 195640623b..517cbfa209 100644 --- a/seed/views/v3/analyses.py +++ b/seed/views/v3/analyses.py @@ -6,8 +6,7 @@ import json import logging -from django.db import connection -from django.db.models import Count, F +from django.db.models import F from django.http import JsonResponse from drf_yasg.utils import swagger_auto_schema from rest_framework import serializers, status, viewsets @@ -315,33 +314,14 @@ def stats(self, request): property__access_level_instance__lft__gte=access_level_instance.lft, property__access_level_instance__rgt__lte=access_level_instance.rgt, ).values_list("state_id", flat=True) - states = PropertyState.objects.filter(id__in=state_ids) columns = ( Column.objects.filter(organization_id=org_id, derived_column=None, table_name="PropertyState") .exclude(column_name__in=EXCLUDED_API_FIELDS) .only("is_extra_data", "column_name") ) - num_of_nonnulls_by_column_name = {c.column_name: 0 for c in columns} - canonical_columns = [c.column_name for c in columns if not c.is_extra_data] extra_data_columns = [c.column_name for c in columns if c.is_extra_data] - - # add non-null counts for extra_data columns - with connection.cursor() as cursor: - non_null_extra_data_counts_query = ( - f'SELECT key, COUNT(*)\n' - f'FROM seed_propertystate, LATERAL JSONB_EACH_TEXT(extra_data) AS each_entry(key, value)\n' - f'WHERE id IN ({", ".join(map(str, state_ids))})\n' - f' AND value IS NOT NULL\n' - f'GROUP BY key;' - ) - cursor.execute(non_null_extra_data_counts_query) - extra_data_counts = dict(cursor.fetchall()) - num_of_nonnulls_by_column_name.update(extra_data_counts) - - # add non-null counts for canonical columns - canonical_counts = states.aggregate(**{col: Count(col) for col in canonical_columns}) - num_of_nonnulls_by_column_name.update(canonical_counts) + num_of_nonnulls_by_column_name = Column.get_num_of_nonnulls_by_column_name(state_ids, PropertyState, columns) return JsonResponse( { @@ -376,30 +356,11 @@ def used_columns(self, request): ).values_list("state_id", flat=True) if state_ids: - states = PropertyState.objects.filter(id__in=state_ids) - columns = Column.objects.filter(organization_id=org_id, derived_column=None, table_name="PropertyState").exclude( column_name__in=EXCLUDED_API_FIELDS ) - num_of_nonnulls_by_column_name = {c.column_name: 0 for c in columns} - canonical_columns = [c.column_name for c in columns if not c.is_extra_data] - - # add non-null counts for extra_data columns - with connection.cursor() as cursor: - non_null_extra_data_counts_query = ( - f'SELECT key, COUNT(*)\n' - f'FROM seed_propertystate, LATERAL JSONB_EACH_TEXT(extra_data) AS each_entry(key, value)\n' - f'WHERE id IN ({", ".join(map(str, state_ids))})\n' - f' AND value IS NOT NULL\n' - f'GROUP BY key;' - ) - cursor.execute(non_null_extra_data_counts_query) - extra_data_counts = dict(cursor.fetchall()) - num_of_nonnulls_by_column_name.update(extra_data_counts) - - # add non-null counts for canonical columns - canonical_counts = states.aggregate(**{col: Count(col) for col in canonical_columns}) - num_of_nonnulls_by_column_name.update(canonical_counts) + + num_of_nonnulls_by_column_name = Column.get_num_of_nonnulls_by_column_name(state_ids, PropertyState, columns) # Taxlots tstate_ids = TaxLotView.objects.filter( @@ -410,29 +371,10 @@ def used_columns(self, request): # add non-null counts for extra_data columns if tstate_ids: - tstates = TaxLotState.objects.filter(id__in=tstate_ids) - tcolumns = Column.objects.filter(organization_id=org_id, derived_column=None, table_name="TaxLotState").exclude( column_name__in=EXCLUDED_API_FIELDS ) - tnum_of_nonnulls_by_column_name = {c.column_name: 0 for c in tcolumns} - tcanonical_columns = [c.column_name for c in tcolumns if not c.is_extra_data] - - with connection.cursor() as cursor: - tnon_null_extra_data_counts_query = ( - f'SELECT key, COUNT(*)\n' - f'FROM seed_taxlotstate, LATERAL JSONB_EACH_TEXT(extra_data) AS each_entry(key, value)\n' - f'WHERE id IN ({", ".join(map(str, tstate_ids))})\n' - f' AND value IS NOT NULL\n' - f'GROUP BY key;' - ) - cursor.execute(tnon_null_extra_data_counts_query) - extra_data_counts = dict(cursor.fetchall()) - tnum_of_nonnulls_by_column_name.update(extra_data_counts) - - # add non-null counts for canonical columns - tcanonical_counts = tstates.aggregate(**{col: Count(col) for col in tcanonical_columns}) - tnum_of_nonnulls_by_column_name.update(tcanonical_counts) + num_of_nonnulls_by_column_name = Column.get_num_of_nonnulls_by_column_name(state_ids, TaxLotState, columns) # properties and taxlots together num_of_nonnulls_by_column_name.update(tnum_of_nonnulls_by_column_name) diff --git a/seed/views/v3/column_list_profiles.py b/seed/views/v3/column_list_profiles.py index 0d40c50426..385c5c8095 100644 --- a/seed/views/v3/column_list_profiles.py +++ b/seed/views/v3/column_list_profiles.py @@ -9,6 +9,7 @@ from django.utils.decorators import method_decorator from drf_yasg.utils import swagger_auto_schema from rest_framework import status +from rest_framework.decorators import action from rest_framework.response import Response from seed.filters import ColumnListProfileFilterBackend @@ -21,7 +22,12 @@ Column, ColumnListProfile, Organization, + PropertyState, + PropertyView, + TaxLotState, + TaxLotView, ) +from seed.models.columns import EXCLUDED_API_FIELDS from seed.serializers.column_list_profiles import ColumnListProfileSerializer from seed.utils.api import OrgValidateMixin from seed.utils.api_schema import AutoSchemaHelper, swagger_auto_schema_org_query_param @@ -183,3 +189,34 @@ def list_comstock_columns(org_id): ) return results + + @has_perm_class("requires_root_member") + @action(detail=True, methods=["PUT"]) + def show_populated(self, request, pk): + column_list_profile = ColumnListProfile.objects.get(pk=pk) + org_id = self.get_organization(self.request) + cycle_id = request.data.get("cycle_id") + inventory_type = request.data.get("inventory_type") + StateTable = PropertyState if inventory_type == "Property" else TaxLotState + ViewTable = PropertyView if inventory_type == "Property" else TaxLotView + + # get all the columns and states we need to query + all_columns = ( + Column.objects.filter(organization_id=org_id, derived_column=None, table_name=StateTable.__name__) + .exclude(column_name__in=EXCLUDED_API_FIELDS) + .only("is_extra_data", "column_name") + ) + state_ids = ViewTable.objects.filter( + cycle_id=cycle_id, + ).values_list("state_id", flat=True) + + # filter for only the populated columns + num_of_nonnulls_by_column_name = Column.get_num_of_nonnulls_by_column_name(state_ids, StateTable, all_columns) + needed_column_names = [col for col, count in num_of_nonnulls_by_column_name.items() if count > 0] + needed_columns = Column.objects.filter(column_name__in=needed_column_names, table_name=StateTable.__name__, organization=org_id) + + # set needed columns in there + column_list_profile.columns.set(needed_columns) + column_list_profile.save() + + return JsonResponse({"status": "success", "data": self.get_serializer(column_list_profile).data}, status=status.HTTP_200_OK)