From 8130e0e11735d0bf0cb7c1e443dcc03fcddf071f Mon Sep 17 00:00:00 2001 From: "Jean Cochrane (Lead developer, DataMade)" Date: Mon, 24 Jun 2019 17:24:16 -0500 Subject: [PATCH 1/4] Factor out 'import_geographies' script from old 'import_data' --- .../management/commands/import_geographies.py | 64 +++++++++++++++++++ .../migrations/0048_auto_20190624_1722.py | 36 +++++++++++ councilmatic_core/models.py | 5 +- 3 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 councilmatic_core/management/commands/import_geographies.py create mode 100644 councilmatic_core/migrations/0048_auto_20190624_1722.py diff --git a/councilmatic_core/management/commands/import_geographies.py b/councilmatic_core/management/commands/import_geographies.py new file mode 100644 index 00000000..fb5ca0f0 --- /dev/null +++ b/councilmatic_core/management/commands/import_geographies.py @@ -0,0 +1,64 @@ +import json + +import requests +from django.core.management.base import BaseCommand +from councilmatic_core import models + +BOUNDARY_SET = ['chicago-wards-2015'] +BOUNDARY_BASE_URL = 'https://ocd.datamade.us' + +session = requests.Session() + + +class Command(BaseCommand): + help = "Import boundary shapefiles for Post entities" + + def get_response(self, url, params=None, timeout=60, **kwargs): + """ + The OCD API has intermittently thrown 502 and 504 errors, so only proceed + when receiving an 'ok' status. + """ + response = self.session.get(url, params=params, timeout=timeout, **kwargs) + + if response.ok: + return response + else: + message = '{url} returned a bad response - {status}'.format( + url=url, + status=response.status_code + ) + raise requests.exceptions.HTTPError('ERROR: {0}'.format(message)) + + def handle(self, *args, **kwargs): + self.stdout.write('Populating boundaries...') + boundaries_populated = 0 + for boundary in BOUNDARY_SET: + bndry_set_url = BOUNDARY_BASE_URL + '/boundaries/' + boundary + + page_res = self.get_response(bndry_set_url + '/?limit=0') + page_json = json.loads(page_res.text) + + for bndry_json in page_json['objects']: + shape_url = BOUNDARY_BASE_URL + bndry_json['url'] + 'shape' + shape_res = self.get_response(shape_url) + if shape_res: + if 'ocd-division' in bndry_json['external_id']: + filters = { + 'division_ocd_id': bndry_json['external_id'] + } + else: + # The API doesn't appear to use an OCD id as external_id, + # so we have to filter on a fragment of the Division ID + # instead of the entire ID + filters = { + 'division_ocd_id__endswith': bndry_json['external_id'] + } + models.Post.objects.filter(**filters).update( + shape=json.loads(shape_res.text) + ) + boundaries_populated += 1 + self.stdout.write( + self.style.SUCCESS( + 'Populated {} boundaries'.format(str(boundaries_populated)) + ) + ) diff --git a/councilmatic_core/migrations/0048_auto_20190624_1722.py b/councilmatic_core/migrations/0048_auto_20190624_1722.py new file mode 100644 index 00000000..24a0b25a --- /dev/null +++ b/councilmatic_core/migrations/0048_auto_20190624_1722.py @@ -0,0 +1,36 @@ +# Generated by Django 2.1.9 on 2019-06-24 22:22 + +import django.contrib.postgres.fields.jsonb +import django.core.files.storage +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0004_auto_20171005_2028'), + ('councilmatic_core', '0047_update_filepath'), + ] + + operations = [ + migrations.DeleteModel( + name='Post', + ), + migrations.AlterField( + model_name='person', + name='headshot', + field=models.FileField(default='images/headshot_placeholder.png', storage=django.core.files.storage.FileSystemStorage(base_url='/', location='/Users/goobzie/datamade/chi-councilmatic'), upload_to='images/headshots'), + ), + migrations.CreateModel( + name='Post', + fields=[ + ('post_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.Post')), + ('shape', django.contrib.postgres.fields.jsonb.JSONField()), + ], + options={ + 'abstract': False, + }, + bases=('core.post',), + ), + ] diff --git a/councilmatic_core/models.py b/councilmatic_core/models.py index f6149716..784edee4 100644 --- a/councilmatic_core/models.py +++ b/councilmatic_core/models.py @@ -2,6 +2,7 @@ import os from django.db import models +from django.contrib.postgres.fields import JSONField from django.core.exceptions import ImproperlyConfigured from django.conf import settings from django.urls import reverse, NoReverseMatch @@ -233,8 +234,6 @@ def link_html(self): class Post(opencivicdata.core.models.Post): - class Meta: - proxy = True organization = ProxyForeignKey( Organization, @@ -243,6 +242,8 @@ class Meta: on_delete=models.CASCADE, ) + shape = JSONField() + @cached_property def current_member(self): membership = self.memberships.filter(end_date_dt__gt=timezone.now())\ From fd471dd7acef7b7bf43328fff0aaa28737285999 Mon Sep 17 00:00:00 2001 From: "Jean Cochrane (Lead developer, DataMade)" Date: Tue, 25 Jun 2019 14:39:44 -0500 Subject: [PATCH 2/4] Fix up import_geographies and rename it to import_shapes --- ...import_geographies.py => import_shapes.py} | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) rename councilmatic_core/management/commands/{import_geographies.py => import_shapes.py} (67%) diff --git a/councilmatic_core/management/commands/import_geographies.py b/councilmatic_core/management/commands/import_shapes.py similarity index 67% rename from councilmatic_core/management/commands/import_geographies.py rename to councilmatic_core/management/commands/import_shapes.py index fb5ca0f0..5e19cf95 100644 --- a/councilmatic_core/management/commands/import_geographies.py +++ b/councilmatic_core/management/commands/import_shapes.py @@ -4,54 +4,50 @@ from django.core.management.base import BaseCommand from councilmatic_core import models -BOUNDARY_SET = ['chicago-wards-2015'] -BOUNDARY_BASE_URL = 'https://ocd.datamade.us' - -session = requests.Session() - class Command(BaseCommand): help = "Import boundary shapefiles for Post entities" - def get_response(self, url, params=None, timeout=60, **kwargs): - """ - The OCD API has intermittently thrown 502 and 504 errors, so only proceed - when receiving an 'ok' status. - """ - response = self.session.get(url, params=params, timeout=timeout, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.session = requests.Session() - if response.ok: - return response - else: - message = '{url} returned a bad response - {status}'.format( - url=url, - status=response.status_code - ) - raise requests.exceptions.HTTPError('ERROR: {0}'.format(message)) + def add_arguments(self, parser): + parser.add_argument( + '--base-url', + help='The base URL for the API to use to retrieve boundaries', + default='https://ocd.datamade.us', + ) + parser.add_argument( + 'boundary_set', + help='One or more slugs of boundary sets to retrieve', + nargs='*', + default=['chicago-wards-2015'], + ) - def handle(self, *args, **kwargs): + def handle(self, *args, **options): self.stdout.write('Populating boundaries...') boundaries_populated = 0 - for boundary in BOUNDARY_SET: - bndry_set_url = BOUNDARY_BASE_URL + '/boundaries/' + boundary + for boundary in options['boundary_set']: + bndry_set_url = options['base_url'] + '/boundaries/' + boundary page_res = self.get_response(bndry_set_url + '/?limit=0') page_json = json.loads(page_res.text) for bndry_json in page_json['objects']: - shape_url = BOUNDARY_BASE_URL + bndry_json['url'] + 'shape' + shape_url = options['base_url'] + bndry_json['url'] + 'shape' shape_res = self.get_response(shape_url) if shape_res: if 'ocd-division' in bndry_json['external_id']: filters = { - 'division_ocd_id': bndry_json['external_id'] + 'division_id': bndry_json['external_id'] } else: # The API doesn't appear to use an OCD id as external_id, # so we have to filter on a fragment of the Division ID # instead of the entire ID filters = { - 'division_ocd_id__endswith': bndry_json['external_id'] + 'division_id__endswith': bndry_json['external_id'] } models.Post.objects.filter(**filters).update( shape=json.loads(shape_res.text) @@ -62,3 +58,20 @@ def handle(self, *args, **kwargs): 'Populated {} boundaries'.format(str(boundaries_populated)) ) ) + + def get_response(self, url, params=None, timeout=60, **kwargs): + """ + The OCD API has intermittently thrown 502 and 504 errors, so only proceed + when receiving an 'ok' status. + """ + response = self.session.get(url, params=params, timeout=timeout, **kwargs) + + if response.ok: + return response + else: + message = '{url} returned a bad response - {status}'.format( + url=url, + status=response.status_code + ) + raise requests.exceptions.HTTPError('ERROR: {0}'.format(message)) + From 77fdebf57ab07a96c23043dab79adff0cb587177 Mon Sep 17 00:00:00 2001 From: "Jean Cochrane (Lead developer, DataMade)" Date: Wed, 26 Jun 2019 14:19:06 -0500 Subject: [PATCH 3/4] Refactor import_shapes command to read shapes from a local file --- .../management/commands/import_shapes.py | 75 +++++-------------- ...to_20190624_1722.py => 0048_post_shape.py} | 4 +- councilmatic_core/models.py | 5 ++ councilmatic_core/signals/handlers.py | 13 +++- councilmatic_core/views.py | 4 +- 5 files changed, 38 insertions(+), 63 deletions(-) rename councilmatic_core/migrations/{0048_auto_20190624_1722.py => 0048_post_shape.py} (80%) diff --git a/councilmatic_core/management/commands/import_shapes.py b/councilmatic_core/management/commands/import_shapes.py index 5e19cf95..1d8c9ffb 100644 --- a/councilmatic_core/management/commands/import_shapes.py +++ b/councilmatic_core/management/commands/import_shapes.py @@ -1,6 +1,5 @@ import json -import requests from django.core.management.base import BaseCommand from councilmatic_core import models @@ -8,70 +7,32 @@ class Command(BaseCommand): help = "Import boundary shapefiles for Post entities" - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.session = requests.Session() - def add_arguments(self, parser): parser.add_argument( - '--base-url', - help='The base URL for the API to use to retrieve boundaries', - default='https://ocd.datamade.us', - ) - parser.add_argument( - 'boundary_set', - help='One or more slugs of boundary sets to retrieve', - nargs='*', - default=['chicago-wards-2015'], + 'shape_file', + help=( + 'The location of the file containing shapes for each Division, ' + 'relative to the project root. The file should be formatted as JSON ' + 'with each key corresponding to a Division ID and each value corresponding' + 'to a GeoJSON Feature object.' + ) ) def handle(self, *args, **options): - self.stdout.write('Populating boundaries...') - boundaries_populated = 0 - for boundary in options['boundary_set']: - bndry_set_url = options['base_url'] + '/boundaries/' + boundary + self.stdout.write('Populating shapes for Posts...') + shapes_populated = 0 + + with open(options['shape_file']) as shapef: + shapes = json.load(shapef) - page_res = self.get_response(bndry_set_url + '/?limit=0') - page_json = json.loads(page_res.text) + for division_id, shape in shapes.items(): + models.Post.objects.filter(division_id=division_id).update( + shape=shape + ) + shapes_populated += 1 - for bndry_json in page_json['objects']: - shape_url = options['base_url'] + bndry_json['url'] + 'shape' - shape_res = self.get_response(shape_url) - if shape_res: - if 'ocd-division' in bndry_json['external_id']: - filters = { - 'division_id': bndry_json['external_id'] - } - else: - # The API doesn't appear to use an OCD id as external_id, - # so we have to filter on a fragment of the Division ID - # instead of the entire ID - filters = { - 'division_id__endswith': bndry_json['external_id'] - } - models.Post.objects.filter(**filters).update( - shape=json.loads(shape_res.text) - ) - boundaries_populated += 1 self.stdout.write( self.style.SUCCESS( - 'Populated {} boundaries'.format(str(boundaries_populated)) + 'Populated {} shapes'.format(str(shapes_populated)) ) ) - - def get_response(self, url, params=None, timeout=60, **kwargs): - """ - The OCD API has intermittently thrown 502 and 504 errors, so only proceed - when receiving an 'ok' status. - """ - response = self.session.get(url, params=params, timeout=timeout, **kwargs) - - if response.ok: - return response - else: - message = '{url} returned a bad response - {status}'.format( - url=url, - status=response.status_code - ) - raise requests.exceptions.HTTPError('ERROR: {0}'.format(message)) - diff --git a/councilmatic_core/migrations/0048_auto_20190624_1722.py b/councilmatic_core/migrations/0048_post_shape.py similarity index 80% rename from councilmatic_core/migrations/0048_auto_20190624_1722.py rename to councilmatic_core/migrations/0048_post_shape.py index 24a0b25a..b011531e 100644 --- a/councilmatic_core/migrations/0048_auto_20190624_1722.py +++ b/councilmatic_core/migrations/0048_post_shape.py @@ -1,4 +1,4 @@ -# Generated by Django 2.1.9 on 2019-06-24 22:22 +# Generated by Django 2.1.9 on 2019-06-26 17:10 import django.contrib.postgres.fields.jsonb import django.core.files.storage @@ -25,7 +25,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Post', fields=[ - ('post_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.Post')), + ('post', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='councilmatic_post', serialize=False, to='core.Post')), ('shape', django.contrib.postgres.fields.jsonb.JSONField()), ], options={ diff --git a/councilmatic_core/models.py b/councilmatic_core/models.py index 784edee4..287b9602 100644 --- a/councilmatic_core/models.py +++ b/councilmatic_core/models.py @@ -235,6 +235,11 @@ def link_html(self): class Post(opencivicdata.core.models.Post): + post = models.OneToOneField(opencivicdata.core.models.Post, + on_delete=models.CASCADE, + related_name='councilmatic_post', + parent_link=True) + organization = ProxyForeignKey( Organization, related_name='posts', diff --git a/councilmatic_core/signals/handlers.py b/councilmatic_core/signals/handlers.py index b5d81441..d039c890 100644 --- a/councilmatic_core/signals/handlers.py +++ b/councilmatic_core/signals/handlers.py @@ -3,14 +3,16 @@ from django.utils.text import slugify, Truncator from opencivicdata.core.models import (Organization as OCDOrganization, - Person as OCDPerson) + Person as OCDPerson, + Post as OCDPost) from opencivicdata.legislative.models import (Event as OCDEvent, Bill as OCDBill) from councilmatic_core.models import (Organization as CouncilmaticOrganization, Person as CouncilmaticPerson, Event as CouncilmaticEvent, - Bill as CouncilmaticBill) + Bill as CouncilmaticBill, + Post as CouncilmaticPost) @receiver(post_save, sender=OCDOrganization) @@ -61,3 +63,10 @@ def create_councilmatic_bill(sender, instance, created, **kwargs): cb = CouncilmaticBill.objects.get(id=instance.id) else: cb = instance.councilmatic_bill + +@receiver(post_save, sender=OCDPost) +def create_councilmatic_post(sender, instance, created, **kwargs): + if created: + cp = CouncilmaticPost(post=instance, + shape={}) + cp.save_base(raw=True) diff --git a/councilmatic_core/views.py b/councilmatic_core/views.py index 81cf0637..766e2f6f 100644 --- a/councilmatic_core/views.py +++ b/councilmatic_core/views.py @@ -195,7 +195,7 @@ def map(self): feature = { 'type': 'Feature', - 'geometry': json.loads(post.shape), + 'geometry': post.shape, 'properties': { 'district': post.label, 'council_member': council_member, @@ -379,7 +379,7 @@ def get_context_data(self, **kwargs): feature = { 'type': 'Feature', - 'geometry': json.loads(person.latest_council_membership.post.shape), + 'geometry': person.latest_council_membership.post.shape, 'properties': { 'district': person.latest_council_membership.post.label, } From b3b7bf8513239d7f99d7e7600352df9fc416c53a Mon Sep 17 00:00:00 2001 From: "Jean Cochrane (Lead developer, DataMade)" Date: Wed, 26 Jun 2019 16:25:29 -0500 Subject: [PATCH 4/4] Store and load Post.shapes as GeoJSON --- .../management/commands/import_shapes.py | 56 ++++++++++++++++--- .../migrations/0048_post_shape.py | 11 +--- councilmatic_core/models.py | 4 +- councilmatic_core/signals/handlers.py | 3 +- councilmatic_core/templates/partials/map.html | 1 - councilmatic_core/views.py | 4 +- 6 files changed, 55 insertions(+), 24 deletions(-) diff --git a/councilmatic_core/management/commands/import_shapes.py b/councilmatic_core/management/commands/import_shapes.py index 1d8c9ffb..56955d79 100644 --- a/councilmatic_core/management/commands/import_shapes.py +++ b/councilmatic_core/management/commands/import_shapes.py @@ -1,6 +1,8 @@ import json -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError +from django.contrib.gis.geos import GEOSGeometry + from councilmatic_core import models @@ -9,12 +11,12 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - 'shape_file', + 'geojson_file', help=( - 'The location of the file containing shapes for each Division, ' - 'relative to the project root. The file should be formatted as JSON ' - 'with each key corresponding to a Division ID and each value corresponding' - 'to a GeoJSON Feature object.' + 'The location of the GeoJSON file containing shapes for each Division, ' + 'relative to the project root. The file should be formatted as a ' + 'GeoJSON FeatureCollection where each Feature A) corresponds to a distinct ' + 'Division and B) has a "division_id" attribute in the "properties" object. ' ) ) @@ -22,12 +24,34 @@ def handle(self, *args, **options): self.stdout.write('Populating shapes for Posts...') shapes_populated = 0 - with open(options['shape_file']) as shapef: + with open(options['geojson_file']) as shapef: shapes = json.load(shapef) - for division_id, shape in shapes.items(): + features = self._get_or_raise( + shapes, + 'features', + 'Could not find the "features" array in the input file.' + ) + + for feature in features: + shape = self._get_or_raise( + feature, + 'geometry', + 'Could not find a "geometry" key in the Feature.' + ) + properties = self._get_or_raise( + feature, + 'properties', + 'Could not find a "properties" key in the Feature.' + ) + division_id = self._get_or_raise( + properties, + 'division_id', + 'Could not find a "division_id" key in the Feature properties.' + ) + models.Post.objects.filter(division_id=division_id).update( - shape=shape + shape=GEOSGeometry(json.dumps(shape)) ) shapes_populated += 1 @@ -36,3 +60,17 @@ def handle(self, *args, **options): 'Populated {} shapes'.format(str(shapes_populated)) ) ) + + def _get_or_raise(self, dct, key, msg): + """ + Check to see if 'dct' has a key corresponding to 'key', and raise an + error if it doesn't. + """ + format_prompt = ( + 'Is the input file formatted as a GeoJSON FeatureCollection ' + 'where each feature has a "division_id" property?' + ) + if not dct.get(key): + raise CommandError(msg + ' ' + format_prompt) + else: + return dct[key] diff --git a/councilmatic_core/migrations/0048_post_shape.py b/councilmatic_core/migrations/0048_post_shape.py index b011531e..af68bb77 100644 --- a/councilmatic_core/migrations/0048_post_shape.py +++ b/councilmatic_core/migrations/0048_post_shape.py @@ -1,6 +1,6 @@ -# Generated by Django 2.1.9 on 2019-06-26 17:10 +# Generated by Django 2.1.9 on 2019-06-26 21:17 -import django.contrib.postgres.fields.jsonb +import django.contrib.gis.db.models.fields import django.core.files.storage from django.db import migrations, models import django.db.models.deletion @@ -17,16 +17,11 @@ class Migration(migrations.Migration): migrations.DeleteModel( name='Post', ), - migrations.AlterField( - model_name='person', - name='headshot', - field=models.FileField(default='images/headshot_placeholder.png', storage=django.core.files.storage.FileSystemStorage(base_url='/', location='/Users/goobzie/datamade/chi-councilmatic'), upload_to='images/headshots'), - ), migrations.CreateModel( name='Post', fields=[ ('post', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, related_name='councilmatic_post', serialize=False, to='core.Post')), - ('shape', django.contrib.postgres.fields.jsonb.JSONField()), + ('shape', django.contrib.gis.db.models.fields.GeometryField(null=True, srid=4326)), ], options={ 'abstract': False, diff --git a/councilmatic_core/models.py b/councilmatic_core/models.py index 287b9602..65defd7c 100644 --- a/councilmatic_core/models.py +++ b/councilmatic_core/models.py @@ -2,7 +2,7 @@ import os from django.db import models -from django.contrib.postgres.fields import JSONField +from django.contrib.gis.db import models as geo_models from django.core.exceptions import ImproperlyConfigured from django.conf import settings from django.urls import reverse, NoReverseMatch @@ -247,7 +247,7 @@ class Post(opencivicdata.core.models.Post): on_delete=models.CASCADE, ) - shape = JSONField() + shape = geo_models.GeometryField(null=True) @cached_property def current_member(self): diff --git a/councilmatic_core/signals/handlers.py b/councilmatic_core/signals/handlers.py index d039c890..e312feb7 100644 --- a/councilmatic_core/signals/handlers.py +++ b/councilmatic_core/signals/handlers.py @@ -67,6 +67,5 @@ def create_councilmatic_bill(sender, instance, created, **kwargs): @receiver(post_save, sender=OCDPost) def create_councilmatic_post(sender, instance, created, **kwargs): if created: - cp = CouncilmaticPost(post=instance, - shape={}) + cp = CouncilmaticPost(post=instance) cp.save_base(raw=True) diff --git a/councilmatic_core/templates/partials/map.html b/councilmatic_core/templates/partials/map.html index 69acde50..9a17fbdf 100644 --- a/councilmatic_core/templates/partials/map.html +++ b/councilmatic_core/templates/partials/map.html @@ -75,7 +75,6 @@ }); layer.on('mouseover', function(e){ - console.log(e.target.feature.properties) infoBox.update(e.target.feature.properties); e.target.setStyle({'fillOpacity': 0.6, 'color': "{{MAP_CONFIG.highlight_color}}"}); }); diff --git a/councilmatic_core/views.py b/councilmatic_core/views.py index 766e2f6f..adcc21a0 100644 --- a/councilmatic_core/views.py +++ b/councilmatic_core/views.py @@ -195,7 +195,7 @@ def map(self): feature = { 'type': 'Feature', - 'geometry': post.shape, + 'geometry': json.loads(post.shape.json), 'properties': { 'district': post.label, 'council_member': council_member, @@ -379,7 +379,7 @@ def get_context_data(self, **kwargs): feature = { 'type': 'Feature', - 'geometry': person.latest_council_membership.post.shape, + 'geometry': json.loads(person.latest_council_membership.post.shape.json), 'properties': { 'district': person.latest_council_membership.post.label, }