From aee5342fe82c662eda563ee72e67df6a43f60973 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 7 Feb 2024 15:45:21 -0500 Subject: [PATCH 01/74] 1569 add dummy committee types for testing --- .../test_efo_mock_data/committee_accounts.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json b/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json index b5e3a1252d..ef3e2608fe 100644 --- a/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json +++ b/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json @@ -237,7 +237,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title10", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "H" }, { "committee_id": "C00100297", @@ -447,7 +448,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title18", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Q" }, { "committee_id": "C00100529", @@ -477,7 +479,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title19", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "W" }, { "committee_id": "C00100537", @@ -507,7 +510,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title20", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Y" }, { "committee_id": "C00100362", From 5dc783cf4d6e01cdc38ada56ded586badb86ee0f Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Thu, 15 Feb 2024 17:07:08 -0500 Subject: [PATCH 02/74] Fix F1M contact lookup to screen out duplicates --- .../fecfiler/contacts/serializers.py | 9 +++++- .../fecfiler/contacts/test_views.py | 1 + django-backend/fecfiler/contacts/views.py | 30 +++++++++++++++++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/contacts/serializers.py b/django-backend/fecfiler/contacts/serializers.py index a6489d1f99..a4331a34c3 100644 --- a/django-backend/fecfiler/contacts/serializers.py +++ b/django-backend/fecfiler/contacts/serializers.py @@ -53,6 +53,9 @@ class ContactSerializer( # Contains the number of transactions linked to the contact transaction_count = IntegerField(required=False) + # Contains the number of reports directly linked to the contact (e.g. F1M) + report_count = IntegerField(required=False) + def get_schema_name(self, data): return f"Contact_{self.contact_value[data.get('type', None)]}" @@ -91,17 +94,21 @@ class Meta: ] ] fields.append("transaction_count") + fields.append("report_count") read_only_fields = [ "uuid", "deleted", "created", "updated", "transaction_count", + "report_count", ] def to_internal_value(self, data): - # Remove the transaction_count because it is an annotated field + # Remove the transaction_count and report_count because they are annotated fields # delivered to the front end. if "transaction_count" in data: del data["transaction_count"] + if "report_count" in data: + del data["report_count"] return super().to_internal_value(data) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index 0d21501cde..422ce293d7 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -176,6 +176,7 @@ def test_committee_lookup_happy_path(self, mock_get): "created": "2022-02-09T00:00:00Z", "updated": "2022-02-09T00:00:00Z", "transaction_count": 0, + "report_count": 0, } ], } diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index 3ed495eaa2..048841e163 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -58,6 +58,14 @@ class ContactViewSet(CommitteeOwnedViewSet): + Count("contact_2_transaction_set") + Count("contact_3_transaction_set"), ) + .annotate( + report_count=Count("contact_affiliated_transaction_set") + + Count("contact_candidate_I_transaction_set") + + Count("contact_candidate_II_transaction_set") + + Count("contact_candidate_III_transaction_set") + + Count("contact_candidate_IV_transaction_set") + + Count("contact_candidate_V_transaction_set") + ) .alias( sort_name=Concat( "name", "last_name", Value(" "), "first_name", output_field=CharField() @@ -98,6 +106,10 @@ def candidate_lookup(self, request): max_fecfile_results, max_fec_results = self.get_max_results(request) office = request.GET.get("office", "") + exclude_fec_ids = request.GET.get("exclude_fec_ids").split(',') \ + if request.GET.get("exclude_fec_ids") else [] + exclude_ids = request.GET.get("exclude_ids").split(',') \ + if request.GET.get("exclude_ids") else [] params = {"q": q, "api_key": FEC_API_KEY} if office: params["office"] = office @@ -121,6 +133,7 @@ def candidate_lookup(self, request): | (Q(full_name_fwd__regex=term) | Q(full_name_bwd__regex=term)) ) ) + .exclude(id__in=exclude_ids) .values() .order_by("-candidate_id") ) @@ -133,7 +146,7 @@ def candidate_lookup(self, request): for fac in fec_api_candidates if not any( fac["candidate_id"] == ffc["candidate_id"] for ffc in fecfile_candidates - ) + ) and fac["candidate_id"] not in exclude_fec_ids ] return_value = { "fec_api_candidates": fec_api_candidates[:max_fec_results], @@ -150,6 +163,10 @@ def committee_lookup(self, request): max_fecfile_results, max_fec_results = self.get_max_results(request) + exclude_fec_ids = request.GET.get("exclude_fec_ids").split(',') \ + if request.GET.get("exclude_fec_ids") else [] + exclude_ids = request.GET.get("exclude_ids").split(',') \ + if request.GET.get("exclude_ids") else [] params = urlencode({"q": q, "api_key": FEC_API_KEY}) json_results = requests.get( FEC_API_COMMITTEE_LOOKUP_ENDPOINT, params=params @@ -160,6 +177,7 @@ def committee_lookup(self, request): .filter( Q(type="COM") & (Q(committee_id__icontains=q) | Q(name__icontains=q)) ) + .exclude(id__in=exclude_ids) .values() .order_by("-committee_id") ) @@ -167,7 +185,9 @@ def committee_lookup(self, request): fec_api_committees = [ fac for fac in fec_api_committees - if not any(fac["id"] == ffc["committee_id"] for ffc in fecfile_committees) + if not any( + fac["id"] == ffc["committee_id"] for ffc in fecfile_committees + ) and fac["committee_id"] not in exclude_fec_ids ] fec_api_committees = fec_api_committees[:max_fec_results] fecfile_committees = fecfile_committees[:max_fecfile_results] @@ -188,6 +208,8 @@ def individual_lookup(self, request): term = (".*" + ".* .*".join(tokens) + ".*").lower() max_fecfile_results, _ = self.get_max_results(request) + exclude_ids = request.GET.get("exclude_ids").split(',') \ + if request.GET.get("exclude_ids") else [] fecfile_individuals = list( self.get_queryset() @@ -199,6 +221,7 @@ def individual_lookup(self, request): Q(type="IND") & (Q(full_name_fwd__regex=term) | Q(full_name_bwd__regex=term)) ) + .exclude(id__in=exclude_ids) .values() .order_by(Lower("last_name").desc())[:max_fecfile_results] ) @@ -215,10 +238,13 @@ def organization_lookup(self, request): return HttpResponseBadRequest() max_fecfile_results, _ = self.get_max_results(request) + exclude_ids = request.GET.get("exclude_ids").split(',') \ + if request.GET.get("exclude_ids") else [] fecfile_organizations = list( self.get_queryset() .filter(Q(type="ORG") & Q(name__icontains=q)) + .exclude(id__in=exclude_ids) .values() .order_by("-name")[:max_fecfile_results] ) From bbce83b7c6507bfc4e8dd0dbf3e1bc0e63486cbd Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Fri, 16 Feb 2024 15:36:48 -0500 Subject: [PATCH 03/74] Create endpoint to remove a member from a committee --- .../fecfiler/committee_accounts/test_views.py | 19 +++++++++++++++++++ .../fecfiler/committee_accounts/urls.py | 5 ++++- .../fecfiler/committee_accounts/views.py | 9 +++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 django-backend/fecfiler/committee_accounts/test_views.py diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py new file mode 100644 index 0000000000..e8942bbaf2 --- /dev/null +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -0,0 +1,19 @@ +from unittest import mock +from fecfiler.user.models import User +from django.test import RequestFactory, TestCase +from .views import CommitteeViewSet + +class CommitteeViewSetTest(TestCase): + fixtures = ["C01234567_user_and_committee"] + + def setUp(self): + self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") + self.factory = RequestFactory() + + def test_remove_member(self): + request = self.factory.get("/api/v1/committees/C87654321/member/test@fec.gov") + request.user = self.user + request.session = {"committee_uuid": "11111111-2222-3333-4444-555555555555"} + request.method = "DELETE" + response = CommitteeViewSet.remove_member(request, "C01234567", "test@fec.gov") + self.assertEqual(response.status_code, 200) \ No newline at end of file diff --git a/django-backend/fecfiler/committee_accounts/urls.py b/django-backend/fecfiler/committee_accounts/urls.py index 83f3df8457..439b865111 100644 --- a/django-backend/fecfiler/committee_accounts/urls.py +++ b/django-backend/fecfiler/committee_accounts/urls.py @@ -7,4 +7,7 @@ router.register(r"", CommitteeViewSet, basename="committees") # The API URLs are now determined automatically by the router. -urlpatterns = [path("committees/", include(router.urls))] +urlpatterns = [ + path("committees/", include(router.urls)), + path('committees//member//', CommitteeViewSet.remove_member, name='remove-member'), + ] diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index c428dfe625..e6cc89ace3 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -5,6 +5,7 @@ from .models import CommitteeAccount from .serializers import CommitteeAccountSerializer, CommitteeMemberSerializer import structlog +from django.http import HttpResponse logger = structlog.get_logger(__name__) @@ -47,6 +48,14 @@ def active(self, request): committee_uuid = request.session["committee_uuid"] committee = self.get_queryset().filter(id=committee_uuid).first() return Response(self.get_serializer(committee).data) + + @action(detail=True, methods=["delete"]) + def remove_member(request, committee_id, member_email): + committee_uuid = request.session["committee_uuid"] + committee = CommitteeAccount.objects.filter(id=committee_uuid).first() + member = committee.members.filter(email=member_email).first() + member.delete() + return HttpResponse('Member removed') class CommitteeOwnedViewSet(viewsets.ModelViewSet): From 2002634ecaa26c8b853f91883f8088954b110952 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 16 Feb 2024 15:42:58 -0500 Subject: [PATCH 04/74] Implements the ability to add new committee members, including pending memberships --- .../fecfiler/committee_accounts/models.py | 10 +++- .../committee_accounts/serializers.py | 49 ++++++++++++++++- .../fecfiler/committee_accounts/views.py | 52 ++++++++++++++++--- .../C01234567_user_and_committee.json | 5 +- django-backend/fecfiler/user/managers.py | 24 +++++++++ django-backend/fecfiler/user/models.py | 3 ++ django-backend/fixtures/e2e-test-data.json | 12 +++-- 7 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 django-backend/fecfiler/user/managers.py diff --git a/django-backend/fecfiler/committee_accounts/models.py b/django-backend/fecfiler/committee_accounts/models.py index 63102ebdce..4d4b0ab90f 100644 --- a/django-backend/fecfiler/committee_accounts/models.py +++ b/django-backend/fecfiler/committee_accounts/models.py @@ -40,8 +40,16 @@ class CommitteeRole(models.TextChoices): COMMITTEE_ADMINISTRATOR = "COMMITTEE_ADMINISTRATOR" REVIEWER = "REVIEWER" + id = models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ) committee_account = models.ForeignKey(CommitteeAccount, on_delete=models.CASCADE) - user = models.ForeignKey(User, on_delete=models.CASCADE) + user = models.ForeignKey(User, null=True, blank=False, on_delete=models.CASCADE) + pending_email = models.EmailField(null=True, blank=True) role = models.CharField( max_length=25, choices=CommitteeRole.choices, null=False, blank=False ) diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index f1f44a37d8..fb9a84ec0f 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -1,4 +1,4 @@ -from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.models import CommitteeAccount, Membership from django.contrib.sessions.exceptions import SuspiciousSession from rest_framework import serializers, relations import structlog @@ -26,6 +26,53 @@ def get_role(self, object): return object.membership_set.get(committee_account_id=committee_id).role +class CommitteeMembershipSerializer(serializers.Serializer): + id = serializers.SerializerMethodField() + email = serializers.SerializerMethodField() + username = serializers.SerializerMethodField() + name = serializers.SerializerMethodField() + role = serializers.CharField() + is_active = serializers.SerializerMethodField() + + def get_id(self, object): + if object.user is not None: + return object.user.id + return object.id + + def get_email(self, object): + if object.user is not None: + return object.user.email + return object.pending_email + + def get_username(self, object): + if object.user is not None: + return object.user.username + return '' + + def get_name(self, object): + if object.user is not None: + return f"{object.user.last_name}, {object.user.first_name}" + return '' + + def get_is_active(self, object): + return object.user is not None + + class Meta: + model = Membership + + def get_fields(): + return [ + f.name + for f in Membership._meta.get_fields() + if f.name + not in [ + "deleted", + "user", + "pending_email" + ] + ] + + class CommitteeOwnedSerializer(serializers.ModelSerializer): """Serializer for CommitteeOwnedModel Inherit this to assign the user's committee as the object's diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index c428dfe625..faf45ac4a3 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -1,9 +1,10 @@ +from fecfiler.user.models import User from rest_framework import viewsets, mixins from django.contrib.sessions.exceptions import SuspiciousSession from rest_framework.decorators import action from rest_framework.response import Response -from .models import CommitteeAccount -from .serializers import CommitteeAccountSerializer, CommitteeMemberSerializer +from .models import CommitteeAccount, Membership +from .serializers import CommitteeAccountSerializer, CommitteeMembershipSerializer import structlog logger = structlog.get_logger(__name__) @@ -19,20 +20,55 @@ def get_queryset(self): @action(detail=True, methods=["get"]) def members(self, request, pk): committee = self.get_object() - serializer_context = {"committee_id": committee.id} - queryset = self.filter_queryset(committee.members.all()) + queryset = Membership.objects.filter(committee_account=committee) page = self.paginate_queryset(queryset) if page is not None: - serializer = CommitteeMemberSerializer( - page, many=True, context=serializer_context + serializer = CommitteeMembershipSerializer( + page, many=True ) return self.get_paginated_response(serializer.data) - serializer = CommitteeMemberSerializer( - queryset, many=True, context=serializer_context + serializer = CommitteeMembershipSerializer( + queryset, many=True ) return Response(serializer.data) + @action(detail=True, methods=["post"]) + def add_member(self, request, pk): + committee = self.get_object() + queryset= Membership.objects.filter(committee_account=committee) + + email = request.data.get('email', None) + role = request.data.get('role', None) + + missing_fields = [] + if email is None or len(email) == 0: + missing_fields.append("email") + + if role is None: + missing_fields.append("role") + + if len(missing_fields) > 0: + return Response(f"Missing fields: {', '.join(missing_fields)}", status=400) + + if role not in Membership.CommitteeRole.choices: + return Response(f"Invalid role", status=400) + + matching_users = User.objects.filter(email=email) + if matching_users.count() > 0: + for user in matching_users: + added_member = committee.members.add(user) + added_member.role = role + added_member.save() + logger.info(f"Added existing user {email} to committee {committee.committee_id}") + else: + Membership( + committee_account=committee, + pending_email=email, + role=request.role + ).save() + logger.info(f"Added pending membership for email {email} for committee {committee.committee_id}") + @action(detail=True, methods=["post"]) def activate(self, request, pk): committee = self.get_object() diff --git a/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json b/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json index 7e551057c1..f157bddebb 100644 --- a/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json +++ b/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json @@ -26,11 +26,12 @@ }, { "model": "committee_accounts.Membership", - "pk": "99", + "pk": "136a21f2-66fe-4d56-89e9-0d1d4612741c", "fields": { "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "11111111-2222-3333-4444-555555555555", - "user_id": "12345678-aaaa-bbbb-cccc-111122223333" + "user_id": "12345678-aaaa-bbbb-cccc-111122223333", + "pending_email": null } } ] \ No newline at end of file diff --git a/django-backend/fecfiler/user/managers.py b/django-backend/fecfiler/user/managers.py new file mode 100644 index 0000000000..8e72ac39c6 --- /dev/null +++ b/django-backend/fecfiler/user/managers.py @@ -0,0 +1,24 @@ +import structlog +from django.contrib.auth.models import UserManager as AbstractUserManager + +logger = structlog.get_logger(__name__) + + +class UserManager(AbstractUserManager): + def create_user(self, user_id, **obj_data): + from fecfiler.committee_accounts.models import Membership + + new_user = super().create_user(user_id, **obj_data) + pending_memberships = Membership.objects.filter( + user=None, + pending_email=obj_data['email'] + ) + + logger.info(f"New User Created: {obj_data['email']} - {pending_memberships.count()} Pending Memberships") + + for new_membership in pending_memberships: + new_membership.user = new_user + new_membership.pending_email = None + new_membership.save() + + return new_user \ No newline at end of file diff --git a/django-backend/fecfiler/user/models.py b/django-backend/fecfiler/user/models.py index ec75b7e607..778a40b2cb 100644 --- a/django-backend/fecfiler/user/models.py +++ b/django-backend/fecfiler/user/models.py @@ -1,6 +1,7 @@ from django.contrib.auth.models import AbstractUser from django.db import models import uuid +from fecfiler.user.managers import UserManager class User(AbstractUser): @@ -17,3 +18,5 @@ class User(AbstractUser): user_permissions = None login_dot_gov = False security_consent_date = models.DateField(null=True, blank=True) + + objects = UserManager() diff --git a/django-backend/fixtures/e2e-test-data.json b/django-backend/fixtures/e2e-test-data.json index dd84be49f2..c8ee874f67 100644 --- a/django-backend/fixtures/e2e-test-data.json +++ b/django-backend/fixtures/e2e-test-data.json @@ -42,20 +42,22 @@ }, { "model": "committee_accounts.Membership", - "pk": "1", + "pk": "3e281c08-2b1f-4cd0-9236-410fe872edb9", "fields": { "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "c94c5d1a-9e73-464d-ad72-b73b5d8667a9", - "user_id": "1cb79553-a0e6-43f2-b182-f102eca4a043" + "user_id": "1cb79553-a0e6-43f2-b182-f102eca4a043", + "pending_email": null } }, { "model": "committee_accounts.Membership", - "pk": "2", + "pk": "2b261104-76a1-4f20-b082-9aafdd7115fe", "fields": { "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "c94c5d1a-9e73-464d-ad72-b73b5d8667a9", - "user_id": "2faed187-f4b5-46b4-a0d9-4a250ac792da" + "user_id": "2faed187-f4b5-46b4-a0d9-4a250ac792da", + "pending_email": null } } -] \ No newline at end of file +] From fd0c9af1aa5a53a4486fed97733cd3b5052655ce Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 16 Feb 2024 16:09:57 -0500 Subject: [PATCH 05/74] Adds new migrations --- ...ding_email_alter_membership_id_and_more.py | 54 +++++++++++++++++++ .../migrations/0004_alter_user_managers.py | 20 +++++++ 2 files changed, 74 insertions(+) create mode 100644 django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py create mode 100644 django-backend/fecfiler/user/migrations/0004_alter_user_managers.py diff --git a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py new file mode 100644 index 0000000000..24c882fc5b --- /dev/null +++ b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py @@ -0,0 +1,54 @@ +# Generated by Django 4.2.7 on 2024-02-16 20:43 + +from django.conf import settings +from django.db import migrations, models +import uuid + + +def delete_pending_memberships(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") # noqa + Membership.objects.filter(user=None).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('committee_accounts', '0002_membership'), + ] + + operations = [ + migrations.AddField( + model_name='membership', + name='pending_email', + field=models.EmailField(blank=True, max_length=254, null=True), + ), + migrations.AddField( + model_name='membership', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=False, serialize=False, unique=True) + ), + migrations.RemoveField( + model_name='membership', + name='id', + ), + migrations.AlterField( + model_name='membership', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True) + ), + migrations.RenameField( + model_name='membership', + old_name='uuid', + new_name='id' + ), + migrations.AlterField( + model_name='membership', + name='user', + field=models.ForeignKey(null=True, on_delete=models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.RunPython( + migrations.RunPython.noop, + delete_pending_memberships + ), + ] diff --git a/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py b/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py new file mode 100644 index 0000000000..dd7809c987 --- /dev/null +++ b/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.7 on 2024-02-16 20:43 + +from django.db import migrations +import fecfiler.user.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('user', '0003_user_security_consent_date'), + ] + + operations = [ + migrations.AlterModelManagers( + name='user', + managers=[ + ('objects', fecfiler.user.managers.UserManager()), + ], + ), + ] From e3a764dd735234d61805ef583edc364177078bd0 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 16 Feb 2024 16:31:26 -0500 Subject: [PATCH 06/74] Makes the new migration reversable --- ...ding_email_alter_membership_id_and_more.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py index 24c882fc5b..7c1f0715cd 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py @@ -9,6 +9,12 @@ def delete_pending_memberships(apps, schema_editor): Membership = apps.get_model("committee_accounts", "Membership") # noqa Membership.objects.filter(user=None).delete() +def generate_new_uuid(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") # noqa + for membership in Membership.objects.all(): + membership.uuid = uuid.uuid4() + membership.save() + class Migration(migrations.Migration): @@ -26,22 +32,26 @@ class Migration(migrations.Migration): migrations.AddField( model_name='membership', name='uuid', - field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=False, serialize=False, unique=True) + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=False, serialize=False, unique=False) + ), + migrations.RunPython( + generate_new_uuid, + migrations.RunPython.noop, ), migrations.RemoveField( model_name='membership', name='id', ), - migrations.AlterField( - model_name='membership', - name='uuid', - field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True) - ), migrations.RenameField( model_name='membership', old_name='uuid', new_name='id' ), + migrations.AlterField( + model_name='membership', + name='id', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True) + ), migrations.AlterField( model_name='membership', name='user', From ec3ad916872206a0774b5cd949d48d992dbdad4a Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Fri, 16 Feb 2024 15:53:47 -0500 Subject: [PATCH 07/74] Fix linting --- .../fecfiler/committee_accounts/test_views.py | 15 +++++++++------ .../fecfiler/committee_accounts/urls.py | 5 +++-- .../fecfiler/committee_accounts/views.py | 7 +++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index e8942bbaf2..c5644a3721 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,19 +1,22 @@ -from unittest import mock from fecfiler.user.models import User from django.test import RequestFactory, TestCase from .views import CommitteeViewSet + class CommitteeViewSetTest(TestCase): fixtures = ["C01234567_user_and_committee"] - + def setUp(self): self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") self.factory = RequestFactory() def test_remove_member(self): - request = self.factory.get("/api/v1/committees/C87654321/member/test@fec.gov") + request = self.factory.get( + "/api/v1/committees/C87654321/member/test@fec.gov") request.user = self.user - request.session = {"committee_uuid": "11111111-2222-3333-4444-555555555555"} + request.session = { + "committee_uuid": "11111111-2222-3333-4444-555555555555"} request.method = "DELETE" - response = CommitteeViewSet.remove_member(request, "C01234567", "test@fec.gov") - self.assertEqual(response.status_code, 200) \ No newline at end of file + response = CommitteeViewSet.remove_member( + self, request, "C01234567", "test@fec.gov") + self.assertEqual(response.status_code, 200) diff --git a/django-backend/fecfiler/committee_accounts/urls.py b/django-backend/fecfiler/committee_accounts/urls.py index 439b865111..95417b2066 100644 --- a/django-backend/fecfiler/committee_accounts/urls.py +++ b/django-backend/fecfiler/committee_accounts/urls.py @@ -9,5 +9,6 @@ # The API URLs are now determined automatically by the router. urlpatterns = [ path("committees/", include(router.urls)), - path('committees//member//', CommitteeViewSet.remove_member, name='remove-member'), - ] + path('committees//member//', + CommitteeViewSet.remove_member, name='remove-member'), +] diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index e6cc89ace3..25c3e4e65d 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -48,9 +48,9 @@ def active(self, request): committee_uuid = request.session["committee_uuid"] committee = self.get_queryset().filter(id=committee_uuid).first() return Response(self.get_serializer(committee).data) - + @action(detail=True, methods=["delete"]) - def remove_member(request, committee_id, member_email): + def remove_member(self, request, committee_id, member_email): committee_uuid = request.session["committee_uuid"] committee = CommitteeAccount.objects.filter(id=committee_uuid).first() member = committee.members.filter(email=member_email).first() @@ -71,6 +71,5 @@ def get_queryset(self): raise SuspiciousSession("session has invalid committee_uuid") queryset = super().get_queryset() structlog.contextvars.bind_contextvars( - committee_id=committee.committee_id, committee_uuid=committee.id - ) + committee_id=committee.committee_id, committee_uuid=committee.id) return queryset.filter(committee_account_id=committee.id) From 02637d27d9a08bf873ac94074cd6ae6bbd9f9d8a Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 16 Feb 2024 17:20:27 -0500 Subject: [PATCH 08/74] CommitteeMemberships have their own viewset --- ...ding_email_alter_membership_id_and_more.py | 11 +++ .../fecfiler/committee_accounts/models.py | 2 + .../committee_accounts/serializers.py | 23 ++--- .../fecfiler/committee_accounts/urls.py | 11 ++- .../fecfiler/committee_accounts/views.py | 84 +++++++++++-------- 5 files changed, 82 insertions(+), 49 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py index 7c1f0715cd..94caaed21e 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py @@ -57,6 +57,17 @@ class Migration(migrations.Migration): name='user', field=models.ForeignKey(null=True, on_delete=models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), ), + migrations.AddField( + model_name='membership', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='membership', + name='updated', + field=models.DateTimeField(auto_now=True), + ), migrations.RunPython( migrations.RunPython.noop, delete_pending_memberships diff --git a/django-backend/fecfiler/committee_accounts/models.py b/django-backend/fecfiler/committee_accounts/models.py index 4d4b0ab90f..fbce1e3393 100644 --- a/django-backend/fecfiler/committee_accounts/models.py +++ b/django-backend/fecfiler/committee_accounts/models.py @@ -53,6 +53,8 @@ class CommitteeRole(models.TextChoices): role = models.CharField( max_length=25, choices=CommitteeRole.choices, null=False, blank=False ) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) class CommitteeOwnedModel(models.Model): diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index fb9a84ec0f..01311ee1ff 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -60,17 +60,20 @@ def get_is_active(self, object): class Meta: model = Membership - def get_fields(): - return [ - f.name - for f in Membership._meta.get_fields() - if f.name - not in [ - "deleted", - "user", - "pending_email" - ] + fields = [ + f.name + for f in Membership._meta.get_fields() + if f.name + not in [ + "deleted", + "user", + "pending_email" ] + ] + read_only_fields = [ + "id", + "created", + ] class CommitteeOwnedSerializer(serializers.ModelSerializer): diff --git a/django-backend/fecfiler/committee_accounts/urls.py b/django-backend/fecfiler/committee_accounts/urls.py index 83f3df8457..c600ca76dc 100644 --- a/django-backend/fecfiler/committee_accounts/urls.py +++ b/django-backend/fecfiler/committee_accounts/urls.py @@ -1,10 +1,13 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import CommitteeViewSet +from .views import CommitteeMembershipViewSet, CommitteeViewSet # Create a router and register our viewsets with it. -router = DefaultRouter() -router.register(r"", CommitteeViewSet, basename="committees") +committee_router = DefaultRouter() +committee_router.register(r"", CommitteeViewSet, basename="committees") + +members_router = DefaultRouter() +members_router.register(r"", CommitteeMembershipViewSet) # The API URLs are now determined automatically by the router. -urlpatterns = [path("committees/", include(router.urls))] +urlpatterns = [path("committees/", include(committee_router.urls)), path("committees/", include(members_router.urls))] diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index faf45ac4a3..40475c3898 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -1,5 +1,5 @@ from fecfiler.user.models import User -from rest_framework import viewsets, mixins +from rest_framework import filters, viewsets, mixins from django.contrib.sessions.exceptions import SuspiciousSession from rest_framework.decorators import action from rest_framework.response import Response @@ -17,6 +17,53 @@ def get_queryset(self): user = self.request.user return CommitteeAccount.objects.filter(members=user) + @action(detail=True, methods=["post"]) + def activate(self, request, pk): + committee = self.get_object() + if not committee: + return Response("Committee could not be activated", status=403) + committee_uuid = committee.id + request.session["committee_uuid"] = str(committee_uuid) + return Response("Committee activated") + + @action(detail=False, methods=["get"]) + def active(self, request): + committee_uuid = request.session["committee_uuid"] + committee = self.get_queryset().filter(id=committee_uuid).first() + return Response(self.get_serializer(committee).data) + + +class CommitteeOwnedViewSet(viewsets.ModelViewSet): + + """ModelViewSet for models using CommitteeOwnedModel + Inherit this view set to filter the queryset by the user's committee + """ + + def get_queryset(self): + committee_uuid = self.request.session["committee_uuid"] + committee = CommitteeAccount.objects.filter(id=committee_uuid).first() + if not committee: + raise SuspiciousSession("session has invalid committee_uuid") + queryset = super().get_queryset() + structlog.contextvars.bind_contextvars( + committee_id=committee.committee_id, committee_uuid=committee.id + ) + return queryset.filter(committee_account_id=committee.id) + + +class CommitteeMembershipViewSet(viewsets.ModelViewSet): + filter_backends = [filters.OrderingFilter] + ordering_fields = [ + "name", + "email", + "role", + "is_active", + "created" + ] + ordering = ["-created"] + + queryset = Membership.objects.all() + @action(detail=True, methods=["get"]) def members(self, request, pk): committee = self.get_object() @@ -67,37 +114,4 @@ def add_member(self, request, pk): pending_email=email, role=request.role ).save() - logger.info(f"Added pending membership for email {email} for committee {committee.committee_id}") - - @action(detail=True, methods=["post"]) - def activate(self, request, pk): - committee = self.get_object() - if not committee: - return Response("Committee could not be activated", status=403) - committee_uuid = committee.id - request.session["committee_uuid"] = str(committee_uuid) - return Response("Committee activated") - - @action(detail=False, methods=["get"]) - def active(self, request): - committee_uuid = request.session["committee_uuid"] - committee = self.get_queryset().filter(id=committee_uuid).first() - return Response(self.get_serializer(committee).data) - - -class CommitteeOwnedViewSet(viewsets.ModelViewSet): - - """ModelViewSet for models using CommitteeOwnedModel - Inherit this view set to filter the queryset by the user's committee - """ - - def get_queryset(self): - committee_uuid = self.request.session["committee_uuid"] - committee = CommitteeAccount.objects.filter(id=committee_uuid).first() - if not committee: - raise SuspiciousSession("session has invalid committee_uuid") - queryset = super().get_queryset() - structlog.contextvars.bind_contextvars( - committee_id=committee.committee_id, committee_uuid=committee.id - ) - return queryset.filter(committee_account_id=committee.id) + logger.info(f"Added pending membership for email {email} for committee {committee.committee_id}") \ No newline at end of file From b882c0c8348d47e03dfbb561973f6a66b3018f02 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 16 Feb 2024 18:07:17 -0500 Subject: [PATCH 09/74] Fixes sorting with new membership viewset --- ...ding_email_alter_membership_id_and_more.py | 3 ++- .../committee_accounts/serializers.py | 2 +- .../fecfiler/committee_accounts/urls.py | 2 +- .../fecfiler/committee_accounts/views.py | 23 ++++++++++++++++++- .../C01234567_user_and_committee.json | 4 +++- django-backend/fixtures/e2e-test-data.json | 8 +++++-- 6 files changed, 35 insertions(+), 7 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py index 94caaed21e..4d5b8e19f0 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py @@ -2,6 +2,7 @@ from django.conf import settings from django.db import migrations, models +from django.utils import timezone import uuid @@ -60,7 +61,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='membership', name='created', - field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + field=models.DateTimeField(auto_now_add=True, default=timezone.now), preserve_default=False, ), migrations.AddField( diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index 01311ee1ff..5f2f17de7e 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -69,7 +69,7 @@ class Meta: "user", "pending_email" ] - ] + ] + ["name", "email"] read_only_fields = [ "id", "created", diff --git a/django-backend/fecfiler/committee_accounts/urls.py b/django-backend/fecfiler/committee_accounts/urls.py index c600ca76dc..edc5bc8746 100644 --- a/django-backend/fecfiler/committee_accounts/urls.py +++ b/django-backend/fecfiler/committee_accounts/urls.py @@ -10,4 +10,4 @@ members_router.register(r"", CommitteeMembershipViewSet) # The API URLs are now determined automatically by the router. -urlpatterns = [path("committees/", include(committee_router.urls)), path("committees/", include(members_router.urls))] +urlpatterns = [path(r"committees/", include(committee_router.urls)), path(r"committee-members/", include(members_router.urls))] diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 40475c3898..317fd3949d 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -5,6 +5,9 @@ from rest_framework.response import Response from .models import CommitteeAccount, Membership from .serializers import CommitteeAccountSerializer, CommitteeMembershipSerializer +from django.db.models.fields import TextField +from django.db.models.functions import Coalesce, Concat +from django.db.models import Q, Value import structlog logger = structlog.get_logger(__name__) @@ -52,6 +55,7 @@ def get_queryset(self): class CommitteeMembershipViewSet(viewsets.ModelViewSet): + serializer_class = CommitteeMembershipSerializer filter_backends = [filters.OrderingFilter] ordering_fields = [ "name", @@ -62,7 +66,24 @@ class CommitteeMembershipViewSet(viewsets.ModelViewSet): ] ordering = ["-created"] - queryset = Membership.objects.all() + queryset = Membership.objects.all().annotate( + name= Coalesce( + Concat( + "user__last_name", + Value(", "), + "user__first_name", + output_field=TextField(), + ), + Value(''), + output_field=TextField() + ), + email=Coalesce( + "user__email", + "pending_email", + output_field=TextField() + ), + is_active=~Q(user=None) + ) @action(detail=True, methods=["get"]) def members(self, request, pk): diff --git a/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json b/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json index f157bddebb..03664d1119 100644 --- a/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json +++ b/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json @@ -31,7 +31,9 @@ "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "11111111-2222-3333-4444-555555555555", "user_id": "12345678-aaaa-bbbb-cccc-111122223333", - "pending_email": null + "pending_email": null, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z" } } ] \ No newline at end of file diff --git a/django-backend/fixtures/e2e-test-data.json b/django-backend/fixtures/e2e-test-data.json index c8ee874f67..4e12ce76e4 100644 --- a/django-backend/fixtures/e2e-test-data.json +++ b/django-backend/fixtures/e2e-test-data.json @@ -47,7 +47,9 @@ "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "c94c5d1a-9e73-464d-ad72-b73b5d8667a9", "user_id": "1cb79553-a0e6-43f2-b182-f102eca4a043", - "pending_email": null + "pending_email": null, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z" } }, { @@ -57,7 +59,9 @@ "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "c94c5d1a-9e73-464d-ad72-b73b5d8667a9", "user_id": "2faed187-f4b5-46b4-a0d9-4a250ac792da", - "pending_email": null + "pending_email": null, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z" } } ] From a3218b742cddcb904338f7be297f6815ad7ce103 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sat, 17 Feb 2024 19:59:10 -0500 Subject: [PATCH 10/74] Fix unit tests --- django-backend/fecfiler/contacts/test_views.py | 8 ++++---- django-backend/fecfiler/contacts/views.py | 2 +- django-backend/fecfiler/transactions/tests/test_views.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index 422ce293d7..00d153f3b2 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -10,10 +10,10 @@ mock_results = { "results": [ - {"name": "LNAME, FNAME I", "id": "P60012143", "office_sought": "P"}, + {"name": "LNAME, FNAME I", "candidate_id": "P60012143", "office_sought": "P"}, { "name": "LNAME, FNAME", - "id": "P60012465", + "candidate_id": "P60012465", "office_sought": "P", }, ] @@ -100,10 +100,10 @@ def test_candidate_lookup_happy_path(self, mock_get): expected_json = { "fec_api_candidates": [ - {"name": "LNAME, FNAME I", "id": "P60012143", "office_sought": "P"}, + {"name": "LNAME, FNAME I", "candidate_id": "P60012143", "office_sought": "P"}, { "name": "LNAME, FNAME", - "id": "P60012465", + "candidate_id": "P60012465", "office_sought": "P", }, ], diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index 048841e163..0109007324 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -187,7 +187,7 @@ def committee_lookup(self, request): for fac in fec_api_committees if not any( fac["id"] == ffc["committee_id"] for ffc in fecfile_committees - ) and fac["committee_id"] not in exclude_fec_ids + ) and fac["id"] not in exclude_fec_ids ] fec_api_committees = fec_api_committees[:max_fec_results] fecfile_committees = fecfile_committees[:max_fecfile_results] diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index 652af35f35..19deee6d75 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -36,13 +36,13 @@ def request(self, payload, params={}): request.session = {"committee_uuid": "11111111-2222-3333-4444-555555555555"} return request - def xtest_save_transaction_pair(self): + def test_save_transaction_pair(self): request = self.request(self.payloads["IN_KIND"]) transaction = TransactionViewSet().save_transaction(request.data, request) self.assertEqual("John", transaction.contact_1.first_name) self.assertEqual("Smith", transaction.contact_1.last_name) - def xtest_update(self): + def test_update(self): request = self.request(self.payloads["IN_KIND"]) transaction = TransactionViewSet().save_transaction(request.data, request) updated_payload = deepcopy(self.payloads["IN_KIND"]) @@ -161,7 +161,7 @@ def test_inherited_election_aggregate(self): transaction = response.data self.assertEqual(transaction.get("calendar_ytd_per_election_office"), 58.00) - def xtest_multisave_transactions(self): + def test_multisave_transactions(self): txn1 = deepcopy(self.payloads["IN_KIND"]) txn1["contributor_last_name"] = "one" txn2 = deepcopy(self.payloads["IN_KIND"]) @@ -188,7 +188,7 @@ def xtest_multisave_transactions(self): self.assertEqual(len(transactions), 3) # self.assertEqual("one", transactions[0]["contributor_last_name"]) - def xtest_reatt_redes_multisave_transactions(self): + def test_reatt_redes_multisave_transactions(self): txn1 = deepcopy(self.payloads["IN_KIND"]) txn1["contributor_last_name"] = "one" txn2 = deepcopy(self.payloads["IN_KIND"]) From e11221894dfff4aacd32f8c9edc3b316bb7a0478 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sat, 17 Feb 2024 20:08:40 -0500 Subject: [PATCH 11/74] Add jinja2 to dependency exception --- .safety.dependency.ignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore index e549625a49..616e71bd9e 100644 --- a/.safety.dependency.ignore +++ b/.safety.dependency.ignore @@ -9,3 +9,4 @@ # 40104 2022-01-15 # 63687 2024-04-01 # gitpython <3.1.41 +64227 2024-04-01 # jinja2 <3.1.3 From 1cb3554f82d145a44d17ef11061ea26359b11766 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sat, 17 Feb 2024 20:16:23 -0500 Subject: [PATCH 12/74] Add flake8 exception --- django-backend/fecfiler/contacts/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index 00d153f3b2..52a4605f16 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -100,7 +100,7 @@ def test_candidate_lookup_happy_path(self, mock_get): expected_json = { "fec_api_candidates": [ - {"name": "LNAME, FNAME I", "candidate_id": "P60012143", "office_sought": "P"}, + {"name": "LNAME, FNAME I", "candidate_id": "P60012143", "office_sought": "P"}, # noqa: E501 { "name": "LNAME, FNAME", "candidate_id": "P60012465", From c55b2820e4f82eca75b2abe8a99b6f87fbcf9e92 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sat, 17 Feb 2024 20:32:04 -0500 Subject: [PATCH 13/74] Add flake8 exception E261 --- django-backend/fecfiler/contacts/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index 52a4605f16..eaa751a623 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -100,7 +100,7 @@ def test_candidate_lookup_happy_path(self, mock_get): expected_json = { "fec_api_candidates": [ - {"name": "LNAME, FNAME I", "candidate_id": "P60012143", "office_sought": "P"}, # noqa: E501 + {"name": "LNAME, FNAME I", "candidate_id": "P60012143", "office_sought": "P"}, # noqa: E501 { "name": "LNAME, FNAME", "candidate_id": "P60012465", From 69b0bc541b6f3d6d2380c49e342ed977325c401a Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 19 Feb 2024 13:26:13 -0500 Subject: [PATCH 14/74] Add exclusion of fec ids to candidate lookup to unit tests --- django-backend/fecfiler/contacts/test_views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index eaa751a623..f5031f1fd5 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -91,7 +91,7 @@ def test_candidate_lookup_no_q(self, mock_get): def test_candidate_lookup_happy_path(self, mock_get): request = self.factory.get( "/api/v1/contacts/candidate_lookup?" - "q=test&max_fecfile_results=5&max_fec_results=5" + "q=test&max_fecfile_results=5&max_fec_results=5&exclude_fec_ids=P60012143" ) request.user = self.user request.session = {"committee_uuid": "11111111-2222-3333-4444-555555555555"} @@ -100,7 +100,6 @@ def test_candidate_lookup_happy_path(self, mock_get): expected_json = { "fec_api_candidates": [ - {"name": "LNAME, FNAME I", "candidate_id": "P60012143", "office_sought": "P"}, # noqa: E501 { "name": "LNAME, FNAME", "candidate_id": "P60012465", From 62a3182ed68f934cea6fa30b0266b22ff46a4adc Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Tue, 20 Feb 2024 09:25:35 -0500 Subject: [PATCH 15/74] Remove unused vulnerable package --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 30d82dac26..7b5ff29f7b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ drf-spectacular==0.24.2 git+https://github.com/fecgov/fecfile-validate@e36a2eff869a6adc97a9e812f028d0b8f18ec3a3#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 -Jinja2==3.1.2 invoke==1.7.3 itypes==1.2.0 MarkupSafe==2.1.1 From 0171a0a27e4a951b8d8af070e87ffc0e0975f1b8 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 20 Feb 2024 11:02:34 -0500 Subject: [PATCH 16/74] 719 upgrate GitPython --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7b5ff29f7b..b233de80f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 git+https://github.com/fecgov/fecfile-validate@e36a2eff869a6adc97a9e812f028d0b8f18ec3a3#egg=fecfile_validate&subdirectory=fecfile_validate_python -GitPython==3.1.35 +GitPython==3.1.42 gunicorn==20.1.0 invoke==1.7.3 itypes==1.2.0 From 7434d4185a6439a8e1018fd4c0629796332f7336 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 20 Feb 2024 11:07:56 -0500 Subject: [PATCH 17/74] 719 remove safety depcheck --- .safety.dependency.ignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore index e549625a49..74de714264 100644 --- a/.safety.dependency.ignore +++ b/.safety.dependency.ignore @@ -8,4 +8,3 @@ # Example: # 40104 2022-01-15 # -63687 2024-04-01 # gitpython <3.1.41 From cd5d63e63d071ff8f289f3903ee2a903be5ffa22 Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Tue, 20 Feb 2024 13:59:48 -0500 Subject: [PATCH 18/74] Update worker count to match production --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e3543a30b9..744b824e98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,4 +12,4 @@ RUN useradd nxgu --no-create-home --home /opt/nxg_fec && chown -R nxgu:nxgu /opt USER nxgu EXPOSE 8080 -ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 --reload"] +ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --reload"] From e2fe37269fedf4f512bada07e5a6aa4b8f763ba7 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 20 Feb 2024 14:15:31 -0500 Subject: [PATCH 19/74] Update count in e2e Dockerfile --- Dockerfile-e2e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-e2e b/Dockerfile-e2e index cc6d7c906a..1f6ed789ca 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -13,4 +13,4 @@ RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu USER nxgu EXPOSE 8080 -ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 --reload"] +ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --reload"] From de49f09e41f808f0dae8d728a3ec3de0fd2c2d8c Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Wed, 21 Feb 2024 15:52:54 -0500 Subject: [PATCH 20/74] Update sphinx to latest --- .circleci/config.yml | 3 --- requirements-test.txt | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2129b41318..514cbecc02 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -254,9 +254,6 @@ workflows: requires: - test - dependency-check - filters: - branches: - only: develop - docs-deploy: requires: - docs-build diff --git a/requirements-test.txt b/requirements-test.txt index c283af60aa..8d4bf66448 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,4 +4,4 @@ coverage==6.3 jsonschema==3.2 safety liccheck==0.6.2 -sphinx==4.5.0 +sphinx==7.2.6 From ba2af345e0f0f904a61edc27b5a6df73959d3e1f Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Wed, 21 Feb 2024 16:05:46 -0500 Subject: [PATCH 21/74] Revert "Update sphinx to latest" This reverts commit de49f09e41f808f0dae8d728a3ec3de0fd2c2d8c. --- .circleci/config.yml | 3 +++ requirements-test.txt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 514cbecc02..2129b41318 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -254,6 +254,9 @@ workflows: requires: - test - dependency-check + filters: + branches: + only: develop - docs-deploy: requires: - docs-build diff --git a/requirements-test.txt b/requirements-test.txt index 8d4bf66448..c283af60aa 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,4 +4,4 @@ coverage==6.3 jsonschema==3.2 safety liccheck==0.6.2 -sphinx==7.2.6 +sphinx==4.5.0 From 60d7beca06ee9e10c399e3a6d8073d8db6649e4b Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Wed, 21 Feb 2024 16:06:42 -0500 Subject: [PATCH 22/74] Reverting docs-build to only develop --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index c283af60aa..8d4bf66448 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,4 +4,4 @@ coverage==6.3 jsonschema==3.2 safety liccheck==0.6.2 -sphinx==4.5.0 +sphinx==7.2.6 From 394c74c87d1760425971b846e51080f1241804dd Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Wed, 21 Feb 2024 17:08:23 -0500 Subject: [PATCH 23/74] Remove listed files --- django-backend/data.json | 1 - django-backend/fixtures/base_data.json | 1 - django-backend/fixtures/base_data_tmp.json | 1 - .../fixtures/committee_base_data.json | 45 --- .../fixtures/committee_updated.json | 1 - django-backend/fixtures/forms_commitee.csv | 30 -- django-backend/scripts/create_new_users.sql | 163 ----------- .../generate_transaction_record_fixture.py | 275 ------------------ django-backend/scripts/postInstall.sh | 4 - sonar-project.properties | 2 +- 10 files changed, 1 insertion(+), 522 deletions(-) delete mode 100644 django-backend/data.json delete mode 100644 django-backend/fixtures/base_data.json delete mode 100644 django-backend/fixtures/base_data_tmp.json delete mode 100644 django-backend/fixtures/committee_base_data.json delete mode 100644 django-backend/fixtures/committee_updated.json delete mode 100644 django-backend/fixtures/forms_commitee.csv delete mode 100644 django-backend/scripts/create_new_users.sql delete mode 100644 django-backend/scripts/generate_transaction_record_fixture.py delete mode 100755 django-backend/scripts/postInstall.sh diff --git a/django-backend/data.json b/django-backend/data.json deleted file mode 100644 index 4b9f0de19d..0000000000 --- a/django-backend/data.json +++ /dev/null @@ -1 +0,0 @@ -{"COMMITTEE_NAME": "Progress PAC", "FILER_FEC_ID_NUMBER": "C00351056", "STREET_1": "PO Box 83142", "CITY": "MD", "STATE": "MD", "ZIP": "20148", "REASON_TYPE": "MSM", "DATE_SIGNED_MM": "04", "DATE_SIGNED_DD": "10", "DATE_SIGNED_YY": "2019", "TREASURER_FULL_NAME": " Craig Stroman ", "TREASURER_NAME": "Craig Stroman", "EF_STAMP": "[Electronically Filed]", "MISCELLANEOUS_TEXT": "
daddfsddsdsfdfddsdsafa
", "STREET_2": ""} \ No newline at end of file diff --git a/django-backend/fixtures/base_data.json b/django-backend/fixtures/base_data.json deleted file mode 100644 index 24c65de994..0000000000 --- a/django-backend/fixtures/base_data.json +++ /dev/null @@ -1 +0,0 @@ -[{"model": "authentication.account", "pk": 1, "fields": {"password": "pbkdf2_sha256$36000$1a12NsHSiM1P$TSQHr/206lhRHsPmS/EFuv2MDaKRnM45QiFQcewrGk8=", "last_login": "2018-09-10T02:21:50.676Z", "is_superuser": false, "email": "test1@test.com", "username": "C01234567", "tagline": "", "created_at": "2018-09-08T18:59:45.552Z", "updated_at": "2018-09-17T02:25:05.164Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T18:59:45.208Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 2, "fields": {"password": "pbkdf2_sha256$36000$T8SFHw3JGtnX$6yVLWtn83IWPXcV7IQXcTk/2SN7DM92vA8cAjNFNAgc=", "last_login": "2018-09-08T19:01:53.412Z", "is_superuser": false, "email": "test2@test.com", "username": "C11234567", "tagline": "", "created_at": "2018-09-08T19:01:52.580Z", "updated_at": "2018-09-17T02:25:05.146Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:01:52.218Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 3, "fields": {"password": "pbkdf2_sha256$36000$wtcx3pv32dgt$tETHKHzI2FPAtVyp2GI/gutqirDwMISwKyWBnEehlVI=", "last_login": "2018-09-08T19:02:33Z", "is_superuser": false, "email": "test3@test.com", "username": "C00220269", "tagline": "", "created_at": "2018-09-08T19:02:33.065Z", "updated_at": "2018-10-29T17:05:28.190Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:02:32Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 4, "fields": {"password": "bcrypt_sha256$$2b$12$Vs6shIWepA7Bc/LVaZu2aujPQxCoY7Pe6epyXHSH9GrPqEJmgEIsW", "last_login": "2018-09-08T19:04:44.282Z", "is_superuser": true, "email": "testtube@test.com", "username": "adminnxg", "tagline": "", "created_at": "2018-09-08T19:04:33.178Z", "updated_at": "2018-09-17T02:25:05.163Z", "is_staff": true, "is_active": true, "date_joined": "2018-09-08T19:04:32.844Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 5, "fields": {"password": "pbkdf2_sha256$36000$ir0uuUb5gNO3$PYv6Px2Hd7uOwNDIIM/29jukiFh4gxMtklXh3lB8atk=", "last_login": "2018-10-03T13:10:32.154Z", "is_superuser": true, "email": "t@t.com", "username": "admin_nxg", "tagline": "", "created_at": "2018-10-03T13:10:17.524Z", "updated_at": "2018-10-03T13:10:17.527Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-03T13:10:17.440Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 6, "fields": {"password": "pbkdf2_sha256$36000$rJqxMw2luNoH$lzKgItcwzhtwdA9qSpiR/6QGpSQyN4u7Yj8UqFn0I3k=", "last_login": "2018-10-29T16:03:59.849Z", "is_superuser": true, "email": "tt@tt.ctt", "username": "tt", "tagline": "", "created_at": "2018-10-29T16:02:39.519Z", "updated_at": "2018-10-29T16:02:39.579Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-29T16:02:39.035Z", "groups": [], "user_permissions": []}}] \ No newline at end of file diff --git a/django-backend/fixtures/base_data_tmp.json b/django-backend/fixtures/base_data_tmp.json deleted file mode 100644 index 5eac9fa07b..0000000000 --- a/django-backend/fixtures/base_data_tmp.json +++ /dev/null @@ -1 +0,0 @@ -[{"model": "authentication.account", "pk": 1, "fields": {"password": "pbkdf2_sha256$36000$1a12NsHSiM1P$TSQHr/206lhRHsPmS/EFuv2MDaKRnM45QiFQcewrGk8=", "last_login": "2018-09-10T02:21:50.676Z", "is_superuser": false, "email": "test1@test.com", "username": "C01234567", "tagline": "", "created_at": "2018-09-08T18:59:45.552Z", "updated_at": "2018-09-17T02:25:05.164Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T18:59:45.208Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 2, "fields": {"password": "pbkdf2_sha256$36000$T8SFHw3JGtnX$6yVLWtn83IWPXcV7IQXcTk/2SN7DM92vA8cAjNFNAgc=", "last_login": "2018-09-08T19:01:53.412Z", "is_superuser": false, "email": "test2@test.com", "username": "C11234567", "tagline": "", "created_at": "2018-09-08T19:01:52.580Z", "updated_at": "2018-09-17T02:25:05.146Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:01:52.218Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 3, "fields": {"password": "pbkdf2_sha256$36000$wtcx3pv32dgt$tETHKHzI2FPAtVyp2GI/gutqirDwMISwKyWBnEehlVI=", "last_login": "2018-09-08T19:02:33Z", "is_superuser": false, "email": "test3@test.com", "username": "C00220269", "tagline": "", "created_at": "2018-09-08T19:02:33.065Z", "updated_at": "2018-10-29T17:05:28.190Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:02:32Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 4, "fields": {"password": "pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=", "last_login": "2018-09-08T19:04:44.282Z", "is_superuser": true, "email": "testtube@test.com", "username": "adminnxg", "tagline": "", "created_at": "2018-09-08T19:04:33.178Z", "updated_at": "2018-09-17T02:25:05.163Z", "is_staff": true, "is_active": true, "date_joined": "2018-09-08T19:04:32.844Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 5, "fields": {"password": "pbkdf2_sha256$36000$ir0uuUb5gNO3$PYv6Px2Hd7uOwNDIIM/29jukiFh4gxMtklXh3lB8atk=", "last_login": "2018-10-03T13:10:32.154Z", "is_superuser": true, "email": "t@t.com", "username": "admin_nxg", "tagline": "", "created_at": "2018-10-03T13:10:17.524Z", "updated_at": "2018-10-03T13:10:17.527Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-03T13:10:17.440Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 6, "fields": {"password": "pbkdf2_sha256$36000$rJqxMw2luNoH$lzKgItcwzhtwdA9qSpiR/6QGpSQyN4u7Yj8UqFn0I3k=", "last_login": "2018-10-29T16:03:59.849Z", "is_superuser": true, "email": "tt@tt.ctt", "username": "tt", "tagline": "", "created_at": "2018-10-29T16:02:39.519Z", "updated_at": "2018-10-29T16:02:39.579Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-29T16:02:39.035Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 8, "fields": {"password": "pbkdf2_sha256$36000$lH8qTbHqorAx$JU7k5EJbvoZ3zQecTyAQqW9JnbWsvK9Shki4AokAI58=", "last_login": null, "is_superuser": false, "email": "jchumley@fec.gov", "username": "C00547349", "tagline": "", "created_at": "2018-11-17T18:14:46.506Z", "updated_at": "2018-11-17T18:14:46.506Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:14:45.968Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 9, "fields": {"password": "pbkdf2_sha256$36000$iz0eGkpfv0Kk$MsWBxN8kxEd1UhbjDccuDEddxf617ZIKHl2aUYbE5hU=", "last_login": null, "is_superuser": false, "email": "DGardner.ctr@fec.gov", "username": "C00010603", "tagline": "", "created_at": "2018-11-17T18:16:31.399Z", "updated_at": "2018-11-17T18:16:31.399Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:16:31.070Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 10, "fields": {"password": "pbkdf2_sha256$36000$zx8BoYBCxmr3$rvhX0Cb8Ob6sFKii3C4fGdkOnqdTv5JaseBLUHQkk9s=", "last_login": null, "is_superuser": false, "email": "RLanz@fec.gov", "username": "C00004036", "tagline": "", "created_at": "2018-11-17T18:26:26.526Z", "updated_at": "2018-11-17T18:26:26.526Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:26:25.939Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 13, "fields": {"password": "pbkdf2_sha256$36000$i52VO7zc5soK$+HCMcJ3EPwGT16PkpIKONq0l5DkWpqbCfuXvuRbUFpY=", "last_login": null, "is_superuser": false, "email": "mkancherla.ctr@fec.gov", "username": "C00690222", "tagline": "", "created_at": "2018-11-17T19:06:12.787Z", "updated_at": "2018-11-17T19:06:12.787Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:06:12.579Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 14, "fields": {"password": "pbkdf2_sha256$36000$z6v6LA7UXwsa$TifYZ6wXrt4L4AYX8BJk15QBM3NjcxTfCu5K4zqD/rw=", "last_login": null, "is_superuser": false, "email": "cmukerjee.ctr@fec.gov", "username": "C00690198", "tagline": "", "created_at": "2018-11-17T19:06:55.319Z", "updated_at": "2018-11-17T19:06:55.319Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:06:54.823Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 15, "fields": {"password": "pbkdf2_sha256$36000$7O3SmWmvZNcA$ljiDe6RXy4xmlVTQPnSnf5jfA8AoTi7fCRfxw+0Rys4=", "last_login": null, "is_superuser": false, "email": "dhardway.ctr@fec.gov", "username": "C00690206", "tagline": "", "created_at": "2018-11-17T19:21:59.287Z", "updated_at": "2018-11-17T19:21:59.287Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:21:58.858Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 16, "fields": {"password": "pbkdf2_sha256$36000$Q8ybNJR8wyQ1$idODgWN0G82McSZwSs1dhFsBtphImy1Oy4eHi4PA1cs=", "last_login": null, "is_superuser": false, "email": "rsanchez.ctr@fec.gov", "username": "C00690099", "tagline": "", "created_at": "2018-11-17T19:24:15.455Z", "updated_at": "2018-11-17T19:24:15.455Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:24:15.004Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 17, "fields": {"password": "pbkdf2_sha256$36000$LGrAedbIrW3u$mwajveqr5llDOU6og8KuTzPadLoNl2lYhjZxU6aItqk=", "last_login": null, "is_superuser": false, "email": "smartinez.ctr@fec.gov", "username": "C00689992", "tagline": "", "created_at": "2018-11-17T19:27:13.890Z", "updated_at": "2018-11-17T19:27:13.890Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:27:13.473Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 18, "fields": {"password": "pbkdf2_sha256$36000$QhOGxXUeHV6W$AkPdL+ptcFtZJxW7zLVH+olXbBY4WoG7/T9Waog/5s4=", "last_login": null, "is_superuser": false, "email": "cstroman.ctr@fec.gov", "username": "C00689968", "tagline": "", "created_at": "2018-11-17T19:29:40.284Z", "updated_at": "2018-11-17T19:29:40.284Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:29:39.774Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 19, "fields": {"password": "pbkdf2_sha256$36000$U9LVMZNdAAVJ$rhwOoc08a6aawfUP0X3Bds/S16i0X60Pbc0oQE7OTak=", "last_login": null, "is_superuser": false, "email": "mbasupally.ctr@fec.gov", "username": "C00689844", "tagline": "", "created_at": "2018-11-17T19:31:33.825Z", "updated_at": "2018-11-17T19:31:33.825Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:31:33.312Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 20, "fields": {"password": "pbkdf2_sha256$36000$lcK3NdwOy47A$/6AnRILnYKHAPittLPFl7eNgl9mojGzyq70jfHPWXaI=", "last_login": null, "is_superuser": false, "email": "rdasaradhi.ctr@fec.gov", "username": "C00689877", "tagline": "", "created_at": "2018-11-17T19:35:59.009Z", "updated_at": "2018-11-17T19:35:59.009Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:35:58.582Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 21, "fields": {"password": "pbkdf2_sha256$36000$5KbCHPDa2YCi$PpPLth/nAKpJoD48aEEKmJPTaWvJ8GfojAmAGZY7YN8=", "last_login": null, "is_superuser": false, "email": "Cris.Daniluk@ctr.salientcrgt.com", "username": "C00689174", "tagline": "", "created_at": "2018-11-19T14:10:59.448Z", "updated_at": "2018-11-19T14:10:59.448Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-19T14:10:59.081Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 22, "fields": {"password": "pbkdf2_sha256$36000$qbDY0zugkfhq$TlH+tdQGMKIs7D9BbyQZUm/hy6wS/bpK0ULLIibfM+c=", "last_login": null, "is_superuser": false, "email": "mahendra.marathe@salientcrgt.com", "username": "C00689224", "tagline": "", "created_at": "2018-11-19T14:21:20.498Z", "updated_at": "2018-11-19T14:21:20.498Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-19T14:21:20.129Z", "groups": [], "user_permissions": []}}] \ No newline at end of file diff --git a/django-backend/fixtures/committee_base_data.json b/django-backend/fixtures/committee_base_data.json deleted file mode 100644 index e3dfc1ef5a..0000000000 --- a/django-backend/fixtures/committee_base_data.json +++ /dev/null @@ -1,45 +0,0 @@ -[{ - "model": "forms.committee", - "pk": 1, - "fields": { - "committeeid": "C01234567", - "committeename": "Test Committee 1", - "street1": "Street 1", - "street2": "Street 2", - "city": "Washington", - "state": "DC", - "zipcode": 91285, - "treasurerlastname": "Smith", - "treasurerfirstname": "John", - "treasurermiddlename": "E", - "treasurerprefix": "Mr", - "treasurersuffix": "Jr.", - "created_at": "2018-09-19T17:34:00.477Z", - "updated_at": "2018-09-19T17:34:00.477Z", - "deleted_at": null, - "isdeleted": false, - "email_on_file": "jsmith@gmail.com" - } -}, { - "model": "forms.committee", - "pk": 2, - "fields": { - "committeeid": "C11234567", - "committeename": "Test Committee 2", - "street1": "Street 1", - "street2": "Street 2", - "city": "Washington", - "state": "DC", - "zipcode": 20001, - "treasurerlastname": "Smith", - "treasurerfirstname": "John", - "treasurermiddlename": "Doe", - "treasurerprefix": "Mr", - "treasurersuffix": "IV", - "created_at": "2018-09-19T17:34:00.479Z", - "updated_at": "2018-09-19T17:34:00.479Z", - "deleted_at": null, - "isdeleted": false, - "email_on_file": "jsmith@outlook.com" - } -}] diff --git a/django-backend/fixtures/committee_updated.json b/django-backend/fixtures/committee_updated.json deleted file mode 100644 index 5eac9fa07b..0000000000 --- a/django-backend/fixtures/committee_updated.json +++ /dev/null @@ -1 +0,0 @@ -[{"model": "authentication.account", "pk": 1, "fields": {"password": "pbkdf2_sha256$36000$1a12NsHSiM1P$TSQHr/206lhRHsPmS/EFuv2MDaKRnM45QiFQcewrGk8=", "last_login": "2018-09-10T02:21:50.676Z", "is_superuser": false, "email": "test1@test.com", "username": "C01234567", "tagline": "", "created_at": "2018-09-08T18:59:45.552Z", "updated_at": "2018-09-17T02:25:05.164Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T18:59:45.208Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 2, "fields": {"password": "pbkdf2_sha256$36000$T8SFHw3JGtnX$6yVLWtn83IWPXcV7IQXcTk/2SN7DM92vA8cAjNFNAgc=", "last_login": "2018-09-08T19:01:53.412Z", "is_superuser": false, "email": "test2@test.com", "username": "C11234567", "tagline": "", "created_at": "2018-09-08T19:01:52.580Z", "updated_at": "2018-09-17T02:25:05.146Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:01:52.218Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 3, "fields": {"password": "pbkdf2_sha256$36000$wtcx3pv32dgt$tETHKHzI2FPAtVyp2GI/gutqirDwMISwKyWBnEehlVI=", "last_login": "2018-09-08T19:02:33Z", "is_superuser": false, "email": "test3@test.com", "username": "C00220269", "tagline": "", "created_at": "2018-09-08T19:02:33.065Z", "updated_at": "2018-10-29T17:05:28.190Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:02:32Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 4, "fields": {"password": "pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=", "last_login": "2018-09-08T19:04:44.282Z", "is_superuser": true, "email": "testtube@test.com", "username": "adminnxg", "tagline": "", "created_at": "2018-09-08T19:04:33.178Z", "updated_at": "2018-09-17T02:25:05.163Z", "is_staff": true, "is_active": true, "date_joined": "2018-09-08T19:04:32.844Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 5, "fields": {"password": "pbkdf2_sha256$36000$ir0uuUb5gNO3$PYv6Px2Hd7uOwNDIIM/29jukiFh4gxMtklXh3lB8atk=", "last_login": "2018-10-03T13:10:32.154Z", "is_superuser": true, "email": "t@t.com", "username": "admin_nxg", "tagline": "", "created_at": "2018-10-03T13:10:17.524Z", "updated_at": "2018-10-03T13:10:17.527Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-03T13:10:17.440Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 6, "fields": {"password": "pbkdf2_sha256$36000$rJqxMw2luNoH$lzKgItcwzhtwdA9qSpiR/6QGpSQyN4u7Yj8UqFn0I3k=", "last_login": "2018-10-29T16:03:59.849Z", "is_superuser": true, "email": "tt@tt.ctt", "username": "tt", "tagline": "", "created_at": "2018-10-29T16:02:39.519Z", "updated_at": "2018-10-29T16:02:39.579Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-29T16:02:39.035Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 8, "fields": {"password": "pbkdf2_sha256$36000$lH8qTbHqorAx$JU7k5EJbvoZ3zQecTyAQqW9JnbWsvK9Shki4AokAI58=", "last_login": null, "is_superuser": false, "email": "jchumley@fec.gov", "username": "C00547349", "tagline": "", "created_at": "2018-11-17T18:14:46.506Z", "updated_at": "2018-11-17T18:14:46.506Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:14:45.968Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 9, "fields": {"password": "pbkdf2_sha256$36000$iz0eGkpfv0Kk$MsWBxN8kxEd1UhbjDccuDEddxf617ZIKHl2aUYbE5hU=", "last_login": null, "is_superuser": false, "email": "DGardner.ctr@fec.gov", "username": "C00010603", "tagline": "", "created_at": "2018-11-17T18:16:31.399Z", "updated_at": "2018-11-17T18:16:31.399Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:16:31.070Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 10, "fields": {"password": "pbkdf2_sha256$36000$zx8BoYBCxmr3$rvhX0Cb8Ob6sFKii3C4fGdkOnqdTv5JaseBLUHQkk9s=", "last_login": null, "is_superuser": false, "email": "RLanz@fec.gov", "username": "C00004036", "tagline": "", "created_at": "2018-11-17T18:26:26.526Z", "updated_at": "2018-11-17T18:26:26.526Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:26:25.939Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 13, "fields": {"password": "pbkdf2_sha256$36000$i52VO7zc5soK$+HCMcJ3EPwGT16PkpIKONq0l5DkWpqbCfuXvuRbUFpY=", "last_login": null, "is_superuser": false, "email": "mkancherla.ctr@fec.gov", "username": "C00690222", "tagline": "", "created_at": "2018-11-17T19:06:12.787Z", "updated_at": "2018-11-17T19:06:12.787Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:06:12.579Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 14, "fields": {"password": "pbkdf2_sha256$36000$z6v6LA7UXwsa$TifYZ6wXrt4L4AYX8BJk15QBM3NjcxTfCu5K4zqD/rw=", "last_login": null, "is_superuser": false, "email": "cmukerjee.ctr@fec.gov", "username": "C00690198", "tagline": "", "created_at": "2018-11-17T19:06:55.319Z", "updated_at": "2018-11-17T19:06:55.319Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:06:54.823Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 15, "fields": {"password": "pbkdf2_sha256$36000$7O3SmWmvZNcA$ljiDe6RXy4xmlVTQPnSnf5jfA8AoTi7fCRfxw+0Rys4=", "last_login": null, "is_superuser": false, "email": "dhardway.ctr@fec.gov", "username": "C00690206", "tagline": "", "created_at": "2018-11-17T19:21:59.287Z", "updated_at": "2018-11-17T19:21:59.287Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:21:58.858Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 16, "fields": {"password": "pbkdf2_sha256$36000$Q8ybNJR8wyQ1$idODgWN0G82McSZwSs1dhFsBtphImy1Oy4eHi4PA1cs=", "last_login": null, "is_superuser": false, "email": "rsanchez.ctr@fec.gov", "username": "C00690099", "tagline": "", "created_at": "2018-11-17T19:24:15.455Z", "updated_at": "2018-11-17T19:24:15.455Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:24:15.004Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 17, "fields": {"password": "pbkdf2_sha256$36000$LGrAedbIrW3u$mwajveqr5llDOU6og8KuTzPadLoNl2lYhjZxU6aItqk=", "last_login": null, "is_superuser": false, "email": "smartinez.ctr@fec.gov", "username": "C00689992", "tagline": "", "created_at": "2018-11-17T19:27:13.890Z", "updated_at": "2018-11-17T19:27:13.890Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:27:13.473Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 18, "fields": {"password": "pbkdf2_sha256$36000$QhOGxXUeHV6W$AkPdL+ptcFtZJxW7zLVH+olXbBY4WoG7/T9Waog/5s4=", "last_login": null, "is_superuser": false, "email": "cstroman.ctr@fec.gov", "username": "C00689968", "tagline": "", "created_at": "2018-11-17T19:29:40.284Z", "updated_at": "2018-11-17T19:29:40.284Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:29:39.774Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 19, "fields": {"password": "pbkdf2_sha256$36000$U9LVMZNdAAVJ$rhwOoc08a6aawfUP0X3Bds/S16i0X60Pbc0oQE7OTak=", "last_login": null, "is_superuser": false, "email": "mbasupally.ctr@fec.gov", "username": "C00689844", "tagline": "", "created_at": "2018-11-17T19:31:33.825Z", "updated_at": "2018-11-17T19:31:33.825Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:31:33.312Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 20, "fields": {"password": "pbkdf2_sha256$36000$lcK3NdwOy47A$/6AnRILnYKHAPittLPFl7eNgl9mojGzyq70jfHPWXaI=", "last_login": null, "is_superuser": false, "email": "rdasaradhi.ctr@fec.gov", "username": "C00689877", "tagline": "", "created_at": "2018-11-17T19:35:59.009Z", "updated_at": "2018-11-17T19:35:59.009Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:35:58.582Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 21, "fields": {"password": "pbkdf2_sha256$36000$5KbCHPDa2YCi$PpPLth/nAKpJoD48aEEKmJPTaWvJ8GfojAmAGZY7YN8=", "last_login": null, "is_superuser": false, "email": "Cris.Daniluk@ctr.salientcrgt.com", "username": "C00689174", "tagline": "", "created_at": "2018-11-19T14:10:59.448Z", "updated_at": "2018-11-19T14:10:59.448Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-19T14:10:59.081Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 22, "fields": {"password": "pbkdf2_sha256$36000$qbDY0zugkfhq$TlH+tdQGMKIs7D9BbyQZUm/hy6wS/bpK0ULLIibfM+c=", "last_login": null, "is_superuser": false, "email": "mahendra.marathe@salientcrgt.com", "username": "C00689224", "tagline": "", "created_at": "2018-11-19T14:21:20.498Z", "updated_at": "2018-11-19T14:21:20.498Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-19T14:21:20.129Z", "groups": [], "user_permissions": []}}] \ No newline at end of file diff --git a/django-backend/fixtures/forms_commitee.csv b/django-backend/fixtures/forms_commitee.csv deleted file mode 100644 index 99af353e30..0000000000 --- a/django-backend/fixtures/forms_commitee.csv +++ /dev/null @@ -1,30 +0,0 @@ -1,C12345678,test1,xyz,abc,Washington,DC,20001,David,Mark,J,Mr,II,2018-09-24 15:54:23.419144+00,2018-09-24 15:54:23.419172+00,,f,- -2,C01234567,test1,xyz,abc,Washington,DC,20001,David,Mark,J,Mr,II,2018-09-25 03:59:41.400669+00,2018-09-25 03:59:41.400697+00,,f,- -5,C01234555,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,Northeast,Washington DC,DC,2002,Michael,David,M,Mr,II,2018-11-16 18:23:10.478856+00,2018-11-16 18:23:10.478889+00,,f,mdavid@test.com -6,C01234555,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,Northeast,Washington DC,DC,2002,Michael,David,M,Mr,II,2018-11-16 18:55:20.639207+00,2018-11-16 18:55:20.639244+00,,f,jchumley@fec.gov -7,C01234555,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,Northeast,Washington DC,DC,2002,Chumley,Jeff,A,Mr,II,2018-11-16 18:57:03.977755+00,2018-11-16 18:57:03.97779+00,,f,jchumley@fec.gov -8,C01234555,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 18,street 1,Washington DC,DC,2003,Lally,Ken,b,Mr,II,2018-11-16 18:58:56.582818+00,2018-11-16 18:58:56.58285+00,,f,klally@fec.gov -9,C01234566,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 18,street 1,Washington DC,DC,2003,Lally,Ken,b,Mr,II,2018-11-16 18:59:49.994864+00,2018-11-16 18:59:49.994895+00,,f,klally@fec.gov -10,C01234555,NATIONAL ASSOCIATION OF MEDICAL COMMITTEE,Street 18,Northeast,Washington DC,DC,2002,Chumley,Jeff,A,Mr,II,2018-11-16 19:01:59.096731+00,2018-11-16 19:01:59.096764+00,,f,jchumley@fec.gov -11,C01234577,NATIONAL ASSOCIATION OF STUDENT COMMITTEE,Street 17,street2,Washington DC,DC,2002,Lanz,Ryan,C,Mr,II,2018-11-16 19:05:32.845853+00,2018-11-16 19:05:32.845887+00,,f,RLanz@fec.gov -12,C01234588,NATIONAL ASSOCIATION OF PROGRAM MANAGEMENT,Street 16,street3,Washington DC,DC,2004,Gardner,Drew,D,Mr,II,2018-11-16 19:08:25.901263+00,2018-11-16 19:08:25.9013+00,,f,DGardner.ctr@fec.gov -13,C01234588,NATIONAL ASSOCIATION OF EVENT MANAGEMENT,Street 16,street3,Washington DC,DC,2004,Gardner,Drew,D,Mr,II,2018-11-16 19:08:48.057127+00,2018-11-16 19:08:48.057161+00,,f,DGardner.ctr@fec.gov -15,C00220269,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,street2,Washington DC,DC,2004,Lally,Ken,B,Mr,II,2018-11-16 20:50:02.01507+00,2018-11-16 20:50:02.015102+00,,f,klally@fec.gov -17,C01234567,NATIONAL ASSOCIATION OF CHAIN DRUG STORES,Street 19,street1,Washington DC,DC,2002,Jinka,Praveen,B,Mr,II,2018-11-17 00:38:16.491029+00,2018-11-17 00:38:16.491066+00,,f,pjinka.ctr@fec.gov -18,C00547349,NEXTGEN CLIMATE ACTION COMMITTEE,Street 19,street1,Washington DC,DC,2002,Chumley,Jeff,B,Mr,II,2018-11-17 18:01:35.856607+00,2018-11-17 18:01:35.856641+00,,f,jchumley@fec.gov -19,C00010603,DNC SERVICES CORP./DEM. NAT'L COMMITTEE,430 SOUTH CAPITOL STREET SE,street1,Washington DC,DC,2002,Gardner,Drew,B,Mr,II,2018-11-17 18:18:59.461612+00,2018-11-17 18:18:59.461648+00,,f,DGardner.ctr@fec.gov -20,C00004036,SEIU COPE (SERVICE EMPLOYEES INTERNATIONAL UNION COMMITTEE ON POLITICAL EDUCATION),1800 MASSACHUSETTS AVE NW,street1,Washington DC,DC,20036,Lanz,Ryan,B,Mr,II,2018-11-17 18:35:25.772953+00,2018-11-17 18:35:25.772983+00,,f,RLanz@fec.gov -22,C00690222,ROCKINGHAM COUNTY DEMOCRATIC COMMITTEE,11830 FORT TURLEY TRAIL,street1,LINVILLE,VA,24010,Kancherla,Madhuri,B,Ms,II,2018-11-17 18:54:16.840261+00,2018-11-17 18:54:16.840296+00,,f,mkancherla.ctr@fec.gov -24,C00690198,ROCKINGHAM COUNTY DEMOCRATIC COMMITTEE,11831 FORT TURLEY TRAIL,street1,Vienna,VA,24010,Mukerjee,Chamila,B,Ms,II,2018-11-17 19:10:31.876374+00,2018-11-17 19:10:31.876404+00,,f,cmukerjee.ctr@fec.gov -25,C00690206,5050 BY 2020 COMMITTEE,PO BOX 1041,street1,ESTERO,FL,24010,Donald,Hardway,B,Mr,II,2018-11-17 19:23:09.303373+00,2018-11-17 19:23:09.303406+00,,f,dhardway.ctr@fec.gov -26,C00690099,"BRADFORD WHITE CORPORATION POLITICAL ACTION COMMITTEE (""BWC PAC"")",725 TALAMORE DRIVE,street1,AMBLER,PA,19002,Sanchez,Roberto,B,Mr,II,2018-11-17 19:25:49.526771+00,2018-11-17 19:25:49.526804+00,,f,rsanchez.ctr@fec.gov -27,C00689992,UNITED STRATEGIES GROUP POLITCAL ACTION COMMITTEE,1708 ELSINORE AVE,street1,HENDERSON,NV,19002,Martinez,Said,B,Mr,II,2018-11-17 19:28:43.559561+00,2018-11-17 19:28:43.559595+00,,f,smartinez.ctr@fec.gov -28,C00689968,PO BOX 751271,1708 ELSINORE AVE,street1,LAS VEGAS,NV,19002,Stroman,Craig,B,Mr,II,2018-11-17 19:30:40.55474+00,2018-11-17 19:30:40.55477+00,,f,cstroman.ctr@fec.gov -29,C00689844,PO BOX 751271,28600 BELLA VISTA PARKWAY,street1,WARRENVILLE,IL,60555,Basupally,Mahi,B,Mr,II,2018-11-17 19:34:20.271653+00,2018-11-17 19:34:20.2717+00,,f,mbasupally.ctr@fec.gov -30,C00689877,COMMITTEE TO ELECT STEVEN DEVALD,2100 LAKESIDE AVE E.,street1,CLEVELAND,OH,60555,Dasaradhi,Ranjith,B,Mr,II,2018-11-17 19:37:39.733053+00,2018-11-17 19:37:39.733095+00,,f,rdasaradhi.ctr@fec.gov -31,C00689174,DIGITAL AGE NEURAL COMMITTEE,23333 RIDGE ROUTE DR #64,street1,LAKE FOREST,CA,92630,Cris,Daniluk,B,Mr,II,2018-11-19 14:16:37.324656+00,2018-11-19 14:16:37.324686+00,,f,Cris.Daniluk@ctr.salientcrgt.com -32,C00689224,DIGITAL AGE NEURAL COMMITTEE,332 W LEE HWY,street1,WARRENTON,VA,92630,Marathe,Mahendra,B,Mr,II,2018-11-19 14:22:54.178217+00,2018-11-19 14:22:54.178248+00,,f,mahendra.marathe@salientcrgt.com -4,C00022368,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,Northeast,Washington DC,DC,2002,Michael,David,M,Mr,II,2018-10-26 19:05:50.217864+00,2018-10-26 19:05:50.217904+00,,f,test@test.com -3,C01234567,test1,xyz,abc,Washington,DC,20001,David,Mark,J,Mr,II,2018-09-26 15:31:18.018227+00,2018-09-26 15:31:18.018263+00,,f,test@test.com -33,C00689224,DIGITAL AGE NEURAL COMMITTEE,332 W LEE HWY,street1,WARRENTON,VA,92630,Marathe,Mahendra,B,Mr,II,2018-11-29 17:37:32.094507+00,2018-11-29 17:37:32.094538+00,,f,mahendra.marathe@salientcrgt.com -34,C01234567,DIGITAL,332 W LEE HWY,street1,WARRENTON,VA,92630,Marathe,Mahendra,B,Mr,II,2018-11-30 19:42:28.665529+00,2018-11-30 19:42:28.665577+00,,f,mahendra.marathe@salientcrgt.com diff --git a/django-backend/scripts/create_new_users.sql b/django-backend/scripts/create_new_users.sql deleted file mode 100644 index 17833c7b87..0000000000 --- a/django-backend/scripts/create_new_users.sql +++ /dev/null @@ -1,163 +0,0 @@ --- Utility script to create new users in the fecfile-online database. --- Edit the sample VALUES() below to create new user accounts. - --- Update authentication_account_id_seq to ensure the index is pointing to the correct value -SELECT setval('authentication_account_id_seq', (SELECT max(id) FROM authentication_account)); - --- To associate a user to additional committee ids Taking an existing committee id from the committee_master table --- ensure the committee table doesn't have an entry in authentication_account table -INSERT into authentication_account( - password, - is_superuser, - email, - username, - cmtee_id, - created_at, - updated_at, - is_staff, - is_active, - date_joined, - tagline, - role, - delete_ind, - contact, - status - ) -VALUES - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','egreen.ctr@fec.gov','C00601534egreen.ctr@fec.gov','C00601534',now(),now(),'f','t',now(),'', 'C_ADMIN','N','2026941307','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','egreen.ctr@fec.gov','C00601229egreen.ctr@fec.gov','C00601229',now(),now(),'f','t',now(),'', 'C_ADMIN','N','2402743764','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','egreen.ctr@fec.gov','C00233361egreen.ctr@fec.gov','C00233361',now(),now(),'f','t',now(),'', 'C_ADMIN','N','2402743764','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','akolan.ctr@fec.gov','C00601609akolan.ctr@fec.gov','C00601609',now(),now(),'f','t',now(),'', 'C_ADMIN','N','5121234567','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','akolan.ctr@fec.gov','C00670430akolan.ctr@fec.gov','C00670430',now(),now(),'f','t',now(),'', 'C_ADMIN','N','5121234567','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','akolan.ctr@fec.gov','C00715672akolan.ctr@fec.gov','C00715672',now(),now(),'f','t',now(),'', 'C_ADMIN','N','5121234567','Registered'); - --- Applies to bot PAC and PTY --- Update forms_committee table with the user information -INSERT INTO public.forms_committee( - committeeid, - committeename, - street1, - street2, - city, - state, - zipcode, - treasurerlastname, - treasurerfirstname, - treasurermiddlename, - treasurerprefix, - treasurersuffix, - created_at, - updated_at, - isdeleted, - email_on_file) - SELECT - cm.cmte_id, - cm.cmte_name, - cm.street_1, - cm.street_2, - cm.city, - cm.state, - 20148, - cm.treasurer_last_name, - cm.treasurer_first_name, - cm.treasurer_middle_name, - cm.treasurer_prefix, - cm.treasurer_suffix, - now(), - now(), - false, - aa.email - FROM committee_master cm - INNER JOIN authentication_account aa ON cm.cmte_id = aa.cmtee_id -WHERE cm.cmte_id in ( - 'C00601534', - 'C00601229', - 'C00233361', - 'C00601609', - 'C00670430', - 'C00715672' - ); - --- Ensure there aren't any due reports associated with the committee ids. -Delete from cmte_current_due_reports -where cmte_id in ( - 'C00601534', - 'C00601229', - 'C00233361', - 'C00601609', - 'C00670430', - 'C00715672' - -); - --- Applies to PAC only using C00000422 as a template -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00601534', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00000422'; -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00601609', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00000422'; -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00601229', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00000422'; -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00670430', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00000422'; - --- Applies to PTY only using C00010603 as a template -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00233361', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00010603'; - -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00715672', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00010603'; - - --- Applies to PAC -update committee_master set cmte_type='Q' where cmte_id in ( - 'C00601534', - 'C00601609', - 'C00601229', - 'C00670430' - ); - --- Applies to PTY -update committee_master set cmte_type='X' where cmte_id in ( - 'C00233361', - 'C00715672' - ); - --- Applies to both PAC and PTY -update committee_master set cmte_dsgn='B', cmte_filing_freq='M' where cmte_id in ( - 'C00601534', - 'C00601229', - 'C00233361', - 'C00601609', - 'C00670430', - 'C00715672' - -); --- PAC with 3L -UPDATE public.committee_master set cmte_type='X', cmte_dsgn='U', cmte_filing_freq='M', cmte_filed_type='4' WHERE cmte_type_category='PAC' AND cmte_id in ( - 'C00601534', - 'C00601229' -); - --- PAC without 3L -UPDATE public.committee_master set cmte_filing_freq='Q' WHERE cmte_type_category='PAC' AND cmte_id in ( - 'C00601609', - 'C00670430' -); diff --git a/django-backend/scripts/generate_transaction_record_fixture.py b/django-backend/scripts/generate_transaction_record_fixture.py deleted file mode 100644 index c561118db8..0000000000 --- a/django-backend/scripts/generate_transaction_record_fixture.py +++ /dev/null @@ -1,275 +0,0 @@ -import argparse -from secrets import choice - -# Generate Transaction Record Fixture -# -# Creates a json file containing valid transaction records -# for the form_type values specified by the user, and with -# randomly generated values for transaction_amount. - - -def get_arguments(): - parser = argparse.ArgumentParser( - prog='Generate Transaction Record Fixture', - description='Creates a json file containing valid transaction records\n' - + 'for the form_type values specified by the user and with\n' - + 'randomly generated values for transaction_amount.' - ) - - parser.add_argument( - 'form_types', - nargs='+', - default=[], - help="form_type values (e.g SA12 or SC/9) separated by spaces" - ) - parser.add_argument( - '--committee_account_id', - default="735db943-9446-462a-9be0-c820baadb622", - help='default value: 735db943-9446-462a-9be0-c820baadb622' - ) - parser.add_argument( - '--report_a_id', - default="b6d60d2d-d926-4e89-ad4b-c47d152a66ae", - help='primary report (default: b6d60d2d-d926-4e89-ad4b-c47d152a66ae)' - ) - parser.add_argument( - '--report_b_id', - default="1406535e-f99f-42c4-97a8-247904b7d297", - help='secondary report used for transactions outside of report dates' - + ' (default: 1406535e-f99f-42c4-97a8-247904b7d297)' - ) - parser.add_argument( - '--report_start_date', - default="2005-01-30", - help='default value: 2005-01-30' - ) - parser.add_argument( - '--report_end_date', - default="2005-02-28", - help='default value: 2005-02-28' - ) - parser.add_argument( - '--transaction_type_identifier', - default="Transaction Type Identifier", - help='default value: Transaction Type Identifier' - ) - parser.add_argument( - '--output_file', - help="Output to a file instead of the console (e.g --output_file=records.txt)") - - return parser.parse_args() - - -def get_schedule(form_type): - schedule = str(form_type[1]).capitalize() # the second char in the form_type - if schedule == "C" and form_type[2] in ["1", "2"]: - schedule = form_type[1:2] - return schedule - - -def get_comment(form_type, test_case): - if test_case == "within_dates": - return f"{form_type} within report dates, should count in Col. A" - if test_case == "within_year": - return f"{form_type} within year, should count in Col. B" - if test_case == "outside_dates": - return f"{form_type} outside of dates, should not count" - if test_case == "within_dates_but_memo": - return f"{form_type} is a memo item, should not count" - return "" - - -def get_amount_field(schedule): - if schedule == "A": - return "contribution_amount" - if schedule == "B": - return "expenditure_amount" - if schedule == "C": - return "loan_amount" - if schedule == "C1": - return "loan_amount" - if schedule == "C2": - return "guaranteed_amount" - if schedule == "D": - return "incurred_amount" - if schedule == "E": - return "expenditure_amount" - return "" - - -def get_date_field(schedule): - if schedule == "A": - return "contribution_date" - if schedule == "B": - return "expenditure_date" - if schedule == "C": - return "loan_incurred_date" - if schedule == "C1": - return "loan_incurred_date" - if schedule == "C2": - return "" - if schedule == "D": - return "" - if schedule == "E": - return "disbursement_date" - return "" - - -def get_date(args, test_case): - report_start = args.report_start_date.split("-") - report_end = args.report_end_date.split("-") - - if "within_dates" in test_case: - return "-".join(report_start) - - if "year" in test_case: - months = set([str(m).rjust(2, "0") for m in range(1, 13)]) - report_start_month = report_start[1] - report_months = range(int(report_start_month), 13) - formatted_report_months = set([str(m).rjust(2, "0") for m in report_months]) - months_before_report = list(months - formatted_report_months) - if len(months_before_report) > 0: - random_month = choice(months_before_report) - else: - random_month = report_start_month - - report_start_day = report_start[2] - if (int(report_start_day) > 1): - random_day = str(choice(range(1, int(report_start_day)))).rjust(2, "0") - else: - random_day = "01" - - if report_start_month == "01" and report_start_day == "01": - raise ValueError( - 'Cannot set any date in same year before provided start date' - + f' ({args.report_start_date})' - ) - - year = report_start[0] - return f"{year}-{random_month}-{random_day}" - - report_years = set([report_start[0], report_end[0]]) - random_year = choice(list(set(range(2000 - 50, 2000 + 50)) - report_years)) - random_month = choice([str(m).rjust(2, "0") for m in range(1, 13)]) - return f"{random_year}-{random_month}-01" - - -def get_date_entry(date_field, date): - if (len(date_field) > 0): - return f''',\n{" "*12}"{date_field}": "{date}"''' - return "" - - -def get_report_id(args, test_case): - if "within_dates" in test_case: - return args.report_a_id - else: - return args.report_b_id - - -def random_hex(length=1): - hex_chars = "0123456789abcdef" - - out_string = "" - while len(out_string) < length: - out_string += choice(hex_chars) - - return out_string - - -def get_id(form_type, record_number, schedule=""): - hex_chars = "0123456789abcdef" - - schedule_str = "" - if(schedule[0].lower() in hex_chars): - schedule_str += schedule.lower() - else: - schedule_str += hex(ord(schedule.lower()))[2:] - - line_number = form_type.split(schedule)[1].strip('/') - schedule_str += "0" + str(line_number) - schedule_str = schedule_str.ljust(8, "0") - - record_str = str(record_number + 1).rjust(12, "0") - - return f"{schedule_str}-{random_hex(4)}-{random_hex(4)}-{random_hex(4)}-{record_str}" - - -def get_records(form_type, args): - records = [] - test_cases = [ - "within_dates", - "within_dates", - "within_year", - "outside_dates", - "within_dates_but_memo" - ] - for record_number in range(len(test_cases)): - test_case = test_cases[record_number] - - comment = get_comment(form_type, test_case) - - schedule = get_schedule(form_type) - schedule_model = f"schedule{schedule.lower()}" - - schedule_id = get_id(form_type, record_number, schedule) - transaction_id = get_id(form_type, record_number, schedule) - - amount_field = get_amount_field(schedule) - amount = choice(range(20, 150)) - - date_field = get_date_field(schedule) - date = get_date(args, test_case) - date_entry = get_date_entry(date_field, date) - - report_id = get_report_id(args, test_case) - - memo_code = str( - test_case == "within_dates_but_memo" - ).lower() # JSON uses lowercase true/false - - schedule_record = f""" {{ - "comment": "{comment}", - "model": "transactions.{schedule_model}", - "fields": {{ - "id": "{schedule_id}", - "{amount_field}": {amount}{date_entry} - }}\n }}""" - - transaction_record = f""" {{ - "comment": "{comment}", - "model": "transactions.transaction", - "fields": {{ - "id": "{transaction_id}", - "schedule_{schedule.lower()}_id": "{schedule_id}", - "committee_account_id": "{args.committee_account_id}", - "report_id": "{report_id}", - "_form_type": "{form_type}", - "memo_code": {memo_code}, - "created": "2022-02-09T00:00:00.000Z", - "updated": "2022-02-09T00:00:00.000Z", - "transaction_type_identifier": {args.transaction_type_identifier} - }}\n }}""" - - records.append(schedule_record) - records.append(transaction_record) - - return records - - -if (__name__ == "__main__"): - args = get_arguments() - - records = [] - for form_type in args.form_types: - records += get_records(form_type, args) - - records_str = ",\n".join(records) - output = "[\n" + records_str + "\n]" - - if not args.output_file: - print(output) - else: - file = open(args.output_file, 'w') - file.write(output) - file.close() diff --git a/django-backend/scripts/postInstall.sh b/django-backend/scripts/postInstall.sh deleted file mode 100755 index f867e7c876..0000000000 --- a/django-backend/scripts/postInstall.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -./node_modules/bower/bin/bower install -./node_modules/gulp/bin/gulp.js diff --git a/sonar-project.properties b/sonar-project.properties index 779e06ea36..67ceb88ef6 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -12,7 +12,7 @@ sonar.sources=django-backend #sonar.sourceEncoding=UTF-8 # Exclude utility script from coverage -sonar.coverage.exclusions=**/json_schema_to_django_model.py,**/migrations/**,**/settings/*.py,**/scripts/generate_transaction_record_fixture.py +sonar.coverage.exclusions=**/json_schema_to_django_model.py,**/migrations/**,**/settings/*.py sonar.python.coverage.reportPaths=coverage-reports/coverage.xml sonar.python.bandit.reportPaths=bandit.out sonar.python.flake8.reportPaths=flake8.out From 4c0373d72a5c062d0b0ebee03326ac25b7436af0 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 21 Feb 2024 17:29:55 -0500 Subject: [PATCH 24/74] Can retrieve all members instead of always being paginated --- .../fecfiler/committee_accounts/views.py | 103 ++++++++++-------- 1 file changed, 59 insertions(+), 44 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 317fd3949d..50986d9d76 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -54,7 +54,7 @@ def get_queryset(self): return queryset.filter(committee_account_id=committee.id) -class CommitteeMembershipViewSet(viewsets.ModelViewSet): +class CommitteeMembershipViewSet(CommitteeOwnedViewSet): serializer_class = CommitteeMembershipSerializer filter_backends = [filters.OrderingFilter] ordering_fields = [ @@ -66,49 +66,49 @@ class CommitteeMembershipViewSet(viewsets.ModelViewSet): ] ordering = ["-created"] - queryset = Membership.objects.all().annotate( - name= Coalesce( - Concat( - "user__last_name", - Value(", "), - "user__first_name", - output_field=TextField(), - ), - Value(''), - output_field=TextField() - ), - email=Coalesce( - "user__email", - "pending_email", - output_field=TextField() - ), - is_active=~Q(user=None) - ) - - @action(detail=True, methods=["get"]) - def members(self, request, pk): - committee = self.get_object() - queryset = Membership.objects.filter(committee_account=committee) - page = self.paginate_queryset(queryset) - if page is not None: - serializer = CommitteeMembershipSerializer( - page, many=True - ) - return self.get_paginated_response(serializer.data) + queryset = Membership.objects.all() - serializer = CommitteeMembershipSerializer( - queryset, many=True + def get_queryset(self): + return super().get_queryset().annotate( + name= Coalesce( + Concat( + "user__last_name", + Value(", "), + "user__first_name", + output_field=TextField(), + ), + Value(''), + output_field=TextField() + ), + email=Coalesce( + "user__email", + "pending_email", + output_field=TextField() + ), + is_active=~Q(user=None) ) + + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + + if 'page' in request.query_params: + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) return Response(serializer.data) - @action(detail=True, methods=["post"]) - def add_member(self, request, pk): - committee = self.get_object() - queryset= Membership.objects.filter(committee_account=committee) + @action(detail=False, methods=["post"], url_path="add-member", url_name="add_member") + def add_member(self, request): + committee_uuid = self.request.session["committee_uuid"] + committee = CommitteeAccount.objects.filter(id=committee_uuid).first() email = request.data.get('email', None) role = request.data.get('role', None) + # Check for necessary fields missing_fields = [] if email is None or len(email) == 0: missing_fields.append("email") @@ -119,20 +119,35 @@ def add_member(self, request, pk): if len(missing_fields) > 0: return Response(f"Missing fields: {', '.join(missing_fields)}", status=400) - if role not in Membership.CommitteeRole.choices: + # Check for valid role + choiceOf = False + for choice in Membership.CommitteeRole.choices: + if role in choice: + choiceOf = True + break + if not choiceOf: return Response(f"Invalid role", status=400) + # Check for pre-existing membership + matching_memberships = self.get_queryset().filter(Q(pending_email=email) | Q(user__email=email)) + if matching_memberships.count() > 0: + return Response(f"This email is already a member", status=400) + + # Create new membership + new_member = None matching_users = User.objects.filter(email=email) if matching_users.count() > 0: for user in matching_users: - added_member = committee.members.add(user) - added_member.role = role - added_member.save() + new_member = committee.members.add(user) + new_member.role = role logger.info(f"Added existing user {email} to committee {committee.committee_id}") else: - Membership( + new_member = Membership( committee_account=committee, pending_email=email, - role=request.role - ).save() - logger.info(f"Added pending membership for email {email} for committee {committee.committee_id}") \ No newline at end of file + role=role + ) + logger.info(f"Added pending membership for email {email} for committee {committee.committee_id}") + + new_member.save() + return Response(CommitteeMembershipSerializer(new_member).data, status=200) \ No newline at end of file From dfd06d32df4c9f67fc47d1cad0f91c096978e0e4 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 21 Feb 2024 17:36:01 -0500 Subject: [PATCH 25/74] Do not delete pending email record after redeeming pending membership --- django-backend/fecfiler/user/managers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django-backend/fecfiler/user/managers.py b/django-backend/fecfiler/user/managers.py index 8e72ac39c6..7721fc8b1c 100644 --- a/django-backend/fecfiler/user/managers.py +++ b/django-backend/fecfiler/user/managers.py @@ -18,7 +18,6 @@ def create_user(self, user_id, **obj_data): for new_membership in pending_memberships: new_membership.user = new_user - new_membership.pending_email = None new_membership.save() return new_user \ No newline at end of file From 9caa4b7117738bf14e5d0a6f3fd16072fb836af1 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 22 Feb 2024 12:26:32 -0500 Subject: [PATCH 26/74] 643 remove django page --- django-backend/fecfiler/urls.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django-backend/fecfiler/urls.py b/django-backend/fecfiler/urls.py index ab3d335fb2..e8bb583302 100644 --- a/django-backend/fecfiler/urls.py +++ b/django-backend/fecfiler/urls.py @@ -16,7 +16,6 @@ def test_celery(request): urlpatterns = [ - re_path(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), re_path( r"^api/schema/", SpectacularAPIView.as_view(api_version="v1"), name="schema" ), From 8cc4f3f32cedc56aa2d9cb9a73e6e8402ce09a3c Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 22 Feb 2024 15:44:04 -0500 Subject: [PATCH 27/74] Fixes linting errors --- ...ding_email_alter_membership_id_and_more.py | 23 +++++++++++-- .../fecfiler/committee_accounts/urls.py | 5 ++- .../fecfiler/committee_accounts/views.py | 34 +++++++++++++------ django-backend/fecfiler/user/managers.py | 8 +++-- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py index 4d5b8e19f0..1e9809d267 100644 --- a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py +++ b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py @@ -10,6 +10,7 @@ def delete_pending_memberships(apps, schema_editor): Membership = apps.get_model("committee_accounts", "Membership") # noqa Membership.objects.filter(user=None).delete() + def generate_new_uuid(apps, schema_editor): Membership = apps.get_model("committee_accounts", "Membership") # noqa for membership in Membership.objects.all(): @@ -33,7 +34,13 @@ class Migration(migrations.Migration): migrations.AddField( model_name='membership', name='uuid', - field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=False, serialize=False, unique=False) + field=models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=False, + serialize=False, + unique=False + ) ), migrations.RunPython( generate_new_uuid, @@ -51,12 +58,22 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='membership', name='id', - field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True) + field=models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True + ) ), migrations.AlterField( model_name='membership', name='user', - field=models.ForeignKey(null=True, on_delete=models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + field=models.ForeignKey( + null=True, + on_delete=models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL + ), ), migrations.AddField( model_name='membership', diff --git a/django-backend/fecfiler/committee_accounts/urls.py b/django-backend/fecfiler/committee_accounts/urls.py index edc5bc8746..c07362f1a3 100644 --- a/django-backend/fecfiler/committee_accounts/urls.py +++ b/django-backend/fecfiler/committee_accounts/urls.py @@ -10,4 +10,7 @@ members_router.register(r"", CommitteeMembershipViewSet) # The API URLs are now determined automatically by the router. -urlpatterns = [path(r"committees/", include(committee_router.urls)), path(r"committee-members/", include(members_router.urls))] +urlpatterns = [ + path(r"committees/", include(committee_router.urls)), + path(r"committee-members/", include(members_router.urls)) +] diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 50986d9d76..07810b34a6 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -70,7 +70,7 @@ class CommitteeMembershipViewSet(CommitteeOwnedViewSet): def get_queryset(self): return super().get_queryset().annotate( - name= Coalesce( + name=Coalesce( Concat( "user__last_name", Value(", "), @@ -120,18 +120,20 @@ def add_member(self, request): return Response(f"Missing fields: {', '.join(missing_fields)}", status=400) # Check for valid role - choiceOf = False + choice_of = False for choice in Membership.CommitteeRole.choices: if role in choice: - choiceOf = True + choice_of = True break - if not choiceOf: - return Response(f"Invalid role", status=400) + if not choice_of: + return Response("Invalid role", status=400) # Check for pre-existing membership - matching_memberships = self.get_queryset().filter(Q(pending_email=email) | Q(user__email=email)) + matching_memberships = self.get_queryset().filter( + Q(pending_email=email) | Q(user__email=email) + ) if matching_memberships.count() > 0: - return Response(f"This email is already a member", status=400) + return Response("This email is already a member", status=400) # Create new membership new_member = None @@ -140,14 +142,26 @@ def add_member(self, request): for user in matching_users: new_member = committee.members.add(user) new_member.role = role - logger.info(f"Added existing user {email} to committee {committee.committee_id}") + logger.info( + f"""Added existing user { + email + } to committee { + committee.committee_id + }""" + ) else: new_member = Membership( committee_account=committee, pending_email=email, role=role ) - logger.info(f"Added pending membership for email {email} for committee {committee.committee_id}") + logger.info( + f"""Added pending membership for email { + email + } for committee { + committee.committee_id + }""" + ) new_member.save() - return Response(CommitteeMembershipSerializer(new_member).data, status=200) \ No newline at end of file + return Response(CommitteeMembershipSerializer(new_member).data, status=200) diff --git a/django-backend/fecfiler/user/managers.py b/django-backend/fecfiler/user/managers.py index 7721fc8b1c..16e4278301 100644 --- a/django-backend/fecfiler/user/managers.py +++ b/django-backend/fecfiler/user/managers.py @@ -14,10 +14,14 @@ def create_user(self, user_id, **obj_data): pending_email=obj_data['email'] ) - logger.info(f"New User Created: {obj_data['email']} - {pending_memberships.count()} Pending Memberships") + logger.info( + f"""New User Created: { + obj_data['email']} - {pending_memberships.count() + } Pending Memberships""" + ) for new_membership in pending_memberships: new_membership.user = new_user new_membership.save() - return new_user \ No newline at end of file + return new_user From 1f3077ec32676471166464d47ab3440d3779c381 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 22 Feb 2024 16:12:15 -0500 Subject: [PATCH 28/74] Adds unit test that checks for automatic redemption of pending memberships upon user creation --- django-backend/fecfiler/user/test_models.py | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 django-backend/fecfiler/user/test_models.py diff --git a/django-backend/fecfiler/user/test_models.py b/django-backend/fecfiler/user/test_models.py new file mode 100644 index 0000000000..796cf84bef --- /dev/null +++ b/django-backend/fecfiler/user/test_models.py @@ -0,0 +1,32 @@ +from django.test import TestCase +from .models import User +from fecfiler.committee_accounts.models import CommitteeAccount, Membership + + +class CommitteeAccountTestCase(TestCase): + fixtures = ["test_committee_accounts"] + + def test_get_contact(self): + committee_account = CommitteeAccount.objects.create( + committee_id="C87654321", + ) + + new_membership = Membership( + committee_account=committee_account, + role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + pending_email="test_1234@test.com" + ) + new_membership.committee_account_id = committee_account.id + new_membership.save() + + self.assertEquals( + Membership.objects.get(pending_email="test_1234@test.com").user, None + ) + + new_user = User.objects.create_user( + email="test_1234@test.com", user_id='5e3c145f-a813-46c7-af5a-5739304acc27' + ) + + self.assertEquals( + Membership.objects.get(pending_email="test_1234@test.com").user, new_user + ) From cd930dd1bb59284759010a5391e1977d10a5f525 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 22 Feb 2024 16:31:24 -0500 Subject: [PATCH 29/74] Renames elements within user model tests --- django-backend/fecfiler/user/test_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/user/test_models.py b/django-backend/fecfiler/user/test_models.py index 796cf84bef..632c55b629 100644 --- a/django-backend/fecfiler/user/test_models.py +++ b/django-backend/fecfiler/user/test_models.py @@ -3,10 +3,10 @@ from fecfiler.committee_accounts.models import CommitteeAccount, Membership -class CommitteeAccountTestCase(TestCase): +class UserModelTestCase(TestCase): fixtures = ["test_committee_accounts"] - def test_get_contact(self): + def test_redeem_pending_membership(self): committee_account = CommitteeAccount.objects.create( committee_id="C87654321", ) From a10cee5602720e42dbe6c6275fc3307415af9a5e Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 23 Feb 2024 13:56:18 -0500 Subject: [PATCH 30/74] Simplifies CommitteeMember serializer --- .../committee_accounts/serializers.py | 65 +++++++------------ 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index 5f2f17de7e..350c00e645 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -12,50 +12,29 @@ class Meta: fields = "__all__" -class CommitteeMemberSerializer(serializers.Serializer): - id = serializers.CharField() - email = serializers.CharField() - username = serializers.CharField() - first_name = serializers.CharField() - last_name = serializers.CharField() - role = serializers.SerializerMethodField() - is_active = serializers.BooleanField() - - def get_role(self, object): - committee_id = self.context.get("committee_id") - return object.membership_set.get(committee_account_id=committee_id).role - - class CommitteeMembershipSerializer(serializers.Serializer): - id = serializers.SerializerMethodField() - email = serializers.SerializerMethodField() - username = serializers.SerializerMethodField() - name = serializers.SerializerMethodField() - role = serializers.CharField() - is_active = serializers.SerializerMethodField() - - def get_id(self, object): - if object.user is not None: - return object.user.id - return object.id - - def get_email(self, object): - if object.user is not None: - return object.user.email - return object.pending_email - - def get_username(self, object): - if object.user is not None: - return object.user.username - return '' - - def get_name(self, object): - if object.user is not None: - return f"{object.user.last_name}, {object.user.first_name}" - return '' - - def get_is_active(self, object): - return object.user is not None + role = serializers.ChoiceField(choices=Membership.CommitteeRole) + + def to_representation(self, instance): + representation = super().to_representation(instance) + + if instance.user is not None: + representation.update({ + 'id': instance.user.id, + 'email': instance.user.email, + 'username': instance.user.username, + 'name': f"{instance.user.last_name}, {instance.user.first_name}", + }) + else: + representation.update({ + 'id': instance.id, + 'email': instance.pending_email, + 'username': '', + 'name': '', + }) + + representation['is_active'] = instance.user is not None + return representation class Meta: model = Membership From e38cf77caf21b72b70a2a08da20fb26ff4129a7b Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 23 Feb 2024 15:06:17 -0500 Subject: [PATCH 31/74] Cleans up addMember() --- .../fecfiler/committee_accounts/views.py | 43 +++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 07810b34a6..079aaf339f 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -133,35 +133,24 @@ def add_member(self, request): Q(pending_email=email) | Q(user__email=email) ) if matching_memberships.count() > 0: - return Response("This email is already a member", status=400) + return Response("This email is taken by an existing membership to this committee", status=400) # Create new membership - new_member = None - matching_users = User.objects.filter(email=email) - if matching_users.count() > 0: - for user in matching_users: - new_member = committee.members.add(user) - new_member.role = role - logger.info( - f"""Added existing user { - email - } to committee { - committee.committee_id - }""" - ) - else: - new_member = Membership( - committee_account=committee, - pending_email=email, - role=role - ) - logger.info( - f"""Added pending membership for email { - email - } for committee { - committee.committee_id - }""" - ) + user = User.objects.filter(email=email).first() + membership_args = { + "committee_account": committee, + "role": role, + "user": user + } + + if user is None: + membership_args["pending_email"] = email + + new_member = Membership(**membership_args) new_member.save() + + member_type = "existing user" if user else "pending membership for" + logger.info(f'Added {member_type} "{email}" to committee {committee}') + return Response(CommitteeMembershipSerializer(new_member).data, status=200) From 0ea2bbf97db02332fd09ef29ea407100f84e134c Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 23 Feb 2024 15:17:05 -0500 Subject: [PATCH 32/74] Further tidying --- django-backend/fecfiler/committee_accounts/views.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 079aaf339f..a3ced720a2 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -142,10 +142,9 @@ def add_member(self, request): "committee_account": committee, "role": role, "user": user - } - - if user is None: - membership_args["pending_email"] = email + } | { + "pending_email": email + } if user is None else {} # Add pending email to args only if there is no user new_member = Membership(**membership_args) new_member.save() From 21de8f8287ba4b16ea57b579b2f0a63ba49e6bce Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 23 Feb 2024 15:21:09 -0500 Subject: [PATCH 33/74] Fixes linting error --- django-backend/fecfiler/committee_accounts/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index a3ced720a2..cf07dcd846 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -133,7 +133,10 @@ def add_member(self, request): Q(pending_email=email) | Q(user__email=email) ) if matching_memberships.count() > 0: - return Response("This email is taken by an existing membership to this committee", status=400) + return Response( + "This email is taken by an existing membership to this committee", + status=400 + ) # Create new membership user = User.objects.filter(email=email).first() From 7749b00386c329e453435912e3b832cf76d8a1f4 Mon Sep 17 00:00:00 2001 From: toddlees Date: Sun, 25 Feb 2024 20:15:22 -0500 Subject: [PATCH 34/74] query committee filings --- .../fecfiler/mock_openfec/__init__.py | 0 django-backend/fecfiler/mock_openfec/apps.py | 5 + .../mock_openfec/management/__init__.py | 0 .../management/commands/_init_.py | 0 .../management/commands/committee_data.json | 13 ++ .../commands/load_committee_data.py | 26 ++++ .../fecfiler/mock_openfec/mock_endpoints.py | 26 ++++ django-backend/fecfiler/mock_openfec/urls.py | 12 ++ django-backend/fecfiler/openfec/views.py | 125 +++++++++++------- django-backend/fecfiler/settings/base.py | 10 +- 10 files changed, 166 insertions(+), 51 deletions(-) create mode 100644 django-backend/fecfiler/mock_openfec/__init__.py create mode 100644 django-backend/fecfiler/mock_openfec/apps.py create mode 100644 django-backend/fecfiler/mock_openfec/management/__init__.py create mode 100644 django-backend/fecfiler/mock_openfec/management/commands/_init_.py create mode 100644 django-backend/fecfiler/mock_openfec/management/commands/committee_data.json create mode 100644 django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py create mode 100644 django-backend/fecfiler/mock_openfec/mock_endpoints.py create mode 100644 django-backend/fecfiler/mock_openfec/urls.py diff --git a/django-backend/fecfiler/mock_openfec/__init__.py b/django-backend/fecfiler/mock_openfec/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/mock_openfec/apps.py b/django-backend/fecfiler/mock_openfec/apps.py new file mode 100644 index 0000000000..e93fbaf6f4 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class MockOpenfecConfig(AppConfig): + name = "fecfiler.mock_openfec" diff --git a/django-backend/fecfiler/mock_openfec/management/__init__.py b/django-backend/fecfiler/mock_openfec/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/mock_openfec/management/commands/_init_.py b/django-backend/fecfiler/mock_openfec/management/commands/_init_.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json b/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json new file mode 100644 index 0000000000..515a2b5e92 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json @@ -0,0 +1,13 @@ +[ + { + "committee_id": "C12345678", + "committee_name": "Test Committee", + "committee_type": "O", + "form_type": "F1", + "pdf_url": "https://docquery.fec.gov/pdf/161/201601019004427161/201601019004427161.pdf", + "receipt_date": "2024-01-01T00:00:00", + "report_year": 2024, + "treasurer_name": "Last, First", + "update_date": "2024-01-01" + } +] \ No newline at end of file diff --git a/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py new file mode 100644 index 0000000000..5555327419 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py @@ -0,0 +1,26 @@ +from django.core.management.base import BaseCommand, CommandError +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL, BASE_DIR +import redis +import os +import json + + +class Command(BaseCommand): + help = "Load mock committee data into redis" + + # def add_arguments(self, parser): + # parser.add_argument("source") + + def handle(self, *args, **options): + if MOCK_OPENFEC_REDIS_URL: + redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) + COMMITTEE_DATA_REDIS_KEY = "COMMITTEE_DATA" + if not options.get("source"): + path = os.path.join( + BASE_DIR, "mock_openfec/management/commands/committee_data.json" + ) + with open(path) as file: + committee_data = file.read() + redis_instance.set(COMMITTEE_DATA_REDIS_KEY, committee_data) + + self.stdout.write(self.style.SUCCESS(f"Successfully loaded committees")) diff --git a/django-backend/fecfiler/mock_openfec/mock_endpoints.py b/django-backend/fecfiler/mock_openfec/mock_endpoints.py new file mode 100644 index 0000000000..d516240237 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/mock_endpoints.py @@ -0,0 +1,26 @@ +from rest_framework.response import Response +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL +import json +import redis + +COMMITTEE_DATA_REDIS_KEY = "COMMITTEE_DATA" +if MOCK_OPENFEC_REDIS_URL: + redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) + + +def query_filings(query, form_type): + if redis_instance: + committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) or "" + committees = json.loads(committee_data) or [] + filtered_committee_data = [ + committee + for committee in committees + if query in committee.get("committee_id") + or query in committee.get("committee_name") + ] + print(filtered_committee_data) + return { # same as api.open.fec.gov + "api_version": "1.0", + "results": filtered_committee_data, + "pagination": {"pages": 1, "per_page": 20, "count": 1, "page": 1}, + } diff --git a/django-backend/fecfiler/mock_openfec/urls.py b/django-backend/fecfiler/mock_openfec/urls.py new file mode 100644 index 0000000000..8faa2abd58 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/urls.py @@ -0,0 +1,12 @@ +from django.urls import path, include +from rest_framework.routers import DefaultRouter +from .views import MockOpenfecViewSet + +# Create a router and register our viewsets with it. +router = DefaultRouter() +router.register(r"mock_openfec", MockOpenfecViewSet, basename="mock_openfec") + +# The API URLs are now determined automatically by the router. +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/django-backend/fecfiler/openfec/views.py b/django-backend/fecfiler/openfec/views.py index 99bebe17c9..77d9397457 100644 --- a/django-backend/fecfiler/openfec/views.py +++ b/django-backend/fecfiler/openfec/views.py @@ -2,8 +2,16 @@ from django.http.response import HttpResponse, HttpResponseServerError from rest_framework.response import Response from rest_framework.decorators import action +from fecfiler.mock_openfec.views import MockOpenfecViewSet +from fecfiler.mock_openfec.mock_endpoints import query_filings import requests -from fecfiler.settings import base +from fecfiler.settings import ( + FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE, + FEC_API_KEY, + MOCK_OPENFEC_URL, + BASE_DIR, + FEC_API, +) import os import json @@ -13,92 +21,113 @@ class OpenfecViewSet(viewsets.ModelViewSet): - @action(detail=True) def committee(self, request, pk=None): - cids_to_override = list(map( - str.strip, base.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE.split(',') - )) if base.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE else [] - cid_to_override = next(( - cid for cid in cids_to_override if cid == pk - ), None) + cids_to_override = ( + list(map(str.strip, FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE.split(","))) + if FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE + else [] + ) + cid_to_override = next((cid for cid in cids_to_override if cid == pk), None) if cid_to_override: mock_committee_account = get_test_efo_mock_committee_account( cid_to_override ) if mock_committee_account: - return Response({ # same as api.open.fec.gov - "api_version": "1.0", - "results": [ - mock_committee_account, - ], - "pagination": { - "pages": 1, - "per_page": 20, - "count": 1, - "page": 1 + return Response( + { # same as api.open.fec.gov + "api_version": "1.0", + "results": [ + mock_committee_account, + ], + "pagination": { + "pages": 1, + "per_page": 20, + "count": 1, + "page": 1, + }, } - }) + ) else: - logger.error("Failed to find mock committee account data for " - "committee id to override: " + cid_to_override - ) + logger.error( + "Failed to find mock committee account data for " + "committee id to override: " + cid_to_override + ) return HttpResponseServerError() else: - resp = requests.get( - f'{base.FEC_API}committee/{pk}/?api_key={base.FEC_API_KEY}') + resp = requests.get(f"{FEC_API}committee/{pk}/?api_key={FEC_API_KEY}") return HttpResponse(resp) @action(detail=True) def filings(self, request, pk=None): - headers = { - 'Content-Type': 'application/json' - } - params = { - 'api_key': base.FEC_API_KEY, - 'committee_id': pk, - 'sort': '-receipt_date' - } + headers = {"Content-Type": "application/json"} + params = {"api_key": FEC_API_KEY, "committee_id": pk, "sort": "-receipt_date"} - resp = requests.get(f'{base.FEC_API}efile/filings/', - headers=headers, params=params) + resp = requests.get(f"{FEC_API}efile/filings/", headers=headers, params=params) retval = get_recent_f1_from_openfec_resp(resp) if not retval: - params['per_page'] = 1 - params['page'] = 1 - params['form_type'] = 'F1' - resp = requests.get(f'{base.FEC_API}filings/', headers=headers, params=params) + params["per_page"] = 1 + params["page"] = 1 + params["form_type"] = "F1" + resp = requests.get(f"{FEC_API}filings/", headers=headers, params=params) retval = get_recent_f1_from_openfec_resp(resp) return Response(retval) + @action(detail=False) + def query_filings(self, request): + query = request.query_params.get("query") + form_type = request.query_params.get("form_type") + if MOCK_OPENFEC_URL: + response = query_filings(query, form_type) + else: + params = { + "api_key": FEC_API_KEY, + "q_filer": query, + "sort": "-receipt_date", + "form_type": form_type, + "most_recent": True, + } + fec_response = requests.get(f"{FEC_API}filings/", params) + response = fec_response.json() + return Response(response) + def get_recent_f1_from_openfec_resp(resp: requests.Response): if resp: try: - filing_list = resp.json()['results'] - return next(( - filing for filing in filing_list if filing['form_type'].startswith('F1') - ), None) + filing_list = resp.json()["results"] + return next( + ( + filing + for filing in filing_list + if filing["form_type"].startswith("F1") + ), + None, + ) except Exception as error: logger.error( - f'Failed to process response from {resp.url} due to error: {str(error)}' + f"Failed to process response from {resp.url} due to error: {str(error)}" ) return None def get_test_efo_mock_committee_account(committee_id): mock_committee_accounts = get_test_efo_mock_committee_accounts() - return next(( - committee for committee in mock_committee_accounts - if committee['committee_id'] == committee_id - ), None) + return next( + ( + committee + for committee in mock_committee_accounts + if committee["committee_id"] == committee_id + ), + None, + ) def get_test_efo_mock_committee_accounts(): mock_committee_accounts_file = "committee_accounts.json" mock_committee_accounts_file_path = os.path.join( - base.BASE_DIR, "openfec/test_efo_mock_data/", mock_committee_accounts_file + BASE_DIR, "openfec/test_efo_mock_data/", mock_committee_accounts_file ) with open(mock_committee_accounts_file_path) as fp: return json.load(fp) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 6a6d180c39..2382872785 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -79,6 +79,7 @@ "fecfiler.web_services", "fecfiler.openfec", "fecfiler.user", + "fecfiler.mock_openfec", ] MIDDLEWARE = [ @@ -235,8 +236,7 @@ def get_logging_config(log_format=LINE): "plain_console": { "()": structlog.stdlib.ProcessorFormatter, "processor": structlog.dev.ConsoleRenderer( - colors=True, - exception_formatter=structlog.dev.rich_traceback + colors=True, exception_formatter=structlog.dev.rich_traceback ), }, "key_value": { @@ -257,7 +257,7 @@ def get_logging_config(log_format=LINE): "formatter": "key_value", "stream": sys.stdout, }, - } + }, } if log_format == LINE: @@ -374,3 +374,7 @@ def get_env_logging_processors(log_format=LINE): ) FEC_API_CANDIDATE_LOOKUP_ENDPOINT = str(FEC_API) + "candidates/" FEC_API_CANDIDATE_ENDPOINT = str(FEC_API) + "candidate/" + +"""MOCK OPENFEC settings""" +MOCK_OPENFEC_URL = env.get_credential("MOCK_OPEN_URL") +MOCK_OPENFEC_REDIS_URL = env.get_credential("REDIS_URL") From 5dfacbbffe9fa569c1dd5ce91db666ebc3b50792 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 08:38:52 -0500 Subject: [PATCH 35/74] create committee account --- .../fecfiler/committee_accounts/views.py | 20 ++++++- django-backend/fecfiler/openfec/views.py | 55 ++++++++----------- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index c428dfe625..1ad3673b72 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -2,8 +2,9 @@ from django.contrib.sessions.exceptions import SuspiciousSession from rest_framework.decorators import action from rest_framework.response import Response -from .models import CommitteeAccount +from .models import CommitteeAccount, Membership from .serializers import CommitteeAccountSerializer, CommitteeMemberSerializer +from fecfiler.openfec.views import retrieve_recent_f1 import structlog logger = structlog.get_logger(__name__) @@ -48,6 +49,23 @@ def active(self, request): committee = self.get_queryset().filter(id=committee_uuid).first() return Response(self.get_serializer(committee).data) + @action(detail=False, methods=["post"]) + def register(self, request): + email = request.user.email + committee_id = request.data.get("committee_id") + if not committee_id: + raise Exception("no committee_id provided") + f1 = retrieve_recent_f1(committee_id) + if not f1 or f1.email != email: + raise Exception("could not register committee") + account = CommitteeAccount.objects.create(committee_id=committee_id) + Membership.objects.create( + committee_account=account, + user=request.user, + role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + ) + return account + class CommitteeOwnedViewSet(viewsets.ModelViewSet): diff --git a/django-backend/fecfiler/openfec/views.py b/django-backend/fecfiler/openfec/views.py index 77d9397457..e3195ad9da 100644 --- a/django-backend/fecfiler/openfec/views.py +++ b/django-backend/fecfiler/openfec/views.py @@ -2,7 +2,6 @@ from django.http.response import HttpResponse, HttpResponseServerError from rest_framework.response import Response from rest_framework.decorators import action -from fecfiler.mock_openfec.views import MockOpenfecViewSet from fecfiler.mock_openfec.mock_endpoints import query_filings import requests from fecfiler.settings import ( @@ -20,7 +19,7 @@ logger = structlog.get_logger(__name__) -class OpenfecViewSet(viewsets.ModelViewSet): +class OpenfecViewSet(viewsets.GenericViewSet): @action(detail=True) def committee(self, request, pk=None): cids_to_override = ( @@ -59,20 +58,8 @@ def committee(self, request, pk=None): return HttpResponse(resp) @action(detail=True) - def filings(self, request, pk=None): - headers = {"Content-Type": "application/json"} - params = {"api_key": FEC_API_KEY, "committee_id": pk, "sort": "-receipt_date"} - - resp = requests.get(f"{FEC_API}efile/filings/", headers=headers, params=params) - retval = get_recent_f1_from_openfec_resp(resp) - if not retval: - params["per_page"] = 1 - params["page"] = 1 - params["form_type"] = "F1" - resp = requests.get(f"{FEC_API}filings/", headers=headers, params=params) - retval = get_recent_f1_from_openfec_resp(resp) - - return Response(retval) + def f1_filing(self, request, pk=None): + return Response(retrieve_recent_f1(pk)) @action(detail=False) def query_filings(self, request): @@ -93,23 +80,25 @@ def query_filings(self, request): return Response(response) -def get_recent_f1_from_openfec_resp(resp: requests.Response): - if resp: - try: - filing_list = resp.json()["results"] - return next( - ( - filing - for filing in filing_list - if filing["form_type"].startswith("F1") - ), - None, - ) - except Exception as error: - logger.error( - f"Failed to process response from {resp.url} due to error: {str(error)}" - ) - return None +def retrieve_recent_f1(committee_id): + """Gets the most recent F1 filing + First checks the realtime enpdpoint for a recent F1 filing. If none is found, a request is + made to a different endpoint that is updated nightly. The realtime endpoint will have + more recent filings, but does not provide filings older than 6 months. + The nightly endpoint keeps a longer history""" + headers = {"Content-Type": "application/json"} + params = { + "api_key": FEC_API_KEY, + "committee_id": committee_id, + "sort": "-receipt_date", + "form_type": "F1", + } + endpoints = [f"{FEC_API}efile/filings/", f"{FEC_API}filings/"] + for endpoint in endpoints: + response = requests.get(endpoint, headers=headers, params=params).json() + results = response["results"] + if len(results) > 0: + return results[0] def get_test_efo_mock_committee_account(committee_id): From 7f6c2af73931e1ff29222fa43a9ecd5c745ad165 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 09:19:32 -0500 Subject: [PATCH 36/74] load ss3 data --- .../fecfiler/committee_accounts/views.py | 14 ++++++++++--- .../management/commands/committee_data.json | 3 ++- .../commands/load_committee_data.py | 20 ++++++++++++------- .../fecfiler/mock_openfec/mock_endpoints.py | 12 ++++++++++- django-backend/fecfiler/openfec/test_views.py | 8 ++++---- django-backend/fecfiler/openfec/views.py | 4 ++-- django-backend/fecfiler/settings/base.py | 5 +++-- docker-compose.yml | 1 + 8 files changed, 47 insertions(+), 20 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 1ad3673b72..0957ed8e8d 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -5,6 +5,8 @@ from .models import CommitteeAccount, Membership from .serializers import CommitteeAccountSerializer, CommitteeMemberSerializer from fecfiler.openfec.views import retrieve_recent_f1 +from fecfiler.mock_openfec.mock_endpoints import recent_f1 +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL import structlog logger = structlog.get_logger(__name__) @@ -55,8 +57,14 @@ def register(self, request): committee_id = request.data.get("committee_id") if not committee_id: raise Exception("no committee_id provided") - f1 = retrieve_recent_f1(committee_id) - if not f1 or f1.email != email: + if MOCK_OPENFEC_REDIS_URL: + f1 = recent_f1(committee_id) + else: + f1 = retrieve_recent_f1(committee_id) + existing_account = CommitteeAccount.objects.filter( + committee_id=committee_id + ).first() + if existing_account or not f1 or f1.get("email") != email: raise Exception("could not register committee") account = CommitteeAccount.objects.create(committee_id=committee_id) Membership.objects.create( @@ -64,7 +72,7 @@ def register(self, request): user=request.user, role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, ) - return account + return Response(account) class CommitteeOwnedViewSet(viewsets.ModelViewSet): diff --git a/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json b/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json index 515a2b5e92..3174e4e764 100644 --- a/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json +++ b/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json @@ -8,6 +8,7 @@ "receipt_date": "2024-01-01T00:00:00", "report_year": 2024, "treasurer_name": "Last, First", - "update_date": "2024-01-01" + "update_date": "2024-01-01", + "email": "test@fec.gov" } ] \ No newline at end of file diff --git a/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py index 5555327419..407ffcee63 100644 --- a/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py +++ b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py @@ -1,26 +1,32 @@ -from django.core.management.base import BaseCommand, CommandError -from fecfiler.settings import MOCK_OPENFEC_REDIS_URL, BASE_DIR +from django.core.management.base import BaseCommand +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL, BASE_DIR, AWS_STORAGE_BUCKET_NAME import redis import os -import json +from fecfiler.s3 import S3_SESSION class Command(BaseCommand): help = "Load mock committee data into redis" - # def add_arguments(self, parser): - # parser.add_argument("source") + def add_arguments(self, parser): + parser.add_argument("--s3") def handle(self, *args, **options): if MOCK_OPENFEC_REDIS_URL: redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) COMMITTEE_DATA_REDIS_KEY = "COMMITTEE_DATA" - if not options.get("source"): + if not options.get("s3"): path = os.path.join( BASE_DIR, "mock_openfec/management/commands/committee_data.json" ) with open(path) as file: committee_data = file.read() - redis_instance.set(COMMITTEE_DATA_REDIS_KEY, committee_data) + else: + s3_object = S3_SESSION.Object( + AWS_STORAGE_BUCKET_NAME, "mock_committee_data.json" + ) + file = s3_object.get()["Body"] + committee_data = file.read() + redis_instance.set(COMMITTEE_DATA_REDIS_KEY, committee_data) self.stdout.write(self.style.SUCCESS(f"Successfully loaded committees")) diff --git a/django-backend/fecfiler/mock_openfec/mock_endpoints.py b/django-backend/fecfiler/mock_openfec/mock_endpoints.py index d516240237..b8291c1529 100644 --- a/django-backend/fecfiler/mock_openfec/mock_endpoints.py +++ b/django-backend/fecfiler/mock_openfec/mock_endpoints.py @@ -18,9 +18,19 @@ def query_filings(query, form_type): if query in committee.get("committee_id") or query in committee.get("committee_name") ] - print(filtered_committee_data) return { # same as api.open.fec.gov "api_version": "1.0", "results": filtered_committee_data, "pagination": {"pages": 1, "per_page": 20, "count": 1, "page": 1}, } + + +def recent_f1(committee_id): + if redis_instance: + committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) or "" + committees = json.loads(committee_data) or [] + return next( + committee + for committee in committees + if committee.get("committee_id") == committee_id + ) diff --git a/django-backend/fecfiler/openfec/test_views.py b/django-backend/fecfiler/openfec/test_views.py index e914f6b48c..4751cf79df 100644 --- a/django-backend/fecfiler/openfec/test_views.py +++ b/django-backend/fecfiler/openfec/test_views.py @@ -54,19 +54,19 @@ def test_get_committee_override_happy_path(self): ) def test_get_filings_invalid_resp(self): - request = self.factory.get("/api/v1/openfec/C00100230/filings/") + request = self.factory.get("/api/v1/openfec/C00100230/f1_filing/") request.user = self.user with patch("fecfiler.openfec.views.requests") as mock_requests: mock_requests.get.return_value = mock_response = Mock() mock_response.status_code = 200 mock_response.json.return_value = None - response = OpenfecViewSet.as_view({"get": "filings"})( + response = OpenfecViewSet.as_view({"get": "f1_filing"})( request, pk="C00100230" ) self.assertEqual(response.status_code, 200) def test_get_filings_happy_path(self): - request = self.factory.get("/api/v1/openfec/C00100230/filings/") + request = self.factory.get("/api/v1/openfec/C00100230/f1_filing/") request.user = self.user with patch("fecfiler.openfec.views.requests") as mock_requests: mock_requests.get.return_value = mock_response = Mock() @@ -75,7 +75,7 @@ def test_get_filings_happy_path(self): mock_response_object["results"] = [{}] mock_response_object["results"][0]["form_type"] = "F1" mock_response.json.return_value = mock_response_object - response = OpenfecViewSet.as_view({"get": "filings"})( + response = OpenfecViewSet.as_view({"get": "f1_filing"})( request, pk="C00100230" ) self.assertEqual(response.status_code, 200) diff --git a/django-backend/fecfiler/openfec/views.py b/django-backend/fecfiler/openfec/views.py index e3195ad9da..726a8dcc25 100644 --- a/django-backend/fecfiler/openfec/views.py +++ b/django-backend/fecfiler/openfec/views.py @@ -7,7 +7,7 @@ from fecfiler.settings import ( FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE, FEC_API_KEY, - MOCK_OPENFEC_URL, + MOCK_OPENFEC_REDIS_URL, BASE_DIR, FEC_API, ) @@ -65,7 +65,7 @@ def f1_filing(self, request, pk=None): def query_filings(self, request): query = request.query_params.get("query") form_type = request.query_params.get("form_type") - if MOCK_OPENFEC_URL: + if MOCK_OPENFEC_REDIS_URL: response = query_filings(query, form_type) else: params = { diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 2382872785..28ef6059d3 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -376,5 +376,6 @@ def get_env_logging_processors(log_format=LINE): FEC_API_CANDIDATE_ENDPOINT = str(FEC_API) + "candidate/" """MOCK OPENFEC settings""" -MOCK_OPENFEC_URL = env.get_credential("MOCK_OPEN_URL") -MOCK_OPENFEC_REDIS_URL = env.get_credential("REDIS_URL") +MOCK_OPENFEC = env.get_credential("MOCK_OPENFEC") +if MOCK_OPENFEC == "redis": + MOCK_OPENFEC_REDIS_URL = env.get_credential("REDIS_URL") diff --git a/docker-compose.yml b/docker-compose.yml index 313eb02fd7..23e35850e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,3 +108,4 @@ services: FEC_API_KEY: FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE: LOG_FORMAT: + MOCK_OPENFEC: 'redis' From 077199451a33c541613152b5f4af8c8deb3204b1 Mon Sep 17 00:00:00 2001 From: Laura Beaufort Date: Mon, 26 Feb 2024 09:32:55 -0500 Subject: [PATCH 37/74] Add notes to requirements file for oidc package --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b233de80f6..af518cb76a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ coreschema==0.0.4 decorator==5.1.1 dj-database-url==1.0.0 dj-static==0.0.6 -Django==4.2.7 +Django==4.2.7 # when updating this, also update requirements/tox versions in fecgov/mozilla_django_oidc django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 From b614924bba91324cde98558c561b8095af688724 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 09:50:49 -0500 Subject: [PATCH 38/74] set MOCK_OPENFEC_REDIS_URL to None when no mock --- django-backend/fecfiler/settings/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 28ef6059d3..f3b17c0f44 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -379,3 +379,5 @@ def get_env_logging_processors(log_format=LINE): MOCK_OPENFEC = env.get_credential("MOCK_OPENFEC") if MOCK_OPENFEC == "redis": MOCK_OPENFEC_REDIS_URL = env.get_credential("REDIS_URL") +else: + MOCK_OPENFEC_REDIS_URL = None From 68cf18154dfba61ebaf97230499fe70ed3547fc2 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 10:21:42 -0500 Subject: [PATCH 39/74] set up manifest --- django-backend/fecfiler/openfec/test_views.py | 14 ++++---- django-backend/fecfiler/openfec/views.py | 32 +++++++++---------- django-backend/fecfiler/settings/base.py | 2 +- docker-compose.yml | 2 +- manifests/manifest-dev-api.yml | 1 + manifests/manifest-prod-api.yml | 1 + manifests/manifest-stage-api.yml | 1 + 7 files changed, 28 insertions(+), 25 deletions(-) diff --git a/django-backend/fecfiler/openfec/test_views.py b/django-backend/fecfiler/openfec/test_views.py index 4751cf79df..835084f386 100644 --- a/django-backend/fecfiler/openfec/test_views.py +++ b/django-backend/fecfiler/openfec/test_views.py @@ -24,9 +24,9 @@ def test_get_committee_no_override(self): self.assertEqual(response.status_code, 200) def test_get_committee_override_data_not_found(self): - with patch("fecfiler.openfec.views.base") as base: - base.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = "C12345678" - base.BASE_DIR = "fecfiler/" + with patch("fecfiler.openfec.views.settings") as settings: + settings.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = "C12345678" + settings.BASE_DIR = "fecfiler/" request = self.factory.get("/api/v1/openfec/C12345678/committee/") request.user = self.user response = OpenfecViewSet.as_view({"get": "committee"})( @@ -35,9 +35,9 @@ def test_get_committee_override_data_not_found(self): self.assertEqual(response.status_code, 500) def test_get_committee_override_happy_path(self): - with patch("fecfiler.openfec.views.base") as base: - base.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = "C00100230" - base.BASE_DIR = "fecfiler/" + with patch("fecfiler.openfec.views.settings") as settings: + settings.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = "C00100230" + settings.BASE_DIR = "fecfiler/" request = self.factory.get("/api/v1/openfec/C00100230/committee/") request.user = self.user response = OpenfecViewSet.as_view({"get": "committee"})( @@ -59,7 +59,7 @@ def test_get_filings_invalid_resp(self): with patch("fecfiler.openfec.views.requests") as mock_requests: mock_requests.get.return_value = mock_response = Mock() mock_response.status_code = 200 - mock_response.json.return_value = None + mock_response.json.return_value = {"results": []} response = OpenfecViewSet.as_view({"get": "f1_filing"})( request, pk="C00100230" ) diff --git a/django-backend/fecfiler/openfec/views.py b/django-backend/fecfiler/openfec/views.py index 726a8dcc25..879ff43508 100644 --- a/django-backend/fecfiler/openfec/views.py +++ b/django-backend/fecfiler/openfec/views.py @@ -4,13 +4,7 @@ from rest_framework.decorators import action from fecfiler.mock_openfec.mock_endpoints import query_filings import requests -from fecfiler.settings import ( - FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE, - FEC_API_KEY, - MOCK_OPENFEC_REDIS_URL, - BASE_DIR, - FEC_API, -) +import fecfiler.settings as settings import os import json @@ -23,8 +17,12 @@ class OpenfecViewSet(viewsets.GenericViewSet): @action(detail=True) def committee(self, request, pk=None): cids_to_override = ( - list(map(str.strip, FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE.split(","))) - if FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE + list( + map( + str.strip, settings.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE.split(",") + ) + ) + if settings.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE else [] ) cid_to_override = next((cid for cid in cids_to_override if cid == pk), None) @@ -54,7 +52,9 @@ def committee(self, request, pk=None): ) return HttpResponseServerError() else: - resp = requests.get(f"{FEC_API}committee/{pk}/?api_key={FEC_API_KEY}") + resp = requests.get( + f"{settings.FEC_API}committee/{pk}/?api_key={settings.FEC_API_KEY}" + ) return HttpResponse(resp) @action(detail=True) @@ -65,17 +65,17 @@ def f1_filing(self, request, pk=None): def query_filings(self, request): query = request.query_params.get("query") form_type = request.query_params.get("form_type") - if MOCK_OPENFEC_REDIS_URL: + if settings.MOCK_OPENFEC_REDIS_URL: response = query_filings(query, form_type) else: params = { - "api_key": FEC_API_KEY, + "api_key": settings.FEC_API_KEY, "q_filer": query, "sort": "-receipt_date", "form_type": form_type, "most_recent": True, } - fec_response = requests.get(f"{FEC_API}filings/", params) + fec_response = requests.get(f"{settings.FEC_API}filings/", params) response = fec_response.json() return Response(response) @@ -88,12 +88,12 @@ def retrieve_recent_f1(committee_id): The nightly endpoint keeps a longer history""" headers = {"Content-Type": "application/json"} params = { - "api_key": FEC_API_KEY, + "api_key": settings.FEC_API_KEY, "committee_id": committee_id, "sort": "-receipt_date", "form_type": "F1", } - endpoints = [f"{FEC_API}efile/filings/", f"{FEC_API}filings/"] + endpoints = [f"{settings.FEC_API}efile/filings/", f"{settings.FEC_API}filings/"] for endpoint in endpoints: response = requests.get(endpoint, headers=headers, params=params).json() results = response["results"] @@ -116,7 +116,7 @@ def get_test_efo_mock_committee_account(committee_id): def get_test_efo_mock_committee_accounts(): mock_committee_accounts_file = "committee_accounts.json" mock_committee_accounts_file_path = os.path.join( - BASE_DIR, "openfec/test_efo_mock_data/", mock_committee_accounts_file + settings.BASE_DIR, "openfec/test_efo_mock_data/", mock_committee_accounts_file ) with open(mock_committee_accounts_file_path) as fp: return json.load(fp) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index f3b17c0f44..9a3b475780 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -377,7 +377,7 @@ def get_env_logging_processors(log_format=LINE): """MOCK OPENFEC settings""" MOCK_OPENFEC = env.get_credential("MOCK_OPENFEC") -if MOCK_OPENFEC == "redis": +if MOCK_OPENFEC == "REDIS": MOCK_OPENFEC_REDIS_URL = env.get_credential("REDIS_URL") else: MOCK_OPENFEC_REDIS_URL = None diff --git a/docker-compose.yml b/docker-compose.yml index 23e35850e3..0a273a41e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,4 +108,4 @@ services: FEC_API_KEY: FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE: LOG_FORMAT: - MOCK_OPENFEC: 'redis' + MOCK_OPENFEC: 'REDIS' diff --git a/manifests/manifest-dev-api.yml b/manifests/manifest-dev-api.yml index 1192a18f54..6ef17e66c2 100644 --- a/manifests/manifest-dev-api.yml +++ b/manifests/manifest-dev-api.yml @@ -25,3 +25,4 @@ applications: LOGIN_REDIRECT_SERVER_URL: https://fecfile-web-api-dev.app.cloud.gov/api/v1/auth/login-redirect LOGOUT_REDIRECT_URL: https://fecfile-web-api-dev.app.cloud.gov/api/v1/auth/logout-redirect LOG_FORMAT: KEY_VALUE + MOCK_OPENFEC: REDIS diff --git a/manifests/manifest-prod-api.yml b/manifests/manifest-prod-api.yml index fc256fa22e..a3d3861c77 100644 --- a/manifests/manifest-prod-api.yml +++ b/manifests/manifest-prod-api.yml @@ -25,3 +25,4 @@ applications: LOGIN_REDIRECT_SERVER_URL: https://fecfile-web-api-prod.app.cloud.gov/api/v1/auth/login-redirect LOGOUT_REDIRECT_URL: https://fecfile-web-api-prod.app.cloud.gov/api/v1/auth/logout-redirect LOG_FORMAT: KEY_VALUE + MOCK_OPENFEC: REDIS diff --git a/manifests/manifest-stage-api.yml b/manifests/manifest-stage-api.yml index 429d3da41e..9a6d1addc6 100644 --- a/manifests/manifest-stage-api.yml +++ b/manifests/manifest-stage-api.yml @@ -25,3 +25,4 @@ applications: LOGIN_REDIRECT_SERVER_URL: https://fecfile-web-api-stage.app.cloud.gov/api/v1/auth/login-redirect LOGOUT_REDIRECT_URL: https://fecfile-web-api-stage.app.cloud.gov/api/v1/auth/logout-redirect LOG_FORMAT: KEY_VALUE + MOCK_OPENFEC: REDIS From da7274bc3a788bdae9b2fb0e86baa4dcd6a2c82f Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 10:30:15 -0500 Subject: [PATCH 40/74] lint --- .../management/commands/load_committee_data.py | 4 ++-- django-backend/fecfiler/mock_openfec/mock_endpoints.py | 1 - django-backend/fecfiler/openfec/views.py | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py index 407ffcee63..a9773e08a2 100644 --- a/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py +++ b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py @@ -3,6 +3,7 @@ import redis import os from fecfiler.s3 import S3_SESSION +from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY class Command(BaseCommand): @@ -14,7 +15,6 @@ def add_arguments(self, parser): def handle(self, *args, **options): if MOCK_OPENFEC_REDIS_URL: redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) - COMMITTEE_DATA_REDIS_KEY = "COMMITTEE_DATA" if not options.get("s3"): path = os.path.join( BASE_DIR, "mock_openfec/management/commands/committee_data.json" @@ -29,4 +29,4 @@ def handle(self, *args, **options): committee_data = file.read() redis_instance.set(COMMITTEE_DATA_REDIS_KEY, committee_data) - self.stdout.write(self.style.SUCCESS(f"Successfully loaded committees")) + self.stdout.write(self.style.SUCCESS("Successfully loaded committees")) diff --git a/django-backend/fecfiler/mock_openfec/mock_endpoints.py b/django-backend/fecfiler/mock_openfec/mock_endpoints.py index b8291c1529..29194304f3 100644 --- a/django-backend/fecfiler/mock_openfec/mock_endpoints.py +++ b/django-backend/fecfiler/mock_openfec/mock_endpoints.py @@ -1,4 +1,3 @@ -from rest_framework.response import Response from fecfiler.settings import MOCK_OPENFEC_REDIS_URL import json import redis diff --git a/django-backend/fecfiler/openfec/views.py b/django-backend/fecfiler/openfec/views.py index 879ff43508..4a62fe8da9 100644 --- a/django-backend/fecfiler/openfec/views.py +++ b/django-backend/fecfiler/openfec/views.py @@ -82,10 +82,10 @@ def query_filings(self, request): def retrieve_recent_f1(committee_id): """Gets the most recent F1 filing - First checks the realtime enpdpoint for a recent F1 filing. If none is found, a request is - made to a different endpoint that is updated nightly. The realtime endpoint will have - more recent filings, but does not provide filings older than 6 months. - The nightly endpoint keeps a longer history""" + First checks the realtime enpdpoint for a recent F1 filing. If none is found, + a request is made to a different endpoint that is updated nightly. + The realtime endpoint will have more recent filings, but does not provide + filings older than 6 months. The nightly endpoint keeps a longer history""" headers = {"Content-Type": "application/json"} params = { "api_key": settings.FEC_API_KEY, From 7c09b253170d8b2801e57c925d5c2d52195dbb67 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 12:14:08 -0500 Subject: [PATCH 41/74] cov --- .../mock_openfec/test_load_committee_data.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 django-backend/fecfiler/mock_openfec/test_load_committee_data.py diff --git a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py new file mode 100644 index 0000000000..62d0eb80b9 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py @@ -0,0 +1,17 @@ +from django.test import TestCase +from django.core.management import call_command +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL +import redis +from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY + + +class LoadTestDataCommandTest(TestCase): + + def setUp(self): + pass + + def test_load_test_data(self): + call_command("load_committee_data") + redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) + committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) + self.assertEqual(len(committee_data), 1) From f12b3e2ddece28096b663091fdd8f9b6d5abdc9f Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 12:38:52 -0500 Subject: [PATCH 42/74] read email from f1: --- .../fecfiler/committee_accounts/views.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 0957ed8e8d..1cbf4f99a4 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -6,7 +6,9 @@ from .serializers import CommitteeAccountSerializer, CommitteeMemberSerializer from fecfiler.openfec.views import retrieve_recent_f1 from fecfiler.mock_openfec.mock_endpoints import recent_f1 +from fecfiler.web_services.dot_fec.dot_fec_serializer import CRLF_STR, FS_STR from fecfiler.settings import MOCK_OPENFEC_REDIS_URL +import requests import structlog logger = structlog.get_logger(__name__) @@ -54,17 +56,30 @@ def active(self, request): @action(detail=False, methods=["post"]) def register(self, request): email = request.user.email + f1_email = None committee_id = request.data.get("committee_id") if not committee_id: raise Exception("no committee_id provided") if MOCK_OPENFEC_REDIS_URL: f1 = recent_f1(committee_id) + f1_email = (f1 or {}).get("email") else: f1 = retrieve_recent_f1(committee_id) + dot_fec_url = (f1 or {}).get("fec_url") + if dot_fec_url: + response = requests.get( + dot_fec_url, + headers={"User-Agent": ""}, + ) + dot_fec_content = response.content.decode("utf-8") + f1_line = dot_fec_content.split("\n")[1] + f1_email = f1_line.split(FS_STR)[11] + existing_account = CommitteeAccount.objects.filter( committee_id=committee_id ).first() - if existing_account or not f1 or f1.get("email") != email: + print(f"existing: {existing_account}, f1_email: {f1_email}, email: {email}") + if existing_account or f1_email != email: raise Exception("could not register committee") account = CommitteeAccount.objects.create(committee_id=committee_id) Membership.objects.create( @@ -76,7 +91,6 @@ def register(self, request): class CommitteeOwnedViewSet(viewsets.ModelViewSet): - """ModelViewSet for models using CommitteeOwnedModel Inherit this view set to filter the queryset by the user's committee """ From dcc0e9702bd890dc69739c03690d18bb3f064b2a Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 12:47:45 -0500 Subject: [PATCH 43/74] correct string --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0a273a41e3..d6040d89b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,4 +108,4 @@ services: FEC_API_KEY: FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE: LOG_FORMAT: - MOCK_OPENFEC: 'REDIS' + MOCK_OPENFEC: REDIS From 5bca3d7a975f72f584915f4faf915f384bb62283 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 12:48:00 -0500 Subject: [PATCH 44/74] add user-agent --- django-backend/fecfiler/committee_accounts/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 1cbf4f99a4..ca5c973583 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -6,7 +6,7 @@ from .serializers import CommitteeAccountSerializer, CommitteeMemberSerializer from fecfiler.openfec.views import retrieve_recent_f1 from fecfiler.mock_openfec.mock_endpoints import recent_f1 -from fecfiler.web_services.dot_fec.dot_fec_serializer import CRLF_STR, FS_STR +from fecfiler.web_services.dot_fec.dot_fec_serializer import FS_STR from fecfiler.settings import MOCK_OPENFEC_REDIS_URL import requests import structlog @@ -69,7 +69,7 @@ def register(self, request): if dot_fec_url: response = requests.get( dot_fec_url, - headers={"User-Agent": ""}, + headers={"User-Agent": "fecfile-api"}, ) dot_fec_content = response.content.decode("utf-8") f1_line = dot_fec_content.split("\n")[1] From 6347a873961c95641a4860b6ee6c3b16d9baad1c Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 12:53:38 -0500 Subject: [PATCH 45/74] see mock_openfec value --- .../fecfiler/mock_openfec/test_load_committee_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py index 62d0eb80b9..5f3bb1da10 100644 --- a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py +++ b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py @@ -1,6 +1,6 @@ from django.test import TestCase from django.core.management import call_command -from fecfiler.settings import MOCK_OPENFEC_REDIS_URL +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL, MOCK_OPENFEC import redis from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY @@ -12,6 +12,7 @@ def setUp(self): def test_load_test_data(self): call_command("load_committee_data") + print(f"MOCK_OPENFEC {MOCK_OPENFEC}") redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) self.assertEqual(len(committee_data), 1) From d67722959202d4e1da614fa2bb39c52d2af4c9ee Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 13:50:01 -0500 Subject: [PATCH 46/74] add redis to circleci --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2129b41318..c08f1c9762 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,6 +50,7 @@ jobs: docker: - image: cimg/python:3.10-node - image: cimg/postgres:12.8 + - image: cimg/redis:6.2.6 steps: - run: @@ -64,6 +65,9 @@ jobs: - checkout + - run: + command: redis-server --port 6379 + - run: name: Create unified requirements so CircleCI can cache them command: | From 77eefb78641d41c633c2cfe90ad3a3b910d88eb4 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 13:52:19 -0500 Subject: [PATCH 47/74] redundant command --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c08f1c9762..0b5592ea04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,9 +65,6 @@ jobs: - checkout - - run: - command: redis-server --port 6379 - - run: name: Create unified requirements so CircleCI can cache them command: | From 19b90519a0d49439626bdad46ffdc069fddb0610 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 14:06:48 -0500 Subject: [PATCH 48/74] remove print --- django-backend/fecfiler/mock_openfec/test_load_committee_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py index 5f3bb1da10..81244e6b6d 100644 --- a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py +++ b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py @@ -12,7 +12,6 @@ def setUp(self): def test_load_test_data(self): call_command("load_committee_data") - print(f"MOCK_OPENFEC {MOCK_OPENFEC}") redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) self.assertEqual(len(committee_data), 1) From bf5b2aa3a7d6e2e5d7817c406b65942be177ff58 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 14:19:32 -0500 Subject: [PATCH 49/74] better test --- .../fecfiler/mock_openfec/test_load_committee_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py index 81244e6b6d..4bc0067c47 100644 --- a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py +++ b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py @@ -2,6 +2,7 @@ from django.core.management import call_command from fecfiler.settings import MOCK_OPENFEC_REDIS_URL, MOCK_OPENFEC import redis +import json from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY @@ -14,4 +15,4 @@ def test_load_test_data(self): call_command("load_committee_data") redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) - self.assertEqual(len(committee_data), 1) + self.assertEqual(len(json.loads(committee_data)), 1) From 0bcf7da31c0a3145bd91f07a35830852cbc0838b Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 14:37:41 -0500 Subject: [PATCH 50/74] even better tests --- .../mock_openfec/test_load_committee_data.py | 2 +- .../mock_openfec/test_mock_endpoints.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 django-backend/fecfiler/mock_openfec/test_mock_endpoints.py diff --git a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py index 4bc0067c47..bd44349616 100644 --- a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py +++ b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py @@ -1,6 +1,6 @@ from django.test import TestCase from django.core.management import call_command -from fecfiler.settings import MOCK_OPENFEC_REDIS_URL, MOCK_OPENFEC +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL import redis import json from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY diff --git a/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py b/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py new file mode 100644 index 0000000000..feabb6c3a0 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py @@ -0,0 +1,22 @@ +from django.test import TestCase +from django.core.management import call_command +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL +from fecfiler.mock_openfec.mock_endpoints import query_filings +import redis +import json +from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY + + +class MockEndpointsTest(TestCase): + + def setUp(self): + pass + + def test_query_filings(self): + call_command("load_committee_data") + response = query_filings("NOT FINDABLE", "F3") + self.assertEqual(len(response["results"]), 0) + response = query_filings("NOT FINDABLE", "F1") + self.assertEqual(len(response["results"]), 0) + response = query_filings("st Com", "F1") + self.assertEqual(len(response["results"]), 1) From e3f76891a740b2d63fb05dfb0a071d00981dae63 Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Mon, 26 Feb 2024 14:38:00 -0500 Subject: [PATCH 51/74] Move code to committee members section and change from using email to pk --- .../fecfiler/committee_accounts/test_views.py | 11 ++++++----- .../fecfiler/committee_accounts/urls.py | 19 ++++++------------- .../fecfiler/committee_accounts/views.py | 19 +++++++++++-------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index c5644a3721..310d01ddd1 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,9 +1,9 @@ from fecfiler.user.models import User from django.test import RequestFactory, TestCase -from .views import CommitteeViewSet +from .views import CommitteeMembershipViewSet -class CommitteeViewSetTest(TestCase): +class CommitteeMemberViewSetTest(TestCase): fixtures = ["C01234567_user_and_committee"] def setUp(self): @@ -12,11 +12,12 @@ def setUp(self): def test_remove_member(self): request = self.factory.get( - "/api/v1/committees/C87654321/member/test@fec.gov") + "/api/v1/committee-members/12345678-aaaa-bbbb-cccc-111122223333/remove-member" + ) request.user = self.user request.session = { "committee_uuid": "11111111-2222-3333-4444-555555555555"} request.method = "DELETE" - response = CommitteeViewSet.remove_member( - self, request, "C01234567", "test@fec.gov") + response = CommitteeMembershipViewSet.remove_member( + self, request, "12345678-aaaa-bbbb-cccc-111122223333") self.assertEqual(response.status_code, 200) diff --git a/django-backend/fecfiler/committee_accounts/urls.py b/django-backend/fecfiler/committee_accounts/urls.py index 97a415b055..fefada1e75 100644 --- a/django-backend/fecfiler/committee_accounts/urls.py +++ b/django-backend/fecfiler/committee_accounts/urls.py @@ -1,19 +1,12 @@ -from django.urls import path, include from rest_framework.routers import DefaultRouter from .views import CommitteeMembershipViewSet, CommitteeViewSet # Create a router and register our viewsets with it. -committee_router = DefaultRouter() -committee_router.register(r"", CommitteeViewSet, basename="committees") - -members_router = DefaultRouter() -members_router.register(r"", CommitteeMembershipViewSet) +router = DefaultRouter() +router.register(r"committees", CommitteeViewSet, basename="committees") +router.register(r"committee-members", CommitteeMembershipViewSet, + basename="committee-members") # The API URLs are now determined automatically by the router. -urlpatterns = [ - path("committees/", include(router.urls)), - path(r"committees/", include(committee_router.urls)), - path('committees//member//', - CommitteeViewSet.remove_member, name='remove-member'), - path(r"committee-members/", include(members_router.urls)) -] +urlpatterns = [] +urlpatterns += router.urls diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 4491972d63..f42338d29e 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -36,14 +36,6 @@ def active(self, request): committee = self.get_queryset().filter(id=committee_uuid).first() return Response(self.get_serializer(committee).data) - @action(detail=True, methods=["delete"]) - def remove_member(self, request, committee_id, member_email): - committee_uuid = request.session["committee_uuid"] - committee = CommitteeAccount.objects.filter(id=committee_uuid).first() - member = committee.members.filter(email=member_email).first() - member.delete() - return HttpResponse('Member removed') - class CommitteeOwnedViewSet(viewsets.ModelViewSet): @@ -164,3 +156,14 @@ def add_member(self, request): logger.info(f'Added {member_type} "{email}" to committee {committee}') return Response(CommitteeMembershipSerializer(new_member).data, status=200) + + @action(detail=True, methods=["delete"], + url_path="remove-member", url_name="remove_member") + def remove_member(self, request, pk): + committee_uuid = request.session["committee_uuid"] + committee = CommitteeAccount.objects.filter(id=committee_uuid).first() + member = committee.members.filter(id=pk).first() + if member is None: + return HttpResponse("No member found", 500) + member.delete() + return HttpResponse('Member removed') From de9398bd213ee107e2297c35f7786acc5e4e0cfd Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 15:01:33 -0500 Subject: [PATCH 52/74] way more coverage --- .../fecfiler/committee_accounts/test_views.py | 29 +++++++++ .../fecfiler/committee_accounts/views.py | 64 ++++++++++--------- .../mock_openfec/test_mock_endpoints.py | 4 -- 3 files changed, 64 insertions(+), 33 deletions(-) create mode 100644 django-backend/fecfiler/committee_accounts/test_views.py diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py new file mode 100644 index 0000000000..cfba4bb325 --- /dev/null +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -0,0 +1,29 @@ +from django.test import TestCase +from fecfiler.committee_accounts.views import register_committee +from fecfiler.user.models import User + + +class CommitteeAccountsViewsTest(TestCase): + + def setUp(self): + self.test_user = User.objects.create(email="test@fec.gov", username="gov") + self.other_user = User.objects.create(email="test@fec.com", username="com") + + def test_register_committee(self): + account = register_committee("C12345678", self.test_user) + self.assertEquals(account.committee_id, "C12345678") + + def test_register_committee_existing(self): + account = register_committee("C12345678", self.test_user) + self.assertEquals(account.committee_id, "C12345678") + self.assertRaises( + Exception, register_committee, committee_id="C12345678", user=self.test_user + ) + + def test_register_committee_mismatch_email(self): + self.assertRaises( + Exception, + register_committee, + committee_id="C12345678", + user=self.other_user, + ) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 62e953eff4..849547e4b9 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -42,38 +42,11 @@ def active(self, request): @action(detail=False, methods=["post"]) def register(self, request): - email = request.user.email - f1_email = None committee_id = request.data.get("committee_id") if not committee_id: raise Exception("no committee_id provided") - if MOCK_OPENFEC_REDIS_URL: - f1 = recent_f1(committee_id) - f1_email = (f1 or {}).get("email") - else: - f1 = retrieve_recent_f1(committee_id) - dot_fec_url = (f1 or {}).get("fec_url") - if dot_fec_url: - response = requests.get( - dot_fec_url, - headers={"User-Agent": "fecfile-api"}, - ) - dot_fec_content = response.content.decode("utf-8") - f1_line = dot_fec_content.split("\n")[1] - f1_email = f1_line.split(FS_STR)[11] - - existing_account = CommitteeAccount.objects.filter( - committee_id=committee_id - ).first() - print(f"existing: {existing_account}, f1_email: {f1_email}, email: {email}") - if existing_account or f1_email != email: - raise Exception("could not register committee") - account = CommitteeAccount.objects.create(committee_id=committee_id) - Membership.objects.create( - committee_account=account, - user=request.user, - role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, - ) + account = register_committee(committee_id, request.user) + return Response(account) @@ -193,3 +166,36 @@ def add_member(self, request): logger.info(f'Added {member_type} "{email}" to committee {committee}') return Response(CommitteeMembershipSerializer(new_member).data, status=200) + + +def register_committee(committee_id, user): + email = user.email + + if MOCK_OPENFEC_REDIS_URL: + f1 = recent_f1(committee_id) + f1_email = (f1 or {}).get("email") + else: + f1 = retrieve_recent_f1(committee_id) + dot_fec_url = (f1 or {}).get("fec_url") + if dot_fec_url: + response = requests.get( + dot_fec_url, + headers={"User-Agent": "fecfile-api"}, + ) + dot_fec_content = response.content.decode("utf-8") + f1_line = dot_fec_content.split("\n")[1] + f1_email = f1_line.split(FS_STR)[11] + + existing_account = CommitteeAccount.objects.filter( + committee_id=committee_id + ).first() + print(f"existing: {existing_account}, f1_email: {f1_email}, email: {email}") + if existing_account or f1_email != email: + raise Exception("could not register committee") + account = CommitteeAccount.objects.create(committee_id=committee_id) + Membership.objects.create( + committee_account=account, + user=user, + role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + ) + return account diff --git a/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py b/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py index feabb6c3a0..a3c2b5299a 100644 --- a/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py +++ b/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py @@ -1,10 +1,6 @@ from django.test import TestCase from django.core.management import call_command -from fecfiler.settings import MOCK_OPENFEC_REDIS_URL from fecfiler.mock_openfec.mock_endpoints import query_filings -import redis -import json -from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY class MockEndpointsTest(TestCase): From 10eaec7790d97055047b073c1ace3a5bc32469b1 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 26 Feb 2024 15:10:05 -0500 Subject: [PATCH 53/74] load data into redis for tests --- django-backend/fecfiler/committee_accounts/test_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index cfba4bb325..149a5f07f2 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,11 +1,13 @@ from django.test import TestCase from fecfiler.committee_accounts.views import register_committee from fecfiler.user.models import User +from django.core.management import call_command class CommitteeAccountsViewsTest(TestCase): def setUp(self): + call_command("load_committee_data") self.test_user = User.objects.create(email="test@fec.gov", username="gov") self.other_user = User.objects.create(email="test@fec.com", username="com") From 87b367dced6297cb14ade1bff3408d039e00d022 Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Mon, 26 Feb 2024 16:04:18 -0500 Subject: [PATCH 54/74] Fix Membership Serializer to point to membership id instead of user id. --- django-backend/fecfiler/committee_accounts/serializers.py | 2 +- django-backend/fecfiler/committee_accounts/views.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index 350c00e645..72c1a3329c 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -20,7 +20,7 @@ def to_representation(self, instance): if instance.user is not None: representation.update({ - 'id': instance.user.id, + 'id': instance.id, 'email': instance.user.email, 'username': instance.user.username, 'name': f"{instance.user.last_name}, {instance.user.first_name}", diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index f42338d29e..5ca2f77030 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -160,10 +160,6 @@ def add_member(self, request): @action(detail=True, methods=["delete"], url_path="remove-member", url_name="remove_member") def remove_member(self, request, pk): - committee_uuid = request.session["committee_uuid"] - committee = CommitteeAccount.objects.filter(id=committee_uuid).first() - member = committee.members.filter(id=pk).first() - if member is None: - return HttpResponse("No member found", 500) + member = self.get_object() member.delete() return HttpResponse('Member removed') From fe2c8d05034bb16d1cf1e11bc79135ffa75f973c Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 26 Feb 2024 16:28:21 -0500 Subject: [PATCH 55/74] Temporarily disable test_remove_member --- django-backend/fecfiler/committee_accounts/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 310d01ddd1..3242c48b96 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -10,7 +10,7 @@ def setUp(self): self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") self.factory = RequestFactory() - def test_remove_member(self): + def xtest_remove_member(self): request = self.factory.get( "/api/v1/committee-members/12345678-aaaa-bbbb-cccc-111122223333/remove-member" ) From c2a0ed211638d588c40e4505988b42bd412eaf11 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 26 Feb 2024 16:47:33 -0500 Subject: [PATCH 56/74] Fix linting issues --- django-backend/fecfiler/committee_accounts/test_views.py | 4 +++- django-backend/fecfiler/committee_accounts/views.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 76ac15fd82..4abe7d4a0e 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,8 +1,10 @@ from django.test import RequestFactory, TestCase -from fecfiler.committee_accounts.views import register_committee, CommitteeMembershipViewSet +from fecfiler.committee_accounts.views import register_committee, \ + CommitteeMembershipViewSet from fecfiler.user.models import User from django.core.management import call_command + class CommitteeAccountsViewsTest(TestCase): def setUp(self): diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 6caf97ae4b..4e45b895a6 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -198,8 +198,8 @@ def register_committee(committee_id, user): role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, ) return account - - + + @action(detail=True, methods=["delete"], url_path="remove-member", url_name="remove_member") def remove_member(self, request, pk): From a61abd95157849804597ceb0cfcb92e0efbbaa42 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 26 Feb 2024 16:52:11 -0500 Subject: [PATCH 57/74] rename function --- django-backend/fecfiler/committee_accounts/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 4abe7d4a0e..401df6a46d 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -39,7 +39,7 @@ def setUp(self): self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") self.factory = RequestFactory() - def xtest_remove_member(self): + def ztest_remove_member(self): request = self.factory.get( "/api/v1/committee-members/12345678-aaaa-bbbb-cccc-111122223333/remove-member" ) From 8a2a5a25e36f6b9a2970854bec4967d3c8ab0440 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 26 Feb 2024 17:08:07 -0500 Subject: [PATCH 58/74] Put remove_member into class --- .../fecfiler/committee_accounts/views.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 4e45b895a6..2c13fea6d6 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -167,6 +167,13 @@ def add_member(self, request): return Response(CommitteeMembershipSerializer(new_member).data, status=200) + @action(detail=True, methods=["delete"], + url_path="remove-member", url_name="remove_member") + def remove_member(self, request, pk): + member = self.get_object() + member.delete() + return HttpResponse('Member removed') + def register_committee(committee_id, user): email = user.email @@ -198,11 +205,3 @@ def register_committee(committee_id, user): role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, ) return account - - -@action(detail=True, methods=["delete"], - url_path="remove-member", url_name="remove_member") -def remove_member(self, request, pk): - member = self.get_object() - member.delete() - return HttpResponse('Member removed') From 12a55ed23e700593d9851275cddf39233fb1f9ea Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Tue, 27 Feb 2024 13:48:33 -0500 Subject: [PATCH 59/74] Update test for removing a committee to be functional --- .../fecfiler/committee_accounts/test_views.py | 49 +++++-------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 401df6a46d..2d6cfb9aaa 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,52 +1,29 @@ from django.test import RequestFactory, TestCase -from fecfiler.committee_accounts.views import register_committee, \ - CommitteeMembershipViewSet +from .views import CommitteeMembershipViewSet +from rest_framework import viewsets from fecfiler.user.models import User -from django.core.management import call_command -class CommitteeAccountsViewsTest(TestCase): - - def setUp(self): - call_command("load_committee_data") - self.test_user = User.objects.create(email="test@fec.gov", username="gov") - self.other_user = User.objects.create(email="test@fec.com", username="com") - - def test_register_committee(self): - account = register_committee("C12345678", self.test_user) - self.assertEquals(account.committee_id, "C12345678") - - def test_register_committee_existing(self): - account = register_committee("C12345678", self.test_user) - self.assertEquals(account.committee_id, "C12345678") - self.assertRaises( - Exception, register_committee, committee_id="C12345678", user=self.test_user - ) - - def test_register_committee_mismatch_email(self): - self.assertRaises( - Exception, - register_committee, - committee_id="C12345678", - user=self.other_user, - ) - - -class CommitteeMemberViewSetTest(TestCase): +class CommitteeMemberViewSetTest(TestCase, viewsets.ModelViewSet): fixtures = ["C01234567_user_and_committee"] def setUp(self): self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") self.factory = RequestFactory() - def ztest_remove_member(self): + def test_remove_member(self): + membership_uuid = "3e281c08-2b1f-4cd0-9236-410fe872edb9" + view = CommitteeMembershipViewSet() request = self.factory.get( - "/api/v1/committee-members/12345678-aaaa-bbbb-cccc-111122223333/remove-member" + "/api/v1/committee-members/{membership_uuid}/remove-member" ) request.user = self.user request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555"} + "committee_uuid": "c94c5d1a-9e73-464d-ad72-b73b5d8667a9"} request.method = "DELETE" - response = CommitteeMembershipViewSet.remove_member( - self, request, "12345678-aaaa-bbbb-cccc-111122223333") + request.query_params = dict() + view.kwargs = {"pk": membership_uuid} + view.request = request + response = view.remove_member( + request, membership_uuid) self.assertEqual(response.status_code, 200) From f36b09266b26da95b0167559f78cfb1ae19757e0 Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Tue, 27 Feb 2024 13:48:56 -0500 Subject: [PATCH 60/74] Revert "Update test for removing a committee to be functional" This reverts commit 12a55ed23e700593d9851275cddf39233fb1f9ea. --- .../fecfiler/committee_accounts/test_views.py | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 2d6cfb9aaa..401df6a46d 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,29 +1,52 @@ from django.test import RequestFactory, TestCase -from .views import CommitteeMembershipViewSet -from rest_framework import viewsets +from fecfiler.committee_accounts.views import register_committee, \ + CommitteeMembershipViewSet from fecfiler.user.models import User +from django.core.management import call_command -class CommitteeMemberViewSetTest(TestCase, viewsets.ModelViewSet): +class CommitteeAccountsViewsTest(TestCase): + + def setUp(self): + call_command("load_committee_data") + self.test_user = User.objects.create(email="test@fec.gov", username="gov") + self.other_user = User.objects.create(email="test@fec.com", username="com") + + def test_register_committee(self): + account = register_committee("C12345678", self.test_user) + self.assertEquals(account.committee_id, "C12345678") + + def test_register_committee_existing(self): + account = register_committee("C12345678", self.test_user) + self.assertEquals(account.committee_id, "C12345678") + self.assertRaises( + Exception, register_committee, committee_id="C12345678", user=self.test_user + ) + + def test_register_committee_mismatch_email(self): + self.assertRaises( + Exception, + register_committee, + committee_id="C12345678", + user=self.other_user, + ) + + +class CommitteeMemberViewSetTest(TestCase): fixtures = ["C01234567_user_and_committee"] def setUp(self): self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") self.factory = RequestFactory() - def test_remove_member(self): - membership_uuid = "3e281c08-2b1f-4cd0-9236-410fe872edb9" - view = CommitteeMembershipViewSet() + def ztest_remove_member(self): request = self.factory.get( - "/api/v1/committee-members/{membership_uuid}/remove-member" + "/api/v1/committee-members/12345678-aaaa-bbbb-cccc-111122223333/remove-member" ) request.user = self.user request.session = { - "committee_uuid": "c94c5d1a-9e73-464d-ad72-b73b5d8667a9"} + "committee_uuid": "11111111-2222-3333-4444-555555555555"} request.method = "DELETE" - request.query_params = dict() - view.kwargs = {"pk": membership_uuid} - view.request = request - response = view.remove_member( - request, membership_uuid) + response = CommitteeMembershipViewSet.remove_member( + self, request, "12345678-aaaa-bbbb-cccc-111122223333") self.assertEqual(response.status_code, 200) From ee2a371dbe57cefc83bf8a35fe85c2e826477d3d Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Tue, 27 Feb 2024 13:51:19 -0500 Subject: [PATCH 61/74] Update test for removing a committee to be functional --- .../fecfiler/committee_accounts/test_views.py | 50 ++++++------------- .../fecfiler/committee_accounts/views.py | 5 +- 2 files changed, 17 insertions(+), 38 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 401df6a46d..53dddcb1f8 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,52 +1,30 @@ +from uuid import UUID from django.test import RequestFactory, TestCase -from fecfiler.committee_accounts.views import register_committee, \ - CommitteeMembershipViewSet +from .views import CommitteeMembershipViewSet +from rest_framework import viewsets from fecfiler.user.models import User -from django.core.management import call_command -class CommitteeAccountsViewsTest(TestCase): - - def setUp(self): - call_command("load_committee_data") - self.test_user = User.objects.create(email="test@fec.gov", username="gov") - self.other_user = User.objects.create(email="test@fec.com", username="com") - - def test_register_committee(self): - account = register_committee("C12345678", self.test_user) - self.assertEquals(account.committee_id, "C12345678") - - def test_register_committee_existing(self): - account = register_committee("C12345678", self.test_user) - self.assertEquals(account.committee_id, "C12345678") - self.assertRaises( - Exception, register_committee, committee_id="C12345678", user=self.test_user - ) - - def test_register_committee_mismatch_email(self): - self.assertRaises( - Exception, - register_committee, - committee_id="C12345678", - user=self.other_user, - ) - - -class CommitteeMemberViewSetTest(TestCase): +class CommitteeMemberViewSetTest(TestCase, viewsets.ModelViewSet): fixtures = ["C01234567_user_and_committee"] def setUp(self): self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") self.factory = RequestFactory() - def ztest_remove_member(self): + def test_remove_member(self): + membership_uuid = UUID("136a21f2-66fe-4d56-89e9-0d1d4612741c") + view = CommitteeMembershipViewSet() request = self.factory.get( - "/api/v1/committee-members/12345678-aaaa-bbbb-cccc-111122223333/remove-member" + "/api/v1/committee-members/{membership_uuid}/remove-member" ) request.user = self.user request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555"} + "committee_uuid": UUID('11111111-2222-3333-4444-555555555555')} request.method = "DELETE" - response = CommitteeMembershipViewSet.remove_member( - self, request, "12345678-aaaa-bbbb-cccc-111122223333") + request.query_params = dict() + view.kwargs = {"pk": membership_uuid} + view.request = request + response = view.remove_member( + request, membership_uuid) self.assertEqual(response.status_code, 200) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 2c13fea6d6..994aa0f937 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -1,3 +1,4 @@ +from uuid import UUID from fecfiler.user.models import User from rest_framework import filters, viewsets, mixins from django.contrib.sessions.exceptions import SuspiciousSession @@ -168,8 +169,8 @@ def add_member(self, request): return Response(CommitteeMembershipSerializer(new_member).data, status=200) @action(detail=True, methods=["delete"], - url_path="remove-member", url_name="remove_member") - def remove_member(self, request, pk): + url_path="remove-member", url_name="remove_member") + def remove_member(self, request, pk: UUID): member = self.get_object() member.delete() return HttpResponse('Member removed') From c0d4b8d967526b90a860f8fe54cf157adb63c515 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 27 Feb 2024 14:13:17 -0500 Subject: [PATCH 62/74] serialize committee account --- .../fecfiler/committee_accounts/views.py | 15 ++++++++++----- .../management/commands/load_committee_data.py | 2 +- .../fecfiler/mock_openfec/mock_endpoints.py | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 2c13fea6d6..804803b7f8 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -48,7 +48,7 @@ def register(self, request): raise Exception("no committee_id provided") account = register_committee(committee_id, request.user) - return Response(account) + return Response(CommitteeAccountSerializer(account).data) class CommitteeOwnedViewSet(viewsets.ModelViewSet): @@ -63,7 +63,8 @@ def get_queryset(self): raise SuspiciousSession("session has invalid committee_uuid") queryset = super().get_queryset() structlog.contextvars.bind_contextvars( - committee_id=committee.committee_id, committee_uuid=committee.id) + committee_id=committee.committee_id, committee_uuid=committee.id + ) return queryset.filter(committee_account_id=committee.id) @@ -167,12 +168,16 @@ def add_member(self, request): return Response(CommitteeMembershipSerializer(new_member).data, status=200) - @action(detail=True, methods=["delete"], - url_path="remove-member", url_name="remove_member") + @action( + detail=True, + methods=["delete"], + url_path="remove-member", + url_name="remove_member", + ) def remove_member(self, request, pk): member = self.get_object() member.delete() - return HttpResponse('Member removed') + return HttpResponse("Member removed") def register_committee(committee_id, user): diff --git a/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py index a9773e08a2..1cbe193672 100644 --- a/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py +++ b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py @@ -10,7 +10,7 @@ class Command(BaseCommand): help = "Load mock committee data into redis" def add_arguments(self, parser): - parser.add_argument("--s3") + parser.add_argument("--s3", action="store_true") def handle(self, *args, **options): if MOCK_OPENFEC_REDIS_URL: diff --git a/django-backend/fecfiler/mock_openfec/mock_endpoints.py b/django-backend/fecfiler/mock_openfec/mock_endpoints.py index 29194304f3..2f7f8bdd72 100644 --- a/django-backend/fecfiler/mock_openfec/mock_endpoints.py +++ b/django-backend/fecfiler/mock_openfec/mock_endpoints.py @@ -14,8 +14,8 @@ def query_filings(query, form_type): filtered_committee_data = [ committee for committee in committees - if query in committee.get("committee_id") - or query in committee.get("committee_name") + if query.upper() in committee.get("committee_id").upper() + or query.upper() in committee.get("committee_name").upper() ] return { # same as api.open.fec.gov "api_version": "1.0", From efb61b16b17163251ac59cde0d88c449e96d1ce4 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 28 Feb 2024 10:47:04 -0500 Subject: [PATCH 63/74] Fixes bug that caused adding an existing user as a new committee member to 500 out --- django-backend/fecfiler/committee_accounts/views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 2c13fea6d6..1272902dbf 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -154,9 +154,7 @@ def add_member(self, request): membership_args = ( {"committee_account": committee, "role": role, "user": user} - | {"pending_email": email} - if user is None - else {} + | ({"pending_email": email} if user is None else {}) ) # Add pending email to args only if there is no user new_member = Membership(**membership_args) From 977d394284eee4fd75ad79cf5d1621fe8e496895 Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Wed, 28 Feb 2024 11:18:42 -0500 Subject: [PATCH 64/74] Fix missing tests --- .../fecfiler/committee_accounts/test_views.py | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 53dddcb1f8..316150d41b 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,11 +1,41 @@ from uuid import UUID from django.test import RequestFactory, TestCase -from .views import CommitteeMembershipViewSet -from rest_framework import viewsets +from fecfiler.committee_accounts.views import register_committee, \ + CommitteeMembershipViewSet from fecfiler.user.models import User +from django.core.management import call_command -class CommitteeMemberViewSetTest(TestCase, viewsets.ModelViewSet): +class CommitteeAccountsViewsTest(TestCase): + + def setUp(self): + call_command("load_committee_data") + self.test_user = User.objects.create( + email="test@fec.gov", username="gov") + self.other_user = User.objects.create( + email="test@fec.com", username="com") + + def test_register_committee(self): + account = register_committee("C12345678", self.test_user) + self.assertEquals(account.committee_id, "C12345678") + + def test_register_committee_existing(self): + account = register_committee("C12345678", self.test_user) + self.assertEquals(account.committee_id, "C12345678") + self.assertRaises( + Exception, register_committee, committee_id="C12345678", user=self.test_user + ) + + def test_register_committee_mismatch_email(self): + self.assertRaises( + Exception, + register_committee, + committee_id="C12345678", + user=self.other_user, + ) + + +class CommitteeMemberViewSetTest(TestCase): fixtures = ["C01234567_user_and_committee"] def setUp(self): From e24df8d29d334e38ec404bcd4b361406f4ebf620 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 28 Feb 2024 13:09:28 -0500 Subject: [PATCH 65/74] Adds unit test coverage for add_members endpoint --- .../fecfiler/committee_accounts/test_views.py | 82 ++++++++++++++++++- .../user/fixtures/unaffiliated_users.json | 34 ++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 django-backend/fecfiler/user/fixtures/unaffiliated_users.json diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 316150d41b..5603be4e9f 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -1,9 +1,13 @@ from uuid import UUID from django.test import RequestFactory, TestCase +from fecfiler.committee_accounts.models import Membership from fecfiler.committee_accounts.views import register_committee, \ CommitteeMembershipViewSet from fecfiler.user.models import User from django.core.management import call_command +import structlog + +logger = structlog.get_logger(__name__) class CommitteeAccountsViewsTest(TestCase): @@ -36,7 +40,7 @@ def test_register_committee_mismatch_email(self): class CommitteeMemberViewSetTest(TestCase): - fixtures = ["C01234567_user_and_committee"] + fixtures = ["C01234567_user_and_committee", "unaffiliated_users"] def setUp(self): self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") @@ -58,3 +62,79 @@ def test_remove_member(self): response = view.remove_member( request, membership_uuid) self.assertEqual(response.status_code, 200) + + def test_add_pending_membership(self): + view = CommitteeMembershipViewSet() + request = self.factory.get("/api/v1/committee-members/add-member") + request.user = self.user + request.session = { + "committee_uuid": UUID('11111111-2222-3333-4444-555555555555') + } + request.method = "POST" + request.data = { + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + "email": "this_email_doesnt_match_any_preexisting_user@test.com" + } + view.request = request + response = view.add_member(request) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.data['email'], + 'this_email_doesnt_match_any_preexisting_user@test.com' + ) + self.assertEqual(response.data['is_active'], False) + + def test_add_membership_for_preexisting_user(self): + view = CommitteeMembershipViewSet() + request = self.factory.get("/api/v1/committee-members/add-member") + request.user = self.user + request.session = { + "committee_uuid": UUID('11111111-2222-3333-4444-555555555555') + } + request.method = "POST" + request.data = { + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + "email": 'test_user_0001@fec.gov' + } + view.request = request + response = view.add_member(request) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['email'], 'test_user_0001@fec.gov') + self.assertEqual(response.data['is_active'], True) + + def test_add_membership_requires_correct_parameters(self): + view = CommitteeMembershipViewSet() + request = self.factory.get("/api/v1/committee-members/add-member") + request.user = self.user + request.session = { + "committee_uuid": UUID('11111111-2222-3333-4444-555555555555') + } + request.method = "POST" + + + request.data = { + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + } + view.request = request + response = view.add_member(request) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, 'Missing fields: email') + + request.data = { + "email": "an_email@fec.gov" + } + view.request = request + response = view.add_member(request) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, 'Missing fields: role') + + request.data = { + "email": "an_email@fec.gov", + "role": "A Random String" + } + view.request = request + response = view.add_member(request) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, 'Invalid role') diff --git a/django-backend/fecfiler/user/fixtures/unaffiliated_users.json b/django-backend/fecfiler/user/fixtures/unaffiliated_users.json new file mode 100644 index 0000000000..3e48b57918 --- /dev/null +++ b/django-backend/fecfiler/user/fixtures/unaffiliated_users.json @@ -0,0 +1,34 @@ +[ + { + "model": "user.user", + "pk": "abcdef12-aaaa-bbbb-cccc-000000000001", + "fields": { + "password": "pbkdf2_sha256$this0is0a0fake0password0it0doesnt0match0anything0at0all0it0just0needs0big=", + "is_superuser": false, + "email": "test_user_0001@fec.gov", + "username": "test_user_0001@fec.gov", + "first_name": "First", + "last_name": "Last", + "last_login": "2022-05-02T13:44:07.324Z", + "is_staff": false, + "is_active": true, + "date_joined": "2020-09-28T15:22:56.477Z" + } + }, + { + "model": "user.user", + "pk": "abcdef12-aaaa-bbbb-cccc-000000000002", + "fields": { + "password": "pbkdf2_sha256$this0is0a0fake0password0it0doesnt0match0anything0at0all0it0just0needs0big=", + "is_superuser": false, + "email": "test_user_0002@fec.gov", + "username": "test_user_0002fec.gov", + "first_name": "First", + "last_name": "Last", + "last_login": "2022-05-02T13:44:07.324Z", + "is_staff": false, + "is_active": true, + "date_joined": "2020-09-28T15:22:56.477Z" + } + } +] \ No newline at end of file From 20cf96e2cd886f29072961e8d4a9eb08a21877d5 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 28 Feb 2024 13:11:26 -0500 Subject: [PATCH 66/74] Removes logger definition since it's not used anymore --- django-backend/fecfiler/committee_accounts/test_views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 5603be4e9f..79142d55bc 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -5,9 +5,6 @@ CommitteeMembershipViewSet from fecfiler.user.models import User from django.core.management import call_command -import structlog - -logger = structlog.get_logger(__name__) class CommitteeAccountsViewsTest(TestCase): From 9fa55242cad3a6cde5b2e55c1b860e32316d1932 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 28 Feb 2024 13:15:23 -0500 Subject: [PATCH 67/74] Removes blank line to make linting happy --- django-backend/fecfiler/committee_accounts/test_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 79142d55bc..1f133bbe76 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -110,7 +110,6 @@ def test_add_membership_requires_correct_parameters(self): } request.method = "POST" - request.data = { "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, } From ec227f0552459b96017d42d4f2bfbf7269889f98 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 28 Feb 2024 13:25:18 -0500 Subject: [PATCH 68/74] Adds one more case --- django-backend/fecfiler/committee_accounts/test_views.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 1f133bbe76..293de43d20 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -134,3 +134,12 @@ def test_add_membership_requires_correct_parameters(self): response = view.add_member(request) self.assertEqual(response.status_code, 400) self.assertEqual(response.data, 'Invalid role') + + request.data = { + "email": "test@fec.gov", + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR + } + view.request = request + response = view.add_member(request) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, "This email is taken by an existing membership to this committee") From 8f31e6da9b9029ce998f0e4f79941952a6baaa49 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 28 Feb 2024 13:30:57 -0500 Subject: [PATCH 69/74] Shortens a line to make linting happy --- django-backend/fecfiler/committee_accounts/test_views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 293de43d20..77725642d4 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -142,4 +142,7 @@ def test_add_membership_requires_correct_parameters(self): view.request = request response = view.add_member(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, "This email is taken by an existing membership to this committee") + self.assertEqual( + response.data, + "This email is taken by an existing membership to this committee" + ) From 39ff25d35a92276db05f22714496ff462bffcc5f Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 28 Feb 2024 13:55:51 -0500 Subject: [PATCH 70/74] Adds a comment to new tests --- django-backend/fecfiler/committee_accounts/test_views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index 77725642d4..e7449d8536 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -83,6 +83,9 @@ def test_add_pending_membership(self): self.assertEqual(response.data['is_active'], False) def test_add_membership_for_preexisting_user(self): + # This test covers a bug found by QA where adding a membership + # for a pre-existing user was triggering a 500 server error + view = CommitteeMembershipViewSet() request = self.factory.get("/api/v1/committee-members/add-member") request.user = self.user From 48db62067e378992e86490f19bbbebbca6e2cb78 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 28 Feb 2024 17:39:56 -0500 Subject: [PATCH 71/74] Add committee type field to test committee accounts --- .../openfec/test_efo_mock_data/committee_accounts.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json b/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json index ef3e2608fe..9145bcc7f4 100644 --- a/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json +++ b/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json @@ -87,7 +87,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title5", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Q" }, { "committee_id": "C00100248", @@ -177,7 +178,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title8", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Q" }, { "committee_id": "C00100271", @@ -358,7 +360,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title15", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Q" }, { "committee_id": "C00100495", From 6f1a1673b621515493954e88d8ec5687e509ddda Mon Sep 17 00:00:00 2001 From: Elaine Krauss <104506225+Elaine-Krauss-TCG@users.noreply.github.com> Date: Mon, 11 Mar 2024 13:09:58 -0400 Subject: [PATCH 72/74] Update .safety.dependency.ignore --- .safety.dependency.ignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore index 74de714264..de10a530ab 100644 --- a/.safety.dependency.ignore +++ b/.safety.dependency.ignore @@ -8,3 +8,5 @@ # Example: # 40104 2022-01-15 # +63687 2024-04-01 # gitpython <3.1.41 +64227 2024-05-01 # jinja2 <3.1.3 From 4bb81bc5929a2d28bac044ae7717ed06492e1ad3 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 11 Mar 2024 14:06:30 -0400 Subject: [PATCH 73/74] Pin mozilla-django-oidc repo to commit made at the time the release branch was made --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index af518cb76a..5eb06a87b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ pytz==2022.6 retry==0.9.2 static3==0.7.0 django-otp==1.1.4 -git+https://github.com/fecgov/mozilla-django-oidc.git@main#egg=mozilla_django_oidc +git+https://github.com/fecgov/mozilla-django-oidc.git@bbf2a4567cf65e08430e513a1c2b628a30bd9979#egg=mozilla_django_oidc zeep==4.2.1 django-structlog[celery]==7.1.0 rich==13.7.0 From e225b5dd725afd7de3c67f44bfdf1dc9c9f93b8f Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 11 Mar 2024 14:12:46 -0400 Subject: [PATCH 74/74] Add django exception to safety check --- .safety.dependency.ignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore index de10a530ab..d0a6b203c7 100644 --- a/.safety.dependency.ignore +++ b/.safety.dependency.ignore @@ -8,5 +8,6 @@ # Example: # 40104 2022-01-15 # -63687 2024-04-01 # gitpython <3.1.41 +63687 2024-05-01 # gitpython <3.1.41 64227 2024-05-01 # jinja2 <3.1.3 +64976 2024-05-01 # django < 4.2.10