diff --git a/.circleci/config.yml b/.circleci/config.yml index 2129b41318..0b5592ea04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,6 +50,7 @@ jobs: docker: - image: cimg/python:3.10-node - image: cimg/postgres:12.8 + - image: cimg/redis:6.2.6 steps: - run: diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore index de10a530ab..d0a6b203c7 100644 --- a/.safety.dependency.ignore +++ b/.safety.dependency.ignore @@ -8,5 +8,6 @@ # Example: # 40104 2022-01-15 # -63687 2024-04-01 # gitpython <3.1.41 +63687 2024-05-01 # gitpython <3.1.41 64227 2024-05-01 # jinja2 <3.1.3 +64976 2024-05-01 # django < 4.2.10 diff --git a/Dockerfile b/Dockerfile index e3543a30b9..744b824e98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,4 +12,4 @@ RUN useradd nxgu --no-create-home --home /opt/nxg_fec && chown -R nxgu:nxgu /opt USER nxgu EXPOSE 8080 -ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 --reload"] +ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --reload"] diff --git a/Dockerfile-e2e b/Dockerfile-e2e index cc6d7c906a..1f6ed789ca 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -13,4 +13,4 @@ RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu USER nxgu EXPOSE 8080 -ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 --reload"] +ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 --reload"] diff --git a/django-backend/data.json b/django-backend/data.json deleted file mode 100644 index 4b9f0de19d..0000000000 --- a/django-backend/data.json +++ /dev/null @@ -1 +0,0 @@ -{"COMMITTEE_NAME": "Progress PAC", "FILER_FEC_ID_NUMBER": "C00351056", "STREET_1": "PO Box 83142", "CITY": "MD", "STATE": "MD", "ZIP": "20148", "REASON_TYPE": "MSM", "DATE_SIGNED_MM": "04", "DATE_SIGNED_DD": "10", "DATE_SIGNED_YY": "2019", "TREASURER_FULL_NAME": " Craig Stroman ", "TREASURER_NAME": "Craig Stroman", "EF_STAMP": "[Electronically Filed]", "MISCELLANEOUS_TEXT": "
daddfsddsdsfdfddsdsafa
", "STREET_2": ""} \ No newline at end of file diff --git a/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py new file mode 100644 index 0000000000..1e9809d267 --- /dev/null +++ b/django-backend/fecfiler/committee_accounts/migrations/0003_membership_pending_email_alter_membership_id_and_more.py @@ -0,0 +1,93 @@ +# Generated by Django 4.2.7 on 2024-02-16 20:43 + +from django.conf import settings +from django.db import migrations, models +from django.utils import timezone +import uuid + + +def delete_pending_memberships(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") # noqa + Membership.objects.filter(user=None).delete() + + +def generate_new_uuid(apps, schema_editor): + Membership = apps.get_model("committee_accounts", "Membership") # noqa + for membership in Membership.objects.all(): + membership.uuid = uuid.uuid4() + membership.save() + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('committee_accounts', '0002_membership'), + ] + + operations = [ + migrations.AddField( + model_name='membership', + name='pending_email', + field=models.EmailField(blank=True, max_length=254, null=True), + ), + migrations.AddField( + model_name='membership', + name='uuid', + field=models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=False, + serialize=False, + unique=False + ) + ), + migrations.RunPython( + generate_new_uuid, + migrations.RunPython.noop, + ), + migrations.RemoveField( + model_name='membership', + name='id', + ), + migrations.RenameField( + model_name='membership', + old_name='uuid', + new_name='id' + ), + migrations.AlterField( + model_name='membership', + name='id', + field=models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True + ) + ), + migrations.AlterField( + model_name='membership', + name='user', + field=models.ForeignKey( + null=True, + on_delete=models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL + ), + ), + migrations.AddField( + model_name='membership', + name='created', + field=models.DateTimeField(auto_now_add=True, default=timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='membership', + name='updated', + field=models.DateTimeField(auto_now=True), + ), + migrations.RunPython( + migrations.RunPython.noop, + delete_pending_memberships + ), + ] diff --git a/django-backend/fecfiler/committee_accounts/models.py b/django-backend/fecfiler/committee_accounts/models.py index 63102ebdce..fbce1e3393 100644 --- a/django-backend/fecfiler/committee_accounts/models.py +++ b/django-backend/fecfiler/committee_accounts/models.py @@ -40,11 +40,21 @@ class CommitteeRole(models.TextChoices): COMMITTEE_ADMINISTRATOR = "COMMITTEE_ADMINISTRATOR" REVIEWER = "REVIEWER" + id = models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + unique=True, + ) committee_account = models.ForeignKey(CommitteeAccount, on_delete=models.CASCADE) - user = models.ForeignKey(User, on_delete=models.CASCADE) + user = models.ForeignKey(User, null=True, blank=False, on_delete=models.CASCADE) + pending_email = models.EmailField(null=True, blank=True) role = models.CharField( max_length=25, choices=CommitteeRole.choices, null=False, blank=False ) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) class CommitteeOwnedModel(models.Model): diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index f1f44a37d8..72c1a3329c 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -1,4 +1,4 @@ -from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.models import CommitteeAccount, Membership from django.contrib.sessions.exceptions import SuspiciousSession from rest_framework import serializers, relations import structlog @@ -12,18 +12,47 @@ class Meta: fields = "__all__" -class CommitteeMemberSerializer(serializers.Serializer): - id = serializers.CharField() - email = serializers.CharField() - username = serializers.CharField() - first_name = serializers.CharField() - last_name = serializers.CharField() - role = serializers.SerializerMethodField() - is_active = serializers.BooleanField() +class CommitteeMembershipSerializer(serializers.Serializer): + role = serializers.ChoiceField(choices=Membership.CommitteeRole) - def get_role(self, object): - committee_id = self.context.get("committee_id") - return object.membership_set.get(committee_account_id=committee_id).role + def to_representation(self, instance): + representation = super().to_representation(instance) + + if instance.user is not None: + representation.update({ + 'id': instance.id, + 'email': instance.user.email, + 'username': instance.user.username, + 'name': f"{instance.user.last_name}, {instance.user.first_name}", + }) + else: + representation.update({ + 'id': instance.id, + 'email': instance.pending_email, + 'username': '', + 'name': '', + }) + + representation['is_active'] = instance.user is not None + return representation + + class Meta: + model = Membership + + fields = [ + f.name + for f in Membership._meta.get_fields() + if f.name + not in [ + "deleted", + "user", + "pending_email" + ] + ] + ["name", "email"] + read_only_fields = [ + "id", + "created", + ] class CommitteeOwnedSerializer(serializers.ModelSerializer): diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py new file mode 100644 index 0000000000..e7449d8536 --- /dev/null +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -0,0 +1,151 @@ +from uuid import UUID +from django.test import RequestFactory, TestCase +from fecfiler.committee_accounts.models import Membership +from fecfiler.committee_accounts.views import register_committee, \ + CommitteeMembershipViewSet +from fecfiler.user.models import User +from django.core.management import call_command + + +class CommitteeAccountsViewsTest(TestCase): + + def setUp(self): + call_command("load_committee_data") + self.test_user = User.objects.create( + email="test@fec.gov", username="gov") + self.other_user = User.objects.create( + email="test@fec.com", username="com") + + def test_register_committee(self): + account = register_committee("C12345678", self.test_user) + self.assertEquals(account.committee_id, "C12345678") + + def test_register_committee_existing(self): + account = register_committee("C12345678", self.test_user) + self.assertEquals(account.committee_id, "C12345678") + self.assertRaises( + Exception, register_committee, committee_id="C12345678", user=self.test_user + ) + + def test_register_committee_mismatch_email(self): + self.assertRaises( + Exception, + register_committee, + committee_id="C12345678", + user=self.other_user, + ) + + +class CommitteeMemberViewSetTest(TestCase): + fixtures = ["C01234567_user_and_committee", "unaffiliated_users"] + + def setUp(self): + self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") + self.factory = RequestFactory() + + def test_remove_member(self): + membership_uuid = UUID("136a21f2-66fe-4d56-89e9-0d1d4612741c") + view = CommitteeMembershipViewSet() + request = self.factory.get( + "/api/v1/committee-members/{membership_uuid}/remove-member" + ) + request.user = self.user + request.session = { + "committee_uuid": UUID('11111111-2222-3333-4444-555555555555')} + request.method = "DELETE" + request.query_params = dict() + view.kwargs = {"pk": membership_uuid} + view.request = request + response = view.remove_member( + request, membership_uuid) + self.assertEqual(response.status_code, 200) + + def test_add_pending_membership(self): + view = CommitteeMembershipViewSet() + request = self.factory.get("/api/v1/committee-members/add-member") + request.user = self.user + request.session = { + "committee_uuid": UUID('11111111-2222-3333-4444-555555555555') + } + request.method = "POST" + request.data = { + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + "email": "this_email_doesnt_match_any_preexisting_user@test.com" + } + view.request = request + response = view.add_member(request) + + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.data['email'], + 'this_email_doesnt_match_any_preexisting_user@test.com' + ) + self.assertEqual(response.data['is_active'], False) + + def test_add_membership_for_preexisting_user(self): + # This test covers a bug found by QA where adding a membership + # for a pre-existing user was triggering a 500 server error + + view = CommitteeMembershipViewSet() + request = self.factory.get("/api/v1/committee-members/add-member") + request.user = self.user + request.session = { + "committee_uuid": UUID('11111111-2222-3333-4444-555555555555') + } + request.method = "POST" + request.data = { + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + "email": 'test_user_0001@fec.gov' + } + view.request = request + response = view.add_member(request) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['email'], 'test_user_0001@fec.gov') + self.assertEqual(response.data['is_active'], True) + + def test_add_membership_requires_correct_parameters(self): + view = CommitteeMembershipViewSet() + request = self.factory.get("/api/v1/committee-members/add-member") + request.user = self.user + request.session = { + "committee_uuid": UUID('11111111-2222-3333-4444-555555555555') + } + request.method = "POST" + + request.data = { + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + } + view.request = request + response = view.add_member(request) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, 'Missing fields: email') + + request.data = { + "email": "an_email@fec.gov" + } + view.request = request + response = view.add_member(request) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, 'Missing fields: role') + + request.data = { + "email": "an_email@fec.gov", + "role": "A Random String" + } + view.request = request + response = view.add_member(request) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data, 'Invalid role') + + request.data = { + "email": "test@fec.gov", + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR + } + view.request = request + response = view.add_member(request) + self.assertEqual(response.status_code, 400) + self.assertEqual( + response.data, + "This email is taken by an existing membership to this committee" + ) diff --git a/django-backend/fecfiler/committee_accounts/urls.py b/django-backend/fecfiler/committee_accounts/urls.py index 83f3df8457..fefada1e75 100644 --- a/django-backend/fecfiler/committee_accounts/urls.py +++ b/django-backend/fecfiler/committee_accounts/urls.py @@ -1,10 +1,12 @@ -from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import CommitteeViewSet +from .views import CommitteeMembershipViewSet, CommitteeViewSet # Create a router and register our viewsets with it. router = DefaultRouter() -router.register(r"", CommitteeViewSet, basename="committees") +router.register(r"committees", CommitteeViewSet, basename="committees") +router.register(r"committee-members", CommitteeMembershipViewSet, + basename="committee-members") # The API URLs are now determined automatically by the router. -urlpatterns = [path("committees/", include(router.urls))] +urlpatterns = [] +urlpatterns += router.urls diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index c428dfe625..c1f46623bc 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -1,10 +1,21 @@ -from rest_framework import viewsets, mixins +from uuid import UUID +from fecfiler.user.models import User +from rest_framework import filters, viewsets, mixins from django.contrib.sessions.exceptions import SuspiciousSession from rest_framework.decorators import action from rest_framework.response import Response -from .models import CommitteeAccount -from .serializers import CommitteeAccountSerializer, CommitteeMemberSerializer +from .models import CommitteeAccount, Membership +from fecfiler.openfec.views import retrieve_recent_f1 +from fecfiler.mock_openfec.mock_endpoints import recent_f1 +from fecfiler.web_services.dot_fec.dot_fec_serializer import FS_STR +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL +import requests +from .serializers import CommitteeAccountSerializer, CommitteeMembershipSerializer +from django.db.models.fields import TextField +from django.db.models.functions import Coalesce, Concat +from django.db.models import Q, Value import structlog +from django.http import HttpResponse logger = structlog.get_logger(__name__) @@ -16,23 +27,6 @@ def get_queryset(self): user = self.request.user return CommitteeAccount.objects.filter(members=user) - @action(detail=True, methods=["get"]) - def members(self, request, pk): - committee = self.get_object() - serializer_context = {"committee_id": committee.id} - queryset = self.filter_queryset(committee.members.all()) - page = self.paginate_queryset(queryset) - if page is not None: - serializer = CommitteeMemberSerializer( - page, many=True, context=serializer_context - ) - return self.get_paginated_response(serializer.data) - - serializer = CommitteeMemberSerializer( - queryset, many=True, context=serializer_context - ) - return Response(serializer.data) - @action(detail=True, methods=["post"]) def activate(self, request, pk): committee = self.get_object() @@ -48,9 +42,17 @@ def active(self, request): committee = self.get_queryset().filter(id=committee_uuid).first() return Response(self.get_serializer(committee).data) + @action(detail=False, methods=["post"]) + def register(self, request): + committee_id = request.data.get("committee_id") + if not committee_id: + raise Exception("no committee_id provided") + account = register_committee(committee_id, request.user) + + return Response(CommitteeAccountSerializer(account).data) -class CommitteeOwnedViewSet(viewsets.ModelViewSet): +class CommitteeOwnedViewSet(viewsets.ModelViewSet): """ModelViewSet for models using CommitteeOwnedModel Inherit this view set to filter the queryset by the user's committee """ @@ -65,3 +67,141 @@ def get_queryset(self): committee_id=committee.committee_id, committee_uuid=committee.id ) return queryset.filter(committee_account_id=committee.id) + + +class CommitteeMembershipViewSet(CommitteeOwnedViewSet): + serializer_class = CommitteeMembershipSerializer + filter_backends = [filters.OrderingFilter] + ordering_fields = ["name", "email", "role", "is_active", "created"] + ordering = ["-created"] + + queryset = Membership.objects.all() + + def get_queryset(self): + return ( + super() + .get_queryset() + .annotate( + name=Coalesce( + Concat( + "user__last_name", + Value(", "), + "user__first_name", + output_field=TextField(), + ), + Value(""), + output_field=TextField(), + ), + email=Coalesce( + "user__email", "pending_email", output_field=TextField() + ), + is_active=~Q(user=None), + ) + ) + + def list(self, request, *args, **kwargs): + queryset = self.filter_queryset(self.get_queryset()) + + if "page" in request.query_params: + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + + @action( + detail=False, methods=["post"], url_path="add-member", url_name="add_member" + ) + def add_member(self, request): + committee_uuid = self.request.session["committee_uuid"] + committee = CommitteeAccount.objects.filter(id=committee_uuid).first() + + email = request.data.get("email", None) + role = request.data.get("role", None) + + # Check for necessary fields + missing_fields = [] + if email is None or len(email) == 0: + missing_fields.append("email") + + if role is None: + missing_fields.append("role") + + if len(missing_fields) > 0: + return Response(f"Missing fields: {', '.join(missing_fields)}", status=400) + + # Check for valid role + choice_of = False + for choice in Membership.CommitteeRole.choices: + if role in choice: + choice_of = True + break + if not choice_of: + return Response("Invalid role", status=400) + + # Check for pre-existing membership + matching_memberships = self.get_queryset().filter( + Q(pending_email=email) | Q(user__email=email) + ) + if matching_memberships.count() > 0: + return Response( + "This email is taken by an existing membership to this committee", + status=400, + ) + + # Create new membership + user = User.objects.filter(email=email).first() + + membership_args = ( + {"committee_account": committee, "role": role, "user": user} + | ({"pending_email": email} if user is None else {}) + ) # Add pending email to args only if there is no user + + new_member = Membership(**membership_args) + new_member.save() + + member_type = "existing user" if user else "pending membership for" + logger.info(f'Added {member_type} "{email}" to committee {committee}') + + return Response(CommitteeMembershipSerializer(new_member).data, status=200) + + @action(detail=True, methods=["delete"], + url_path="remove-member", url_name="remove_member") + def remove_member(self, request, pk: UUID): + member = self.get_object() + member.delete() + return HttpResponse("Member removed") + + +def register_committee(committee_id, user): + email = user.email + + if MOCK_OPENFEC_REDIS_URL: + f1 = recent_f1(committee_id) + f1_email = (f1 or {}).get("email") + else: + f1 = retrieve_recent_f1(committee_id) + dot_fec_url = (f1 or {}).get("fec_url") + if dot_fec_url: + response = requests.get( + dot_fec_url, + headers={"User-Agent": "fecfile-api"}, + ) + dot_fec_content = response.content.decode("utf-8") + f1_line = dot_fec_content.split("\n")[1] + f1_email = f1_line.split(FS_STR)[11] + + existing_account = CommitteeAccount.objects.filter( + committee_id=committee_id + ).first() + if existing_account or f1_email != email: + raise Exception("could not register committee") + account = CommitteeAccount.objects.create(committee_id=committee_id) + Membership.objects.create( + committee_account=account, + user=user, + role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + ) + return account diff --git a/django-backend/fecfiler/contacts/serializers.py b/django-backend/fecfiler/contacts/serializers.py index a6489d1f99..a4331a34c3 100644 --- a/django-backend/fecfiler/contacts/serializers.py +++ b/django-backend/fecfiler/contacts/serializers.py @@ -53,6 +53,9 @@ class ContactSerializer( # Contains the number of transactions linked to the contact transaction_count = IntegerField(required=False) + # Contains the number of reports directly linked to the contact (e.g. F1M) + report_count = IntegerField(required=False) + def get_schema_name(self, data): return f"Contact_{self.contact_value[data.get('type', None)]}" @@ -91,17 +94,21 @@ class Meta: ] ] fields.append("transaction_count") + fields.append("report_count") read_only_fields = [ "uuid", "deleted", "created", "updated", "transaction_count", + "report_count", ] def to_internal_value(self, data): - # Remove the transaction_count because it is an annotated field + # Remove the transaction_count and report_count because they are annotated fields # delivered to the front end. if "transaction_count" in data: del data["transaction_count"] + if "report_count" in data: + del data["report_count"] return super().to_internal_value(data) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index 0d21501cde..f5031f1fd5 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -10,10 +10,10 @@ mock_results = { "results": [ - {"name": "LNAME, FNAME I", "id": "P60012143", "office_sought": "P"}, + {"name": "LNAME, FNAME I", "candidate_id": "P60012143", "office_sought": "P"}, { "name": "LNAME, FNAME", - "id": "P60012465", + "candidate_id": "P60012465", "office_sought": "P", }, ] @@ -91,7 +91,7 @@ def test_candidate_lookup_no_q(self, mock_get): def test_candidate_lookup_happy_path(self, mock_get): request = self.factory.get( "/api/v1/contacts/candidate_lookup?" - "q=test&max_fecfile_results=5&max_fec_results=5" + "q=test&max_fecfile_results=5&max_fec_results=5&exclude_fec_ids=P60012143" ) request.user = self.user request.session = {"committee_uuid": "11111111-2222-3333-4444-555555555555"} @@ -100,10 +100,9 @@ def test_candidate_lookup_happy_path(self, mock_get): expected_json = { "fec_api_candidates": [ - {"name": "LNAME, FNAME I", "id": "P60012143", "office_sought": "P"}, { "name": "LNAME, FNAME", - "id": "P60012465", + "candidate_id": "P60012465", "office_sought": "P", }, ], @@ -176,6 +175,7 @@ def test_committee_lookup_happy_path(self, mock_get): "created": "2022-02-09T00:00:00Z", "updated": "2022-02-09T00:00:00Z", "transaction_count": 0, + "report_count": 0, } ], } diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index 3ed495eaa2..0109007324 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -58,6 +58,14 @@ class ContactViewSet(CommitteeOwnedViewSet): + Count("contact_2_transaction_set") + Count("contact_3_transaction_set"), ) + .annotate( + report_count=Count("contact_affiliated_transaction_set") + + Count("contact_candidate_I_transaction_set") + + Count("contact_candidate_II_transaction_set") + + Count("contact_candidate_III_transaction_set") + + Count("contact_candidate_IV_transaction_set") + + Count("contact_candidate_V_transaction_set") + ) .alias( sort_name=Concat( "name", "last_name", Value(" "), "first_name", output_field=CharField() @@ -98,6 +106,10 @@ def candidate_lookup(self, request): max_fecfile_results, max_fec_results = self.get_max_results(request) office = request.GET.get("office", "") + exclude_fec_ids = request.GET.get("exclude_fec_ids").split(',') \ + if request.GET.get("exclude_fec_ids") else [] + exclude_ids = request.GET.get("exclude_ids").split(',') \ + if request.GET.get("exclude_ids") else [] params = {"q": q, "api_key": FEC_API_KEY} if office: params["office"] = office @@ -121,6 +133,7 @@ def candidate_lookup(self, request): | (Q(full_name_fwd__regex=term) | Q(full_name_bwd__regex=term)) ) ) + .exclude(id__in=exclude_ids) .values() .order_by("-candidate_id") ) @@ -133,7 +146,7 @@ def candidate_lookup(self, request): for fac in fec_api_candidates if not any( fac["candidate_id"] == ffc["candidate_id"] for ffc in fecfile_candidates - ) + ) and fac["candidate_id"] not in exclude_fec_ids ] return_value = { "fec_api_candidates": fec_api_candidates[:max_fec_results], @@ -150,6 +163,10 @@ def committee_lookup(self, request): max_fecfile_results, max_fec_results = self.get_max_results(request) + exclude_fec_ids = request.GET.get("exclude_fec_ids").split(',') \ + if request.GET.get("exclude_fec_ids") else [] + exclude_ids = request.GET.get("exclude_ids").split(',') \ + if request.GET.get("exclude_ids") else [] params = urlencode({"q": q, "api_key": FEC_API_KEY}) json_results = requests.get( FEC_API_COMMITTEE_LOOKUP_ENDPOINT, params=params @@ -160,6 +177,7 @@ def committee_lookup(self, request): .filter( Q(type="COM") & (Q(committee_id__icontains=q) | Q(name__icontains=q)) ) + .exclude(id__in=exclude_ids) .values() .order_by("-committee_id") ) @@ -167,7 +185,9 @@ def committee_lookup(self, request): fec_api_committees = [ fac for fac in fec_api_committees - if not any(fac["id"] == ffc["committee_id"] for ffc in fecfile_committees) + if not any( + fac["id"] == ffc["committee_id"] for ffc in fecfile_committees + ) and fac["id"] not in exclude_fec_ids ] fec_api_committees = fec_api_committees[:max_fec_results] fecfile_committees = fecfile_committees[:max_fecfile_results] @@ -188,6 +208,8 @@ def individual_lookup(self, request): term = (".*" + ".* .*".join(tokens) + ".*").lower() max_fecfile_results, _ = self.get_max_results(request) + exclude_ids = request.GET.get("exclude_ids").split(',') \ + if request.GET.get("exclude_ids") else [] fecfile_individuals = list( self.get_queryset() @@ -199,6 +221,7 @@ def individual_lookup(self, request): Q(type="IND") & (Q(full_name_fwd__regex=term) | Q(full_name_bwd__regex=term)) ) + .exclude(id__in=exclude_ids) .values() .order_by(Lower("last_name").desc())[:max_fecfile_results] ) @@ -215,10 +238,13 @@ def organization_lookup(self, request): return HttpResponseBadRequest() max_fecfile_results, _ = self.get_max_results(request) + exclude_ids = request.GET.get("exclude_ids").split(',') \ + if request.GET.get("exclude_ids") else [] fecfile_organizations = list( self.get_queryset() .filter(Q(type="ORG") & Q(name__icontains=q)) + .exclude(id__in=exclude_ids) .values() .order_by("-name")[:max_fecfile_results] ) diff --git a/django-backend/fecfiler/mock_openfec/__init__.py b/django-backend/fecfiler/mock_openfec/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/mock_openfec/apps.py b/django-backend/fecfiler/mock_openfec/apps.py new file mode 100644 index 0000000000..e93fbaf6f4 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class MockOpenfecConfig(AppConfig): + name = "fecfiler.mock_openfec" diff --git a/django-backend/fecfiler/mock_openfec/management/__init__.py b/django-backend/fecfiler/mock_openfec/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/mock_openfec/management/commands/_init_.py b/django-backend/fecfiler/mock_openfec/management/commands/_init_.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json b/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json new file mode 100644 index 0000000000..3174e4e764 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/management/commands/committee_data.json @@ -0,0 +1,14 @@ +[ + { + "committee_id": "C12345678", + "committee_name": "Test Committee", + "committee_type": "O", + "form_type": "F1", + "pdf_url": "https://docquery.fec.gov/pdf/161/201601019004427161/201601019004427161.pdf", + "receipt_date": "2024-01-01T00:00:00", + "report_year": 2024, + "treasurer_name": "Last, First", + "update_date": "2024-01-01", + "email": "test@fec.gov" + } +] \ No newline at end of file diff --git a/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py new file mode 100644 index 0000000000..1cbe193672 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/management/commands/load_committee_data.py @@ -0,0 +1,32 @@ +from django.core.management.base import BaseCommand +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL, BASE_DIR, AWS_STORAGE_BUCKET_NAME +import redis +import os +from fecfiler.s3 import S3_SESSION +from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY + + +class Command(BaseCommand): + help = "Load mock committee data into redis" + + def add_arguments(self, parser): + parser.add_argument("--s3", action="store_true") + + def handle(self, *args, **options): + if MOCK_OPENFEC_REDIS_URL: + redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) + if not options.get("s3"): + path = os.path.join( + BASE_DIR, "mock_openfec/management/commands/committee_data.json" + ) + with open(path) as file: + committee_data = file.read() + else: + s3_object = S3_SESSION.Object( + AWS_STORAGE_BUCKET_NAME, "mock_committee_data.json" + ) + file = s3_object.get()["Body"] + committee_data = file.read() + redis_instance.set(COMMITTEE_DATA_REDIS_KEY, committee_data) + + self.stdout.write(self.style.SUCCESS("Successfully loaded committees")) diff --git a/django-backend/fecfiler/mock_openfec/mock_endpoints.py b/django-backend/fecfiler/mock_openfec/mock_endpoints.py new file mode 100644 index 0000000000..2f7f8bdd72 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/mock_endpoints.py @@ -0,0 +1,35 @@ +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL +import json +import redis + +COMMITTEE_DATA_REDIS_KEY = "COMMITTEE_DATA" +if MOCK_OPENFEC_REDIS_URL: + redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) + + +def query_filings(query, form_type): + if redis_instance: + committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) or "" + committees = json.loads(committee_data) or [] + filtered_committee_data = [ + committee + for committee in committees + if query.upper() in committee.get("committee_id").upper() + or query.upper() in committee.get("committee_name").upper() + ] + return { # same as api.open.fec.gov + "api_version": "1.0", + "results": filtered_committee_data, + "pagination": {"pages": 1, "per_page": 20, "count": 1, "page": 1}, + } + + +def recent_f1(committee_id): + if redis_instance: + committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) or "" + committees = json.loads(committee_data) or [] + return next( + committee + for committee in committees + if committee.get("committee_id") == committee_id + ) diff --git a/django-backend/fecfiler/mock_openfec/test_load_committee_data.py b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py new file mode 100644 index 0000000000..bd44349616 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/test_load_committee_data.py @@ -0,0 +1,18 @@ +from django.test import TestCase +from django.core.management import call_command +from fecfiler.settings import MOCK_OPENFEC_REDIS_URL +import redis +import json +from fecfiler.mock_openfec.mock_endpoints import COMMITTEE_DATA_REDIS_KEY + + +class LoadTestDataCommandTest(TestCase): + + def setUp(self): + pass + + def test_load_test_data(self): + call_command("load_committee_data") + redis_instance = redis.Redis.from_url(MOCK_OPENFEC_REDIS_URL) + committee_data = redis_instance.get(COMMITTEE_DATA_REDIS_KEY) + self.assertEqual(len(json.loads(committee_data)), 1) diff --git a/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py b/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py new file mode 100644 index 0000000000..a3c2b5299a --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/test_mock_endpoints.py @@ -0,0 +1,18 @@ +from django.test import TestCase +from django.core.management import call_command +from fecfiler.mock_openfec.mock_endpoints import query_filings + + +class MockEndpointsTest(TestCase): + + def setUp(self): + pass + + def test_query_filings(self): + call_command("load_committee_data") + response = query_filings("NOT FINDABLE", "F3") + self.assertEqual(len(response["results"]), 0) + response = query_filings("NOT FINDABLE", "F1") + self.assertEqual(len(response["results"]), 0) + response = query_filings("st Com", "F1") + self.assertEqual(len(response["results"]), 1) diff --git a/django-backend/fecfiler/mock_openfec/urls.py b/django-backend/fecfiler/mock_openfec/urls.py new file mode 100644 index 0000000000..8faa2abd58 --- /dev/null +++ b/django-backend/fecfiler/mock_openfec/urls.py @@ -0,0 +1,12 @@ +from django.urls import path, include +from rest_framework.routers import DefaultRouter +from .views import MockOpenfecViewSet + +# Create a router and register our viewsets with it. +router = DefaultRouter() +router.register(r"mock_openfec", MockOpenfecViewSet, basename="mock_openfec") + +# The API URLs are now determined automatically by the router. +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json b/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json index b5e3a1252d..9145bcc7f4 100644 --- a/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json +++ b/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json @@ -87,7 +87,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title5", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Q" }, { "committee_id": "C00100248", @@ -177,7 +178,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title8", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Q" }, { "committee_id": "C00100271", @@ -237,7 +239,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title10", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "H" }, { "committee_id": "C00100297", @@ -357,7 +360,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title15", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Q" }, { "committee_id": "C00100495", @@ -447,7 +451,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title18", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Q" }, { "committee_id": "C00100529", @@ -477,7 +482,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title19", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "W" }, { "committee_id": "C00100537", @@ -507,7 +513,8 @@ "custodian_state": "AL", "custodian_zip": "12345", "custodian_name_title": "test_title20", - "custodian_phone": "8175212197" + "custodian_phone": "8175212197", + "committee_type": "Y" }, { "committee_id": "C00100362", diff --git a/django-backend/fecfiler/openfec/test_views.py b/django-backend/fecfiler/openfec/test_views.py index e914f6b48c..835084f386 100644 --- a/django-backend/fecfiler/openfec/test_views.py +++ b/django-backend/fecfiler/openfec/test_views.py @@ -24,9 +24,9 @@ def test_get_committee_no_override(self): self.assertEqual(response.status_code, 200) def test_get_committee_override_data_not_found(self): - with patch("fecfiler.openfec.views.base") as base: - base.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = "C12345678" - base.BASE_DIR = "fecfiler/" + with patch("fecfiler.openfec.views.settings") as settings: + settings.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = "C12345678" + settings.BASE_DIR = "fecfiler/" request = self.factory.get("/api/v1/openfec/C12345678/committee/") request.user = self.user response = OpenfecViewSet.as_view({"get": "committee"})( @@ -35,9 +35,9 @@ def test_get_committee_override_data_not_found(self): self.assertEqual(response.status_code, 500) def test_get_committee_override_happy_path(self): - with patch("fecfiler.openfec.views.base") as base: - base.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = "C00100230" - base.BASE_DIR = "fecfiler/" + with patch("fecfiler.openfec.views.settings") as settings: + settings.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = "C00100230" + settings.BASE_DIR = "fecfiler/" request = self.factory.get("/api/v1/openfec/C00100230/committee/") request.user = self.user response = OpenfecViewSet.as_view({"get": "committee"})( @@ -54,19 +54,19 @@ def test_get_committee_override_happy_path(self): ) def test_get_filings_invalid_resp(self): - request = self.factory.get("/api/v1/openfec/C00100230/filings/") + request = self.factory.get("/api/v1/openfec/C00100230/f1_filing/") request.user = self.user with patch("fecfiler.openfec.views.requests") as mock_requests: mock_requests.get.return_value = mock_response = Mock() mock_response.status_code = 200 - mock_response.json.return_value = None - response = OpenfecViewSet.as_view({"get": "filings"})( + mock_response.json.return_value = {"results": []} + response = OpenfecViewSet.as_view({"get": "f1_filing"})( request, pk="C00100230" ) self.assertEqual(response.status_code, 200) def test_get_filings_happy_path(self): - request = self.factory.get("/api/v1/openfec/C00100230/filings/") + request = self.factory.get("/api/v1/openfec/C00100230/f1_filing/") request.user = self.user with patch("fecfiler.openfec.views.requests") as mock_requests: mock_requests.get.return_value = mock_response = Mock() @@ -75,7 +75,7 @@ def test_get_filings_happy_path(self): mock_response_object["results"] = [{}] mock_response_object["results"][0]["form_type"] = "F1" mock_response.json.return_value = mock_response_object - response = OpenfecViewSet.as_view({"get": "filings"})( + response = OpenfecViewSet.as_view({"get": "f1_filing"})( request, pk="C00100230" ) self.assertEqual(response.status_code, 200) diff --git a/django-backend/fecfiler/openfec/views.py b/django-backend/fecfiler/openfec/views.py index 99bebe17c9..4a62fe8da9 100644 --- a/django-backend/fecfiler/openfec/views.py +++ b/django-backend/fecfiler/openfec/views.py @@ -2,8 +2,9 @@ from django.http.response import HttpResponse, HttpResponseServerError from rest_framework.response import Response from rest_framework.decorators import action +from fecfiler.mock_openfec.mock_endpoints import query_filings import requests -from fecfiler.settings import base +import fecfiler.settings as settings import os import json @@ -12,93 +13,110 @@ logger = structlog.get_logger(__name__) -class OpenfecViewSet(viewsets.ModelViewSet): - +class OpenfecViewSet(viewsets.GenericViewSet): @action(detail=True) def committee(self, request, pk=None): - cids_to_override = list(map( - str.strip, base.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE.split(',') - )) if base.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE else [] - cid_to_override = next(( - cid for cid in cids_to_override if cid == pk - ), None) + cids_to_override = ( + list( + map( + str.strip, settings.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE.split(",") + ) + ) + if settings.FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE + else [] + ) + cid_to_override = next((cid for cid in cids_to_override if cid == pk), None) if cid_to_override: mock_committee_account = get_test_efo_mock_committee_account( cid_to_override ) if mock_committee_account: - return Response({ # same as api.open.fec.gov - "api_version": "1.0", - "results": [ - mock_committee_account, - ], - "pagination": { - "pages": 1, - "per_page": 20, - "count": 1, - "page": 1 + return Response( + { # same as api.open.fec.gov + "api_version": "1.0", + "results": [ + mock_committee_account, + ], + "pagination": { + "pages": 1, + "per_page": 20, + "count": 1, + "page": 1, + }, } - }) + ) else: - logger.error("Failed to find mock committee account data for " - "committee id to override: " + cid_to_override - ) + logger.error( + "Failed to find mock committee account data for " + "committee id to override: " + cid_to_override + ) return HttpResponseServerError() else: resp = requests.get( - f'{base.FEC_API}committee/{pk}/?api_key={base.FEC_API_KEY}') + f"{settings.FEC_API}committee/{pk}/?api_key={settings.FEC_API_KEY}" + ) return HttpResponse(resp) @action(detail=True) - def filings(self, request, pk=None): - headers = { - 'Content-Type': 'application/json' - } - params = { - 'api_key': base.FEC_API_KEY, - 'committee_id': pk, - 'sort': '-receipt_date' - } - - resp = requests.get(f'{base.FEC_API}efile/filings/', - headers=headers, params=params) - retval = get_recent_f1_from_openfec_resp(resp) - if not retval: - params['per_page'] = 1 - params['page'] = 1 - params['form_type'] = 'F1' - resp = requests.get(f'{base.FEC_API}filings/', headers=headers, params=params) - retval = get_recent_f1_from_openfec_resp(resp) + def f1_filing(self, request, pk=None): + return Response(retrieve_recent_f1(pk)) - return Response(retval) + @action(detail=False) + def query_filings(self, request): + query = request.query_params.get("query") + form_type = request.query_params.get("form_type") + if settings.MOCK_OPENFEC_REDIS_URL: + response = query_filings(query, form_type) + else: + params = { + "api_key": settings.FEC_API_KEY, + "q_filer": query, + "sort": "-receipt_date", + "form_type": form_type, + "most_recent": True, + } + fec_response = requests.get(f"{settings.FEC_API}filings/", params) + response = fec_response.json() + return Response(response) -def get_recent_f1_from_openfec_resp(resp: requests.Response): - if resp: - try: - filing_list = resp.json()['results'] - return next(( - filing for filing in filing_list if filing['form_type'].startswith('F1') - ), None) - except Exception as error: - logger.error( - f'Failed to process response from {resp.url} due to error: {str(error)}' - ) - return None +def retrieve_recent_f1(committee_id): + """Gets the most recent F1 filing + First checks the realtime enpdpoint for a recent F1 filing. If none is found, + a request is made to a different endpoint that is updated nightly. + The realtime endpoint will have more recent filings, but does not provide + filings older than 6 months. The nightly endpoint keeps a longer history""" + headers = {"Content-Type": "application/json"} + params = { + "api_key": settings.FEC_API_KEY, + "committee_id": committee_id, + "sort": "-receipt_date", + "form_type": "F1", + } + endpoints = [f"{settings.FEC_API}efile/filings/", f"{settings.FEC_API}filings/"] + for endpoint in endpoints: + response = requests.get(endpoint, headers=headers, params=params).json() + results = response["results"] + if len(results) > 0: + return results[0] def get_test_efo_mock_committee_account(committee_id): mock_committee_accounts = get_test_efo_mock_committee_accounts() - return next(( - committee for committee in mock_committee_accounts - if committee['committee_id'] == committee_id - ), None) + return next( + ( + committee + for committee in mock_committee_accounts + if committee["committee_id"] == committee_id + ), + None, + ) def get_test_efo_mock_committee_accounts(): mock_committee_accounts_file = "committee_accounts.json" mock_committee_accounts_file_path = os.path.join( - base.BASE_DIR, "openfec/test_efo_mock_data/", mock_committee_accounts_file + settings.BASE_DIR, "openfec/test_efo_mock_data/", mock_committee_accounts_file ) with open(mock_committee_accounts_file_path) as fp: return json.load(fp) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 6a6d180c39..9a3b475780 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -79,6 +79,7 @@ "fecfiler.web_services", "fecfiler.openfec", "fecfiler.user", + "fecfiler.mock_openfec", ] MIDDLEWARE = [ @@ -235,8 +236,7 @@ def get_logging_config(log_format=LINE): "plain_console": { "()": structlog.stdlib.ProcessorFormatter, "processor": structlog.dev.ConsoleRenderer( - colors=True, - exception_formatter=structlog.dev.rich_traceback + colors=True, exception_formatter=structlog.dev.rich_traceback ), }, "key_value": { @@ -257,7 +257,7 @@ def get_logging_config(log_format=LINE): "formatter": "key_value", "stream": sys.stdout, }, - } + }, } if log_format == LINE: @@ -374,3 +374,10 @@ def get_env_logging_processors(log_format=LINE): ) FEC_API_CANDIDATE_LOOKUP_ENDPOINT = str(FEC_API) + "candidates/" FEC_API_CANDIDATE_ENDPOINT = str(FEC_API) + "candidate/" + +"""MOCK OPENFEC settings""" +MOCK_OPENFEC = env.get_credential("MOCK_OPENFEC") +if MOCK_OPENFEC == "REDIS": + MOCK_OPENFEC_REDIS_URL = env.get_credential("REDIS_URL") +else: + MOCK_OPENFEC_REDIS_URL = None diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index 652af35f35..19deee6d75 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -36,13 +36,13 @@ def request(self, payload, params={}): request.session = {"committee_uuid": "11111111-2222-3333-4444-555555555555"} return request - def xtest_save_transaction_pair(self): + def test_save_transaction_pair(self): request = self.request(self.payloads["IN_KIND"]) transaction = TransactionViewSet().save_transaction(request.data, request) self.assertEqual("John", transaction.contact_1.first_name) self.assertEqual("Smith", transaction.contact_1.last_name) - def xtest_update(self): + def test_update(self): request = self.request(self.payloads["IN_KIND"]) transaction = TransactionViewSet().save_transaction(request.data, request) updated_payload = deepcopy(self.payloads["IN_KIND"]) @@ -161,7 +161,7 @@ def test_inherited_election_aggregate(self): transaction = response.data self.assertEqual(transaction.get("calendar_ytd_per_election_office"), 58.00) - def xtest_multisave_transactions(self): + def test_multisave_transactions(self): txn1 = deepcopy(self.payloads["IN_KIND"]) txn1["contributor_last_name"] = "one" txn2 = deepcopy(self.payloads["IN_KIND"]) @@ -188,7 +188,7 @@ def xtest_multisave_transactions(self): self.assertEqual(len(transactions), 3) # self.assertEqual("one", transactions[0]["contributor_last_name"]) - def xtest_reatt_redes_multisave_transactions(self): + def test_reatt_redes_multisave_transactions(self): txn1 = deepcopy(self.payloads["IN_KIND"]) txn1["contributor_last_name"] = "one" txn2 = deepcopy(self.payloads["IN_KIND"]) diff --git a/django-backend/fecfiler/urls.py b/django-backend/fecfiler/urls.py index ab3d335fb2..e8bb583302 100644 --- a/django-backend/fecfiler/urls.py +++ b/django-backend/fecfiler/urls.py @@ -16,7 +16,6 @@ def test_celery(request): urlpatterns = [ - re_path(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), re_path( r"^api/schema/", SpectacularAPIView.as_view(api_version="v1"), name="schema" ), diff --git a/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json b/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json index 7e551057c1..03664d1119 100644 --- a/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json +++ b/django-backend/fecfiler/user/fixtures/C01234567_user_and_committee.json @@ -26,11 +26,14 @@ }, { "model": "committee_accounts.Membership", - "pk": "99", + "pk": "136a21f2-66fe-4d56-89e9-0d1d4612741c", "fields": { "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "11111111-2222-3333-4444-555555555555", - "user_id": "12345678-aaaa-bbbb-cccc-111122223333" + "user_id": "12345678-aaaa-bbbb-cccc-111122223333", + "pending_email": null, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z" } } ] \ No newline at end of file diff --git a/django-backend/fecfiler/user/fixtures/unaffiliated_users.json b/django-backend/fecfiler/user/fixtures/unaffiliated_users.json new file mode 100644 index 0000000000..3e48b57918 --- /dev/null +++ b/django-backend/fecfiler/user/fixtures/unaffiliated_users.json @@ -0,0 +1,34 @@ +[ + { + "model": "user.user", + "pk": "abcdef12-aaaa-bbbb-cccc-000000000001", + "fields": { + "password": "pbkdf2_sha256$this0is0a0fake0password0it0doesnt0match0anything0at0all0it0just0needs0big=", + "is_superuser": false, + "email": "test_user_0001@fec.gov", + "username": "test_user_0001@fec.gov", + "first_name": "First", + "last_name": "Last", + "last_login": "2022-05-02T13:44:07.324Z", + "is_staff": false, + "is_active": true, + "date_joined": "2020-09-28T15:22:56.477Z" + } + }, + { + "model": "user.user", + "pk": "abcdef12-aaaa-bbbb-cccc-000000000002", + "fields": { + "password": "pbkdf2_sha256$this0is0a0fake0password0it0doesnt0match0anything0at0all0it0just0needs0big=", + "is_superuser": false, + "email": "test_user_0002@fec.gov", + "username": "test_user_0002fec.gov", + "first_name": "First", + "last_name": "Last", + "last_login": "2022-05-02T13:44:07.324Z", + "is_staff": false, + "is_active": true, + "date_joined": "2020-09-28T15:22:56.477Z" + } + } +] \ No newline at end of file diff --git a/django-backend/fecfiler/user/managers.py b/django-backend/fecfiler/user/managers.py new file mode 100644 index 0000000000..16e4278301 --- /dev/null +++ b/django-backend/fecfiler/user/managers.py @@ -0,0 +1,27 @@ +import structlog +from django.contrib.auth.models import UserManager as AbstractUserManager + +logger = structlog.get_logger(__name__) + + +class UserManager(AbstractUserManager): + def create_user(self, user_id, **obj_data): + from fecfiler.committee_accounts.models import Membership + + new_user = super().create_user(user_id, **obj_data) + pending_memberships = Membership.objects.filter( + user=None, + pending_email=obj_data['email'] + ) + + logger.info( + f"""New User Created: { + obj_data['email']} - {pending_memberships.count() + } Pending Memberships""" + ) + + for new_membership in pending_memberships: + new_membership.user = new_user + new_membership.save() + + return new_user diff --git a/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py b/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py new file mode 100644 index 0000000000..dd7809c987 --- /dev/null +++ b/django-backend/fecfiler/user/migrations/0004_alter_user_managers.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.7 on 2024-02-16 20:43 + +from django.db import migrations +import fecfiler.user.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('user', '0003_user_security_consent_date'), + ] + + operations = [ + migrations.AlterModelManagers( + name='user', + managers=[ + ('objects', fecfiler.user.managers.UserManager()), + ], + ), + ] diff --git a/django-backend/fecfiler/user/models.py b/django-backend/fecfiler/user/models.py index ec75b7e607..778a40b2cb 100644 --- a/django-backend/fecfiler/user/models.py +++ b/django-backend/fecfiler/user/models.py @@ -1,6 +1,7 @@ from django.contrib.auth.models import AbstractUser from django.db import models import uuid +from fecfiler.user.managers import UserManager class User(AbstractUser): @@ -17,3 +18,5 @@ class User(AbstractUser): user_permissions = None login_dot_gov = False security_consent_date = models.DateField(null=True, blank=True) + + objects = UserManager() diff --git a/django-backend/fecfiler/user/test_models.py b/django-backend/fecfiler/user/test_models.py new file mode 100644 index 0000000000..632c55b629 --- /dev/null +++ b/django-backend/fecfiler/user/test_models.py @@ -0,0 +1,32 @@ +from django.test import TestCase +from .models import User +from fecfiler.committee_accounts.models import CommitteeAccount, Membership + + +class UserModelTestCase(TestCase): + fixtures = ["test_committee_accounts"] + + def test_redeem_pending_membership(self): + committee_account = CommitteeAccount.objects.create( + committee_id="C87654321", + ) + + new_membership = Membership( + committee_account=committee_account, + role=Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + pending_email="test_1234@test.com" + ) + new_membership.committee_account_id = committee_account.id + new_membership.save() + + self.assertEquals( + Membership.objects.get(pending_email="test_1234@test.com").user, None + ) + + new_user = User.objects.create_user( + email="test_1234@test.com", user_id='5e3c145f-a813-46c7-af5a-5739304acc27' + ) + + self.assertEquals( + Membership.objects.get(pending_email="test_1234@test.com").user, new_user + ) diff --git a/django-backend/fixtures/base_data.json b/django-backend/fixtures/base_data.json deleted file mode 100644 index 24c65de994..0000000000 --- a/django-backend/fixtures/base_data.json +++ /dev/null @@ -1 +0,0 @@ -[{"model": "authentication.account", "pk": 1, "fields": {"password": "pbkdf2_sha256$36000$1a12NsHSiM1P$TSQHr/206lhRHsPmS/EFuv2MDaKRnM45QiFQcewrGk8=", "last_login": "2018-09-10T02:21:50.676Z", "is_superuser": false, "email": "test1@test.com", "username": "C01234567", "tagline": "", "created_at": "2018-09-08T18:59:45.552Z", "updated_at": "2018-09-17T02:25:05.164Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T18:59:45.208Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 2, "fields": {"password": "pbkdf2_sha256$36000$T8SFHw3JGtnX$6yVLWtn83IWPXcV7IQXcTk/2SN7DM92vA8cAjNFNAgc=", "last_login": "2018-09-08T19:01:53.412Z", "is_superuser": false, "email": "test2@test.com", "username": "C11234567", "tagline": "", "created_at": "2018-09-08T19:01:52.580Z", "updated_at": "2018-09-17T02:25:05.146Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:01:52.218Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 3, "fields": {"password": "pbkdf2_sha256$36000$wtcx3pv32dgt$tETHKHzI2FPAtVyp2GI/gutqirDwMISwKyWBnEehlVI=", "last_login": "2018-09-08T19:02:33Z", "is_superuser": false, "email": "test3@test.com", "username": "C00220269", "tagline": "", "created_at": "2018-09-08T19:02:33.065Z", "updated_at": "2018-10-29T17:05:28.190Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:02:32Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 4, "fields": {"password": "bcrypt_sha256$$2b$12$Vs6shIWepA7Bc/LVaZu2aujPQxCoY7Pe6epyXHSH9GrPqEJmgEIsW", "last_login": "2018-09-08T19:04:44.282Z", "is_superuser": true, "email": "testtube@test.com", "username": "adminnxg", "tagline": "", "created_at": "2018-09-08T19:04:33.178Z", "updated_at": "2018-09-17T02:25:05.163Z", "is_staff": true, "is_active": true, "date_joined": "2018-09-08T19:04:32.844Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 5, "fields": {"password": "pbkdf2_sha256$36000$ir0uuUb5gNO3$PYv6Px2Hd7uOwNDIIM/29jukiFh4gxMtklXh3lB8atk=", "last_login": "2018-10-03T13:10:32.154Z", "is_superuser": true, "email": "t@t.com", "username": "admin_nxg", "tagline": "", "created_at": "2018-10-03T13:10:17.524Z", "updated_at": "2018-10-03T13:10:17.527Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-03T13:10:17.440Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 6, "fields": {"password": "pbkdf2_sha256$36000$rJqxMw2luNoH$lzKgItcwzhtwdA9qSpiR/6QGpSQyN4u7Yj8UqFn0I3k=", "last_login": "2018-10-29T16:03:59.849Z", "is_superuser": true, "email": "tt@tt.ctt", "username": "tt", "tagline": "", "created_at": "2018-10-29T16:02:39.519Z", "updated_at": "2018-10-29T16:02:39.579Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-29T16:02:39.035Z", "groups": [], "user_permissions": []}}] \ No newline at end of file diff --git a/django-backend/fixtures/base_data_tmp.json b/django-backend/fixtures/base_data_tmp.json deleted file mode 100644 index 5eac9fa07b..0000000000 --- a/django-backend/fixtures/base_data_tmp.json +++ /dev/null @@ -1 +0,0 @@ -[{"model": "authentication.account", "pk": 1, "fields": {"password": "pbkdf2_sha256$36000$1a12NsHSiM1P$TSQHr/206lhRHsPmS/EFuv2MDaKRnM45QiFQcewrGk8=", "last_login": "2018-09-10T02:21:50.676Z", "is_superuser": false, "email": "test1@test.com", "username": "C01234567", "tagline": "", "created_at": "2018-09-08T18:59:45.552Z", "updated_at": "2018-09-17T02:25:05.164Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T18:59:45.208Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 2, "fields": {"password": "pbkdf2_sha256$36000$T8SFHw3JGtnX$6yVLWtn83IWPXcV7IQXcTk/2SN7DM92vA8cAjNFNAgc=", "last_login": "2018-09-08T19:01:53.412Z", "is_superuser": false, "email": "test2@test.com", "username": "C11234567", "tagline": "", "created_at": "2018-09-08T19:01:52.580Z", "updated_at": "2018-09-17T02:25:05.146Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:01:52.218Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 3, "fields": {"password": "pbkdf2_sha256$36000$wtcx3pv32dgt$tETHKHzI2FPAtVyp2GI/gutqirDwMISwKyWBnEehlVI=", "last_login": "2018-09-08T19:02:33Z", "is_superuser": false, "email": "test3@test.com", "username": "C00220269", "tagline": "", "created_at": "2018-09-08T19:02:33.065Z", "updated_at": "2018-10-29T17:05:28.190Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:02:32Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 4, "fields": {"password": "pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=", "last_login": "2018-09-08T19:04:44.282Z", "is_superuser": true, "email": "testtube@test.com", "username": "adminnxg", "tagline": "", "created_at": "2018-09-08T19:04:33.178Z", "updated_at": "2018-09-17T02:25:05.163Z", "is_staff": true, "is_active": true, "date_joined": "2018-09-08T19:04:32.844Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 5, "fields": {"password": "pbkdf2_sha256$36000$ir0uuUb5gNO3$PYv6Px2Hd7uOwNDIIM/29jukiFh4gxMtklXh3lB8atk=", "last_login": "2018-10-03T13:10:32.154Z", "is_superuser": true, "email": "t@t.com", "username": "admin_nxg", "tagline": "", "created_at": "2018-10-03T13:10:17.524Z", "updated_at": "2018-10-03T13:10:17.527Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-03T13:10:17.440Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 6, "fields": {"password": "pbkdf2_sha256$36000$rJqxMw2luNoH$lzKgItcwzhtwdA9qSpiR/6QGpSQyN4u7Yj8UqFn0I3k=", "last_login": "2018-10-29T16:03:59.849Z", "is_superuser": true, "email": "tt@tt.ctt", "username": "tt", "tagline": "", "created_at": "2018-10-29T16:02:39.519Z", "updated_at": "2018-10-29T16:02:39.579Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-29T16:02:39.035Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 8, "fields": {"password": "pbkdf2_sha256$36000$lH8qTbHqorAx$JU7k5EJbvoZ3zQecTyAQqW9JnbWsvK9Shki4AokAI58=", "last_login": null, "is_superuser": false, "email": "jchumley@fec.gov", "username": "C00547349", "tagline": "", "created_at": "2018-11-17T18:14:46.506Z", "updated_at": "2018-11-17T18:14:46.506Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:14:45.968Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 9, "fields": {"password": "pbkdf2_sha256$36000$iz0eGkpfv0Kk$MsWBxN8kxEd1UhbjDccuDEddxf617ZIKHl2aUYbE5hU=", "last_login": null, "is_superuser": false, "email": "DGardner.ctr@fec.gov", "username": "C00010603", "tagline": "", "created_at": "2018-11-17T18:16:31.399Z", "updated_at": "2018-11-17T18:16:31.399Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:16:31.070Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 10, "fields": {"password": "pbkdf2_sha256$36000$zx8BoYBCxmr3$rvhX0Cb8Ob6sFKii3C4fGdkOnqdTv5JaseBLUHQkk9s=", "last_login": null, "is_superuser": false, "email": "RLanz@fec.gov", "username": "C00004036", "tagline": "", "created_at": "2018-11-17T18:26:26.526Z", "updated_at": "2018-11-17T18:26:26.526Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:26:25.939Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 13, "fields": {"password": "pbkdf2_sha256$36000$i52VO7zc5soK$+HCMcJ3EPwGT16PkpIKONq0l5DkWpqbCfuXvuRbUFpY=", "last_login": null, "is_superuser": false, "email": "mkancherla.ctr@fec.gov", "username": "C00690222", "tagline": "", "created_at": "2018-11-17T19:06:12.787Z", "updated_at": "2018-11-17T19:06:12.787Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:06:12.579Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 14, "fields": {"password": "pbkdf2_sha256$36000$z6v6LA7UXwsa$TifYZ6wXrt4L4AYX8BJk15QBM3NjcxTfCu5K4zqD/rw=", "last_login": null, "is_superuser": false, "email": "cmukerjee.ctr@fec.gov", "username": "C00690198", "tagline": "", "created_at": "2018-11-17T19:06:55.319Z", "updated_at": "2018-11-17T19:06:55.319Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:06:54.823Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 15, "fields": {"password": "pbkdf2_sha256$36000$7O3SmWmvZNcA$ljiDe6RXy4xmlVTQPnSnf5jfA8AoTi7fCRfxw+0Rys4=", "last_login": null, "is_superuser": false, "email": "dhardway.ctr@fec.gov", "username": "C00690206", "tagline": "", "created_at": "2018-11-17T19:21:59.287Z", "updated_at": "2018-11-17T19:21:59.287Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:21:58.858Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 16, "fields": {"password": "pbkdf2_sha256$36000$Q8ybNJR8wyQ1$idODgWN0G82McSZwSs1dhFsBtphImy1Oy4eHi4PA1cs=", "last_login": null, "is_superuser": false, "email": "rsanchez.ctr@fec.gov", "username": "C00690099", "tagline": "", "created_at": "2018-11-17T19:24:15.455Z", "updated_at": "2018-11-17T19:24:15.455Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:24:15.004Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 17, "fields": {"password": "pbkdf2_sha256$36000$LGrAedbIrW3u$mwajveqr5llDOU6og8KuTzPadLoNl2lYhjZxU6aItqk=", "last_login": null, "is_superuser": false, "email": "smartinez.ctr@fec.gov", "username": "C00689992", "tagline": "", "created_at": "2018-11-17T19:27:13.890Z", "updated_at": "2018-11-17T19:27:13.890Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:27:13.473Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 18, "fields": {"password": "pbkdf2_sha256$36000$QhOGxXUeHV6W$AkPdL+ptcFtZJxW7zLVH+olXbBY4WoG7/T9Waog/5s4=", "last_login": null, "is_superuser": false, "email": "cstroman.ctr@fec.gov", "username": "C00689968", "tagline": "", "created_at": "2018-11-17T19:29:40.284Z", "updated_at": "2018-11-17T19:29:40.284Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:29:39.774Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 19, "fields": {"password": "pbkdf2_sha256$36000$U9LVMZNdAAVJ$rhwOoc08a6aawfUP0X3Bds/S16i0X60Pbc0oQE7OTak=", "last_login": null, "is_superuser": false, "email": "mbasupally.ctr@fec.gov", "username": "C00689844", "tagline": "", "created_at": "2018-11-17T19:31:33.825Z", "updated_at": "2018-11-17T19:31:33.825Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:31:33.312Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 20, "fields": {"password": "pbkdf2_sha256$36000$lcK3NdwOy47A$/6AnRILnYKHAPittLPFl7eNgl9mojGzyq70jfHPWXaI=", "last_login": null, "is_superuser": false, "email": "rdasaradhi.ctr@fec.gov", "username": "C00689877", "tagline": "", "created_at": "2018-11-17T19:35:59.009Z", "updated_at": "2018-11-17T19:35:59.009Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:35:58.582Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 21, "fields": {"password": "pbkdf2_sha256$36000$5KbCHPDa2YCi$PpPLth/nAKpJoD48aEEKmJPTaWvJ8GfojAmAGZY7YN8=", "last_login": null, "is_superuser": false, "email": "Cris.Daniluk@ctr.salientcrgt.com", "username": "C00689174", "tagline": "", "created_at": "2018-11-19T14:10:59.448Z", "updated_at": "2018-11-19T14:10:59.448Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-19T14:10:59.081Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 22, "fields": {"password": "pbkdf2_sha256$36000$qbDY0zugkfhq$TlH+tdQGMKIs7D9BbyQZUm/hy6wS/bpK0ULLIibfM+c=", "last_login": null, "is_superuser": false, "email": "mahendra.marathe@salientcrgt.com", "username": "C00689224", "tagline": "", "created_at": "2018-11-19T14:21:20.498Z", "updated_at": "2018-11-19T14:21:20.498Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-19T14:21:20.129Z", "groups": [], "user_permissions": []}}] \ No newline at end of file diff --git a/django-backend/fixtures/committee_base_data.json b/django-backend/fixtures/committee_base_data.json deleted file mode 100644 index e3dfc1ef5a..0000000000 --- a/django-backend/fixtures/committee_base_data.json +++ /dev/null @@ -1,45 +0,0 @@ -[{ - "model": "forms.committee", - "pk": 1, - "fields": { - "committeeid": "C01234567", - "committeename": "Test Committee 1", - "street1": "Street 1", - "street2": "Street 2", - "city": "Washington", - "state": "DC", - "zipcode": 91285, - "treasurerlastname": "Smith", - "treasurerfirstname": "John", - "treasurermiddlename": "E", - "treasurerprefix": "Mr", - "treasurersuffix": "Jr.", - "created_at": "2018-09-19T17:34:00.477Z", - "updated_at": "2018-09-19T17:34:00.477Z", - "deleted_at": null, - "isdeleted": false, - "email_on_file": "jsmith@gmail.com" - } -}, { - "model": "forms.committee", - "pk": 2, - "fields": { - "committeeid": "C11234567", - "committeename": "Test Committee 2", - "street1": "Street 1", - "street2": "Street 2", - "city": "Washington", - "state": "DC", - "zipcode": 20001, - "treasurerlastname": "Smith", - "treasurerfirstname": "John", - "treasurermiddlename": "Doe", - "treasurerprefix": "Mr", - "treasurersuffix": "IV", - "created_at": "2018-09-19T17:34:00.479Z", - "updated_at": "2018-09-19T17:34:00.479Z", - "deleted_at": null, - "isdeleted": false, - "email_on_file": "jsmith@outlook.com" - } -}] diff --git a/django-backend/fixtures/committee_updated.json b/django-backend/fixtures/committee_updated.json deleted file mode 100644 index 5eac9fa07b..0000000000 --- a/django-backend/fixtures/committee_updated.json +++ /dev/null @@ -1 +0,0 @@ -[{"model": "authentication.account", "pk": 1, "fields": {"password": "pbkdf2_sha256$36000$1a12NsHSiM1P$TSQHr/206lhRHsPmS/EFuv2MDaKRnM45QiFQcewrGk8=", "last_login": "2018-09-10T02:21:50.676Z", "is_superuser": false, "email": "test1@test.com", "username": "C01234567", "tagline": "", "created_at": "2018-09-08T18:59:45.552Z", "updated_at": "2018-09-17T02:25:05.164Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T18:59:45.208Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 2, "fields": {"password": "pbkdf2_sha256$36000$T8SFHw3JGtnX$6yVLWtn83IWPXcV7IQXcTk/2SN7DM92vA8cAjNFNAgc=", "last_login": "2018-09-08T19:01:53.412Z", "is_superuser": false, "email": "test2@test.com", "username": "C11234567", "tagline": "", "created_at": "2018-09-08T19:01:52.580Z", "updated_at": "2018-09-17T02:25:05.146Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:01:52.218Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 3, "fields": {"password": "pbkdf2_sha256$36000$wtcx3pv32dgt$tETHKHzI2FPAtVyp2GI/gutqirDwMISwKyWBnEehlVI=", "last_login": "2018-09-08T19:02:33Z", "is_superuser": false, "email": "test3@test.com", "username": "C00220269", "tagline": "", "created_at": "2018-09-08T19:02:33.065Z", "updated_at": "2018-10-29T17:05:28.190Z", "is_staff": false, "is_active": true, "date_joined": "2018-09-08T19:02:32Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 4, "fields": {"password": "pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=", "last_login": "2018-09-08T19:04:44.282Z", "is_superuser": true, "email": "testtube@test.com", "username": "adminnxg", "tagline": "", "created_at": "2018-09-08T19:04:33.178Z", "updated_at": "2018-09-17T02:25:05.163Z", "is_staff": true, "is_active": true, "date_joined": "2018-09-08T19:04:32.844Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 5, "fields": {"password": "pbkdf2_sha256$36000$ir0uuUb5gNO3$PYv6Px2Hd7uOwNDIIM/29jukiFh4gxMtklXh3lB8atk=", "last_login": "2018-10-03T13:10:32.154Z", "is_superuser": true, "email": "t@t.com", "username": "admin_nxg", "tagline": "", "created_at": "2018-10-03T13:10:17.524Z", "updated_at": "2018-10-03T13:10:17.527Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-03T13:10:17.440Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 6, "fields": {"password": "pbkdf2_sha256$36000$rJqxMw2luNoH$lzKgItcwzhtwdA9qSpiR/6QGpSQyN4u7Yj8UqFn0I3k=", "last_login": "2018-10-29T16:03:59.849Z", "is_superuser": true, "email": "tt@tt.ctt", "username": "tt", "tagline": "", "created_at": "2018-10-29T16:02:39.519Z", "updated_at": "2018-10-29T16:02:39.579Z", "is_staff": true, "is_active": true, "date_joined": "2018-10-29T16:02:39.035Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 8, "fields": {"password": "pbkdf2_sha256$36000$lH8qTbHqorAx$JU7k5EJbvoZ3zQecTyAQqW9JnbWsvK9Shki4AokAI58=", "last_login": null, "is_superuser": false, "email": "jchumley@fec.gov", "username": "C00547349", "tagline": "", "created_at": "2018-11-17T18:14:46.506Z", "updated_at": "2018-11-17T18:14:46.506Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:14:45.968Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 9, "fields": {"password": "pbkdf2_sha256$36000$iz0eGkpfv0Kk$MsWBxN8kxEd1UhbjDccuDEddxf617ZIKHl2aUYbE5hU=", "last_login": null, "is_superuser": false, "email": "DGardner.ctr@fec.gov", "username": "C00010603", "tagline": "", "created_at": "2018-11-17T18:16:31.399Z", "updated_at": "2018-11-17T18:16:31.399Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:16:31.070Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 10, "fields": {"password": "pbkdf2_sha256$36000$zx8BoYBCxmr3$rvhX0Cb8Ob6sFKii3C4fGdkOnqdTv5JaseBLUHQkk9s=", "last_login": null, "is_superuser": false, "email": "RLanz@fec.gov", "username": "C00004036", "tagline": "", "created_at": "2018-11-17T18:26:26.526Z", "updated_at": "2018-11-17T18:26:26.526Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T18:26:25.939Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 13, "fields": {"password": "pbkdf2_sha256$36000$i52VO7zc5soK$+HCMcJ3EPwGT16PkpIKONq0l5DkWpqbCfuXvuRbUFpY=", "last_login": null, "is_superuser": false, "email": "mkancherla.ctr@fec.gov", "username": "C00690222", "tagline": "", "created_at": "2018-11-17T19:06:12.787Z", "updated_at": "2018-11-17T19:06:12.787Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:06:12.579Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 14, "fields": {"password": "pbkdf2_sha256$36000$z6v6LA7UXwsa$TifYZ6wXrt4L4AYX8BJk15QBM3NjcxTfCu5K4zqD/rw=", "last_login": null, "is_superuser": false, "email": "cmukerjee.ctr@fec.gov", "username": "C00690198", "tagline": "", "created_at": "2018-11-17T19:06:55.319Z", "updated_at": "2018-11-17T19:06:55.319Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:06:54.823Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 15, "fields": {"password": "pbkdf2_sha256$36000$7O3SmWmvZNcA$ljiDe6RXy4xmlVTQPnSnf5jfA8AoTi7fCRfxw+0Rys4=", "last_login": null, "is_superuser": false, "email": "dhardway.ctr@fec.gov", "username": "C00690206", "tagline": "", "created_at": "2018-11-17T19:21:59.287Z", "updated_at": "2018-11-17T19:21:59.287Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:21:58.858Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 16, "fields": {"password": "pbkdf2_sha256$36000$Q8ybNJR8wyQ1$idODgWN0G82McSZwSs1dhFsBtphImy1Oy4eHi4PA1cs=", "last_login": null, "is_superuser": false, "email": "rsanchez.ctr@fec.gov", "username": "C00690099", "tagline": "", "created_at": "2018-11-17T19:24:15.455Z", "updated_at": "2018-11-17T19:24:15.455Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:24:15.004Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 17, "fields": {"password": "pbkdf2_sha256$36000$LGrAedbIrW3u$mwajveqr5llDOU6og8KuTzPadLoNl2lYhjZxU6aItqk=", "last_login": null, "is_superuser": false, "email": "smartinez.ctr@fec.gov", "username": "C00689992", "tagline": "", "created_at": "2018-11-17T19:27:13.890Z", "updated_at": "2018-11-17T19:27:13.890Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:27:13.473Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 18, "fields": {"password": "pbkdf2_sha256$36000$QhOGxXUeHV6W$AkPdL+ptcFtZJxW7zLVH+olXbBY4WoG7/T9Waog/5s4=", "last_login": null, "is_superuser": false, "email": "cstroman.ctr@fec.gov", "username": "C00689968", "tagline": "", "created_at": "2018-11-17T19:29:40.284Z", "updated_at": "2018-11-17T19:29:40.284Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:29:39.774Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 19, "fields": {"password": "pbkdf2_sha256$36000$U9LVMZNdAAVJ$rhwOoc08a6aawfUP0X3Bds/S16i0X60Pbc0oQE7OTak=", "last_login": null, "is_superuser": false, "email": "mbasupally.ctr@fec.gov", "username": "C00689844", "tagline": "", "created_at": "2018-11-17T19:31:33.825Z", "updated_at": "2018-11-17T19:31:33.825Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:31:33.312Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 20, "fields": {"password": "pbkdf2_sha256$36000$lcK3NdwOy47A$/6AnRILnYKHAPittLPFl7eNgl9mojGzyq70jfHPWXaI=", "last_login": null, "is_superuser": false, "email": "rdasaradhi.ctr@fec.gov", "username": "C00689877", "tagline": "", "created_at": "2018-11-17T19:35:59.009Z", "updated_at": "2018-11-17T19:35:59.009Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-17T19:35:58.582Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 21, "fields": {"password": "pbkdf2_sha256$36000$5KbCHPDa2YCi$PpPLth/nAKpJoD48aEEKmJPTaWvJ8GfojAmAGZY7YN8=", "last_login": null, "is_superuser": false, "email": "Cris.Daniluk@ctr.salientcrgt.com", "username": "C00689174", "tagline": "", "created_at": "2018-11-19T14:10:59.448Z", "updated_at": "2018-11-19T14:10:59.448Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-19T14:10:59.081Z", "groups": [], "user_permissions": []}}, {"model": "authentication.account", "pk": 22, "fields": {"password": "pbkdf2_sha256$36000$qbDY0zugkfhq$TlH+tdQGMKIs7D9BbyQZUm/hy6wS/bpK0ULLIibfM+c=", "last_login": null, "is_superuser": false, "email": "mahendra.marathe@salientcrgt.com", "username": "C00689224", "tagline": "", "created_at": "2018-11-19T14:21:20.498Z", "updated_at": "2018-11-19T14:21:20.498Z", "is_staff": false, "is_active": true, "date_joined": "2018-11-19T14:21:20.129Z", "groups": [], "user_permissions": []}}] \ No newline at end of file diff --git a/django-backend/fixtures/e2e-test-data.json b/django-backend/fixtures/e2e-test-data.json index dd84be49f2..4e12ce76e4 100644 --- a/django-backend/fixtures/e2e-test-data.json +++ b/django-backend/fixtures/e2e-test-data.json @@ -42,20 +42,26 @@ }, { "model": "committee_accounts.Membership", - "pk": "1", + "pk": "3e281c08-2b1f-4cd0-9236-410fe872edb9", "fields": { "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "c94c5d1a-9e73-464d-ad72-b73b5d8667a9", - "user_id": "1cb79553-a0e6-43f2-b182-f102eca4a043" + "user_id": "1cb79553-a0e6-43f2-b182-f102eca4a043", + "pending_email": null, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z" } }, { "model": "committee_accounts.Membership", - "pk": "2", + "pk": "2b261104-76a1-4f20-b082-9aafdd7115fe", "fields": { "role": "COMMITTEE_ADMINISTRATOR", "committee_account_id": "c94c5d1a-9e73-464d-ad72-b73b5d8667a9", - "user_id": "2faed187-f4b5-46b4-a0d9-4a250ac792da" + "user_id": "2faed187-f4b5-46b4-a0d9-4a250ac792da", + "pending_email": null, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z" } } -] \ No newline at end of file +] diff --git a/django-backend/fixtures/forms_commitee.csv b/django-backend/fixtures/forms_commitee.csv deleted file mode 100644 index 99af353e30..0000000000 --- a/django-backend/fixtures/forms_commitee.csv +++ /dev/null @@ -1,30 +0,0 @@ -1,C12345678,test1,xyz,abc,Washington,DC,20001,David,Mark,J,Mr,II,2018-09-24 15:54:23.419144+00,2018-09-24 15:54:23.419172+00,,f,- -2,C01234567,test1,xyz,abc,Washington,DC,20001,David,Mark,J,Mr,II,2018-09-25 03:59:41.400669+00,2018-09-25 03:59:41.400697+00,,f,- -5,C01234555,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,Northeast,Washington DC,DC,2002,Michael,David,M,Mr,II,2018-11-16 18:23:10.478856+00,2018-11-16 18:23:10.478889+00,,f,mdavid@test.com -6,C01234555,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,Northeast,Washington DC,DC,2002,Michael,David,M,Mr,II,2018-11-16 18:55:20.639207+00,2018-11-16 18:55:20.639244+00,,f,jchumley@fec.gov -7,C01234555,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,Northeast,Washington DC,DC,2002,Chumley,Jeff,A,Mr,II,2018-11-16 18:57:03.977755+00,2018-11-16 18:57:03.97779+00,,f,jchumley@fec.gov -8,C01234555,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 18,street 1,Washington DC,DC,2003,Lally,Ken,b,Mr,II,2018-11-16 18:58:56.582818+00,2018-11-16 18:58:56.58285+00,,f,klally@fec.gov -9,C01234566,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 18,street 1,Washington DC,DC,2003,Lally,Ken,b,Mr,II,2018-11-16 18:59:49.994864+00,2018-11-16 18:59:49.994895+00,,f,klally@fec.gov -10,C01234555,NATIONAL ASSOCIATION OF MEDICAL COMMITTEE,Street 18,Northeast,Washington DC,DC,2002,Chumley,Jeff,A,Mr,II,2018-11-16 19:01:59.096731+00,2018-11-16 19:01:59.096764+00,,f,jchumley@fec.gov -11,C01234577,NATIONAL ASSOCIATION OF STUDENT COMMITTEE,Street 17,street2,Washington DC,DC,2002,Lanz,Ryan,C,Mr,II,2018-11-16 19:05:32.845853+00,2018-11-16 19:05:32.845887+00,,f,RLanz@fec.gov -12,C01234588,NATIONAL ASSOCIATION OF PROGRAM MANAGEMENT,Street 16,street3,Washington DC,DC,2004,Gardner,Drew,D,Mr,II,2018-11-16 19:08:25.901263+00,2018-11-16 19:08:25.9013+00,,f,DGardner.ctr@fec.gov -13,C01234588,NATIONAL ASSOCIATION OF EVENT MANAGEMENT,Street 16,street3,Washington DC,DC,2004,Gardner,Drew,D,Mr,II,2018-11-16 19:08:48.057127+00,2018-11-16 19:08:48.057161+00,,f,DGardner.ctr@fec.gov -15,C00220269,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,street2,Washington DC,DC,2004,Lally,Ken,B,Mr,II,2018-11-16 20:50:02.01507+00,2018-11-16 20:50:02.015102+00,,f,klally@fec.gov -17,C01234567,NATIONAL ASSOCIATION OF CHAIN DRUG STORES,Street 19,street1,Washington DC,DC,2002,Jinka,Praveen,B,Mr,II,2018-11-17 00:38:16.491029+00,2018-11-17 00:38:16.491066+00,,f,pjinka.ctr@fec.gov -18,C00547349,NEXTGEN CLIMATE ACTION COMMITTEE,Street 19,street1,Washington DC,DC,2002,Chumley,Jeff,B,Mr,II,2018-11-17 18:01:35.856607+00,2018-11-17 18:01:35.856641+00,,f,jchumley@fec.gov -19,C00010603,DNC SERVICES CORP./DEM. NAT'L COMMITTEE,430 SOUTH CAPITOL STREET SE,street1,Washington DC,DC,2002,Gardner,Drew,B,Mr,II,2018-11-17 18:18:59.461612+00,2018-11-17 18:18:59.461648+00,,f,DGardner.ctr@fec.gov -20,C00004036,SEIU COPE (SERVICE EMPLOYEES INTERNATIONAL UNION COMMITTEE ON POLITICAL EDUCATION),1800 MASSACHUSETTS AVE NW,street1,Washington DC,DC,20036,Lanz,Ryan,B,Mr,II,2018-11-17 18:35:25.772953+00,2018-11-17 18:35:25.772983+00,,f,RLanz@fec.gov -22,C00690222,ROCKINGHAM COUNTY DEMOCRATIC COMMITTEE,11830 FORT TURLEY TRAIL,street1,LINVILLE,VA,24010,Kancherla,Madhuri,B,Ms,II,2018-11-17 18:54:16.840261+00,2018-11-17 18:54:16.840296+00,,f,mkancherla.ctr@fec.gov -24,C00690198,ROCKINGHAM COUNTY DEMOCRATIC COMMITTEE,11831 FORT TURLEY TRAIL,street1,Vienna,VA,24010,Mukerjee,Chamila,B,Ms,II,2018-11-17 19:10:31.876374+00,2018-11-17 19:10:31.876404+00,,f,cmukerjee.ctr@fec.gov -25,C00690206,5050 BY 2020 COMMITTEE,PO BOX 1041,street1,ESTERO,FL,24010,Donald,Hardway,B,Mr,II,2018-11-17 19:23:09.303373+00,2018-11-17 19:23:09.303406+00,,f,dhardway.ctr@fec.gov -26,C00690099,"BRADFORD WHITE CORPORATION POLITICAL ACTION COMMITTEE (""BWC PAC"")",725 TALAMORE DRIVE,street1,AMBLER,PA,19002,Sanchez,Roberto,B,Mr,II,2018-11-17 19:25:49.526771+00,2018-11-17 19:25:49.526804+00,,f,rsanchez.ctr@fec.gov -27,C00689992,UNITED STRATEGIES GROUP POLITCAL ACTION COMMITTEE,1708 ELSINORE AVE,street1,HENDERSON,NV,19002,Martinez,Said,B,Mr,II,2018-11-17 19:28:43.559561+00,2018-11-17 19:28:43.559595+00,,f,smartinez.ctr@fec.gov -28,C00689968,PO BOX 751271,1708 ELSINORE AVE,street1,LAS VEGAS,NV,19002,Stroman,Craig,B,Mr,II,2018-11-17 19:30:40.55474+00,2018-11-17 19:30:40.55477+00,,f,cstroman.ctr@fec.gov -29,C00689844,PO BOX 751271,28600 BELLA VISTA PARKWAY,street1,WARRENVILLE,IL,60555,Basupally,Mahi,B,Mr,II,2018-11-17 19:34:20.271653+00,2018-11-17 19:34:20.2717+00,,f,mbasupally.ctr@fec.gov -30,C00689877,COMMITTEE TO ELECT STEVEN DEVALD,2100 LAKESIDE AVE E.,street1,CLEVELAND,OH,60555,Dasaradhi,Ranjith,B,Mr,II,2018-11-17 19:37:39.733053+00,2018-11-17 19:37:39.733095+00,,f,rdasaradhi.ctr@fec.gov -31,C00689174,DIGITAL AGE NEURAL COMMITTEE,23333 RIDGE ROUTE DR #64,street1,LAKE FOREST,CA,92630,Cris,Daniluk,B,Mr,II,2018-11-19 14:16:37.324656+00,2018-11-19 14:16:37.324686+00,,f,Cris.Daniluk@ctr.salientcrgt.com -32,C00689224,DIGITAL AGE NEURAL COMMITTEE,332 W LEE HWY,street1,WARRENTON,VA,92630,Marathe,Mahendra,B,Mr,II,2018-11-19 14:22:54.178217+00,2018-11-19 14:22:54.178248+00,,f,mahendra.marathe@salientcrgt.com -4,C00022368,"NATIONAL ASSOCIATION OF CHAIN DRUG STORES, INC. POLITICAL ACTION COMMITTEE",Street 17,Northeast,Washington DC,DC,2002,Michael,David,M,Mr,II,2018-10-26 19:05:50.217864+00,2018-10-26 19:05:50.217904+00,,f,test@test.com -3,C01234567,test1,xyz,abc,Washington,DC,20001,David,Mark,J,Mr,II,2018-09-26 15:31:18.018227+00,2018-09-26 15:31:18.018263+00,,f,test@test.com -33,C00689224,DIGITAL AGE NEURAL COMMITTEE,332 W LEE HWY,street1,WARRENTON,VA,92630,Marathe,Mahendra,B,Mr,II,2018-11-29 17:37:32.094507+00,2018-11-29 17:37:32.094538+00,,f,mahendra.marathe@salientcrgt.com -34,C01234567,DIGITAL,332 W LEE HWY,street1,WARRENTON,VA,92630,Marathe,Mahendra,B,Mr,II,2018-11-30 19:42:28.665529+00,2018-11-30 19:42:28.665577+00,,f,mahendra.marathe@salientcrgt.com diff --git a/django-backend/scripts/create_new_users.sql b/django-backend/scripts/create_new_users.sql deleted file mode 100644 index 17833c7b87..0000000000 --- a/django-backend/scripts/create_new_users.sql +++ /dev/null @@ -1,163 +0,0 @@ --- Utility script to create new users in the fecfile-online database. --- Edit the sample VALUES() below to create new user accounts. - --- Update authentication_account_id_seq to ensure the index is pointing to the correct value -SELECT setval('authentication_account_id_seq', (SELECT max(id) FROM authentication_account)); - --- To associate a user to additional committee ids Taking an existing committee id from the committee_master table --- ensure the committee table doesn't have an entry in authentication_account table -INSERT into authentication_account( - password, - is_superuser, - email, - username, - cmtee_id, - created_at, - updated_at, - is_staff, - is_active, - date_joined, - tagline, - role, - delete_ind, - contact, - status - ) -VALUES - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','egreen.ctr@fec.gov','C00601534egreen.ctr@fec.gov','C00601534',now(),now(),'f','t',now(),'', 'C_ADMIN','N','2026941307','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','egreen.ctr@fec.gov','C00601229egreen.ctr@fec.gov','C00601229',now(),now(),'f','t',now(),'', 'C_ADMIN','N','2402743764','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','egreen.ctr@fec.gov','C00233361egreen.ctr@fec.gov','C00233361',now(),now(),'f','t',now(),'', 'C_ADMIN','N','2402743764','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','akolan.ctr@fec.gov','C00601609akolan.ctr@fec.gov','C00601609',now(),now(),'f','t',now(),'', 'C_ADMIN','N','5121234567','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','akolan.ctr@fec.gov','C00670430akolan.ctr@fec.gov','C00670430',now(),now(),'f','t',now(),'', 'C_ADMIN','N','5121234567','Registered'), - ('pbkdf2_sha256$36000$gZSXSbOJiHED$NKQPaMEHncW8lw0y8vvRvXD/3w4vs9xXWiTvjdFAUiM=','f','akolan.ctr@fec.gov','C00715672akolan.ctr@fec.gov','C00715672',now(),now(),'f','t',now(),'', 'C_ADMIN','N','5121234567','Registered'); - --- Applies to bot PAC and PTY --- Update forms_committee table with the user information -INSERT INTO public.forms_committee( - committeeid, - committeename, - street1, - street2, - city, - state, - zipcode, - treasurerlastname, - treasurerfirstname, - treasurermiddlename, - treasurerprefix, - treasurersuffix, - created_at, - updated_at, - isdeleted, - email_on_file) - SELECT - cm.cmte_id, - cm.cmte_name, - cm.street_1, - cm.street_2, - cm.city, - cm.state, - 20148, - cm.treasurer_last_name, - cm.treasurer_first_name, - cm.treasurer_middle_name, - cm.treasurer_prefix, - cm.treasurer_suffix, - now(), - now(), - false, - aa.email - FROM committee_master cm - INNER JOIN authentication_account aa ON cm.cmte_id = aa.cmtee_id -WHERE cm.cmte_id in ( - 'C00601534', - 'C00601229', - 'C00233361', - 'C00601609', - 'C00670430', - 'C00715672' - ); - --- Ensure there aren't any due reports associated with the committee ids. -Delete from cmte_current_due_reports -where cmte_id in ( - 'C00601534', - 'C00601229', - 'C00233361', - 'C00601609', - 'C00670430', - 'C00715672' - -); - --- Applies to PAC only using C00000422 as a template -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00601534', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00000422'; -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00601609', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00000422'; -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00601229', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00000422'; -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00670430', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00000422'; - --- Applies to PTY only using C00010603 as a template -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00233361', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00010603'; - -INSERT INTO public.cmte_current_due_reports( - cmte_id, form_type, report_type, due_date, last_update_date) -select 'C00715672', form_type, report_type, due_date, last_update_date -from public.cmte_current_due_reports -where cmte_id ='C00010603'; - - --- Applies to PAC -update committee_master set cmte_type='Q' where cmte_id in ( - 'C00601534', - 'C00601609', - 'C00601229', - 'C00670430' - ); - --- Applies to PTY -update committee_master set cmte_type='X' where cmte_id in ( - 'C00233361', - 'C00715672' - ); - --- Applies to both PAC and PTY -update committee_master set cmte_dsgn='B', cmte_filing_freq='M' where cmte_id in ( - 'C00601534', - 'C00601229', - 'C00233361', - 'C00601609', - 'C00670430', - 'C00715672' - -); --- PAC with 3L -UPDATE public.committee_master set cmte_type='X', cmte_dsgn='U', cmte_filing_freq='M', cmte_filed_type='4' WHERE cmte_type_category='PAC' AND cmte_id in ( - 'C00601534', - 'C00601229' -); - --- PAC without 3L -UPDATE public.committee_master set cmte_filing_freq='Q' WHERE cmte_type_category='PAC' AND cmte_id in ( - 'C00601609', - 'C00670430' -); diff --git a/django-backend/scripts/generate_transaction_record_fixture.py b/django-backend/scripts/generate_transaction_record_fixture.py deleted file mode 100644 index c561118db8..0000000000 --- a/django-backend/scripts/generate_transaction_record_fixture.py +++ /dev/null @@ -1,275 +0,0 @@ -import argparse -from secrets import choice - -# Generate Transaction Record Fixture -# -# Creates a json file containing valid transaction records -# for the form_type values specified by the user, and with -# randomly generated values for transaction_amount. - - -def get_arguments(): - parser = argparse.ArgumentParser( - prog='Generate Transaction Record Fixture', - description='Creates a json file containing valid transaction records\n' - + 'for the form_type values specified by the user and with\n' - + 'randomly generated values for transaction_amount.' - ) - - parser.add_argument( - 'form_types', - nargs='+', - default=[], - help="form_type values (e.g SA12 or SC/9) separated by spaces" - ) - parser.add_argument( - '--committee_account_id', - default="735db943-9446-462a-9be0-c820baadb622", - help='default value: 735db943-9446-462a-9be0-c820baadb622' - ) - parser.add_argument( - '--report_a_id', - default="b6d60d2d-d926-4e89-ad4b-c47d152a66ae", - help='primary report (default: b6d60d2d-d926-4e89-ad4b-c47d152a66ae)' - ) - parser.add_argument( - '--report_b_id', - default="1406535e-f99f-42c4-97a8-247904b7d297", - help='secondary report used for transactions outside of report dates' - + ' (default: 1406535e-f99f-42c4-97a8-247904b7d297)' - ) - parser.add_argument( - '--report_start_date', - default="2005-01-30", - help='default value: 2005-01-30' - ) - parser.add_argument( - '--report_end_date', - default="2005-02-28", - help='default value: 2005-02-28' - ) - parser.add_argument( - '--transaction_type_identifier', - default="Transaction Type Identifier", - help='default value: Transaction Type Identifier' - ) - parser.add_argument( - '--output_file', - help="Output to a file instead of the console (e.g --output_file=records.txt)") - - return parser.parse_args() - - -def get_schedule(form_type): - schedule = str(form_type[1]).capitalize() # the second char in the form_type - if schedule == "C" and form_type[2] in ["1", "2"]: - schedule = form_type[1:2] - return schedule - - -def get_comment(form_type, test_case): - if test_case == "within_dates": - return f"{form_type} within report dates, should count in Col. A" - if test_case == "within_year": - return f"{form_type} within year, should count in Col. B" - if test_case == "outside_dates": - return f"{form_type} outside of dates, should not count" - if test_case == "within_dates_but_memo": - return f"{form_type} is a memo item, should not count" - return "" - - -def get_amount_field(schedule): - if schedule == "A": - return "contribution_amount" - if schedule == "B": - return "expenditure_amount" - if schedule == "C": - return "loan_amount" - if schedule == "C1": - return "loan_amount" - if schedule == "C2": - return "guaranteed_amount" - if schedule == "D": - return "incurred_amount" - if schedule == "E": - return "expenditure_amount" - return "" - - -def get_date_field(schedule): - if schedule == "A": - return "contribution_date" - if schedule == "B": - return "expenditure_date" - if schedule == "C": - return "loan_incurred_date" - if schedule == "C1": - return "loan_incurred_date" - if schedule == "C2": - return "" - if schedule == "D": - return "" - if schedule == "E": - return "disbursement_date" - return "" - - -def get_date(args, test_case): - report_start = args.report_start_date.split("-") - report_end = args.report_end_date.split("-") - - if "within_dates" in test_case: - return "-".join(report_start) - - if "year" in test_case: - months = set([str(m).rjust(2, "0") for m in range(1, 13)]) - report_start_month = report_start[1] - report_months = range(int(report_start_month), 13) - formatted_report_months = set([str(m).rjust(2, "0") for m in report_months]) - months_before_report = list(months - formatted_report_months) - if len(months_before_report) > 0: - random_month = choice(months_before_report) - else: - random_month = report_start_month - - report_start_day = report_start[2] - if (int(report_start_day) > 1): - random_day = str(choice(range(1, int(report_start_day)))).rjust(2, "0") - else: - random_day = "01" - - if report_start_month == "01" and report_start_day == "01": - raise ValueError( - 'Cannot set any date in same year before provided start date' - + f' ({args.report_start_date})' - ) - - year = report_start[0] - return f"{year}-{random_month}-{random_day}" - - report_years = set([report_start[0], report_end[0]]) - random_year = choice(list(set(range(2000 - 50, 2000 + 50)) - report_years)) - random_month = choice([str(m).rjust(2, "0") for m in range(1, 13)]) - return f"{random_year}-{random_month}-01" - - -def get_date_entry(date_field, date): - if (len(date_field) > 0): - return f''',\n{" "*12}"{date_field}": "{date}"''' - return "" - - -def get_report_id(args, test_case): - if "within_dates" in test_case: - return args.report_a_id - else: - return args.report_b_id - - -def random_hex(length=1): - hex_chars = "0123456789abcdef" - - out_string = "" - while len(out_string) < length: - out_string += choice(hex_chars) - - return out_string - - -def get_id(form_type, record_number, schedule=""): - hex_chars = "0123456789abcdef" - - schedule_str = "" - if(schedule[0].lower() in hex_chars): - schedule_str += schedule.lower() - else: - schedule_str += hex(ord(schedule.lower()))[2:] - - line_number = form_type.split(schedule)[1].strip('/') - schedule_str += "0" + str(line_number) - schedule_str = schedule_str.ljust(8, "0") - - record_str = str(record_number + 1).rjust(12, "0") - - return f"{schedule_str}-{random_hex(4)}-{random_hex(4)}-{random_hex(4)}-{record_str}" - - -def get_records(form_type, args): - records = [] - test_cases = [ - "within_dates", - "within_dates", - "within_year", - "outside_dates", - "within_dates_but_memo" - ] - for record_number in range(len(test_cases)): - test_case = test_cases[record_number] - - comment = get_comment(form_type, test_case) - - schedule = get_schedule(form_type) - schedule_model = f"schedule{schedule.lower()}" - - schedule_id = get_id(form_type, record_number, schedule) - transaction_id = get_id(form_type, record_number, schedule) - - amount_field = get_amount_field(schedule) - amount = choice(range(20, 150)) - - date_field = get_date_field(schedule) - date = get_date(args, test_case) - date_entry = get_date_entry(date_field, date) - - report_id = get_report_id(args, test_case) - - memo_code = str( - test_case == "within_dates_but_memo" - ).lower() # JSON uses lowercase true/false - - schedule_record = f""" {{ - "comment": "{comment}", - "model": "transactions.{schedule_model}", - "fields": {{ - "id": "{schedule_id}", - "{amount_field}": {amount}{date_entry} - }}\n }}""" - - transaction_record = f""" {{ - "comment": "{comment}", - "model": "transactions.transaction", - "fields": {{ - "id": "{transaction_id}", - "schedule_{schedule.lower()}_id": "{schedule_id}", - "committee_account_id": "{args.committee_account_id}", - "report_id": "{report_id}", - "_form_type": "{form_type}", - "memo_code": {memo_code}, - "created": "2022-02-09T00:00:00.000Z", - "updated": "2022-02-09T00:00:00.000Z", - "transaction_type_identifier": {args.transaction_type_identifier} - }}\n }}""" - - records.append(schedule_record) - records.append(transaction_record) - - return records - - -if (__name__ == "__main__"): - args = get_arguments() - - records = [] - for form_type in args.form_types: - records += get_records(form_type, args) - - records_str = ",\n".join(records) - output = "[\n" + records_str + "\n]" - - if not args.output_file: - print(output) - else: - file = open(args.output_file, 'w') - file.write(output) - file.close() diff --git a/django-backend/scripts/postInstall.sh b/django-backend/scripts/postInstall.sh deleted file mode 100755 index f867e7c876..0000000000 --- a/django-backend/scripts/postInstall.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -./node_modules/bower/bin/bower install -./node_modules/gulp/bin/gulp.js diff --git a/docker-compose.yml b/docker-compose.yml index 313eb02fd7..d6040d89b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -108,3 +108,4 @@ services: FEC_API_KEY: FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE: LOG_FORMAT: + MOCK_OPENFEC: REDIS diff --git a/manifests/manifest-dev-api.yml b/manifests/manifest-dev-api.yml index 1192a18f54..6ef17e66c2 100644 --- a/manifests/manifest-dev-api.yml +++ b/manifests/manifest-dev-api.yml @@ -25,3 +25,4 @@ applications: LOGIN_REDIRECT_SERVER_URL: https://fecfile-web-api-dev.app.cloud.gov/api/v1/auth/login-redirect LOGOUT_REDIRECT_URL: https://fecfile-web-api-dev.app.cloud.gov/api/v1/auth/logout-redirect LOG_FORMAT: KEY_VALUE + MOCK_OPENFEC: REDIS diff --git a/manifests/manifest-prod-api.yml b/manifests/manifest-prod-api.yml index fc256fa22e..a3d3861c77 100644 --- a/manifests/manifest-prod-api.yml +++ b/manifests/manifest-prod-api.yml @@ -25,3 +25,4 @@ applications: LOGIN_REDIRECT_SERVER_URL: https://fecfile-web-api-prod.app.cloud.gov/api/v1/auth/login-redirect LOGOUT_REDIRECT_URL: https://fecfile-web-api-prod.app.cloud.gov/api/v1/auth/logout-redirect LOG_FORMAT: KEY_VALUE + MOCK_OPENFEC: REDIS diff --git a/manifests/manifest-stage-api.yml b/manifests/manifest-stage-api.yml index 429d3da41e..9a6d1addc6 100644 --- a/manifests/manifest-stage-api.yml +++ b/manifests/manifest-stage-api.yml @@ -25,3 +25,4 @@ applications: LOGIN_REDIRECT_SERVER_URL: https://fecfile-web-api-stage.app.cloud.gov/api/v1/auth/login-redirect LOGOUT_REDIRECT_URL: https://fecfile-web-api-stage.app.cloud.gov/api/v1/auth/logout-redirect LOG_FORMAT: KEY_VALUE + MOCK_OPENFEC: REDIS diff --git a/requirements-test.txt b/requirements-test.txt index c283af60aa..8d4bf66448 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -4,4 +4,4 @@ coverage==6.3 jsonschema==3.2 safety liccheck==0.6.2 -sphinx==4.5.0 +sphinx==7.2.6 diff --git a/requirements.txt b/requirements.txt index 30d82dac26..5eb06a87b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,15 +6,14 @@ coreschema==0.0.4 decorator==5.1.1 dj-database-url==1.0.0 dj-static==0.0.6 -Django==4.2.7 +Django==4.2.7 # when updating this, also update requirements/tox versions in fecgov/mozilla_django_oidc django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 git+https://github.com/fecgov/fecfile-validate@e36a2eff869a6adc97a9e812f028d0b8f18ec3a3#egg=fecfile_validate&subdirectory=fecfile_validate_python -GitPython==3.1.35 +GitPython==3.1.42 gunicorn==20.1.0 -Jinja2==3.1.2 invoke==1.7.3 itypes==1.2.0 MarkupSafe==2.1.1 @@ -24,7 +23,7 @@ pytz==2022.6 retry==0.9.2 static3==0.7.0 django-otp==1.1.4 -git+https://github.com/fecgov/mozilla-django-oidc.git@main#egg=mozilla_django_oidc +git+https://github.com/fecgov/mozilla-django-oidc.git@bbf2a4567cf65e08430e513a1c2b628a30bd9979#egg=mozilla_django_oidc zeep==4.2.1 django-structlog[celery]==7.1.0 rich==13.7.0 diff --git a/sonar-project.properties b/sonar-project.properties index 779e06ea36..67ceb88ef6 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -12,7 +12,7 @@ sonar.sources=django-backend #sonar.sourceEncoding=UTF-8 # Exclude utility script from coverage -sonar.coverage.exclusions=**/json_schema_to_django_model.py,**/migrations/**,**/settings/*.py,**/scripts/generate_transaction_record_fixture.py +sonar.coverage.exclusions=**/json_schema_to_django_model.py,**/migrations/**,**/settings/*.py sonar.python.coverage.reportPaths=coverage-reports/coverage.xml sonar.python.bandit.reportPaths=bandit.out sonar.python.flake8.reportPaths=flake8.out