From ceaf4c8122e753a1cbe4ce9214cf1f8ccbe335e2 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 21 Nov 2022 15:14:20 -0500 Subject: [PATCH 01/65] 498 dev/local testing --- django-backend/fecfiler/scha_transactions/managers.py | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index e10c432241..7c15d3aae2 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -38,6 +38,7 @@ def get_itemization_clause(self): "INDIVIDUAL_NATIONAL_PARTY_CONVENTION_JF_TRANSFER_MEMO", "INDIVIDUAL_RECOUNT_RECEIPT", "JF_TRANSFER_NATIONAL_PARTY_CONVENTION_ACCOUNT", + "JF_TRANSFER_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", "OFFSET_TO_OPERATING_EXPENDITURES", "OTHER_RECEIPT", "TRIBAL_RECEIPT", diff --git a/requirements.txt b/requirements.txt index c00d75a79a..9307863adc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@9f6f85442b2e4ae130c040fc2eb4259a925c8589#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@7685703504c59ca6d1953827d027362d161255e0#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 2467759ab2d0fbb4a3b3ab2a3f1963fba29ca4b7 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 23 Nov 2022 15:12:41 -0500 Subject: [PATCH 02/65] always itemize transactions when agg is below $0 --- django-backend/fecfiler/scha_transactions/managers.py | 1 + django-backend/fecfiler/scha_transactions/test_managers.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index bc8b4b1479..4bf63313c9 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -55,6 +55,7 @@ def get_itemization_clause(self): "INDIVIDUAL_RECEIPT_NON_CONTRIBUTION_ACCOUNT", ] return Case( + When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), When( transaction_type_identifier__in=over_two_hundred_types, then=Q(contribution_aggregate__gt=Value(Decimal(200))), diff --git a/django-backend/fecfiler/scha_transactions/test_managers.py b/django-backend/fecfiler/scha_transactions/test_managers.py index 9e61a221af..4abab65f6a 100644 --- a/django-backend/fecfiler/scha_transactions/test_managers.py +++ b/django-backend/fecfiler/scha_transactions/test_managers.py @@ -30,7 +30,7 @@ def test_aggregate_three(self): def test_aggregate_negative_offset(self): scha_transaction = SchATransaction.objects.get(transaction_id="5") self.assertEquals(scha_transaction.contribution_aggregate, Decimal("-111.11")) - self.assertFalse(scha_transaction.itemized) + self.assertTrue(scha_transaction.itemized) def test_aggregate_itemize_jf_transfer(self): scha_transaction = SchATransaction.objects.get(transaction_id="6") From 73bf4ab8b16a342d45e9868babf10ad53ab6b3cb Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 23 Nov 2022 15:48:31 -0500 Subject: [PATCH 03/65] Added new schema PAC_NATIONAL_PARTY_RECOUNT_ACCOUNT --- django-backend/fecfiler/scha_transactions/managers.py | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index bc8b4b1479..68fd91a4a5 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -45,6 +45,7 @@ def get_itemization_clause(self): "TRIBAL_NATIONAL_PARTY_CONVENTION_JF_TRANSFER_MEMO", "TRIBAL_RECOUNT_RECEIPT", "PAC_NATIONAL_PARTY_CONVENTION_JF_TRANSFER_MEMO", + "PAC_NATIONAL_PARTY_RECOUNT_ACCOUNT", "PAC_RECOUNT_RECEIPT", "PARTY_RECOUNT_RECEIPT", "BUSINESS_LABOR_NON_CONTRIBUTION_ACCOUNT", diff --git a/requirements.txt b/requirements.txt index c00d75a79a..768022ae2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@9f6f85442b2e4ae130c040fc2eb4259a925c8589#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@b59145c812a64fc5f2af94241262e5963ec2c3d4#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 126eea63ea41fc50d9453588ea1550ee9863c670 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 29 Nov 2022 17:59:56 -0500 Subject: [PATCH 04/65] Update validate commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9307863adc..17d47c0d2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@7685703504c59ca6d1953827d027362d161255e0#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@46c5352d35bd217b6bf7817dbbe639fed78538a1#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From abcacb079a9fbd254409a0afd351ae082c51dda3 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 30 Nov 2022 14:24:15 -0500 Subject: [PATCH 05/65] use case for report_code_label --- .../f3x_summaries/report_codes/__init__.py | 0 .../f3x_summaries/report_codes/views.py | 29 +++++++ .../fecfiler/f3x_summaries/serializers.py | 14 ++-- .../fecfiler/f3x_summaries/views.py | 80 +++++++++++-------- 4 files changed, 80 insertions(+), 43 deletions(-) create mode 100644 django-backend/fecfiler/f3x_summaries/report_codes/__init__.py create mode 100644 django-backend/fecfiler/f3x_summaries/report_codes/views.py diff --git a/django-backend/fecfiler/f3x_summaries/report_codes/__init__.py b/django-backend/fecfiler/f3x_summaries/report_codes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django-backend/fecfiler/f3x_summaries/report_codes/views.py b/django-backend/fecfiler/f3x_summaries/report_codes/views.py new file mode 100644 index 0000000000..a6a5272be1 --- /dev/null +++ b/django-backend/fecfiler/f3x_summaries/report_codes/views.py @@ -0,0 +1,29 @@ +from django.db.models import Case, Value, When + +report_code_label_mapping = Case( + When(report_code="Q1", then=Value("APRIL 15 (Q1)")), + When(report_code="Q2", then=Value("JULY 15 (Q2)")), + When(report_code="Q3", then=Value("OCTOBER 15 (Q3)")), + When(report_code="YE", then=Value("JANUARY 31 (YE)")), + When(report_code="TER", then=Value("TERMINATION (TER)")), + When(report_code="MY", then=Value("JULY 31 (MY)")), + When(report_code="12G", then=Value("GENERAL (12G)")), + When(report_code="12P", then=Value("PRIMARY (12P)")), + When(report_code="12R", then=Value("RUNOFF (12R)")), + When(report_code="12S", then=Value("SPECIAL (12S)")), + When(report_code="12C", then=Value("CONVENTION (12C)")), + When(report_code="30G", then=Value("GENERAL (30G)")), + When(report_code="30R", then=Value("RUNOFF (30R)")), + When(report_code="30S", then=Value("SPECIAL (30S)")), + When(report_code="M2", then=Value("FEBRUARY 20 (M2)")), + When(report_code="M3", then=Value("MARCH 30 (M3)")), + When(report_code="M4", then=Value("APRIL 20 (M4)")), + When(report_code="M5", then=Value("MAY 20 (M5)")), + When(report_code="M6", then=Value("JUNE 20 (M6)")), + When(report_code="M7", then=Value("JULY 20 (M7)")), + When(report_code="M8", then=Value("AUGUST 20 (M8)")), + When(report_code="M9", then=Value("SEPTEMBER 20 (M9)")), + When(report_code="M10", then=Value("OCTOBER 20 (M10)")), + When(report_code="M11", then=Value("NOVEMBER 20 (M11)")), + When(report_code="M12", then=Value("DECEMBER 20 (M12)")), +) diff --git a/django-backend/fecfiler/f3x_summaries/serializers.py b/django-backend/fecfiler/f3x_summaries/serializers.py index ebb62fda5d..0660eeddf9 100644 --- a/django-backend/fecfiler/f3x_summaries/serializers.py +++ b/django-backend/fecfiler/f3x_summaries/serializers.py @@ -3,7 +3,7 @@ ModelSerializer, SlugRelatedField, EmailField, - CharField + CharField, ) from fecfiler.committee_accounts.serializers import CommitteeOwnedSerializer from fecfiler.web_services.serializers import ( @@ -48,6 +48,9 @@ class F3XSummarySerializer(CommitteeOwnedSerializer, FecSchemaValidatorSerialize report_status = CharField( read_only=True, ) + report_code_label = CharField( + read_only=True, + ) class Meta: model = F3XSummary @@ -63,13 +66,8 @@ class Meta: "uploadsubmission", "webprintsubmission", ] - ] + ["report_status"] - read_only_fields = [ - "id", - "deleted", - "created", - "updated", - ] + ] + ["report_status", "report_code_label"] + read_only_fields = ["id", "deleted", "created", "updated"] foreign_key_fields = {"report_code": "report_code"} diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 40e4f0451e..8d38322d01 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -6,6 +6,7 @@ from rest_framework.mixins import ListModelMixin from fecfiler.committee_accounts.views import CommitteeOwnedViewSet from .models import F3XSummary, ReportCodeLabel +from .report_codes.views import report_code_label_mapping from fecfiler.scha_transactions.models import SchATransaction from fecfiler.web_services.models import FECSubmissionState, FECStatus from fecfiler.memo_text.models import MemoText @@ -27,49 +28,58 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): in CommitteeOwnedViewSet's implementation of get_queryset() """ - queryset = F3XSummary.objects.select_related("report_code").annotate( - report_status=Case( - When(upload_submission=None, then=Value('In-Progress')), - When( - upload_submission__fecfile_task_state=FECSubmissionState.INITIALIZING, - then=Value('Submitted') + queryset = ( + F3XSummary.objects.select_related("report_code") + .annotate(report_code_label=report_code_label_mapping) + .annotate( + report_status=Case( + When(upload_submission=None, then=Value("In-Progress")), + When( + upload_submission__fecfile_task_state=FECSubmissionState.INITIALIZING, + then=Value("Submitted"), + ), + When( + upload_submission__fecfile_task_state=FECSubmissionState.CREATING_FILE, + then=Value("Submitted"), + ), + When( + upload_submission__fecfile_task_state=FECSubmissionState.SUBMITTING, + then=Value("Submitted"), + ), + When( + upload_submission__fecfile_task_state=FECSubmissionState.FAILED, + then=Value("Failed"), + ), + When( + upload_submission__fec_status=FECStatus.ACCEPTED, + then=Value("Submitted"), + ), + When( + upload_submission__fec_status=FECStatus.PROCESSING, + then=Value("Submitted"), + ), + When( + upload_submission__fec_status=FECStatus.REJECTED, + then=Value("Rejected"), + ), + When(upload_submission__fec_status=None, then=Value("In-Progress")), + When(upload_submission__fec_status="", then=Value("In-Progress")), ), - When( - upload_submission__fecfile_task_state=FECSubmissionState.CREATING_FILE, - then=Value('Submitted') - ), - When( - upload_submission__fecfile_task_state=FECSubmissionState.SUBMITTING, - then=Value('Submitted') - ), - When( - upload_submission__fecfile_task_state=FECSubmissionState.FAILED, - then=Value('Failed') - ), - When( - upload_submission__fec_status=FECStatus.ACCEPTED, - then=Value('Submitted') - ), - When( - upload_submission__fec_status=FECStatus.PROCESSING, - then=Value('Submitted') - ), - When( - upload_submission__fec_status=FECStatus.REJECTED, - then=Value('Rejected') - ), - When(upload_submission__fec_status=None, then=Value('In-Progress')), - When(upload_submission__fec_status='', then=Value('In-Progress')), ) - ).all() + .all() + ) """Join on report code labels""" serializer_class = F3XSummarySerializer permission_classes = [] filter_backends = [filters.OrderingFilter] ordering_fields = [ - "form_type", "report_code__label", "coverage_through_date", - "upload_submission__fec_status", "submission_status" + "form_type", + "report_code__label", + "report_code_label", + "coverage_through_date", + "upload_submission__fec_status", + "submission_status", ] ordering = ["form_type"] From c325829636f5bf6ea4a2152c9e9a3629b64a96d2 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 30 Nov 2022 14:50:22 -0500 Subject: [PATCH 06/65] separates report and report code label table --- .../fixtures/report_code_labels.json | 177 ------------------ .../fecfiler/f3x_summaries/models.py | 9 +- .../fecfiler/f3x_summaries/serializers.py | 9 - .../fecfiler/f3x_summaries/views.py | 3 +- 4 files changed, 2 insertions(+), 196 deletions(-) delete mode 100644 django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json diff --git a/django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json b/django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json deleted file mode 100644 index 53e3943d84..0000000000 --- a/django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json +++ /dev/null @@ -1,177 +0,0 @@ -[ -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "APRIL 15 (Q1)", - "report_code": "Q1" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "JULY 15 (Q2)", - "report_code": "Q2" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "OCTOBER 15 (Q3)", - "report_code": "Q3" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "JANUARY 31 (YE)", - "report_code": "YE" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "TERMINATION (TER)", - "report_code": "TER" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "JULY 31 (MY)", - "report_code": "MY" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "GENERAL (12G)", - "report_code": "12G" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "PRIMARY (12P)", - "report_code": "12P" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "RUNOFF (12R)", - "report_code": "12R" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "SPECIAL (12S)", - "report_code": "12S" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "CONVENTION (12C)", - "report_code": "12C" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "GENERAL (30G)", - "report_code": "30G" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "RUNOFF (30R)", - "report_code": "30R" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "SPECIAL (30S)", - "report_code": "30S" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "FEBRUARY 20 (M2)", - "report_code": "M2" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "MARCH 30 (M3)", - "report_code": "M3" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "APRIL 20 (M4)", - "report_code": "M4" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "MAY 20 (M5)", - "report_code": "M5" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "JUNE 20 (M6)", - "report_code": "M6" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "JULY 20 (M7)", - "report_code": "M7" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "AUGUST 20 (M8)", - "report_code": "M8" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "SEPTEMBER 20 (M9)", - "report_code": "M9" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "OCTOBER 20 (M10)", - "report_code": "M10" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "NOVEMBER 20 (M11)", - "report_code": "M11" - } -}, -{ - "model": "f3x_summaries.reportcodelabel", - "fields": { - "label": "DECEMBER 20 (M12)", - "report_code": "M12" - } -} -] diff --git a/django-backend/fecfiler/f3x_summaries/models.py b/django-backend/fecfiler/f3x_summaries/models.py index 8052ab77ce..f72f175031 100644 --- a/django-backend/fecfiler/f3x_summaries/models.py +++ b/django-backend/fecfiler/f3x_summaries/models.py @@ -48,14 +48,7 @@ class F3XSummary(SoftDeleteModel, CommitteeOwnedModel): ) state = models.TextField(null=True, blank=True) zip = models.TextField(null=True, blank=True) - report_code = models.ForeignKey( - ReportCodeLabel, - models.SET_NULL, - null=True, - blank=True, - to_field="report_code", - db_column="report_code", - ) + report_code = models.TextField(null=True, blank=True) election_code = models.TextField(null=True, blank=True) date_of_election = models.DateField(null=True, blank=True) state_of_election = models.TextField(null=True, blank=True) diff --git a/django-backend/fecfiler/f3x_summaries/serializers.py b/django-backend/fecfiler/f3x_summaries/serializers.py index 0660eeddf9..cfb6416055 100644 --- a/django-backend/fecfiler/f3x_summaries/serializers.py +++ b/django-backend/fecfiler/f3x_summaries/serializers.py @@ -1,7 +1,6 @@ from .models import F3XSummary, ReportCodeLabel from rest_framework.serializers import ( ModelSerializer, - SlugRelatedField, EmailField, CharField, ) @@ -18,13 +17,6 @@ class F3XSummarySerializer(CommitteeOwnedSerializer, FecSchemaValidatorSerializerMixin): schema_name = "F3X" - report_code = SlugRelatedField( - many=False, - required=False, - read_only=False, - slug_field="report_code", - queryset=ReportCodeLabel.objects.all(), - ) confirmation_email_1 = EmailField( max_length=44, min_length=None, @@ -68,7 +60,6 @@ class Meta: ] ] + ["report_status", "report_code_label"] read_only_fields = ["id", "deleted", "created", "updated"] - foreign_key_fields = {"report_code": "report_code"} class ReportCodeLabelSerializer(ModelSerializer): diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 8d38322d01..356c132aed 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -29,8 +29,7 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): """ queryset = ( - F3XSummary.objects.select_related("report_code") - .annotate(report_code_label=report_code_label_mapping) + F3XSummary.objects.annotate(report_code_label=report_code_label_mapping) .annotate( report_status=Case( When(upload_submission=None, then=Value("In-Progress")), From 224e364ac3b4c08c899b4828f802e58214caba6c Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 30 Nov 2022 15:18:58 -0500 Subject: [PATCH 07/65] remove report code label model --- .../migrations/0026_auto_20221130_1459.py | 21 +++++++++++++++++++ .../fecfiler/f3x_summaries/models.py | 12 ----------- .../fecfiler/f3x_summaries/serializers.py | 9 +------- django-backend/fecfiler/f3x_summaries/urls.py | 6 +----- .../fecfiler/f3x_summaries/views.py | 11 ++-------- 5 files changed, 25 insertions(+), 34 deletions(-) create mode 100644 django-backend/fecfiler/f3x_summaries/migrations/0026_auto_20221130_1459.py diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0026_auto_20221130_1459.py b/django-backend/fecfiler/f3x_summaries/migrations/0026_auto_20221130_1459.py new file mode 100644 index 0000000000..03b1d73701 --- /dev/null +++ b/django-backend/fecfiler/f3x_summaries/migrations/0026_auto_20221130_1459.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.12 on 2022-11-30 19:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('f3x_summaries', '0025_auto_20220915_1309'), + ] + + operations = [ + migrations.AlterField( + model_name='f3xsummary', + name='report_code', + field=models.TextField(blank=True, null=True), + ), + migrations.DeleteModel( + name='ReportCodeLabel', + ), + ] diff --git a/django-backend/fecfiler/f3x_summaries/models.py b/django-backend/fecfiler/f3x_summaries/models.py index f72f175031..9ea7ee47c2 100644 --- a/django-backend/fecfiler/f3x_summaries/models.py +++ b/django-backend/fecfiler/f3x_summaries/models.py @@ -8,18 +8,6 @@ logger = logging.getLogger(__name__) -class ReportCodeLabel(models.Model): - label = models.TextField(null=True, blank=True) - report_code = models.TextField(null=True, blank=True, unique=True) - - def __str__(self): - return self.label - - class Meta: - db_table = "report_code_labels" - indexes = [models.Index(fields=["report_code"])] - - class F3XSummary(SoftDeleteModel, CommitteeOwnedModel): """Generated model from json schema""" diff --git a/django-backend/fecfiler/f3x_summaries/serializers.py b/django-backend/fecfiler/f3x_summaries/serializers.py index cfb6416055..398ad2b6b4 100644 --- a/django-backend/fecfiler/f3x_summaries/serializers.py +++ b/django-backend/fecfiler/f3x_summaries/serializers.py @@ -1,6 +1,5 @@ -from .models import F3XSummary, ReportCodeLabel +from .models import F3XSummary from rest_framework.serializers import ( - ModelSerializer, EmailField, CharField, ) @@ -60,9 +59,3 @@ class Meta: ] ] + ["report_status", "report_code_label"] read_only_fields = ["id", "deleted", "created", "updated"] - - -class ReportCodeLabelSerializer(ModelSerializer): - class Meta: - model = ReportCodeLabel - fields = ("label", "report_code") diff --git a/django-backend/fecfiler/f3x_summaries/urls.py b/django-backend/fecfiler/f3x_summaries/urls.py index 9d4cf69267..26f721684d 100644 --- a/django-backend/fecfiler/f3x_summaries/urls.py +++ b/django-backend/fecfiler/f3x_summaries/urls.py @@ -1,14 +1,10 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import F3XSummaryViewSet, ReportCodeLabelViewSet +from .views import F3XSummaryViewSet # Create a router and register our viewsets with it. router = DefaultRouter() router.register(r"f3x-summaries", F3XSummaryViewSet, basename="f3x-summaries") -router.register( - r"report-code-labels", - ReportCodeLabelViewSet, - basename="report-code-labels") # The API URLs are now determined automatically by the router. urlpatterns = [ diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 356c132aed..12a1816d67 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -5,13 +5,13 @@ from rest_framework.viewsets import GenericViewSet from rest_framework.mixins import ListModelMixin from fecfiler.committee_accounts.views import CommitteeOwnedViewSet -from .models import F3XSummary, ReportCodeLabel +from .models import F3XSummary from .report_codes.views import report_code_label_mapping from fecfiler.scha_transactions.models import SchATransaction from fecfiler.web_services.models import FECSubmissionState, FECStatus from fecfiler.memo_text.models import MemoText from fecfiler.web_services.models import DotFEC, UploadSubmission, WebPrintSubmission -from .serializers import F3XSummarySerializer, ReportCodeLabelSerializer +from .serializers import F3XSummarySerializer from django.db.models import Case, Value, When import logging @@ -74,7 +74,6 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): filter_backends = [filters.OrderingFilter] ordering_fields = [ "form_type", - "report_code__label", "report_code_label", "coverage_through_date", "upload_submission__fec_status", @@ -133,12 +132,6 @@ def hard_delete_reports(self, request): return Response(f"Deleted {report_count} Reports") -class ReportCodeLabelViewSet(GenericViewSet, ListModelMixin): - queryset = ReportCodeLabel.objects.all() - serializer_class = ReportCodeLabelSerializer - pagination_class = None - - class ReportViewMixin(GenericViewSet): def get_queryset(self): report_id = ( From 0c403e86f01cafd9bd6ebe6e027c7fcbbcbb3522 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 30 Nov 2022 16:01:08 -0500 Subject: [PATCH 08/65] consolidate report status --- .../fecfiler/f3x_summaries/views.py | 60 +++++++------------ 1 file changed, 23 insertions(+), 37 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 12a1816d67..237218a633 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -12,12 +12,33 @@ from fecfiler.memo_text.models import MemoText from fecfiler.web_services.models import DotFEC, UploadSubmission, WebPrintSubmission from .serializers import F3XSummarySerializer -from django.db.models import Case, Value, When +from django.db.models import Case, Value, When, Q import logging logger = logging.getLogger(__name__) +def get_status_mapping(): + """returns Django Case that determines report status based on upload submission""" + in_progress = Q(upload_submission__fec_status=None) | Q(upload_submission=None) + submitted = Q( + upload_submission__fecfile_task_state__in=[ + FECSubmissionState.INITIALIZING, + FECSubmissionState.CREATING_FILE, + FECSubmissionState.SUBMITTING, + ] + ) | Q(upload_submission__fec_status__in=[FECStatus.ACCEPTED, FECStatus.PROCESSING]) + failed = Q(upload_submission__fecfile_task_state=FECSubmissionState.FAILED) + rejected = Q(upload_submission__fec_status=FECStatus.REJECTED) + + return Case( + When(in_progress, then=Value("In-Progress")), + When(submitted, then=Value("Submitted")), + When(failed, then=Value("Failed")), + When(rejected, then=Value("Rejected")), + ) + + class F3XSummaryViewSet(CommitteeOwnedViewSet): """ This viewset automatically provides `list`, `create`, `retrieve`, @@ -30,44 +51,9 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): queryset = ( F3XSummary.objects.annotate(report_code_label=report_code_label_mapping) - .annotate( - report_status=Case( - When(upload_submission=None, then=Value("In-Progress")), - When( - upload_submission__fecfile_task_state=FECSubmissionState.INITIALIZING, - then=Value("Submitted"), - ), - When( - upload_submission__fecfile_task_state=FECSubmissionState.CREATING_FILE, - then=Value("Submitted"), - ), - When( - upload_submission__fecfile_task_state=FECSubmissionState.SUBMITTING, - then=Value("Submitted"), - ), - When( - upload_submission__fecfile_task_state=FECSubmissionState.FAILED, - then=Value("Failed"), - ), - When( - upload_submission__fec_status=FECStatus.ACCEPTED, - then=Value("Submitted"), - ), - When( - upload_submission__fec_status=FECStatus.PROCESSING, - then=Value("Submitted"), - ), - When( - upload_submission__fec_status=FECStatus.REJECTED, - then=Value("Rejected"), - ), - When(upload_submission__fec_status=None, then=Value("In-Progress")), - When(upload_submission__fec_status="", then=Value("In-Progress")), - ), - ) + .annotate(report_status=get_status_mapping()) .all() ) - """Join on report code labels""" serializer_class = F3XSummarySerializer permission_classes = [] From 77cc544f95ccda440a716ff2c1e641a805c46b9c Mon Sep 17 00:00:00 2001 From: toddlees Date: Thu, 1 Dec 2022 13:21:40 -0500 Subject: [PATCH 09/65] pass through fixture load if it's not there --- .../migrations/0010_auto_20220525_1157.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py index e2cb4e0475..8511313ff1 100644 --- a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py +++ b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py @@ -8,12 +8,11 @@ def forwards_func(apps, schema_editor): original_apps = serializers.python.apps serializers.python.apps = apps - fixture_file = 'fecfiler/f3x_summaries/fixtures/report_code_labels.json' - fixture = open(fixture_file) - objects = serializers.deserialize('json', fixture) - for obj in objects: - obj.save() - fixture.close() + fixture_file = "fecfiler/f3x_summaries/fixtures/report_code_labels.json" + with open(fixture_file) as fixture: + objects = serializers.deserialize("json", fixture) + for obj in objects: + obj.save() serializers.python.apps = original_apps From fa3f1a838481710653b18652998ce26640a14b85 Mon Sep 17 00:00:00 2001 From: toddlees Date: Thu, 1 Dec 2022 13:32:35 -0500 Subject: [PATCH 10/65] try catch fixture --- .../migrations/0010_auto_20220525_1157.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py index 8511313ff1..6e72b6d746 100644 --- a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py +++ b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py @@ -9,10 +9,13 @@ def forwards_func(apps, schema_editor): original_apps = serializers.python.apps serializers.python.apps = apps fixture_file = "fecfiler/f3x_summaries/fixtures/report_code_labels.json" - with open(fixture_file) as fixture: - objects = serializers.deserialize("json", fixture) - for obj in objects: - obj.save() + try: + with open(fixture_file) as fixture: + objects = serializers.deserialize("json", fixture) + for obj in objects: + obj.save() + except: + print("fixture could not be read") serializers.python.apps = original_apps From 3f217d7967f13b87fd191bcf643b776953448ea2 Mon Sep 17 00:00:00 2001 From: toddlees Date: Thu, 1 Dec 2022 13:36:46 -0500 Subject: [PATCH 11/65] update downstream migration --- ...014_patch_report_code_label_MY_20220808_1122.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0014_patch_report_code_label_MY_20220808_1122.py b/django-backend/fecfiler/f3x_summaries/migrations/0014_patch_report_code_label_MY_20220808_1122.py index 22840bd4fe..1218d4aaf3 100644 --- a/django-backend/fecfiler/f3x_summaries/migrations/0014_patch_report_code_label_MY_20220808_1122.py +++ b/django-backend/fecfiler/f3x_summaries/migrations/0014_patch_report_code_label_MY_20220808_1122.py @@ -5,20 +5,22 @@ def patch_report_code_label(apps, schema_editor): report_code_label_model = apps.get_model("f3x_summaries", "ReportCodeLabel") # noqa - errant_object = report_code_label_model.objects.get(report_code="MY") - errant_object.label = "JULY 31 (MY)" - errant_object.save() + try: + errant_object = report_code_label_model.objects.get(report_code="MY") + errant_object.label = "JULY 31 (MY)" + errant_object.save() + except: + print("report code MY not found") class Migration(migrations.Migration): dependencies = [ - ('f3x_summaries', '0013_auto_20220807_0743'), + ("f3x_summaries", "0013_auto_20220807_0743"), ] operations = [ migrations.RunPython( - code=patch_report_code_label, - reverse_code=migrations.RunPython.noop + code=patch_report_code_label, reverse_code=migrations.RunPython.noop ) ] From 5914a7ba6569e9fca8858034dec90835eb2aca5a Mon Sep 17 00:00:00 2001 From: toddlees Date: Thu, 1 Dec 2022 13:44:02 -0500 Subject: [PATCH 12/65] i'm just not gonna fight the migrations right now --- .../fixtures/report_code_labels.json | 177 ++++++++++++++++++ .../migrations/0010_auto_20220525_1157.py | 12 +- ...atch_report_code_label_MY_20220808_1122.py | 9 +- 3 files changed, 185 insertions(+), 13 deletions(-) create mode 100644 django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json diff --git a/django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json b/django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json new file mode 100644 index 0000000000..4e0b30d3e6 --- /dev/null +++ b/django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json @@ -0,0 +1,177 @@ +[ + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "APRIL 15 (Q1)", + "report_code": "Q1" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "JULY 15 (Q2)", + "report_code": "Q2" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "OCTOBER 15 (Q3)", + "report_code": "Q3" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "JANUARY 31 (YE)", + "report_code": "YE" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "TERMINATION (TER)", + "report_code": "TER" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "JULY 31 (MY)", + "report_code": "MY" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "GENERAL (12G)", + "report_code": "12G" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "PRIMARY (12P)", + "report_code": "12P" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "RUNOFF (12R)", + "report_code": "12R" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "SPECIAL (12S)", + "report_code": "12S" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "CONVENTION (12C)", + "report_code": "12C" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "GENERAL (30G)", + "report_code": "30G" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "RUNOFF (30R)", + "report_code": "30R" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "SPECIAL (30S)", + "report_code": "30S" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "FEBRUARY 20 (M2)", + "report_code": "M2" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "MARCH 30 (M3)", + "report_code": "M3" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "APRIL 20 (M4)", + "report_code": "M4" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "MAY 20 (M5)", + "report_code": "M5" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "JUNE 20 (M6)", + "report_code": "M6" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "JULY 20 (M7)", + "report_code": "M7" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "AUGUST 20 (M8)", + "report_code": "M8" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "SEPTEMBER 20 (M9)", + "report_code": "M9" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "OCTOBER 20 (M10)", + "report_code": "M10" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "NOVEMBER 20 (M11)", + "report_code": "M11" + } + }, + { + "model": "f3x_summaries.reportcodelabel", + "fields": { + "label": "DECEMBER 20 (M12)", + "report_code": "M12" + } + } +] \ No newline at end of file diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py index 6e72b6d746..deab5f6eda 100644 --- a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py +++ b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py @@ -9,13 +9,11 @@ def forwards_func(apps, schema_editor): original_apps = serializers.python.apps serializers.python.apps = apps fixture_file = "fecfiler/f3x_summaries/fixtures/report_code_labels.json" - try: - with open(fixture_file) as fixture: - objects = serializers.deserialize("json", fixture) - for obj in objects: - obj.save() - except: - print("fixture could not be read") + fixture = open(fixture_file) + objects = serializers.deserialize("json", fixture) + for obj in objects: + obj.save() + fixture.close() serializers.python.apps = original_apps diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0014_patch_report_code_label_MY_20220808_1122.py b/django-backend/fecfiler/f3x_summaries/migrations/0014_patch_report_code_label_MY_20220808_1122.py index 1218d4aaf3..8c7eaa70b9 100644 --- a/django-backend/fecfiler/f3x_summaries/migrations/0014_patch_report_code_label_MY_20220808_1122.py +++ b/django-backend/fecfiler/f3x_summaries/migrations/0014_patch_report_code_label_MY_20220808_1122.py @@ -5,12 +5,9 @@ def patch_report_code_label(apps, schema_editor): report_code_label_model = apps.get_model("f3x_summaries", "ReportCodeLabel") # noqa - try: - errant_object = report_code_label_model.objects.get(report_code="MY") - errant_object.label = "JULY 31 (MY)" - errant_object.save() - except: - print("report code MY not found") + errant_object = report_code_label_model.objects.get(report_code="MY") + errant_object.label = "JULY 31 (MY)" + errant_object.save() class Migration(migrations.Migration): From dd5b0c4ccdf43e6d50b6d98f55c416d9b9a5c3f7 Mon Sep 17 00:00:00 2001 From: toddlees Date: Thu, 1 Dec 2022 13:47:17 -0500 Subject: [PATCH 13/65] unused import --- django-backend/fecfiler/f3x_summaries/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 237218a633..8534ffdd45 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -3,7 +3,6 @@ from rest_framework.response import Response from rest_framework.decorators import action from rest_framework.viewsets import GenericViewSet -from rest_framework.mixins import ListModelMixin from fecfiler.committee_accounts.views import CommitteeOwnedViewSet from .models import F3XSummary from .report_codes.views import report_code_label_mapping From 9f6176ac0e50af47f71cd31f18636171fc736dca Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 1 Dec 2022 16:27:09 -0500 Subject: [PATCH 14/65] WIP updating to Django 4.1 --- .../authentication/authenticate_login.py | 7 ++-- .../fecfiler/authentication/models.py | 2 +- .../fecfiler/authentication/token.py | 26 +++++------- manifest-dev.yml | 2 +- manifest-prod.yml | 2 +- manifest-stage.yml | 2 +- requirements.txt | 40 +++++++++---------- 7 files changed, 38 insertions(+), 43 deletions(-) diff --git a/django-backend/fecfiler/authentication/authenticate_login.py b/django-backend/fecfiler/authentication/authenticate_login.py index dc83a17b0e..6e4efa20c0 100644 --- a/django-backend/fecfiler/authentication/authenticate_login.py +++ b/django-backend/fecfiler/authentication/authenticate_login.py @@ -6,13 +6,12 @@ ) from rest_framework import permissions, status, views, viewsets from rest_framework.response import Response -from rest_framework_jwt.settings import api_settings -from fecfiler.authentication.token import jwt_payload_handler from .models import Account from datetime import datetime from django.http import JsonResponse from .serializers import AccountSerializer from .permissions import IsAccountOwner +import json import logging from fecfiler.settings import E2E_TESTING_LOGIN @@ -32,11 +31,13 @@ def handle_invalid_login(username): "message": "ID/Password combination invalid.", }, status=401) +def jwt_encode_handler(payload): + return json.dump(payload) + def handle_valid_login(account): update_last_login_time(account) payload = jwt_payload_handler(account) - jwt_encode_handler = api_settings.JWT_ENCODE_HANDLER token = jwt_encode_handler(payload) logger.debug("Successful login: {}".format(account)) diff --git a/django-backend/fecfiler/authentication/models.py b/django-backend/fecfiler/authentication/models.py index e4bd7070dd..7ebd561361 100644 --- a/django-backend/fecfiler/authentication/models.py +++ b/django-backend/fecfiler/authentication/models.py @@ -4,7 +4,7 @@ BaseUserManager, ) from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.utils import timezone from django.core.mail import send_mail diff --git a/django-backend/fecfiler/authentication/token.py b/django-backend/fecfiler/authentication/token.py index cd89074ff9..87dde68487 100644 --- a/django-backend/fecfiler/authentication/token.py +++ b/django-backend/fecfiler/authentication/token.py @@ -3,8 +3,9 @@ from datetime import datetime from fecfiler.settings import SECRET_KEY import jwt -from rest_framework_jwt.compat import get_username_field, get_username -from rest_framework_jwt.settings import api_settings, settings +from rest_framework_simplejwt.models.TokenUser import get_username +from rest_framework_simplejwt.settings import api_settings +from rest_framework_jwt.settings import settings import logging from urllib.parse import urlencode @@ -33,35 +34,28 @@ def generate_username(uuid): def jwt_payload_handler(user): - username_field = get_username_field() username = get_username(user) - warnings.warn( - "The following fields will be removed in the future: " - "`email` and `user_id`. ", - DeprecationWarning, - ) - payload = { "user_id": user.pk, "email": user.email, "username": username, "role": user.role, - "exp": datetime.utcnow() + api_settings.JWT_EXPIRATION_DELTA, + "exp": datetime.utcnow() + api_settings.ACCESS_TOKEN_LIFETIME, } - payload[username_field] = username + payload["username_field"] = username # Include original issued at time for a brand new token, # to allow token refresh - if api_settings.JWT_ALLOW_REFRESH: + if api_settings.ROTATE_REFRESH_TOKENS: payload["orig_iat"] = timegm(datetime.utcnow().utctimetuple()) - if api_settings.JWT_AUDIENCE is not None: - payload["aud"] = api_settings.JWT_AUDIENCE + if api_settings.AUDIENCE is not None: + payload["aud"] = api_settings.AUDIENCE - if api_settings.JWT_ISSUER is not None: - payload["iss"] = api_settings.JWT_ISSUER + if api_settings.ISSUER is not None: + payload["iss"] = api_settings.ISSUER return payload diff --git a/manifest-dev.yml b/manifest-dev.yml index 3d1e1d9805..3ba4a3177b 100644 --- a/manifest-dev.yml +++ b/manifest-dev.yml @@ -18,7 +18,7 @@ applications: DISABLE_COLLECTSTATIC: 1 DJANGO_SETTINGS_MODULE: fecfiler.settings.production CORS_ALLOWED_ORIGINS: https://fecfile-web-app-dev.app.cloud.gov - CSRF_TRUSTED_ORIGINS: fecfile-web-app-dev.app.cloud.gov + CSRF_TRUSTED_ORIGINS: https://fecfile-web-app-dev.app.cloud.gov FFAPI_COOKIE_DOMAIN: app.cloud.gov FRONTEND_URL: fecfile-web-app-dev.app.cloud.gov LOGIN_REDIRECT_CLIENT_URL: https://fecfile-web-app-dev.app.cloud.gov diff --git a/manifest-prod.yml b/manifest-prod.yml index e3829a6843..f24873bd04 100644 --- a/manifest-prod.yml +++ b/manifest-prod.yml @@ -16,7 +16,7 @@ applications: - fecfile-api-creds-prod env: CORS_ALLOWED_ORIGINS: https://fecfile-web-app-prod.app.cloud.gov - CSRF_TRUSTED_ORIGINS: fecfile-web-app-prod.app.cloud.gov + CSRF_TRUSTED_ORIGINS: https://fecfile-web-app-prod.app.cloud.gov DISABLE_COLLECTSTATIC: 1 DJANGO_SETTINGS_MODULE: fecfiler.settings.production FFAPI_COOKIE_DOMAIN: app.cloud.gov diff --git a/manifest-stage.yml b/manifest-stage.yml index 5f92eeadfb..6f546c6345 100644 --- a/manifest-stage.yml +++ b/manifest-stage.yml @@ -16,7 +16,7 @@ applications: - fecfile-api-creds-stage env: CORS_ALLOWED_ORIGINS: https://fecfile-web-app-stage.app.cloud.gov - CSRF_TRUSTED_ORIGINS: fecfile-web-app-stage.app.cloud.gov + CSRF_TRUSTED_ORIGINS: https://fecfile-web-app-stage.app.cloud.gov DISABLE_COLLECTSTATIC: 1 DJANGO_SETTINGS_MODULE: fecfiler.settings.production FFAPI_COOKIE_DOMAIN: app.cloud.gov diff --git a/requirements.txt b/requirements.txt index 17d47c0d2c..cfe38b6502 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,30 +1,30 @@ -boto3==1.24.34 +boto3==1.26.20 celery[redis]==5.2.7 -cfenv==0.5.2 +cfenv==0.5.3 coreapi==2.3.3 coreschema==0.0.4 -decorator==4.3.0 -dj-database-url==0.3.0 +decorator==5.1.1 +dj-database-url==1.0.0 dj-static==0.0.6 -Django==3.2.12 -django-cors-headers==3.11.0 -django-storages==1.12.3 -djangorestframework==3.13.1 -djangorestframework-jwt==1.11.0 -drf-spectacular==0.21.2 +Django==4.1.3 +django-cors-headers==3.13.0 +django-storages==1.13.1 +djangorestframework==3.14.0 +djangorestframework-simplejwt[crypto]==5.2.2 +drf-spectacular==0.24.2 git+https://github.com/fecgov/fecfile-validate@46c5352d35bd217b6bf7817dbbe639fed78538a1#egg=fecfile_validate&subdirectory=fecfile_validate_python -GitPython==3.1.27 -gunicorn==19.10.0 +GitPython==3.1.29 +gunicorn==20.1.0 Jinja2==3.1.2 -invoke==1.7.0 -itypes==1.1.0 +invoke==1.7.3 +itypes==1.2.0 MarkupSafe==2.1.1 openapi-codec==1.3.2 -psycopg2-binary==2.9.3 -PyJWT==1.6.4 -pytz==2021.3 +psycopg2-binary==2.9.5 +PyJWT==2.6.0 +pytz==2022.6 retry==0.9.2 -static3==0.6.1 -django-otp==0.9.3 +static3==0.7.0 +django-otp==1.1.4 git+https://github.com/fecgov/mozilla-django-oidc.git@main#egg=mozilla_django_oidc -zeep==4.1.0 +zeep==4.2.1 From af487310447d349d2e95f9073402fa203b14b572 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 2 Dec 2022 11:13:50 -0500 Subject: [PATCH 15/65] Updated dates to depenency exceptions --- .safety.dependency.ignore | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore index 106c584cd5..940fad95da 100644 --- a/.safety.dependency.ignore +++ b/.safety.dependency.ignore @@ -8,9 +8,9 @@ # Example: # 40104 2022-01-15 # -48040 2022-12-01 # django -48041 2022-12-01 # django -48542 2022-12-01 # pyjwt -49733 2022-12-01 # django -50454 2022-12-01 # django -51340 2022-12-01 # django +48040 2022-12-31 # django +48041 2022-12-31 # django +48542 2022-12-31 # pyjwt +49733 2022-12-31 # django +50454 2022-12-31 # django +51340 2022-12-31 # django From 9f75cee4961ffc086cf53746fd22c268b76af44a Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Fri, 2 Dec 2022 11:43:51 -0500 Subject: [PATCH 16/65] 761 update compose cfg for circle e2e --- Dockerfile-e2e | 16 ++++++++++++++++ Worker_Dockerfile-e2e | 11 +++++++++++ docker-compose.yml | 6 +++--- 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 Dockerfile-e2e create mode 100644 Worker_Dockerfile-e2e diff --git a/Dockerfile-e2e b/Dockerfile-e2e new file mode 100644 index 0000000000..9430dfee9a --- /dev/null +++ b/Dockerfile-e2e @@ -0,0 +1,16 @@ +FROM python:3.8 +ENV PYTHONUNBUFFERED=1 + +RUN mkdir /opt/nxg_fec_e2e +WORKDIR /opt/nxg_fec_e2e +ADD requirements.txt /opt +ADD django-backend /opt/nxg_fec_e2e/ +RUN pip3 install -r /opt/requirements.txt + +RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime + +RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu /opt/nxg_fec_e2e +USER nxgu + +EXPOSE 8080 +ENTRYPOINT ["/bin/sh", "-c", "python wait_for_db.py && python manage.py migrate && python manage.py loaddata fixtures/e2e-test-data.json && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 10 -t 200 --reload"] diff --git a/Worker_Dockerfile-e2e b/Worker_Dockerfile-e2e new file mode 100644 index 0000000000..9ac8ea0bb7 --- /dev/null +++ b/Worker_Dockerfile-e2e @@ -0,0 +1,11 @@ +FROM python:3.8 +ENV PYTHONUNBUFFERED=1 + +RUN mkdir /opt/nxg_fec_e2e +WORKDIR /opt/nxg_fec_e2e +ADD requirements.txt /opt +RUN pip3 install -r /opt/requirements.txt + +RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime + +ENTRYPOINT ["/bin/sh", "-c", "celery -A fecfiler worker --loglevel=info"] diff --git a/docker-compose.yml b/docker-compose.yml index 8a7e370b1b..0f27a33e1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: db: build: context: './db' - dockerfile: ${DOCKERFILE:-Dockerfile} + dockerfile: ${DB_DOCKERFILE:-Dockerfile} args: ENCRYPTION_PASSWORD: ${ENCRYPTION_PASSWORD} image: fecfile-db @@ -28,7 +28,7 @@ services: api-worker: build: context: './' - dockerfile: './Worker_Dockerfile' + dockerfile: '${WORKER_DOCKERFILE:-Worker_Dockerfile}' image: fecfile-celery-worker container_name: fecfile-celery-worker volumes: @@ -63,7 +63,7 @@ services: api: build: context: './' - dockerfile: './Dockerfile' + dockerfile: '${API_DOCKERFILE:-Dockerfile}' image: fecfile-api container_name: fecfile-api volumes: From c2ae5353542d634d5861de75289591e3c316eff0 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Fri, 2 Dec 2022 12:36:09 -0500 Subject: [PATCH 17/65] 761 e2e dockerfile fix --- Worker_Dockerfile-e2e | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Worker_Dockerfile-e2e b/Worker_Dockerfile-e2e index 9ac8ea0bb7..e200c74038 100644 --- a/Worker_Dockerfile-e2e +++ b/Worker_Dockerfile-e2e @@ -4,8 +4,9 @@ ENV PYTHONUNBUFFERED=1 RUN mkdir /opt/nxg_fec_e2e WORKDIR /opt/nxg_fec_e2e ADD requirements.txt /opt +ADD django-backend /opt/nxg_fec_e2e/ RUN pip3 install -r /opt/requirements.txt RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime -ENTRYPOINT ["/bin/sh", "-c", "celery -A fecfiler worker --loglevel=info"] +ENTRYPOINT ["/bin/sh", "-c", "cd django-backend && celery -A fecfiler worker --loglevel=info"] From 39c9fabb8b44b03086ac6cf55101bf7cf05bcd10 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Fri, 2 Dec 2022 12:42:22 -0500 Subject: [PATCH 18/65] added debug --- Worker_Dockerfile-e2e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Worker_Dockerfile-e2e b/Worker_Dockerfile-e2e index e200c74038..43861abb92 100644 --- a/Worker_Dockerfile-e2e +++ b/Worker_Dockerfile-e2e @@ -9,4 +9,4 @@ RUN pip3 install -r /opt/requirements.txt RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime -ENTRYPOINT ["/bin/sh", "-c", "cd django-backend && celery -A fecfiler worker --loglevel=info"] +ENTRYPOINT ["/bin/sh", "-c", "pwd && whoami && ls -la && celery -A fecfiler worker --loglevel=info"] From 21dd166854c62a53aacf7885693b76e38517b964 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 2 Dec 2022 15:47:41 -0500 Subject: [PATCH 19/65] Legacy Login "works," kinda --- .../authentication/authenticate_login.py | 3 ++- django-backend/fecfiler/authentication/token.py | 17 ++++++++++------- django-backend/fecfiler/settings/base.py | 4 +++- django-backend/fecfiler/urls.py | 9 ++++++--- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/django-backend/fecfiler/authentication/authenticate_login.py b/django-backend/fecfiler/authentication/authenticate_login.py index 6e4efa20c0..64a0007371 100644 --- a/django-backend/fecfiler/authentication/authenticate_login.py +++ b/django-backend/fecfiler/authentication/authenticate_login.py @@ -8,6 +8,7 @@ from rest_framework.response import Response from .models import Account from datetime import datetime +from .token import jwt_payload_handler from django.http import JsonResponse from .serializers import AccountSerializer from .permissions import IsAccountOwner @@ -32,7 +33,7 @@ def handle_invalid_login(username): }, status=401) def jwt_encode_handler(payload): - return json.dump(payload) + return json.dumps(payload) def handle_valid_login(account): diff --git a/django-backend/fecfiler/authentication/token.py b/django-backend/fecfiler/authentication/token.py index 87dde68487..5fa64f4329 100644 --- a/django-backend/fecfiler/authentication/token.py +++ b/django-backend/fecfiler/authentication/token.py @@ -3,9 +3,9 @@ from datetime import datetime from fecfiler.settings import SECRET_KEY import jwt -from rest_framework_simplejwt.models.TokenUser import get_username +from rest_framework_simplejwt.models import TokenUser from rest_framework_simplejwt.settings import api_settings -from rest_framework_jwt.settings import settings +#from rest_framework_jwt.settings import settings import logging from urllib.parse import urlencode @@ -13,8 +13,10 @@ def login_dot_gov_logout(request): - client_id = settings.OIDC_RP_CLIENT_ID - post_logout_redirect_uri = settings.LOGOUT_REDIRECT_URL + #client_id = settings.OIDC_RP_CLIENT_ID + client_id = "" + #post_logout_redirect_uri = settings.LOGOUT_REDIRECT_URL + post_logout_redirect_uri = "/" state = request.get_signed_cookie('oidc_state') params = { @@ -23,7 +25,8 @@ def login_dot_gov_logout(request): 'state': state, } query = urlencode(params) - op_logout_url = settings.OIDC_OP_LOGOUT_ENDPOINT + #op_logout_url = settings.OIDC_OP_LOGOUT_ENDPOINT + op_logout_url = "/" redirect_url = '{url}?{query}'.format(url=op_logout_url, query=query) return redirect_url @@ -34,14 +37,14 @@ def generate_username(uuid): def jwt_payload_handler(user): - username = get_username(user) + username = TokenUser.get_username(user) payload = { "user_id": user.pk, "email": user.email, "username": username, "role": user.role, - "exp": datetime.utcnow() + api_settings.ACCESS_TOKEN_LIFETIME, +# "exp": datetime.utcnow() + api_settings.ACCESS_TOKEN_LIFETIME, } payload["username_field"] = username diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index f6a043f439..6ddd62ea93 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -66,6 +66,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "rest_framework", + "rest_framework_simplejwt", "drf_spectacular", "corsheaders", "storages", @@ -230,7 +231,8 @@ "rest_framework.permissions.IsAuthenticatedOrReadOnly", ), "DEFAULT_AUTHENTICATION_CLASSES": ( - "rest_framework_jwt.authentication.JSONWebTokenAuthentication", + 'rest_framework_simplejwt.authentication.JWTAuthentication', +# "rest_framework_jwt.authentication.JSONWebTokenAuthentication", "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.BasicAuthentication", ), diff --git a/django-backend/fecfiler/urls.py b/django-backend/fecfiler/urls.py index c88847c155..ca5c9b8616 100644 --- a/django-backend/fecfiler/urls.py +++ b/django-backend/fecfiler/urls.py @@ -2,7 +2,8 @@ from django.urls import re_path from rest_framework.decorators import api_view from rest_framework.response import Response -from rest_framework_jwt.views import obtain_jwt_token, refresh_jwt_token +#from rest_framework_jwt.views import obtain_jwt_token, refresh_jwt_token +from rest_framework_simplejwt.views import TokenObtainPairView, TokenRefreshView from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView from .authentication.authenticate_login import LogoutView @@ -32,8 +33,10 @@ def test_celery(request): re_path(BASE_V1_URL, include("fecfiler.scha_transactions.urls")), re_path(BASE_V1_URL, include("fecfiler.memo_text.urls")), re_path(r"^api/v1/auth/logout/$", LogoutView.as_view(), name="logout"), - re_path(r"^api/v1/token/obtain$", obtain_jwt_token), - re_path(r"^api/v1/token/refresh$", refresh_jwt_token), +# re_path(r"^api/v1/token/obtain$", obtain_jwt_token), +# re_path(r"^api/v1/token/refresh$", refresh_jwt_token), + re_path(r"^api/v1/token/obtain$", TokenObtainPairView.as_view(), name='token_obtain_pair'), + re_path(r"^api/v1/token/refresh$", TokenRefreshView.as_view(), name='token_refresh'), re_path(BASE_V1_URL, include("fecfiler.triage.urls")), re_path(BASE_V1_URL, include("fecfiler.authentication.urls")), re_path(BASE_V1_URL, include("fecfiler.web_services.urls")), From bd1965dade90e96a7b072b24840a1b76ec619b4a Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Fri, 2 Dec 2022 18:28:33 -0500 Subject: [PATCH 20/65] updated e2e dockerfile --- Dockerfile-e2e | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Dockerfile-e2e b/Dockerfile-e2e index 9430dfee9a..28bba4c68e 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -8,6 +8,16 @@ ADD django-backend /opt/nxg_fec_e2e/ RUN pip3 install -r /opt/requirements.txt RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime +RUN apt-get update +RUN apt-get install -y wget +RUN wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +RUN apt-get install -y ./google-chrome-stable_current_amd64.deb +RUN apt-get install -y \ + software-properties-common \ + npm +RUN npm install npm@latest -g && \ + npm install n -g && \ + n latest RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu /opt/nxg_fec_e2e USER nxgu From 4cf5c00d91a53be426f78bbdd1d76995e0f0a5af Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Fri, 2 Dec 2022 19:04:06 -0500 Subject: [PATCH 21/65] removed changes --- Dockerfile-e2e | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Dockerfile-e2e b/Dockerfile-e2e index 28bba4c68e..9430dfee9a 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -8,16 +8,6 @@ ADD django-backend /opt/nxg_fec_e2e/ RUN pip3 install -r /opt/requirements.txt RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime -RUN apt-get update -RUN apt-get install -y wget -RUN wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb -RUN apt-get install -y ./google-chrome-stable_current_amd64.deb -RUN apt-get install -y \ - software-properties-common \ - npm -RUN npm install npm@latest -g && \ - npm install n -g && \ - n latest RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu /opt/nxg_fec_e2e USER nxgu From bcf5bc1e6199329fcdce295f24c2453d62d2ee25 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 5 Dec 2022 16:54:24 -0500 Subject: [PATCH 22/65] WIP updating to Django 4.1 --- .../fecfiler/authentication/authenticate_login.py | 11 ++++++----- django-backend/fecfiler/authentication/token.py | 9 +++++---- django-backend/fecfiler/committee_accounts/views.py | 5 +++++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/django-backend/fecfiler/authentication/authenticate_login.py b/django-backend/fecfiler/authentication/authenticate_login.py index 64a0007371..e0ad8bd528 100644 --- a/django-backend/fecfiler/authentication/authenticate_login.py +++ b/django-backend/fecfiler/authentication/authenticate_login.py @@ -6,15 +6,17 @@ ) from rest_framework import permissions, status, views, viewsets from rest_framework.response import Response + +from fecfiler.settings import SECRET_KEY, E2E_TESTING_LOGIN from .models import Account from datetime import datetime from .token import jwt_payload_handler +from rest_framework_simplejwt.tokens import RefreshToken +import jwt from django.http import JsonResponse from .serializers import AccountSerializer from .permissions import IsAccountOwner -import json import logging -from fecfiler.settings import E2E_TESTING_LOGIN logger = logging.getLogger(__name__) @@ -33,8 +35,7 @@ def handle_invalid_login(username): }, status=401) def jwt_encode_handler(payload): - return json.dumps(payload) - + return jwt.encode(payload, key=SECRET_KEY, algorithm="HS256") def handle_valid_login(account): update_last_login_time(account) @@ -46,7 +47,7 @@ def handle_valid_login(account): "is_allowed": True, "committee_id": account.cmtee_id, "email": account.email, - "token": token, + "token": str(token), }, status=200, safe=False) diff --git a/django-backend/fecfiler/authentication/token.py b/django-backend/fecfiler/authentication/token.py index 5fa64f4329..f0c931dc29 100644 --- a/django-backend/fecfiler/authentication/token.py +++ b/django-backend/fecfiler/authentication/token.py @@ -1,11 +1,12 @@ import warnings from calendar import timegm -from datetime import datetime +from datetime import datetime, timedelta from fecfiler.settings import SECRET_KEY import jwt from rest_framework_simplejwt.models import TokenUser from rest_framework_simplejwt.settings import api_settings #from rest_framework_jwt.settings import settings +from rest_framework_simplejwt.authentication import JWTAuthentication import logging from urllib.parse import urlencode @@ -39,16 +40,15 @@ def generate_username(uuid): def jwt_payload_handler(user): username = TokenUser.get_username(user) + print("\n\n\nUser:", user, "\n\n\n") payload = { "user_id": user.pk, "email": user.email, "username": username, "role": user.role, -# "exp": datetime.utcnow() + api_settings.ACCESS_TOKEN_LIFETIME, + "exp": datetime.utcnow() + timedelta(minutes=30), } - payload["username_field"] = username - # Include original issued at time for a brand new token, # to allow token refresh if api_settings.ROTATE_REFRESH_TOKENS: @@ -68,6 +68,7 @@ def verify_token(token_received): "verify_exp": True, # Skipping expiration date check "verify_aud": False, } # Skipping audience check + print("\n\n\nPayload received:",payload,"\n\n\n") payload = jwt.decode( token_received, key=SECRET_KEY, algorithms="HS256", options=options ) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 03031d3217..78db8c44af 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -10,6 +10,11 @@ class CommitteeOwnedViewSet(viewsets.ModelViewSet): """ def get_queryset(self): + print( + '\n\n\n', + self.request.__dict__.keys(), + '\n\n\n' + ) committee_id = self.request.user.cmtee_id queryset = super().get_queryset() return queryset.filter(committee_account__committee_id=committee_id) From 33fc0f482ac9491b09f4226d2e929573345f9afc Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 6 Dec 2022 11:34:26 -0500 Subject: [PATCH 23/65] 761 cleanup --- Worker_Dockerfile-e2e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Worker_Dockerfile-e2e b/Worker_Dockerfile-e2e index 43861abb92..e593c50b77 100644 --- a/Worker_Dockerfile-e2e +++ b/Worker_Dockerfile-e2e @@ -9,4 +9,4 @@ RUN pip3 install -r /opt/requirements.txt RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime -ENTRYPOINT ["/bin/sh", "-c", "pwd && whoami && ls -la && celery -A fecfiler worker --loglevel=info"] +ENTRYPOINT ["/bin/sh", "-c", "celery -A fecfiler worker --loglevel=info"] From 3f18c28b39e0abd808d117e35cb8803b279a0776 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 6 Dec 2022 12:58:25 -0500 Subject: [PATCH 24/65] rename db dockerfile env --- django-backend/fecfiler/settings/e2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/settings/e2e.py b/django-backend/fecfiler/settings/e2e.py index 3af58f98ec..cb30fcaa2e 100644 --- a/django-backend/fecfiler/settings/e2e.py +++ b/django-backend/fecfiler/settings/e2e.py @@ -21,7 +21,7 @@ } # E2E Testing Login API -os.environ["DOCKERFILE"] = "Dockerfile-e2e" +os.environ["DB_DOCKERFILE"] = "Dockerfile-e2e" E2E_TESTING_LOGIN = True try: From f6d602cfbf4dac40d410c91163a8988cb91e6b1b Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 7 Dec 2022 10:34:33 -0500 Subject: [PATCH 25/65] Restores login to working order --- .../authentication/authenticate_login.py | 12 ++++++------ .../fecfiler/authentication/permissions.py | 4 ++-- django-backend/fecfiler/authentication/views.py | 8 +++++--- .../fecfiler/committee_accounts/serializers.py | 4 +++- .../fecfiler/committee_accounts/views.py | 9 +++------ django-backend/fecfiler/settings/base.py | 16 +--------------- .../fecfiler/web_services/serializers.py | 4 +++- django-backend/fecfiler/web_services/views.py | 4 +++- 8 files changed, 26 insertions(+), 35 deletions(-) diff --git a/django-backend/fecfiler/authentication/authenticate_login.py b/django-backend/fecfiler/authentication/authenticate_login.py index e0ad8bd528..23f5fa244f 100644 --- a/django-backend/fecfiler/authentication/authenticate_login.py +++ b/django-backend/fecfiler/authentication/authenticate_login.py @@ -37,19 +37,19 @@ def handle_invalid_login(username): def jwt_encode_handler(payload): return jwt.encode(payload, key=SECRET_KEY, algorithm="HS256") -def handle_valid_login(account): +def handle_valid_login(request, account): update_last_login_time(account) - payload = jwt_payload_handler(account) - token = jwt_encode_handler(payload) + request.session['user_id'] = account.pk logger.debug("Successful login: {}".format(account)) return JsonResponse({ "is_allowed": True, "committee_id": account.cmtee_id, "email": account.email, - "token": str(token), }, status=200, safe=False) +def get_logged_in_user(request): + return Account.objects.get(pk=request.session.get('user_id')) @api_view(["POST", "GET"]) @authentication_classes([]) @@ -68,7 +68,7 @@ def authenticate_login(request): ) # Returns an account if the username is found and the password is valid if account: - return handle_valid_login(account) + return handle_valid_login(request, account) else: return handle_invalid_login(username) @@ -79,7 +79,7 @@ class AccountViewSet(viewsets.ModelViewSet): def get_queryset(self): queryset = Account.objects.all() - queryset = queryset.filter(self.request.user) + queryset = queryset.filter(get_logged_in_user(self.request)) serializer_class = AccountSerializer(Account, many=True) return JsonResponse(serializer_class.data, safe=False) diff --git a/django-backend/fecfiler/authentication/permissions.py b/django-backend/fecfiler/authentication/permissions.py index c14e4ad312..fc12e166fc 100644 --- a/django-backend/fecfiler/authentication/permissions.py +++ b/django-backend/fecfiler/authentication/permissions.py @@ -3,6 +3,6 @@ class IsAccountOwner(permissions.BasePermission): def has_object_permission(self, request, view, account): - if request.user: - return account == request.user + if request.session.get('user_id'): + return account.pk == request.session.get('user_id') return False diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index 69f2eefd2b..4d70d5cb6e 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -1,5 +1,6 @@ from django.views.generic import View from django.http import HttpResponseRedirect +from .authenticate_login import get_logged_in_user from fecfiler.settings import ( LOGIN_REDIRECT_CLIENT_URL, @@ -44,20 +45,21 @@ class AccountViewSet(GenericViewSet, ListModelMixin): def get_queryset(self): queryset = Account.objects.annotate( name=Concat('last_name', Value(', '), 'first_name', output_field=CharField()) - ).filter(cmtee_id=self.request.user.cmtee_id).all() + ).filter(cmtee_id=get_logged_in_user(self.request).cmtee_id).all() return queryset class LoginDotGovSuccessSpaRedirect(View): def get(self, request, *args, **kwargs): + user = get_logged_in_user(request) redirect = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) redirect.set_cookie(FFAPI_COMMITTEE_ID_COOKIE_NAME, - request.user.cmtee_id, + user.cmtee_id, domain=FFAPI_COOKIE_DOMAIN, secure=True) redirect.set_cookie(FFAPI_EMAIL_COOKIE_NAME, - request.user.email, + user.email, domain=FFAPI_COOKIE_DOMAIN, secure=True) return redirect diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index 116855a12e..6dfff94252 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -1,4 +1,5 @@ from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.authentication.authenticate_login import get_logged_in_user from rest_framework import serializers, relations import logging @@ -20,7 +21,8 @@ def to_internal_value(self, data): CommitteeAccount as the owner of the object """ request = self.context["request"] - committee_id = request.user.cmtee_id + user = get_logged_in_user(request) + committee_id = user.cmtee_id committee = CommitteeAccount.objects.get(committee_id=committee_id) data["committee_account"] = committee.id return super().to_internal_value(data) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 78db8c44af..8820d857eb 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -1,4 +1,5 @@ from rest_framework import viewsets +from fecfiler.authentication.authenticate_login import get_logged_in_user import logging logger = logging.getLogger(__name__) @@ -10,11 +11,7 @@ class CommitteeOwnedViewSet(viewsets.ModelViewSet): """ def get_queryset(self): - print( - '\n\n\n', - self.request.__dict__.keys(), - '\n\n\n' - ) - committee_id = self.request.user.cmtee_id + user = get_logged_in_user(self.request) + committee_id = user.cmtee_id queryset = super().get_queryset() return queryset.filter(committee_account__committee_id=committee_id) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 6ddd62ea93..d8a11dcc03 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -34,12 +34,6 @@ LOGIN_TIMEOUT_TIME = 15 LOGIN_MAX_RETRY = 3 -OTP_MAX_RETRY = 20 -OTP_DIGIT = 6 -OTP_TIME_EXPIRY = 300 -OTP_TIMEOUT_TIME = 30 -OTP_DISABLE = True -OTP_DEFAULT_PASSCODE = "111111" JWT_PASSWORD_EXPIRY = 1800 # SECURITY WARNING: keep the secret key used in production secret! @@ -66,7 +60,6 @@ "django.contrib.messages", "django.contrib.staticfiles", "rest_framework", - "rest_framework_simplejwt", "drf_spectacular", "corsheaders", "storages", @@ -153,7 +146,7 @@ }, ] -# OIDC settings start +# OpenID Connect settings start AUTHENTICATION_BACKENDS = ( "django.contrib.auth.backends.ModelBackend", "mozilla_django_oidc.auth.OIDCAuthenticationBackend", @@ -231,7 +224,6 @@ "rest_framework.permissions.IsAuthenticatedOrReadOnly", ), "DEFAULT_AUTHENTICATION_CLASSES": ( - 'rest_framework_simplejwt.authentication.JWTAuthentication', # "rest_framework_jwt.authentication.JSONWebTokenAuthentication", "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.BasicAuthentication", @@ -241,12 +233,6 @@ "PAGE_SIZE": 10, } -JWT_AUTH = { - "JWT_ALLOW_REFRESH": True, - "JWT_EXPIRATION_DELTA": datetime.timedelta(seconds=3600), - "JWT_PAYLOAD_HANDLER": "fecfiler.authentication.token.jwt_payload_handler", -} - LOGGING = { "version": 1, "disable_existing_loggers": False, diff --git a/django-backend/fecfiler/web_services/serializers.py b/django-backend/fecfiler/web_services/serializers.py index 5ad86f51fb..9eb068a5f6 100644 --- a/django-backend/fecfiler/web_services/serializers.py +++ b/django-backend/fecfiler/web_services/serializers.py @@ -1,6 +1,7 @@ from rest_framework import serializers from fecfiler.web_services.models import UploadSubmission, WebPrintSubmission from fecfiler.f3x_summaries.models import F3XSummary +from fecfiler.authentication.authenticate_login import get_logged_in_user class ReportIdSerializer(serializers.Serializer): @@ -8,7 +9,8 @@ class ReportIdSerializer(serializers.Serializer): def validate(self, data): request = self.context["request"] - committee_id = request.user.cmtee_id + user = get_logged_in_user(request) + committee_id = user.cmtee_id f3x_summary_result = F3XSummary.objects.filter( id=data["report_id"], committee_account__committee_id=committee_id ) diff --git a/django-backend/fecfiler/web_services/views.py b/django-backend/fecfiler/web_services/views.py index 48121fd74c..2b6d82d47b 100644 --- a/django-backend/fecfiler/web_services/views.py +++ b/django-backend/fecfiler/web_services/views.py @@ -12,6 +12,7 @@ from .renderers import DotFECRenderer from .web_service_storage import get_file from .models import DotFEC, UploadSubmission, WebPrintSubmission +from fecfiler.authentication.authenticate_login import get_logged_in_user import logging @@ -52,7 +53,8 @@ def get_dot_fec(self, request, report_id): """Download the most recent .FEC created for a report Currently only useful for testing purposes """ - committee_id = request.user.cmtee_id + user = get_logged_in_user(request) + committee_id = user.cmtee_id dot_fec_record = DotFEC.objects.filter( report__id=report_id, report__committee_account__committee_id=committee_id ).order_by("-file_name") From ebea29e0664be2ff396c015efed683931393ace7 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 7 Dec 2022 15:16:48 -0500 Subject: [PATCH 26/65] Added annotation for contact transaction count --- .../fecfiler/contacts/serializers.py | 23 +++++++++++-------- django-backend/fecfiler/contacts/views.py | 22 ++++++++++-------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/django-backend/fecfiler/contacts/serializers.py b/django-backend/fecfiler/contacts/serializers.py index b73b9ba17c..12907d12f3 100644 --- a/django-backend/fecfiler/contacts/serializers.py +++ b/django-backend/fecfiler/contacts/serializers.py @@ -2,6 +2,7 @@ from fecfiler.committee_accounts.serializers import CommitteeOwnedSerializer from fecfiler.validation import serializers +from rest_framework.serializers import IntegerField from .models import Contact @@ -12,12 +13,12 @@ class ContactSerializer( serializers.FecSchemaValidatorSerializerMixin, CommitteeOwnedSerializer ): contact_value = dict( - COM="Committee", - IND="Individual", - ORG="Organization", - CAN="Candidate", + COM="Committee", IND="Individual", ORG="Organization", CAN="Candidate", ) + # Contains the number of transactions linked to the contact + transaction_count = IntegerField(required=False) + def get_schema_name(self, data): return f"Contact_{self.contact_value[data.get('type', None)]}" @@ -26,15 +27,19 @@ class Meta: fields = [ f.name for f in Contact._meta.get_fields() - if f.name - not in [ - "deleted", - "schatransaction", - ] + if f.name not in ["deleted", "schatransaction",] ] + fields.append("transaction_count") read_only_fields = [ "uuid", "deleted", "created", "updated", + "transaction_count", ] + + def to_internal_value(self, data): + # Remove the transactin_count because it is an annotated field + # delivered to the front end. + del data["transaction_count"] + return super().to_internal_value(data) diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index a44305c35f..b9fb0c9080 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -3,7 +3,7 @@ from urllib.parse import urlencode import requests -from django.db.models import CharField, Q, Value +from django.db.models import CharField, Q, Value, Count from django.db.models.functions import Concat, Lower from django.http import HttpResponseBadRequest, JsonResponse from fecfiler.committee_accounts.views import CommitteeOwnedViewSet @@ -33,7 +33,11 @@ class ContactViewSet(CommitteeOwnedViewSet): The queryset will be further limmited by the user's committee in CommitteeOwnedViewSet's implementation of get_queryset() """ - queryset = Contact.objects.all().order_by("-created") + queryset = ( + Contact.objects.annotate(transaction_count=Count("schatransaction")) + .all() + .order_by("-created") + ) @action(detail=False) def committee_lookup(self, request): @@ -52,12 +56,7 @@ def committee_lookup(self, request): max_allowed_results, ) - query_params = urlencode( - { - "q": q, - "api_key": FEC_API_KEY, - } - ) + query_params = urlencode({"q": q, "api_key": FEC_API_KEY,}) url = "{url}?{query_params}".format( url=FEC_API_COMMITTEE_LOOKUP_ENDPOINT, query_params=query_params ) @@ -72,8 +71,11 @@ def committee_lookup(self, request): .order_by("-committee_id") ) fec_api_committees = json_results.get("results", []) - fec_api_committees = [fac for fac in fec_api_committees if not any( - fac["id"] == ffc["committee_id"] for ffc in fecfile_committees)] + fec_api_committees = [ + fac + for fac in fec_api_committees + if not any(fac["id"] == ffc["committee_id"] for ffc in fecfile_committees) + ] fec_api_committees = fec_api_committees[:max_fec_results] fecfile_committees = fecfile_committees[:max_fecfile_results] return_value = { From 004f6c5d0091dc507421aa0003420a5abad69bc5 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 7 Dec 2022 16:03:47 -0500 Subject: [PATCH 27/65] Add key existence check to transaction_count field --- django-backend/fecfiler/contacts/serializers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/contacts/serializers.py b/django-backend/fecfiler/contacts/serializers.py index 12907d12f3..f9019a2749 100644 --- a/django-backend/fecfiler/contacts/serializers.py +++ b/django-backend/fecfiler/contacts/serializers.py @@ -41,5 +41,6 @@ class Meta: def to_internal_value(self, data): # Remove the transactin_count because it is an annotated field # delivered to the front end. - del data["transaction_count"] + if "transaction_count" in data: + del data["transaction_count"] return super().to_internal_value(data) From d995e5be0c50a9edef0488a3a6e40b9bb8620ae7 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 7 Dec 2022 16:07:41 -0500 Subject: [PATCH 28/65] Fixed lint issues --- django-backend/fecfiler/contacts/serializers.py | 2 +- django-backend/fecfiler/contacts/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/contacts/serializers.py b/django-backend/fecfiler/contacts/serializers.py index f9019a2749..d7758bf9bd 100644 --- a/django-backend/fecfiler/contacts/serializers.py +++ b/django-backend/fecfiler/contacts/serializers.py @@ -27,7 +27,7 @@ class Meta: fields = [ f.name for f in Contact._meta.get_fields() - if f.name not in ["deleted", "schatransaction",] + if f.name not in ["deleted", "schatransaction"] ] fields.append("transaction_count") read_only_fields = [ diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index b9fb0c9080..831585a23b 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -56,7 +56,7 @@ def committee_lookup(self, request): max_allowed_results, ) - query_params = urlencode({"q": q, "api_key": FEC_API_KEY,}) + query_params = urlencode({"q": q, "api_key": FEC_API_KEY}) url = "{url}?{query_params}".format( url=FEC_API_COMMITTEE_LOOKUP_ENDPOINT, query_params=query_params ) From 8735374f3bd128e37f5e1ca191826c394c189001 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 8 Dec 2022 13:59:34 -0500 Subject: [PATCH 29/65] Login.gov works --- .../fecfiler/authentication/authenticate_login.py | 11 +++++++++-- django-backend/fecfiler/authentication/views.py | 6 +++--- .../fecfiler/committee_accounts/serializers.py | 5 +++++ django-backend/fecfiler/settings/base.py | 2 +- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/authentication/authenticate_login.py b/django-backend/fecfiler/authentication/authenticate_login.py index 23f5fa244f..2722e6db7b 100644 --- a/django-backend/fecfiler/authentication/authenticate_login.py +++ b/django-backend/fecfiler/authentication/authenticate_login.py @@ -10,7 +10,6 @@ from fecfiler.settings import SECRET_KEY, E2E_TESTING_LOGIN from .models import Account from datetime import datetime -from .token import jwt_payload_handler from rest_framework_simplejwt.tokens import RefreshToken import jwt from django.http import JsonResponse @@ -46,10 +45,18 @@ def handle_valid_login(request, account): "is_allowed": True, "committee_id": account.cmtee_id, "email": account.email, + "login_dot_gov": False, }, status=200, safe=False) def get_logged_in_user(request): - return Account.objects.get(pk=request.session.get('user_id')) + user_id = -1 + if (request.user.pk): + user_id = request.user.pk + else: + user_id = request.session.get('user_id') + + if (user_id and user_id >= 0): + return Account.objects.get(pk=user_id) @api_view(["POST", "GET"]) @authentication_classes([]) diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index 4d70d5cb6e..cc6e763400 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -52,14 +52,14 @@ def get_queryset(self): class LoginDotGovSuccessSpaRedirect(View): def get(self, request, *args, **kwargs): - user = get_logged_in_user(request) + request.session['user_id'] = request.user.pk redirect = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) redirect.set_cookie(FFAPI_COMMITTEE_ID_COOKIE_NAME, - user.cmtee_id, + request.user.cmtee_id, domain=FFAPI_COOKIE_DOMAIN, secure=True) redirect.set_cookie(FFAPI_EMAIL_COOKIE_NAME, - user.email, + request.user.email, domain=FFAPI_COOKIE_DOMAIN, secure=True) return redirect diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index 6dfff94252..25d8587900 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -22,6 +22,11 @@ def to_internal_value(self, data): """ request = self.context["request"] user = get_logged_in_user(request) + print( + '\n\n\n', + 'Committee ID:', user.pk, user.cmtee_id, + '\n\n\n' + ) committee_id = user.cmtee_id committee = CommitteeAccount.objects.get(committee_id=committee_id) data["committee_account"] = committee.id diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index d8a11dcc03..ee97444b41 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -198,7 +198,7 @@ OIDC_OP_LOGOUT_URL_METHOD = "fecfiler.authentication.token.login_dot_gov_logout" OIDC_USERNAME_ALGO = "fecfiler.authentication.token.generate_username" -# OIDC settings end +# OpenID Connect settings end LANGUAGE_CODE = "en-us" TIME_ZONE = "America/New_York" From bdbe0c513f74944e6790cef498d148efb750fea0 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 8 Dec 2022 16:27:38 -0500 Subject: [PATCH 30/65] Adds a custom middleware and stops using a get_logged_in_user() method --- .../fecfiler/authentication/authenticate_login.py | 12 +----------- django-backend/fecfiler/authentication/middleware.py | 11 +++++++++++ django-backend/fecfiler/authentication/views.py | 3 +-- .../fecfiler/committee_accounts/serializers.py | 9 +-------- django-backend/fecfiler/committee_accounts/views.py | 4 +--- django-backend/fecfiler/settings/base.py | 1 + django-backend/fecfiler/web_services/serializers.py | 4 +--- django-backend/fecfiler/web_services/views.py | 6 +++--- 8 files changed, 20 insertions(+), 30 deletions(-) create mode 100644 django-backend/fecfiler/authentication/middleware.py diff --git a/django-backend/fecfiler/authentication/authenticate_login.py b/django-backend/fecfiler/authentication/authenticate_login.py index 2722e6db7b..2de1554f93 100644 --- a/django-backend/fecfiler/authentication/authenticate_login.py +++ b/django-backend/fecfiler/authentication/authenticate_login.py @@ -48,16 +48,6 @@ def handle_valid_login(request, account): "login_dot_gov": False, }, status=200, safe=False) -def get_logged_in_user(request): - user_id = -1 - if (request.user.pk): - user_id = request.user.pk - else: - user_id = request.session.get('user_id') - - if (user_id and user_id >= 0): - return Account.objects.get(pk=user_id) - @api_view(["POST", "GET"]) @authentication_classes([]) @permission_classes([]) @@ -86,7 +76,7 @@ class AccountViewSet(viewsets.ModelViewSet): def get_queryset(self): queryset = Account.objects.all() - queryset = queryset.filter(get_logged_in_user(self.request)) + queryset = queryset.filter(self.request.user) serializer_class = AccountSerializer(Account, many=True) return JsonResponse(serializer_class.data, safe=False) diff --git a/django-backend/fecfiler/authentication/middleware.py b/django-backend/fecfiler/authentication/middleware.py new file mode 100644 index 0000000000..6da310c9ed --- /dev/null +++ b/django-backend/fecfiler/authentication/middleware.py @@ -0,0 +1,11 @@ +from .models import Account + +class AuthMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if (request.session and request.session.get('user_id')): + request.user = Account.objects.get(pk=request.session.get('user_id')) + + return self.get_response(request) \ No newline at end of file diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index cc6e763400..f311d2abb7 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -1,6 +1,5 @@ from django.views.generic import View from django.http import HttpResponseRedirect -from .authenticate_login import get_logged_in_user from fecfiler.settings import ( LOGIN_REDIRECT_CLIENT_URL, @@ -45,7 +44,7 @@ class AccountViewSet(GenericViewSet, ListModelMixin): def get_queryset(self): queryset = Account.objects.annotate( name=Concat('last_name', Value(', '), 'first_name', output_field=CharField()) - ).filter(cmtee_id=get_logged_in_user(self.request).cmtee_id).all() + ).filter(cmtee_id=self.request.user.cmtee_id).all() return queryset diff --git a/django-backend/fecfiler/committee_accounts/serializers.py b/django-backend/fecfiler/committee_accounts/serializers.py index 25d8587900..116855a12e 100644 --- a/django-backend/fecfiler/committee_accounts/serializers.py +++ b/django-backend/fecfiler/committee_accounts/serializers.py @@ -1,5 +1,4 @@ from fecfiler.committee_accounts.models import CommitteeAccount -from fecfiler.authentication.authenticate_login import get_logged_in_user from rest_framework import serializers, relations import logging @@ -21,13 +20,7 @@ def to_internal_value(self, data): CommitteeAccount as the owner of the object """ request = self.context["request"] - user = get_logged_in_user(request) - print( - '\n\n\n', - 'Committee ID:', user.pk, user.cmtee_id, - '\n\n\n' - ) - committee_id = user.cmtee_id + committee_id = request.user.cmtee_id committee = CommitteeAccount.objects.get(committee_id=committee_id) data["committee_account"] = committee.id return super().to_internal_value(data) diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 8820d857eb..03031d3217 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -1,5 +1,4 @@ from rest_framework import viewsets -from fecfiler.authentication.authenticate_login import get_logged_in_user import logging logger = logging.getLogger(__name__) @@ -11,7 +10,6 @@ class CommitteeOwnedViewSet(viewsets.ModelViewSet): """ def get_queryset(self): - user = get_logged_in_user(self.request) - committee_id = user.cmtee_id + committee_id = self.request.user.cmtee_id queryset = super().get_queryset() return queryset.filter(committee_account__committee_id=committee_id) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index ee97444b41..2a3ed5e15b 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -86,6 +86,7 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + "fecfiler.authentication.middleware.AuthMiddleware" ] TEMPLATES = [ diff --git a/django-backend/fecfiler/web_services/serializers.py b/django-backend/fecfiler/web_services/serializers.py index 9eb068a5f6..5ad86f51fb 100644 --- a/django-backend/fecfiler/web_services/serializers.py +++ b/django-backend/fecfiler/web_services/serializers.py @@ -1,7 +1,6 @@ from rest_framework import serializers from fecfiler.web_services.models import UploadSubmission, WebPrintSubmission from fecfiler.f3x_summaries.models import F3XSummary -from fecfiler.authentication.authenticate_login import get_logged_in_user class ReportIdSerializer(serializers.Serializer): @@ -9,8 +8,7 @@ class ReportIdSerializer(serializers.Serializer): def validate(self, data): request = self.context["request"] - user = get_logged_in_user(request) - committee_id = user.cmtee_id + committee_id = request.user.cmtee_id f3x_summary_result = F3XSummary.objects.filter( id=data["report_id"], committee_account__committee_id=committee_id ) diff --git a/django-backend/fecfiler/web_services/views.py b/django-backend/fecfiler/web_services/views.py index 2b6d82d47b..c0070afcda 100644 --- a/django-backend/fecfiler/web_services/views.py +++ b/django-backend/fecfiler/web_services/views.py @@ -12,7 +12,6 @@ from .renderers import DotFECRenderer from .web_service_storage import get_file from .models import DotFEC, UploadSubmission, WebPrintSubmission -from fecfiler.authentication.authenticate_login import get_logged_in_user import logging @@ -25,6 +24,8 @@ class WebServicesViewSet(viewsets.ViewSet): retrieve thier statuses and results """ + permission_classes = [] + @action( detail=False, methods=["post"], @@ -53,8 +54,7 @@ def get_dot_fec(self, request, report_id): """Download the most recent .FEC created for a report Currently only useful for testing purposes """ - user = get_logged_in_user(request) - committee_id = user.cmtee_id + committee_id = request.user.cmtee_id dot_fec_record = DotFEC.objects.filter( report__id=report_id, report__committee_account__committee_id=committee_id ).order_by("-file_name") From 2168c29b5e56f143337ebc2225bdcdb4bfce52bb Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 8 Dec 2022 18:15:40 -0500 Subject: [PATCH 31/65] WIP unbreaking things --- .../fecfiler/authentication/middleware.py | 4 ++-- django-backend/fecfiler/settings/base.py | 22 +------------------ django-backend/fecfiler/web_services/views.py | 2 -- 3 files changed, 3 insertions(+), 25 deletions(-) diff --git a/django-backend/fecfiler/authentication/middleware.py b/django-backend/fecfiler/authentication/middleware.py index 6da310c9ed..0c2a5b3498 100644 --- a/django-backend/fecfiler/authentication/middleware.py +++ b/django-backend/fecfiler/authentication/middleware.py @@ -7,5 +7,5 @@ def __init__(self, get_response): def __call__(self, request): if (request.session and request.session.get('user_id')): request.user = Account.objects.get(pk=request.session.get('user_id')) - - return self.get_response(request) \ No newline at end of file + + return self.get_response(request) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 2a3ed5e15b..cd0e8ad84a 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -3,7 +3,6 @@ """ import os -import datetime import dj_database_url import requests @@ -80,8 +79,8 @@ MIDDLEWARE = [ "corsheaders.middleware.CorsMiddleware", "django.middleware.security.SecurityMiddleware", - "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", @@ -129,24 +128,6 @@ DEFAULT_AUTO_FIELD = "django.db.models.AutoField" -# Password validation -# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa - }, - { - "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", - }, - { - "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", - }, -] - # OpenID Connect settings start AUTHENTICATION_BACKENDS = ( "django.contrib.auth.backends.ModelBackend", @@ -225,7 +206,6 @@ "rest_framework.permissions.IsAuthenticatedOrReadOnly", ), "DEFAULT_AUTHENTICATION_CLASSES": ( -# "rest_framework_jwt.authentication.JSONWebTokenAuthentication", "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.BasicAuthentication", ), diff --git a/django-backend/fecfiler/web_services/views.py b/django-backend/fecfiler/web_services/views.py index c0070afcda..48121fd74c 100644 --- a/django-backend/fecfiler/web_services/views.py +++ b/django-backend/fecfiler/web_services/views.py @@ -24,8 +24,6 @@ class WebServicesViewSet(viewsets.ViewSet): retrieve thier statuses and results """ - permission_classes = [] - @action( detail=False, methods=["post"], From bdd83be882c38a8d5500a9b6433ace9064afe781 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 9 Dec 2022 11:05:53 -0500 Subject: [PATCH 32/65] Updates settings --- django-backend/fecfiler/settings/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index cd0e8ad84a..1aa6788f88 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -20,7 +20,6 @@ TEMPLATE_DEBUG = DEBUG CSRF_COOKIE_DOMAIN = env.get_credential("FFAPI_COOKIE_DOMAIN") - CSRF_TRUSTED_ORIGINS = [ env.get_credential("CSRF_TRUSTED_ORIGINS", "http://localhost:4200") ] @@ -33,7 +32,6 @@ LOGIN_TIMEOUT_TIME = 15 LOGIN_MAX_RETRY = 3 -JWT_PASSWORD_EXPIRY = 1800 # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = env.get_credential("DJANGO_SECRET_KEY", get_random_string(50)) @@ -71,8 +69,6 @@ "fecfiler.soft_delete", "fecfiler.validation", "fecfiler.web_services", - "django_otp", - "django_otp.plugins.otp_totp", "fecfiler.triage", ] From 279bb9e51a442c8e3df0c08199dfac3798ca26a6 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 12 Dec 2022 10:05:47 -0500 Subject: [PATCH 33/65] Restores Login.gov to working order at the cost of the legacy login --- django-backend/fecfiler/settings/base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 1aa6788f88..1d6b9c199e 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -75,13 +75,13 @@ MIDDLEWARE = [ "corsheaders.middleware.CorsMiddleware", "django.middleware.security.SecurityMiddleware", - "django.middleware.common.CommonMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - "fecfiler.authentication.middleware.AuthMiddleware" + #"fecfiler.authentication.middleware.AuthMiddleware" ] TEMPLATES = [ @@ -125,10 +125,10 @@ DEFAULT_AUTO_FIELD = "django.db.models.AutoField" # OpenID Connect settings start -AUTHENTICATION_BACKENDS = ( +AUTHENTICATION_BACKENDS = [ "django.contrib.auth.backends.ModelBackend", "mozilla_django_oidc.auth.OIDCAuthenticationBackend", -) +] OIDC_CREATE_USER = True OIDC_STORE_ID_TOKEN = True From 3984a9b619a1a7a12d694ea85b6d2aca19f8e320 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 12 Dec 2022 12:33:08 -0500 Subject: [PATCH 34/65] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 17d47c0d2c..daa4f20a87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@46c5352d35bd217b6bf7817dbbe639fed78538a1#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@5f1327403ef2e94fa4eb5603c01cc0e8b61f2bb7#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 2b0b460a8ffd9c4d5963d7664c62da4835501a19 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 12 Dec 2022 12:37:44 -0500 Subject: [PATCH 35/65] Adds new transactions to itemization rules --- django-backend/fecfiler/scha_transactions/managers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 3378d6089c..07161ae8ce 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -55,6 +55,8 @@ def get_itemization_clause(self): "INDIVIDUAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "TRIBAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "INDIVIDUAL_RECEIPT_NON_CONTRIBUTION_ACCOUNT", + "TRIBAL_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", + "TRIBAL_NATIONAL_PARTY_CONVENTION_ACCOUNT", ] return Case( When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), From bfc17bfac616a38438b0f357e578cc7cda841533 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 12 Dec 2022 14:29:22 -0500 Subject: [PATCH 36/65] 223 initial dev/local testing --- django-backend/fecfiler/contacts/views.py | 1 - django-backend/fecfiler/f3x_summaries/views.py | 1 - django-backend/fecfiler/scha_transactions/views.py | 1 - django-backend/fecfiler/settings/base.py | 5 ++--- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index a44305c35f..1824298c00 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -27,7 +27,6 @@ class ContactViewSet(CommitteeOwnedViewSet): """ serializer_class = ContactSerializer - permission_classes = [] """Note that this ViewSet inherits from CommitteeOwnedViewSet The queryset will be further limmited by the user's committee diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 8534ffdd45..78d569b869 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -55,7 +55,6 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): ) serializer_class = F3XSummarySerializer - permission_classes = [] filter_backends = [filters.OrderingFilter] ordering_fields = [ "form_type", diff --git a/django-backend/fecfiler/scha_transactions/views.py b/django-backend/fecfiler/scha_transactions/views.py index 76261ee3ad..3df4dcffde 100644 --- a/django-backend/fecfiler/scha_transactions/views.py +++ b/django-backend/fecfiler/scha_transactions/views.py @@ -43,7 +43,6 @@ class SchATransactionViewSet(CommitteeOwnedViewSet, ReportViewMixin): ) serializer_class = SchATransactionParentSerializer - permission_classes = [] filter_backends = [filters.OrderingFilter] ordering_fields = [ "id", diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index f6a043f439..38f8df8568 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -54,7 +54,7 @@ # Application definition -SESSION_COOKIE_AGE = 15 * 60 # Inactivity timeout +SESSION_COOKIE_AGE = 30 * 60 # Inactivity timeout SESSION_SAVE_EVERY_REQUEST = True INSTALLED_APPS = [ @@ -227,10 +227,9 @@ REST_FRAMEWORK = { "DEFAULT_PERMISSION_CLASSES": ( - "rest_framework.permissions.IsAuthenticatedOrReadOnly", + "rest_framework.permissions.IsAuthenticated", ), "DEFAULT_AUTHENTICATION_CLASSES": ( - "rest_framework_jwt.authentication.JSONWebTokenAuthentication", "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.BasicAuthentication", ), From f7d215e187d269f2206ec581b536014ee7d9bdef Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 12 Dec 2022 17:33:40 -0500 Subject: [PATCH 37/65] Adds two new transaction types --- django-backend/fecfiler/scha_transactions/managers.py | 2 ++ requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 3378d6089c..b924b9130c 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -55,6 +55,8 @@ def get_itemization_clause(self): "INDIVIDUAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "TRIBAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "INDIVIDUAL_RECEIPT_NON_CONTRIBUTION_ACCOUNT", + "INDIVIDUAL_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", + "PAC_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", ] return Case( When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), diff --git a/requirements.txt b/requirements.txt index 17d47c0d2c..c2ff9f55e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@46c5352d35bd217b6bf7817dbbe639fed78538a1#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@d34264a0f5a31e86787ea90eadf2ba677d83eedc#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From fbd2714bf99a5b7c5acb36581fade3258e73ef15 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 12 Dec 2022 17:40:32 -0500 Subject: [PATCH 38/65] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c2ff9f55e8..7892aac457 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@d34264a0f5a31e86787ea90eadf2ba677d83eedc#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@07bde10c131504ea2d80bbccc951e98dd2a88cd9#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 9895537791ee391c05826cc25da8ef3f61d3b4e9 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 13 Dec 2022 12:55:41 -0500 Subject: [PATCH 39/65] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 17d47c0d2c..26c9671bba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@46c5352d35bd217b6bf7817dbbe639fed78538a1#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@44bc12b36156c7ef8d2a974def91358c0a438ace#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 9c5d2cc2b6a16e3d873b1696a0dfc22de00a6905 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 13 Dec 2022 12:55:57 -0500 Subject: [PATCH 40/65] Adds new transactions to itemization rules --- django-backend/fecfiler/scha_transactions/managers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 3378d6089c..90c8c16d76 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -55,6 +55,8 @@ def get_itemization_clause(self): "INDIVIDUAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "TRIBAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "INDIVIDUAL_RECEIPT_NON_CONTRIBUTION_ACCOUNT", + "PAC_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", + "PARTY_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", ] return Case( When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), From 29365cb75bd6e1ac3f4b7bb009253d99fc41fc59 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 13 Dec 2022 13:47:30 -0500 Subject: [PATCH 41/65] Refactored code in authentication app to consolidate into views.py --- .../authentication/authenticate_login.py | 112 ----------- .../fecfiler/authentication/middleware.py | 11 -- .../fecfiler/authentication/permissions.py | 8 - .../fecfiler/authentication/serializers.py | 3 - .../fecfiler/authentication/token.py | 87 -------- .../fecfiler/authentication/urls.py | 23 ++- .../fecfiler/authentication/views.py | 185 ++++++++++++++++-- django-backend/fecfiler/settings/base.py | 14 +- django-backend/fecfiler/urls.py | 15 +- requirements.txt | 2 +- 10 files changed, 187 insertions(+), 273 deletions(-) delete mode 100644 django-backend/fecfiler/authentication/authenticate_login.py delete mode 100644 django-backend/fecfiler/authentication/middleware.py delete mode 100644 django-backend/fecfiler/authentication/permissions.py delete mode 100644 django-backend/fecfiler/authentication/token.py diff --git a/django-backend/fecfiler/authentication/authenticate_login.py b/django-backend/fecfiler/authentication/authenticate_login.py deleted file mode 100644 index 2de1554f93..0000000000 --- a/django-backend/fecfiler/authentication/authenticate_login.py +++ /dev/null @@ -1,112 +0,0 @@ -from django.contrib.auth import authenticate, logout -from rest_framework.decorators import ( - authentication_classes, - permission_classes, - api_view, -) -from rest_framework import permissions, status, views, viewsets -from rest_framework.response import Response - -from fecfiler.settings import SECRET_KEY, E2E_TESTING_LOGIN -from .models import Account -from datetime import datetime -from rest_framework_simplejwt.tokens import RefreshToken -import jwt -from django.http import JsonResponse -from .serializers import AccountSerializer -from .permissions import IsAccountOwner -import logging - -logger = logging.getLogger(__name__) - - -def update_last_login_time(account): - account.last_login = datetime.now() - account.save() - - -def handle_invalid_login(username): - logger.debug("Unauthorized login attempt: {}".format(username)) - return JsonResponse({ - "is_allowed": False, - "status": "Unauthorized", - "message": "ID/Password combination invalid.", - }, status=401) - -def jwt_encode_handler(payload): - return jwt.encode(payload, key=SECRET_KEY, algorithm="HS256") - -def handle_valid_login(request, account): - update_last_login_time(account) - request.session['user_id'] = account.pk - - logger.debug("Successful login: {}".format(account)) - return JsonResponse({ - "is_allowed": True, - "committee_id": account.cmtee_id, - "email": account.email, - "login_dot_gov": False, - }, status=200, safe=False) - -@api_view(["POST", "GET"]) -@authentication_classes([]) -@permission_classes([]) -def authenticate_login(request): - if request.method == "GET": - return JsonResponse({"endpoint_available": E2E_TESTING_LOGIN}) - - if not E2E_TESTING_LOGIN: - return JsonResponse(status=405, safe=False) - - username = request.data.get("username", None) - password = request.data.get("password", None) - account = authenticate( - request=request, username=username, password=password - ) # Returns an account if the username is found and the password is valid - - if account: - return handle_valid_login(request, account) - else: - return handle_invalid_login(username) - - -class AccountViewSet(viewsets.ModelViewSet): - lookup_field = "username" - serializer_class = AccountSerializer - - def get_queryset(self): - queryset = Account.objects.all() - queryset = queryset.filter(self.request.user) - serializer_class = AccountSerializer(Account, many=True) - return JsonResponse(serializer_class.data, safe=False) - - def get_permissions(self): - if self.request.method in permissions.SAFE_METHODS: - return (permissions.AllowAny(),) - if self.request.method == "POST": - return (permissions.AllowAny(),) - return permissions.IsAuthenticated(), IsAccountOwner() - - def create(self, request): - serializer = self.serializer_class(data=request.data) - - if serializer.is_valid(): - Account.objects.create_user(**serializer.validated_data) - return Response(serializer.validated_data, status=status.HTTP_201_CREATED) - - return Response( - { - "status": "Bad request", - "message": "Account could not be created with received data.", - "details": str(serializer.errors), - }, - status=status.HTTP_400_BAD_REQUEST, - ) - - -class LogoutView(views.APIView): - permission_classes = (permissions.IsAuthenticated,) - - def post(self, request, format=None): - logout(request) - return Response({}, status=status.HTTP_204_NO_CONTENT) diff --git a/django-backend/fecfiler/authentication/middleware.py b/django-backend/fecfiler/authentication/middleware.py deleted file mode 100644 index 0c2a5b3498..0000000000 --- a/django-backend/fecfiler/authentication/middleware.py +++ /dev/null @@ -1,11 +0,0 @@ -from .models import Account - -class AuthMiddleware: - def __init__(self, get_response): - self.get_response = get_response - - def __call__(self, request): - if (request.session and request.session.get('user_id')): - request.user = Account.objects.get(pk=request.session.get('user_id')) - - return self.get_response(request) diff --git a/django-backend/fecfiler/authentication/permissions.py b/django-backend/fecfiler/authentication/permissions.py deleted file mode 100644 index fc12e166fc..0000000000 --- a/django-backend/fecfiler/authentication/permissions.py +++ /dev/null @@ -1,8 +0,0 @@ -from rest_framework import permissions - - -class IsAccountOwner(permissions.BasePermission): - def has_object_permission(self, request, view, account): - if request.session.get('user_id'): - return account.pk == request.session.get('user_id') - return False diff --git a/django-backend/fecfiler/authentication/serializers.py b/django-backend/fecfiler/authentication/serializers.py index ca0030d677..2bb0bdb610 100644 --- a/django-backend/fecfiler/authentication/serializers.py +++ b/django-backend/fecfiler/authentication/serializers.py @@ -4,9 +4,6 @@ class AccountSerializer(serializers.ModelSerializer): - # password = serializers.CharField(write_only=True, required=False) - # confirm_password = serializers.CharField(write_only=True, required=False) - class Meta: model = Account fields = ( diff --git a/django-backend/fecfiler/authentication/token.py b/django-backend/fecfiler/authentication/token.py deleted file mode 100644 index f0c931dc29..0000000000 --- a/django-backend/fecfiler/authentication/token.py +++ /dev/null @@ -1,87 +0,0 @@ -import warnings -from calendar import timegm -from datetime import datetime, timedelta -from fecfiler.settings import SECRET_KEY -import jwt -from rest_framework_simplejwt.models import TokenUser -from rest_framework_simplejwt.settings import api_settings -#from rest_framework_jwt.settings import settings -from rest_framework_simplejwt.authentication import JWTAuthentication -import logging -from urllib.parse import urlencode - -logger = logging.getLogger(__name__) - - -def login_dot_gov_logout(request): - #client_id = settings.OIDC_RP_CLIENT_ID - client_id = "" - #post_logout_redirect_uri = settings.LOGOUT_REDIRECT_URL - post_logout_redirect_uri = "/" - state = request.get_signed_cookie('oidc_state') - - params = { - 'client_id': client_id, - 'post_logout_redirect_uri': post_logout_redirect_uri, - 'state': state, - } - query = urlencode(params) - #op_logout_url = settings.OIDC_OP_LOGOUT_ENDPOINT - op_logout_url = "/" - redirect_url = '{url}?{query}'.format(url=op_logout_url, query=query) - - return redirect_url - - -def generate_username(uuid): - return uuid - - -def jwt_payload_handler(user): - username = TokenUser.get_username(user) - - print("\n\n\nUser:", user, "\n\n\n") - payload = { - "user_id": user.pk, - "email": user.email, - "username": username, - "role": user.role, - "exp": datetime.utcnow() + timedelta(minutes=30), - } - - # Include original issued at time for a brand new token, - # to allow token refresh - if api_settings.ROTATE_REFRESH_TOKENS: - payload["orig_iat"] = timegm(datetime.utcnow().utctimetuple()) - - if api_settings.AUDIENCE is not None: - payload["aud"] = api_settings.AUDIENCE - - if api_settings.ISSUER is not None: - payload["iss"] = api_settings.ISSUER - - return payload - - -def verify_token(token_received): - options = { - "verify_exp": True, # Skipping expiration date check - "verify_aud": False, - } # Skipping audience check - print("\n\n\nPayload received:",payload,"\n\n\n") - payload = jwt.decode( - token_received, key=SECRET_KEY, algorithms="HS256", options=options - ) - return payload - - -def token_verification(request): - try: - token_received = request.headers["token"] - payload = verify_token(token_received) - return payload - except Exception as e: - logger.debug( - "exception occurred while generating token for email option.", str(e) - ) - raise e diff --git a/django-backend/fecfiler/authentication/urls.py b/django-backend/fecfiler/authentication/urls.py index 318c9470d0..8b300a71cf 100644 --- a/django-backend/fecfiler/authentication/urls.py +++ b/django-backend/fecfiler/authentication/urls.py @@ -1,23 +1,28 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import AccountViewSet -from .authenticate_login import authenticate_login, LogoutView from .views import ( LoginDotGovSuccessSpaRedirect, - LoginDotGovSuccessLogoutSpaRedirect + LoginDotGovSuccessLogoutSpaRedirect, + authenticate_login, + LogoutView, ) # Create a router and register our viewsets with it. router = DefaultRouter() -router.register(r"committee/users", AccountViewSet, basename="committee/users") # The API URLs are now determined automatically by the router. urlpatterns = [ path("auth/logout/", LogoutView.as_view(), name="logout"), - path("auth/login-redirect", LoginDotGovSuccessSpaRedirect.as_view(), - name="login-redirect"), - path("auth/logout-redirect", LoginDotGovSuccessLogoutSpaRedirect.as_view(), - name="logout-redirect"), + path( + "auth/login-redirect", + LoginDotGovSuccessSpaRedirect.as_view(), + name="login-redirect", + ), + path( + "auth/logout-redirect", + LoginDotGovSuccessLogoutSpaRedirect.as_view(), + name="logout-redirect", + ), path("", include(router.urls)), - path("user/login/authenticate", authenticate_login, name="login_authenticate") + path("user/login/authenticate", authenticate_login, name="login_authenticate"), ] diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index f311d2abb7..31f8aa5b0a 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -1,20 +1,33 @@ from django.views.generic import View from django.http import HttpResponseRedirect - +from django.contrib.auth import authenticate, logout +from rest_framework.decorators import ( + authentication_classes, + permission_classes, + api_view, +) from fecfiler.settings import ( LOGIN_REDIRECT_CLIENT_URL, FFAPI_COMMITTEE_ID_COOKIE_NAME, FFAPI_EMAIL_COOKIE_NAME, FFAPI_COOKIE_DOMAIN, + OIDC_RP_CLIENT_ID, + LOGOUT_REDIRECT_URL, + OIDC_OP_LOGOUT_ENDPOINT, + E2E_TESTING_LOGIN, ) -from rest_framework import filters +from rest_framework.response import Response +from rest_framework import filters, permissions, views, status from rest_framework.viewsets import GenericViewSet from rest_framework.mixins import ListModelMixin from django.db.models import Value, CharField from django.db.models.functions import Concat from .models import Account from .serializers import AccountSerializer +from urllib.parse import urlencode +from datetime import datetime +from django.http import JsonResponse import logging logger = logging.getLogger(__name__) @@ -28,6 +41,7 @@ class AccountViewSet(GenericViewSet, ListModelMixin): of a user object versus other objects. (IE - having a "cmtee_id" field instead of "committee_id") """ + serializer_class = AccountSerializer filter_backends = [filters.OrderingFilter] ordering_fields = [ @@ -37,40 +51,169 @@ class AccountViewSet(GenericViewSet, ListModelMixin): "email", "role", "is_active", - "name" + "name", ] ordering = ["name"] def get_queryset(self): - queryset = Account.objects.annotate( - name=Concat('last_name', Value(', '), 'first_name', output_field=CharField()) - ).filter(cmtee_id=self.request.user.cmtee_id).all() + queryset = ( + Account.objects.annotate( + name=Concat( + "last_name", Value(", "), "first_name", output_field=CharField() + ) + ) + .filter(cmtee_id=self.request.user.cmtee_id) + .all() + ) return queryset +# class AccountViewSet(viewsets.ModelViewSet): +# lookup_field = "username" +# serializer_class = AccountSerializer + +# def get_queryset(self): +# queryset = Account.objects.all() +# queryset = queryset.filter(self.request.user) +# serializer_class = AccountSerializer(Account, many=True) +# return JsonResponse(serializer_class.data, safe=False) + +# def get_permissions(self): +# if self.request.method in permissions.SAFE_METHODS: +# return (permissions.AllowAny(),) +# if self.request.method == "POST": +# return (permissions.AllowAny(),) +# return permissions.IsAuthenticated(), IsAccountOwner() + +# def create(self, request): +# serializer = self.serializer_class(data=request.data) + +# if serializer.is_valid(): +# Account.objects.create_user(**serializer.validated_data) +# return Response(serializer.validated_data, status=status.HTTP_201_CREATED) + +# return Response( +# { +# "status": "Bad request", +# "message": "Account could not be created with received data.", +# "details": str(serializer.errors), +# }, +# status=status.HTTP_400_BAD_REQUEST, +# ) + + class LoginDotGovSuccessSpaRedirect(View): def get(self, request, *args, **kwargs): - request.session['user_id'] = request.user.pk + request.session["user_id"] = request.user.pk redirect = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) - redirect.set_cookie(FFAPI_COMMITTEE_ID_COOKIE_NAME, - request.user.cmtee_id, - domain=FFAPI_COOKIE_DOMAIN, - secure=True) - redirect.set_cookie(FFAPI_EMAIL_COOKIE_NAME, - request.user.email, - domain=FFAPI_COOKIE_DOMAIN, - secure=True) + redirect.set_cookie( + FFAPI_COMMITTEE_ID_COOKIE_NAME, + request.user.cmtee_id, + domain=FFAPI_COOKIE_DOMAIN, + secure=True, + ) + redirect.set_cookie( + FFAPI_EMAIL_COOKIE_NAME, + request.user.email, + domain=FFAPI_COOKIE_DOMAIN, + secure=True, + ) return redirect class LoginDotGovSuccessLogoutSpaRedirect(View): def get(self, request, *args, **kwargs): response = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) - response.delete_cookie(FFAPI_COMMITTEE_ID_COOKIE_NAME, - domain=FFAPI_COOKIE_DOMAIN) - response.delete_cookie(FFAPI_EMAIL_COOKIE_NAME, - domain=FFAPI_COOKIE_DOMAIN) - response.delete_cookie('csrftoken', - domain=FFAPI_COOKIE_DOMAIN) + response.delete_cookie( + FFAPI_COMMITTEE_ID_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN + ) + response.delete_cookie(FFAPI_EMAIL_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) + response.delete_cookie("csrftoken", domain=FFAPI_COOKIE_DOMAIN) return response + + +class IsAccountOwner(permissions.BasePermission): + def has_object_permission(self, request, view, account): + if request.session.get("user_id"): + return account.pk == request.session.get("user_id") + return False + + +def login_dot_gov_logout(request): + client_id = OIDC_RP_CLIENT_ID + post_logout_redirect_uri = LOGOUT_REDIRECT_URL + state = request.get_signed_cookie("oidc_state") + + params = { + "client_id": client_id, + "post_logout_redirect_uri": post_logout_redirect_uri, + "state": state, + } + query = urlencode(params) + op_logout_url = OIDC_OP_LOGOUT_ENDPOINT + redirect_url = "{url}?{query}".format(url=op_logout_url, query=query) + + return redirect_url + + +def generate_username(uuid): + return uuid + + +def update_last_login_time(account): + account.last_login = datetime.now() + account.save() + + +def handle_invalid_login(username): + logger.debug("Unauthorized login attempt: {}".format(username)) + return JsonResponse( + { + "is_allowed": False, + "status": "Unauthorized", + "message": "ID/Password combination invalid.", + }, + status=401, + ) + + +def handle_valid_login(account): + update_last_login_time(account) + + logger.debug("Successful login: {}".format(account)) + return JsonResponse( + {"is_allowed": True, "committee_id": account.cmtee_id, "email": account.email}, + status=200, + safe=False, + ) + + +@api_view(["POST", "GET"]) +@authentication_classes([]) +@permission_classes([]) +def authenticate_login(request): + if request.method == "GET": + return JsonResponse({"endpoint_available": E2E_TESTING_LOGIN}) + + if not E2E_TESTING_LOGIN: + return JsonResponse(status=405, safe=False) + + username = request.data.get("username", None) + password = request.data.get("password", None) + account = authenticate( + request=request, username=username, password=password + ) # Returns an account if the username is found and the password is valid + + if account: + return handle_valid_login(account) + else: + return handle_invalid_login(username) + + +class LogoutView(views.APIView): + permission_classes = (permissions.IsAuthenticated,) + + def post(self, request, format=None): + logout(request) + return Response({}, status=status.HTTP_204_NO_CONTENT) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 1d6b9c199e..a17a45b508 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -81,7 +81,6 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - #"fecfiler.authentication.middleware.AuthMiddleware" ] TEMPLATES = [ @@ -173,9 +172,9 @@ "acr_values": "http://idmanagement.gov/ns/assurance/ial/1" } -OIDC_OP_LOGOUT_URL_METHOD = "fecfiler.authentication.token.login_dot_gov_logout" +OIDC_OP_LOGOUT_URL_METHOD = "fecfiler.authentication.views.login_dot_gov_logout" -OIDC_USERNAME_ALGO = "fecfiler.authentication.token.generate_username" +OIDC_USERNAME_ALGO = "fecfiler.authentication.views.generate_username" # OpenID Connect settings end LANGUAGE_CODE = "en-us" @@ -217,14 +216,9 @@ "standard": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"}, }, "handlers": { - "default": { - "class": "logging.StreamHandler", - "formatter": "standard", - }, - }, - "loggers": { - "": {"handlers": ["default"], "level": "INFO", "propagate": True}, + "default": {"class": "logging.StreamHandler", "formatter": "standard",}, }, + "loggers": {"": {"handlers": ["default"], "level": "INFO", "propagate": True},}, } """Celery configurations diff --git a/django-backend/fecfiler/urls.py b/django-backend/fecfiler/urls.py index ca5c9b8616..25779b64f9 100644 --- a/django-backend/fecfiler/urls.py +++ b/django-backend/fecfiler/urls.py @@ -2,12 +2,8 @@ from django.urls import re_path from rest_framework.decorators import api_view from rest_framework.response import Response -#from rest_framework_jwt.views import obtain_jwt_token, refresh_jwt_token -from rest_framework_simplejwt.views import TokenObtainPairView, TokenRefreshView from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView -from .authentication.authenticate_login import LogoutView - BASE_V1_URL = r"^api/v1/" @@ -21,7 +17,9 @@ def test_celery(request): urlpatterns = [ re_path(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), - re_path(r"^api/schema/", SpectacularAPIView.as_view(api_version="v1"), name="schema"), + re_path( + r"^api/schema/", SpectacularAPIView.as_view(api_version="v1"), name="schema" + ), re_path( r"^api/docs/", SpectacularSwaggerView.as_view( @@ -32,14 +30,9 @@ def test_celery(request): re_path(BASE_V1_URL, include("fecfiler.f3x_summaries.urls")), re_path(BASE_V1_URL, include("fecfiler.scha_transactions.urls")), re_path(BASE_V1_URL, include("fecfiler.memo_text.urls")), - re_path(r"^api/v1/auth/logout/$", LogoutView.as_view(), name="logout"), -# re_path(r"^api/v1/token/obtain$", obtain_jwt_token), -# re_path(r"^api/v1/token/refresh$", refresh_jwt_token), - re_path(r"^api/v1/token/obtain$", TokenObtainPairView.as_view(), name='token_obtain_pair'), - re_path(r"^api/v1/token/refresh$", TokenRefreshView.as_view(), name='token_refresh'), re_path(BASE_V1_URL, include("fecfiler.triage.urls")), re_path(BASE_V1_URL, include("fecfiler.authentication.urls")), re_path(BASE_V1_URL, include("fecfiler.web_services.urls")), - re_path(r"^oidc/", include('mozilla_django_oidc.urls')), + re_path(r"^oidc/", include("mozilla_django_oidc.urls")), re_path(r"^celery-test/", test_celery), ] diff --git a/requirements.txt b/requirements.txt index cfe38b6502..1170714a29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ itypes==1.2.0 MarkupSafe==2.1.1 openapi-codec==1.3.2 psycopg2-binary==2.9.5 -PyJWT==2.6.0 +# PyJWT==2.6.0 pytz==2022.6 retry==0.9.2 static3==0.7.0 From 959c175a73e286672b3f8898689afc17a162823c Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 13 Dec 2022 14:48:26 -0500 Subject: [PATCH 42/65] Adds new transaction --- django-backend/fecfiler/scha_transactions/managers.py | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 3378d6089c..6392908790 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -55,6 +55,7 @@ def get_itemization_clause(self): "INDIVIDUAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "TRIBAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "INDIVIDUAL_RECEIPT_NON_CONTRIBUTION_ACCOUNT", + "INDIVIDUAL_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", ] return Case( When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), diff --git a/requirements.txt b/requirements.txt index 17d47c0d2c..1716e918d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@46c5352d35bd217b6bf7817dbbe639fed78538a1#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@7ee51a73bda7eb00ca54996810de2c53ef8c9933#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 96e6c9df15771f97a71becc8e9a04e92dbe0d8bd Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 13 Dec 2022 15:24:45 -0500 Subject: [PATCH 43/65] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7892aac457..082c866780 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@07bde10c131504ea2d80bbccc951e98dd2a88cd9#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@fee507bae11f700bc22e04aa707f2f9df00d616d#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 2a6059d451a6c140ed92b49f00b0469dd5c8b007 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 13 Dec 2022 15:41:12 -0500 Subject: [PATCH 44/65] 207 fix no sessionid set on fecfile login --- django-backend/fecfiler/authentication/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index 31f8aa5b0a..c774480d7a 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -1,6 +1,6 @@ from django.views.generic import View from django.http import HttpResponseRedirect -from django.contrib.auth import authenticate, logout +from django.contrib.auth import authenticate, logout, login from rest_framework.decorators import ( authentication_classes, permission_classes, @@ -206,6 +206,7 @@ def authenticate_login(request): ) # Returns an account if the username is found and the password is valid if account: + login(request, account) return handle_valid_login(account) else: return handle_invalid_login(username) From 7877e28343d98a5677f143478cf2c8af9cf0a54b Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 14 Dec 2022 10:34:42 -0500 Subject: [PATCH 45/65] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0a22c3d80..c120dfd2a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@5c00ad6031fbfbe0f6475b7284b9b29309e79a48#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@4fe2dd95ec2985685caea861987a4d5075603280#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 3ba807211496b219e8ec175505d302962bd7e276 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 14 Dec 2022 11:21:14 -0500 Subject: [PATCH 46/65] 207 cr fixes --- .../authentication/test_authentication.py | 29 ----------- .../fecfiler/authentication/test_token.py | 34 ------------- .../fecfiler/authentication/test_views.py | 51 +++++++++++++++++++ .../fecfiler/authentication/views.py | 34 ------------- requirements.txt | 2 - 5 files changed, 51 insertions(+), 99 deletions(-) delete mode 100644 django-backend/fecfiler/authentication/test_authentication.py delete mode 100644 django-backend/fecfiler/authentication/test_token.py create mode 100644 django-backend/fecfiler/authentication/test_views.py diff --git a/django-backend/fecfiler/authentication/test_authentication.py b/django-backend/fecfiler/authentication/test_authentication.py deleted file mode 100644 index 0dd56474ba..0000000000 --- a/django-backend/fecfiler/authentication/test_authentication.py +++ /dev/null @@ -1,29 +0,0 @@ -from django.test import RequestFactory, TestCase -from fecfiler.authentication.models import Account -from fecfiler.authentication.authenticate_login import ( - update_last_login_time, - handle_invalid_login, - handle_valid_login, -) - - -class TestAuthentication(TestCase): - fixtures = ["test_accounts"] - acc = None - - def setUp(self): - self.factory = RequestFactory() - self.acc = Account.objects.get(email="unit_tester@test.com") - - def test_update_login_time(self): - prev_time = self.acc.last_login - update_last_login_time(self.acc) - self.assertNotEqual(self.acc.last_login, prev_time) - - def test_invalid_login(self): - resp = handle_invalid_login("random_username") - self.assertEqual(resp.status_code, 401) - - def test_valid_login(self): - resp = handle_valid_login(self.acc) - self.assertEqual(resp.status_code, 200) diff --git a/django-backend/fecfiler/authentication/test_token.py b/django-backend/fecfiler/authentication/test_token.py deleted file mode 100644 index d513fcadec..0000000000 --- a/django-backend/fecfiler/authentication/test_token.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -from unittest.mock import Mock - - -from django.test import RequestFactory - -from fecfiler.authentication.token import (login_dot_gov_logout, - generate_username) - - -class TestToken(unittest.TestCase): - - def setUp(self): - self.factory = RequestFactory() - - def test_login_dot_gov_logout_happy_path(self): - test_state = 'test_state' - - mock_request = Mock() - mock_request.session = Mock() - mock_request.get_signed_cookie.return_value = test_state - - retval = login_dot_gov_logout(mock_request) - self.maxDiff = None - self.assertEqual(retval, ('https://idp.int.identitysandbox.gov' - '/openid_connect/logout?' - 'client_id=None' - '&post_logout_redirect_uri=None' - '&state=test_state')) - - def test_generate_username(self): - test_uuid = 'test_uuid' - retval = generate_username(test_uuid) - self.assertEqual(test_uuid, retval) diff --git a/django-backend/fecfiler/authentication/test_views.py b/django-backend/fecfiler/authentication/test_views.py new file mode 100644 index 0000000000..95175a470b --- /dev/null +++ b/django-backend/fecfiler/authentication/test_views.py @@ -0,0 +1,51 @@ +from unittest.mock import Mock + +from django.test import RequestFactory, TestCase +from fecfiler.authentication.models import Account +from fecfiler.authentication.views import (handle_invalid_login, + handle_valid_login, + update_last_login_time) + +from .views import generate_username, login_dot_gov_logout + + +class AuthenticationTest(TestCase): + fixtures = ["test_accounts"] + acc = None + + def setUp(self): + self.factory = RequestFactory() + self.acc = Account.objects.get(email="unit_tester@test.com") + + def test_login_dot_gov_logout_happy_path(self): + test_state = 'test_state' + + mock_request = Mock() + mock_request.session = Mock() + mock_request.get_signed_cookie.return_value = test_state + + retval = login_dot_gov_logout(mock_request) + self.maxDiff = None + self.assertEqual(retval, ('https://idp.int.identitysandbox.gov' + '/openid_connect/logout?' + 'client_id=None' + '&post_logout_redirect_uri=None' + '&state=test_state')) + + def test_generate_username(self): + test_uuid = 'test_uuid' + retval = generate_username(test_uuid) + self.assertEqual(test_uuid, retval) + + def test_update_login_time(self): + prev_time = self.acc.last_login + update_last_login_time(self.acc) + self.assertNotEqual(self.acc.last_login, prev_time) + + def test_invalid_login(self): + resp = handle_invalid_login("random_username") + self.assertEqual(resp.status_code, 401) + + def test_valid_login(self): + resp = handle_valid_login(self.acc) + self.assertEqual(resp.status_code, 200) diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index c774480d7a..c01fdb48d9 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -69,40 +69,6 @@ def get_queryset(self): return queryset -# class AccountViewSet(viewsets.ModelViewSet): -# lookup_field = "username" -# serializer_class = AccountSerializer - -# def get_queryset(self): -# queryset = Account.objects.all() -# queryset = queryset.filter(self.request.user) -# serializer_class = AccountSerializer(Account, many=True) -# return JsonResponse(serializer_class.data, safe=False) - -# def get_permissions(self): -# if self.request.method in permissions.SAFE_METHODS: -# return (permissions.AllowAny(),) -# if self.request.method == "POST": -# return (permissions.AllowAny(),) -# return permissions.IsAuthenticated(), IsAccountOwner() - -# def create(self, request): -# serializer = self.serializer_class(data=request.data) - -# if serializer.is_valid(): -# Account.objects.create_user(**serializer.validated_data) -# return Response(serializer.validated_data, status=status.HTTP_201_CREATED) - -# return Response( -# { -# "status": "Bad request", -# "message": "Account could not be created with received data.", -# "details": str(serializer.errors), -# }, -# status=status.HTTP_400_BAD_REQUEST, -# ) - - class LoginDotGovSuccessSpaRedirect(View): def get(self, request, *args, **kwargs): request.session["user_id"] = request.user.pk diff --git a/requirements.txt b/requirements.txt index f1930e7d91..2848451419 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ Django==4.1.3 django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 -djangorestframework-simplejwt[crypto]==5.2.2 drf-spectacular==0.24.2 git+https://github.com/fecgov/fecfile-validate@5c00ad6031fbfbe0f6475b7284b9b29309e79a48#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.29 @@ -21,7 +20,6 @@ itypes==1.2.0 MarkupSafe==2.1.1 openapi-codec==1.3.2 psycopg2-binary==2.9.5 -# PyJWT==2.6.0 pytz==2022.6 retry==0.9.2 static3==0.7.0 From 5f2f44409b8c8766f2c7ddcee6ad3519faeb1dc7 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 14 Dec 2022 16:20:38 -0500 Subject: [PATCH 47/65] Fixed lint errors --- django-backend/fecfiler/authentication/views.py | 7 ------- django-backend/fecfiler/settings/base.py | 4 +--- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index c01fdb48d9..47b646e829 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -99,13 +99,6 @@ def get(self, request, *args, **kwargs): return response -class IsAccountOwner(permissions.BasePermission): - def has_object_permission(self, request, view, account): - if request.session.get("user_id"): - return account.pk == request.session.get("user_id") - return False - - def login_dot_gov_logout(request): client_id = OIDC_RP_CLIENT_ID post_logout_redirect_uri = LOGOUT_REDIRECT_URL diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index b163ae0b2f..e86219a265 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -197,9 +197,7 @@ STATICFILES_LOCATION = "static" REST_FRAMEWORK = { - "DEFAULT_PERMISSION_CLASSES": ( - "rest_framework.permissions.IsAuthenticated", - ), + "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), "DEFAULT_AUTHENTICATION_CLASSES": ( "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.BasicAuthentication", From 46bd655821b08a66e794fbb7a8f8d1e13e93ca10 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 14 Dec 2022 16:24:02 -0500 Subject: [PATCH 48/65] Fixed lint errors --- django-backend/fecfiler/settings/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index e86219a265..26fe6d08b3 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -214,9 +214,9 @@ "standard": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"}, }, "handlers": { - "default": {"class": "logging.StreamHandler", "formatter": "standard",}, + "default": {"class": "logging.StreamHandler", "formatter": "standard"}, }, - "loggers": {"": {"handlers": ["default"], "level": "INFO", "propagate": True},}, + "loggers": {"": {"handlers": ["default"], "level": "INFO", "propagate": True}}, } """Celery configurations From 8b5d4f93df7b0b5dc3c2c2b997ff38ffb1f60dd5 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 14 Dec 2022 17:19:26 -0500 Subject: [PATCH 49/65] Update validate commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c120dfd2a6..f24a5d2540 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@4fe2dd95ec2985685caea861987a4d5075603280#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@e5cbc8d70d654e2b098843f97f2d362abf7fa5ef#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 5535331261e2e8bf470cc1f8ad9736796da60767 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 15 Dec 2022 10:30:48 -0500 Subject: [PATCH 50/65] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0a22c3d80..6ac96636b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.12.3 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@5c00ad6031fbfbe0f6475b7284b9b29309e79a48#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@ca5507de8d2bfdae57fb7922f1b98373250897ab#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==3.1.2 From 9de5a9467e6cc533ce476a52d4f149539f6d7d60 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 15 Dec 2022 10:41:25 -0500 Subject: [PATCH 51/65] Adds new transaction to itemization rules --- django-backend/fecfiler/scha_transactions/managers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 90c9d98ac1..1d839bc48c 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -62,6 +62,7 @@ def get_itemization_clause(self): "TRIBAL_NATIONAL_PARTY_CONVENTION_ACCOUNT", "INDIVIDUAL_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", "PAC_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", + "INDIVIDUAL_NATIONAL_PARTY_CONVENTION_ACCOUNT", ] return Case( When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), From 170e86396de0e94321a070cc7e11fc76bf89675e Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 15 Dec 2022 14:18:42 -0500 Subject: [PATCH 52/65] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 426cc6bcf8..d4aec9f897 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@e5cbc8d70d654e2b098843f97f2d362abf7fa5ef#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@5526d5b13394911ea45ce4e0c033758ff845c52a#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.29 gunicorn==20.1.0 Jinja2==3.1.2 From 9c97ec98aef5b18e01e5f765cc123a32ff3e24bd Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 15 Dec 2022 14:29:26 -0500 Subject: [PATCH 53/65] Adds new transaction to itemization rules --- django-backend/fecfiler/scha_transactions/managers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 90c9d98ac1..dba4f6ed5b 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -62,6 +62,7 @@ def get_itemization_clause(self): "TRIBAL_NATIONAL_PARTY_CONVENTION_ACCOUNT", "INDIVIDUAL_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", "PAC_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", + "PARTY_NATIONAL_PARTY_CONVENTION_ACCOUNT", ] return Case( When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), From a9c99d15d6aca24943a762f1a226cba4abb871fe Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Thu, 15 Dec 2022 15:04:42 -0500 Subject: [PATCH 54/65] Rearanged authentication viewset code --- .../fecfiler/authentication/urls.py | 23 +--- .../fecfiler/authentication/views.py | 123 +++++++++--------- 2 files changed, 64 insertions(+), 82 deletions(-) diff --git a/django-backend/fecfiler/authentication/urls.py b/django-backend/fecfiler/authentication/urls.py index 8b300a71cf..4867b01740 100644 --- a/django-backend/fecfiler/authentication/urls.py +++ b/django-backend/fecfiler/authentication/urls.py @@ -1,28 +1,15 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import ( - LoginDotGovSuccessSpaRedirect, - LoginDotGovSuccessLogoutSpaRedirect, - authenticate_login, - LogoutView, -) +from .views import AccountViewSet, authenticate_login # Create a router and register our viewsets with it. router = DefaultRouter() # The API URLs are now determined automatically by the router. urlpatterns = [ - path("auth/logout/", LogoutView.as_view(), name="logout"), - path( - "auth/login-redirect", - LoginDotGovSuccessSpaRedirect.as_view(), - name="login-redirect", - ), - path( - "auth/logout-redirect", - LoginDotGovSuccessLogoutSpaRedirect.as_view(), - name="logout-redirect", - ), + path("auth/logout/", AccountViewSet.as_view({"post": "logout"})), + path("auth/login-redirect", AccountViewSet.as_view({"get": "login_redirect"}),), + path("auth/logout-redirect", AccountViewSet.as_view({"get": "logout_redirect"}),), path("", include(router.urls)), - path("user/login/authenticate", authenticate_login, name="login_authenticate"), + path("user/login/authenticate", authenticate_login), ] diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index 47b646e829..e193f4b575 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -1,9 +1,9 @@ -from django.views.generic import View from django.http import HttpResponseRedirect from django.contrib.auth import authenticate, logout, login from rest_framework.decorators import ( authentication_classes, permission_classes, + action, api_view, ) from fecfiler.settings import ( @@ -18,7 +18,7 @@ ) from rest_framework.response import Response -from rest_framework import filters, permissions, views, status +from rest_framework import filters, status from rest_framework.viewsets import GenericViewSet from rest_framework.mixins import ListModelMixin from django.db.models import Value, CharField @@ -33,6 +33,55 @@ logger = logging.getLogger(__name__) +def login_dot_gov_logout(request): + client_id = OIDC_RP_CLIENT_ID + post_logout_redirect_uri = LOGOUT_REDIRECT_URL + state = request.get_signed_cookie("oidc_state") + + params = { + "client_id": client_id, + "post_logout_redirect_uri": post_logout_redirect_uri, + "state": state, + } + query = urlencode(params) + op_logout_url = OIDC_OP_LOGOUT_ENDPOINT + redirect_url = "{url}?{query}".format(url=op_logout_url, query=query) + + return redirect_url + + +def generate_username(uuid): + return uuid + + +def update_last_login_time(account): + account.last_login = datetime.now() + account.save() + + +def handle_valid_login(account): + update_last_login_time(account) + + logger.debug("Successful login: {}".format(account)) + return JsonResponse( + {"is_allowed": True, "committee_id": account.cmtee_id, "email": account.email}, + status=200, + safe=False, + ) + + +def handle_invalid_login(username): + logger.debug("Unauthorized login attempt: {}".format(username)) + return JsonResponse( + { + "is_allowed": False, + "status": "Unauthorized", + "message": "ID/Password combination invalid.", + }, + status=401, + ) + + class AccountViewSet(GenericViewSet, ListModelMixin): """ The Account ViewSet allows the user to retrieve the users in the same committee @@ -68,9 +117,8 @@ def get_queryset(self): return queryset - -class LoginDotGovSuccessSpaRedirect(View): - def get(self, request, *args, **kwargs): + @action(detail=True, methods=["get"]) + def login_redirect(self, request): request.session["user_id"] = request.user.pk redirect = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) redirect.set_cookie( @@ -87,9 +135,13 @@ def get(self, request, *args, **kwargs): ) return redirect + @action(detail=True, methods=["post"]) + def logout(self, request): + logout(request) + return Response({}, status=status.HTTP_204_NO_CONTENT) -class LoginDotGovSuccessLogoutSpaRedirect(View): - def get(self, request, *args, **kwargs): + @action(detail=True, methods=["get"]) + def logout_redirect(self, request): response = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) response.delete_cookie( FFAPI_COMMITTEE_ID_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN @@ -99,55 +151,6 @@ def get(self, request, *args, **kwargs): return response -def login_dot_gov_logout(request): - client_id = OIDC_RP_CLIENT_ID - post_logout_redirect_uri = LOGOUT_REDIRECT_URL - state = request.get_signed_cookie("oidc_state") - - params = { - "client_id": client_id, - "post_logout_redirect_uri": post_logout_redirect_uri, - "state": state, - } - query = urlencode(params) - op_logout_url = OIDC_OP_LOGOUT_ENDPOINT - redirect_url = "{url}?{query}".format(url=op_logout_url, query=query) - - return redirect_url - - -def generate_username(uuid): - return uuid - - -def update_last_login_time(account): - account.last_login = datetime.now() - account.save() - - -def handle_invalid_login(username): - logger.debug("Unauthorized login attempt: {}".format(username)) - return JsonResponse( - { - "is_allowed": False, - "status": "Unauthorized", - "message": "ID/Password combination invalid.", - }, - status=401, - ) - - -def handle_valid_login(account): - update_last_login_time(account) - - logger.debug("Successful login: {}".format(account)) - return JsonResponse( - {"is_allowed": True, "committee_id": account.cmtee_id, "email": account.email}, - status=200, - safe=False, - ) - - @api_view(["POST", "GET"]) @authentication_classes([]) @permission_classes([]) @@ -169,11 +172,3 @@ def authenticate_login(request): return handle_valid_login(account) else: return handle_invalid_login(username) - - -class LogoutView(views.APIView): - permission_classes = (permissions.IsAuthenticated,) - - def post(self, request, format=None): - logout(request) - return Response({}, status=status.HTTP_204_NO_CONTENT) From 7080b4788559226378dd27d915d237b1a8b54bdf Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 15 Dec 2022 15:39:01 -0500 Subject: [PATCH 55/65] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 426cc6bcf8..528aaef6c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@e5cbc8d70d654e2b098843f97f2d362abf7fa5ef#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@b2e0a5ba1ec24843744f4024fbe0728bfd54839e#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.29 gunicorn==20.1.0 Jinja2==3.1.2 From 33720bfd9068224d08cf84bc6f8c05c9343deaaa Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 15 Dec 2022 15:49:43 -0500 Subject: [PATCH 56/65] Adds new transaction to itemization rules --- django-backend/fecfiler/scha_transactions/managers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 90c9d98ac1..796c3a23f3 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -62,6 +62,7 @@ def get_itemization_clause(self): "TRIBAL_NATIONAL_PARTY_CONVENTION_ACCOUNT", "INDIVIDUAL_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", "PAC_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", + "TRIBAL_NATIONAL_PARTY_RECOUNT_ACCOUNT", ] return Case( When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), From b5e0b9fcecf01be4a2e7b3c89dce15fccac605cd Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 16 Dec 2022 10:46:48 -0500 Subject: [PATCH 57/65] Fixed permissions for authentication endpoints --- .../fecfiler/authentication/urls.py | 15 +- .../fecfiler/authentication/views.py | 134 +++++++++--------- 2 files changed, 78 insertions(+), 71 deletions(-) diff --git a/django-backend/fecfiler/authentication/urls.py b/django-backend/fecfiler/authentication/urls.py index 4867b01740..655c73cdc9 100644 --- a/django-backend/fecfiler/authentication/urls.py +++ b/django-backend/fecfiler/authentication/urls.py @@ -1,15 +1,20 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import AccountViewSet, authenticate_login +from .views import ( + authenticate_login, + authenticate_logout, + login_redirect, + logout_redirect, +) # Create a router and register our viewsets with it. router = DefaultRouter() # The API URLs are now determined automatically by the router. urlpatterns = [ - path("auth/logout/", AccountViewSet.as_view({"post": "logout"})), - path("auth/login-redirect", AccountViewSet.as_view({"get": "login_redirect"}),), - path("auth/logout-redirect", AccountViewSet.as_view({"get": "logout_redirect"}),), - path("", include(router.urls)), path("user/login/authenticate", authenticate_login), + path("auth/logout", authenticate_logout), + path("auth/login-redirect", login_redirect), + path("auth/logout-redirect", logout_redirect), + path("", include(router.urls)), ] diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index e193f4b575..d82d836cde 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -3,7 +3,6 @@ from rest_framework.decorators import ( authentication_classes, permission_classes, - action, api_view, ) from fecfiler.settings import ( @@ -33,6 +32,42 @@ logger = logging.getLogger(__name__) +class AccountViewSet(GenericViewSet, ListModelMixin): + """ + The Account ViewSet allows the user to retrieve the users in the same committee + + The CommitteeOwnedViewset could not be inherited due to the different structure + of a user object versus other objects. + (IE - having a "cmtee_id" field instead of "committee_id") + """ + + serializer_class = AccountSerializer + filter_backends = [filters.OrderingFilter] + ordering_fields = [ + "last_name", + "first_name", + "id", + "email", + "role", + "is_active", + "name", + ] + ordering = ["name"] + + def get_queryset(self): + queryset = ( + Account.objects.annotate( + name=Concat( + "last_name", Value(", "), "first_name", output_field=CharField() + ) + ) + .filter(cmtee_id=self.request.user.cmtee_id) + .all() + ) + + return queryset + + def login_dot_gov_logout(request): client_id = OIDC_RP_CLIENT_ID post_logout_redirect_uri = LOGOUT_REDIRECT_URL @@ -82,73 +117,32 @@ def handle_invalid_login(username): ) -class AccountViewSet(GenericViewSet, ListModelMixin): - """ - The Account ViewSet allows the user to retrieve the users in the same committee - - The CommitteeOwnedViewset could not be inherited due to the different structure - of a user object versus other objects. - (IE - having a "cmtee_id" field instead of "committee_id") - """ - - serializer_class = AccountSerializer - filter_backends = [filters.OrderingFilter] - ordering_fields = [ - "last_name", - "first_name", - "id", - "email", - "role", - "is_active", - "name", - ] - ordering = ["name"] - - def get_queryset(self): - queryset = ( - Account.objects.annotate( - name=Concat( - "last_name", Value(", "), "first_name", output_field=CharField() - ) - ) - .filter(cmtee_id=self.request.user.cmtee_id) - .all() - ) +@api_view(["GET"]) +def login_redirect(request): + request.session["user_id"] = request.user.pk + redirect = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) + redirect.set_cookie( + FFAPI_COMMITTEE_ID_COOKIE_NAME, + request.user.cmtee_id, + domain=FFAPI_COOKIE_DOMAIN, + secure=True, + ) + redirect.set_cookie( + FFAPI_EMAIL_COOKIE_NAME, + request.user.email, + domain=FFAPI_COOKIE_DOMAIN, + secure=True, + ) + return redirect - return queryset - @action(detail=True, methods=["get"]) - def login_redirect(self, request): - request.session["user_id"] = request.user.pk - redirect = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) - redirect.set_cookie( - FFAPI_COMMITTEE_ID_COOKIE_NAME, - request.user.cmtee_id, - domain=FFAPI_COOKIE_DOMAIN, - secure=True, - ) - redirect.set_cookie( - FFAPI_EMAIL_COOKIE_NAME, - request.user.email, - domain=FFAPI_COOKIE_DOMAIN, - secure=True, - ) - return redirect - - @action(detail=True, methods=["post"]) - def logout(self, request): - logout(request) - return Response({}, status=status.HTTP_204_NO_CONTENT) - - @action(detail=True, methods=["get"]) - def logout_redirect(self, request): - response = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) - response.delete_cookie( - FFAPI_COMMITTEE_ID_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN - ) - response.delete_cookie(FFAPI_EMAIL_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) - response.delete_cookie("csrftoken", domain=FFAPI_COOKIE_DOMAIN) - return response +@api_view(["GET"]) +def logout_redirect(request): + response = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) + response.delete_cookie(FFAPI_COMMITTEE_ID_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) + response.delete_cookie(FFAPI_EMAIL_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) + response.delete_cookie("csrftoken", domain=FFAPI_COOKIE_DOMAIN) + return response @api_view(["POST", "GET"]) @@ -172,3 +166,11 @@ def authenticate_login(request): return handle_valid_login(account) else: return handle_invalid_login(username) + + +@api_view(["GET"]) +@authentication_classes([]) +@permission_classes([]) +def authenticate_logout(request): + logout(request) + return Response({}, status=status.HTTP_204_NO_CONTENT) From ae6aef16b70cc125e64049b78021bbcea3fbc1d8 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 16 Dec 2022 11:06:16 -0500 Subject: [PATCH 58/65] Corrected security hotspots reported by SonarCloud --- django-backend/fecfiler/authentication/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index d82d836cde..45a86f7a5b 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -1,5 +1,6 @@ from django.http import HttpResponseRedirect from django.contrib.auth import authenticate, logout, login +from django.views.decorators.http import require_http_methods from rest_framework.decorators import ( authentication_classes, permission_classes, @@ -118,6 +119,7 @@ def handle_invalid_login(username): @api_view(["GET"]) +@require_http_methods(["GET"]) def login_redirect(request): request.session["user_id"] = request.user.pk redirect = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) @@ -137,6 +139,7 @@ def login_redirect(request): @api_view(["GET"]) +@require_http_methods(["GET"]) def logout_redirect(request): response = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) response.delete_cookie(FFAPI_COMMITTEE_ID_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) @@ -145,9 +148,10 @@ def logout_redirect(request): return response -@api_view(["POST", "GET"]) +@api_view(["GET", "POST"]) @authentication_classes([]) @permission_classes([]) +@require_http_methods(["GET", "POST"]) def authenticate_login(request): if request.method == "GET": return JsonResponse({"endpoint_available": E2E_TESTING_LOGIN}) @@ -171,6 +175,7 @@ def authenticate_login(request): @api_view(["GET"]) @authentication_classes([]) @permission_classes([]) +@require_http_methods(["GET"]) def authenticate_logout(request): logout(request) return Response({}, status=status.HTTP_204_NO_CONTENT) From aa8e9708ceccce802f959c8881a9679a0677af6c Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Fri, 16 Dec 2022 11:31:16 -0500 Subject: [PATCH 59/65] 527 initial dev --- django-backend/fecfiler/scha_transactions/managers.py | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 90c9d98ac1..46ac833e1a 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -56,6 +56,7 @@ def get_itemization_clause(self): "TRIBAL_NATIONAL_PARTY_RECOUNT_JF_TRANSFER_MEMO", "INDIVIDUAL_RECEIPT_NON_CONTRIBUTION_ACCOUNT", "INDIVIDUAL_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", + "PAC_NATIONAL_PARTY_CONVENTION_ACCOUNT", "PAC_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", "PARTY_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", "TRIBAL_NATIONAL_PARTY_HEADQUARTERS_ACCOUNT", diff --git a/requirements.txt b/requirements.txt index 426cc6bcf8..57616cdf3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@e5cbc8d70d654e2b098843f97f2d362abf7fa5ef#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@cb6b39d5f07fe4bcc5581e9f66842cabc04c2133#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.29 gunicorn==20.1.0 Jinja2==3.1.2 From 0e01dc1fcb15c34c9ab94f5be8c30fdd2dcbfe7e Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 16 Dec 2022 11:54:10 -0500 Subject: [PATCH 60/65] Adds the Individual National Party Recount Account receipt --- django-backend/fecfiler/scha_transactions/managers.py | 1 + requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/scha_transactions/managers.py b/django-backend/fecfiler/scha_transactions/managers.py index 90c9d98ac1..1a3fda4f43 100644 --- a/django-backend/fecfiler/scha_transactions/managers.py +++ b/django-backend/fecfiler/scha_transactions/managers.py @@ -62,6 +62,7 @@ def get_itemization_clause(self): "TRIBAL_NATIONAL_PARTY_CONVENTION_ACCOUNT", "INDIVIDUAL_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", "PAC_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", + "INDIVIDUAL_NATIONAL_PARTY_RECOUNT_ACCOUNT", ] return Case( When(contribution_aggregate__lt=Value(Decimal(0)), then=Value(True)), diff --git a/requirements.txt b/requirements.txt index 426cc6bcf8..ee1d9232ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@e5cbc8d70d654e2b098843f97f2d362abf7fa5ef#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@0d5165bec29aa8237a5c6a9cdd51dc3f16024f70#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.29 gunicorn==20.1.0 Jinja2==3.1.2 From 1e6bb5cb50e09ad18eb0cdc40e163e86ea1f2a8d Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 19 Dec 2022 15:05:54 -0500 Subject: [PATCH 61/65] initial dev --- django-backend/fecfiler/authentication/views.py | 12 +++++++++--- django-backend/fecfiler/settings/base.py | 1 + django-backend/fecfiler/utils.py | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 django-backend/fecfiler/utils.py diff --git a/django-backend/fecfiler/authentication/views.py b/django-backend/fecfiler/authentication/views.py index 45a86f7a5b..1ef3c82936 100644 --- a/django-backend/fecfiler/authentication/views.py +++ b/django-backend/fecfiler/authentication/views.py @@ -118,6 +118,13 @@ def handle_invalid_login(username): ) +def delete_user_logged_in_cookies(response): + response.delete_cookie(FFAPI_COMMITTEE_ID_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) + response.delete_cookie(FFAPI_EMAIL_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) + response.delete_cookie("oidc_state", domain=FFAPI_COOKIE_DOMAIN) + response.delete_cookie("csrftoken", domain=FFAPI_COOKIE_DOMAIN) + + @api_view(["GET"]) @require_http_methods(["GET"]) def login_redirect(request): @@ -140,11 +147,10 @@ def login_redirect(request): @api_view(["GET"]) @require_http_methods(["GET"]) +@permission_classes([]) def logout_redirect(request): response = HttpResponseRedirect(LOGIN_REDIRECT_CLIENT_URL) - response.delete_cookie(FFAPI_COMMITTEE_ID_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) - response.delete_cookie(FFAPI_EMAIL_COOKIE_NAME, domain=FFAPI_COOKIE_DOMAIN) - response.delete_cookie("csrftoken", domain=FFAPI_COOKIE_DOMAIN) + delete_user_logged_in_cookies(response) return response diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 26fe6d08b3..dc03f16f87 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -205,6 +205,7 @@ "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", "PAGE_SIZE": 10, + 'EXCEPTION_HANDLER': 'fecfiler.utils.custom_exception_handler', } LOGGING = { diff --git a/django-backend/fecfiler/utils.py b/django-backend/fecfiler/utils.py new file mode 100644 index 0000000000..1d93d2df44 --- /dev/null +++ b/django-backend/fecfiler/utils.py @@ -0,0 +1,14 @@ +from fecfiler.authentication.views import delete_user_logged_in_cookies +from rest_framework.views import exception_handler + + +def custom_exception_handler(exc, context): + # Call REST framework's default exception handler first, + # to get the standard error response. + response = exception_handler(exc, context) + + # Now add the HTTP status code to the response. + if response is not None and response.status_code == 403: + delete_user_logged_in_cookies(response) + + return response From bd8a1e5e2bd27fb62204532d23c4d1de99d71936 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 19 Dec 2022 15:14:42 -0500 Subject: [PATCH 62/65] testing --- django-backend/fecfiler/settings/base.py | 2 +- tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index dc03f16f87..8d4fa7a874 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -45,7 +45,7 @@ # Application definition -SESSION_COOKIE_AGE = 30 * 60 # Inactivity timeout +SESSION_COOKIE_AGE = 30 # Inactivity timeout SESSION_SAVE_EVERY_REQUEST = True INSTALLED_APPS = [ diff --git a/tasks.py b/tasks.py index b950dbda14..d266a2a187 100644 --- a/tasks.py +++ b/tasks.py @@ -45,7 +45,7 @@ def _detect_space(repo, branch=None): DEPLOY_RULES = ( ("prod", lambda _, branch: branch == "main"), ("stage", lambda _, branch: branch.startswith("release")), - ("dev", lambda _, branch: branch == "develop"), + ("dev", lambda _, branch: branch == "login-dot-gov-logout-is-authenticated-fix"), ) From 40966991e0a580ab988fc238ab7ab65e6210821e Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 19 Dec 2022 17:18:09 -0500 Subject: [PATCH 63/65] added tests --- .../fecfiler/authentication/test_views.py | 18 ++++++++++++++++-- django-backend/fecfiler/contacts/test_views.py | 8 ++++++++ django-backend/fecfiler/settings/base.py | 2 +- tasks.py | 2 +- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/authentication/test_views.py b/django-backend/fecfiler/authentication/test_views.py index 95175a470b..431f1452fd 100644 --- a/django-backend/fecfiler/authentication/test_views.py +++ b/django-backend/fecfiler/authentication/test_views.py @@ -1,12 +1,14 @@ from unittest.mock import Mock - from django.test import RequestFactory, TestCase from fecfiler.authentication.models import Account from fecfiler.authentication.views import (handle_invalid_login, handle_valid_login, update_last_login_time) -from .views import generate_username, login_dot_gov_logout +from .views import (generate_username, + login_dot_gov_logout, + login_redirect, + logout_redirect) class AuthenticationTest(TestCase): @@ -14,6 +16,7 @@ class AuthenticationTest(TestCase): acc = None def setUp(self): + self.user = Account.objects.get(cmtee_id="C12345678") self.factory = RequestFactory() self.acc = Account.objects.get(email="unit_tester@test.com") @@ -32,6 +35,17 @@ def test_login_dot_gov_logout_happy_path(self): '&post_logout_redirect_uri=None' '&state=test_state')) + def test_login_dot_gov_login_redirect(self): + request = self.factory.get("/") + request.user = self.user + request.session = {} + retval = login_redirect(request) + self.assertEqual(retval.status_code, 302) + + def test_login_dot_gov_logout_redirect(self): + retval = logout_redirect(self.factory.get('/')) + self.assertEqual(retval.status_code, 302) + def test_generate_username(self): test_uuid = 'test_uuid' retval = generate_username(test_uuid) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index a48c18f589..c78b73611b 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -33,6 +33,14 @@ def setUp(self): self.user = Account.objects.get(cmtee_id="C12345678") self.factory = RequestFactory() + def test_committee_lookup_no_auth(self): + self.assertEqual(True, True) + request = self.factory.get("/api/v1/contacts/committee_lookup") + + response = ContactViewSet.as_view({"get": "committee_lookup"})(request) + + self.assertEqual(response.status_code, 403) + @mock.patch("requests.get", side_effect=mocked_requests_get) def test_committee_lookup_no_q(self, mock_get): self.assertEqual(True, True) diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 8d4fa7a874..dc03f16f87 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -45,7 +45,7 @@ # Application definition -SESSION_COOKIE_AGE = 30 # Inactivity timeout +SESSION_COOKIE_AGE = 30 * 60 # Inactivity timeout SESSION_SAVE_EVERY_REQUEST = True INSTALLED_APPS = [ diff --git a/tasks.py b/tasks.py index d266a2a187..b950dbda14 100644 --- a/tasks.py +++ b/tasks.py @@ -45,7 +45,7 @@ def _detect_space(repo, branch=None): DEPLOY_RULES = ( ("prod", lambda _, branch: branch == "main"), ("stage", lambda _, branch: branch.startswith("release")), - ("dev", lambda _, branch: branch == "login-dot-gov-logout-is-authenticated-fix"), + ("dev", lambda _, branch: branch == "develop"), ) From 1a3389897ed29647f37e29b1bb15fa612ef27fbe Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 19 Dec 2022 17:24:16 -0500 Subject: [PATCH 64/65] added documentation --- django-backend/fecfiler/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/utils.py b/django-backend/fecfiler/utils.py index 1d93d2df44..79cf36f4f7 100644 --- a/django-backend/fecfiler/utils.py +++ b/django-backend/fecfiler/utils.py @@ -7,7 +7,11 @@ def custom_exception_handler(exc, context): # to get the standard error response. response = exception_handler(exc, context) - # Now add the HTTP status code to the response. + # Delete user cookies on forbidden http response. + # this will ensure that when the user is redirected + # to the login page due to the 403, any cookies + # (such as indicating committee id) are removed to + # allow for a clean new login. if response is not None and response.status_code == 403: delete_user_logged_in_cookies(response) From b7b24f670a73084fbd07a7ed4974273bfa47fb20 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 6 Jan 2023 15:02:01 -0500 Subject: [PATCH 65/65] update gitpython --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1908fa5a6f..0695f62b9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 git+https://github.com/fecgov/fecfile-validate@f1fc910feab28ffe6b5fa9f2f9ee028fbb587e22#egg=fecfile_validate&subdirectory=fecfile_validate_python -GitPython==3.1.29 +GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 invoke==1.7.3