From af7da1470a96fd7d34db7856c8f34b61953b22bb Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Tue, 6 Feb 2024 13:14:53 -0500 Subject: [PATCH 01/11] Add reports table migration --- .../1970cbb30227_create_reports_table.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 alembic/versions/1970cbb30227_create_reports_table.py diff --git a/alembic/versions/1970cbb30227_create_reports_table.py b/alembic/versions/1970cbb30227_create_reports_table.py new file mode 100644 index 0000000..8b1c47c --- /dev/null +++ b/alembic/versions/1970cbb30227_create_reports_table.py @@ -0,0 +1,40 @@ +"""Create reports table + +Revision ID: 1970cbb30227 +Revises: c5596492c87b +Create Date: 2024-02-06 13:10:12.248041 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import func +from sqlalchemy.dialects.postgresql import JSONB + + +# revision identifiers, used by Alembic. +revision: str = '1970cbb30227' +down_revision: Union[str, None] = 'c5596492c87b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "reports", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("created_at", sa.DateTime, server_default=func.now(), nullable=False), + sa.Column( + "updated_at", + sa.DateTime, + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ), + sa.Column("data", JSONB, nullable=False), + sa.Column("data_submission_id", sa.Integer, sa.ForeignKey("data_submissions.id")), + ) + +def downgrade() -> None: + op.drop_table("reports") From 604d027d8651cd886faebabbdc8f5ea8fc5cca2b Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Tue, 6 Feb 2024 13:24:33 -0500 Subject: [PATCH 02/11] Rethink report persistence --- .../1970cbb30227_create_reports_table.py | 40 ------------------- .../851709d3a162_add_report_column.py | 30 ++++++++++++++ 2 files changed, 30 insertions(+), 40 deletions(-) delete mode 100644 alembic/versions/1970cbb30227_create_reports_table.py create mode 100644 alembic/versions/851709d3a162_add_report_column.py diff --git a/alembic/versions/1970cbb30227_create_reports_table.py b/alembic/versions/1970cbb30227_create_reports_table.py deleted file mode 100644 index 8b1c47c..0000000 --- a/alembic/versions/1970cbb30227_create_reports_table.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Create reports table - -Revision ID: 1970cbb30227 -Revises: c5596492c87b -Create Date: 2024-02-06 13:10:12.248041 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.sql import func -from sqlalchemy.dialects.postgresql import JSONB - - -# revision identifiers, used by Alembic. -revision: str = '1970cbb30227' -down_revision: Union[str, None] = 'c5596492c87b' -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "reports", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column("created_at", sa.DateTime, server_default=func.now(), nullable=False), - sa.Column( - "updated_at", - sa.DateTime, - server_default=func.now(), - onupdate=func.now(), - nullable=False, - ), - sa.Column("data", JSONB, nullable=False), - sa.Column("data_submission_id", sa.Integer, sa.ForeignKey("data_submissions.id")), - ) - -def downgrade() -> None: - op.drop_table("reports") diff --git a/alembic/versions/851709d3a162_add_report_column.py b/alembic/versions/851709d3a162_add_report_column.py new file mode 100644 index 0000000..fe6103a --- /dev/null +++ b/alembic/versions/851709d3a162_add_report_column.py @@ -0,0 +1,30 @@ +"""Add report column + +Revision ID: 851709d3a162 +Revises: c5596492c87b +Create Date: 2024-02-06 13:22:25.266733 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB + + +# revision identifiers, used by Alembic. +revision: str = '851709d3a162' +down_revision: Union[str, None] = 'c5596492c87b' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade(): + op.add_column( + "data_submissions", + sa.Column("report", JSONB), + ) + + +def add_column(): + op.drop_column("data_submissions", "report") From 9fde948ed018fa28d5726aa7478a2cd28159e328 Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Tue, 6 Feb 2024 13:51:15 -0500 Subject: [PATCH 03/11] Update entity and sqlalchemy models --- alembic/versions/851709d3a162_add_report_column.py | 8 ++++---- nad_ch/domain/entities.py | 5 +++++ nad_ch/infrastructure/database.py | 7 ++++++- tests/domain/test_entities.py | 13 +++++++++++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/alembic/versions/851709d3a162_add_report_column.py b/alembic/versions/851709d3a162_add_report_column.py index fe6103a..cd1f465 100644 --- a/alembic/versions/851709d3a162_add_report_column.py +++ b/alembic/versions/851709d3a162_add_report_column.py @@ -9,12 +9,12 @@ from alembic import op import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.types import JSON # revision identifiers, used by Alembic. -revision: str = '851709d3a162' -down_revision: Union[str, None] = 'c5596492c87b' +revision: str = "851709d3a162" +down_revision: Union[str, None] = "c5596492c87b" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -22,7 +22,7 @@ def upgrade(): op.add_column( "data_submissions", - sa.Column("report", JSONB), + sa.Column("report", JSON, nullable=True), ) diff --git a/nad_ch/domain/entities.py b/nad_ch/domain/entities.py index 4a54fda..94cccbb 100644 --- a/nad_ch/domain/entities.py +++ b/nad_ch/domain/entities.py @@ -30,11 +30,13 @@ def __init__( self, filename: str, provider: DataProvider, + report=None, id: int = None, ): super().__init__(id) self.filename = filename self.provider = provider + self.report = report def __repr__(self): return f"DataSubmission \ @@ -55,3 +57,6 @@ def generate_filename(file_path: str, provider: DataProvider) -> str: _, file_extension = os.path.splitext(file_path) filename = f"{formatted_provider_name}_{datetime_str}{file_extension}" return filename + + def has_report(self) -> bool: + return False if self.report is None or not self.report else True diff --git a/nad_ch/infrastructure/database.py b/nad_ch/infrastructure/database.py index 4d267e2..d3548e3 100644 --- a/nad_ch/infrastructure/database.py +++ b/nad_ch/infrastructure/database.py @@ -2,6 +2,7 @@ from sqlalchemy import Column, Integer, String, create_engine, ForeignKey, DateTime from sqlalchemy.orm import sessionmaker, declarative_base, relationship, Session from sqlalchemy.sql import func +from sqlalchemy.types import JSON import contextlib from nad_ch.domain.entities import DataProvider, DataSubmission from nad_ch.domain.repositories import DataProviderRepository, DataSubmissionRepository @@ -76,6 +77,7 @@ class DataSubmissionModel(CommonBase): filename = Column(String) data_provider_id = Column(Integer, ForeignKey("data_providers.id")) + report = Column(JSON) data_provider = relationship("DataProviderModel", back_populates="data_submissions") @@ -84,12 +86,15 @@ def from_entity(submission): model = DataSubmissionModel( id=submission.id, filename=submission.filename, + report=submission.report, data_provider_id=submission.provider.id, ) return model def to_entity(self, provider: DataProvider): - entity = DataSubmission(id=self.id, filename=self.filename, provider=provider) + entity = DataSubmission( + id=self.id, filename=self.filename, report=self.report, provider=provider + ) if self.created_at is not None: entity.set_created_at(self.created_at) diff --git a/tests/domain/test_entities.py b/tests/domain/test_entities.py index 81d510a..ce15653 100644 --- a/tests/domain/test_entities.py +++ b/tests/domain/test_entities.py @@ -9,3 +9,16 @@ def test_data_submission_generates_filename(): assert filename.startswith("some_provider_") assert todays_date in filename assert filename.endswith(".zip") + + +def test_data_submission_knows_if_it_has_a_report(): + report_data = {"key1": "value1", "key2": "value2"} + submission = DataSubmission( + "someupload.zip", DataProvider("Some provider"), report_data + ) + assert submission.has_report() == True + + +def test_data_submission_knows_if_it_does_not_have_a_report(): + submission = DataSubmission("someupload.zip", DataProvider("Some provider")) + assert submission.has_report() == False From e9f4357e4dde62302d1fcd7b6976393576fec729 Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Tue, 6 Feb 2024 16:00:11 -0500 Subject: [PATCH 04/11] Fix repostiories --- nad_ch/application/dtos.py | 10 +++++++++- nad_ch/application/interfaces.py | 4 +++- nad_ch/application/use_cases.py | 6 ++++-- nad_ch/config/development_local.py | 6 +++--- nad_ch/config/development_remote.py | 6 +++--- nad_ch/domain/repositories.py | 3 +++ nad_ch/infrastructure/database.py | 31 ++++++++++++++++++----------- nad_ch/infrastructure/task_queue.py | 16 ++++++++++----- tests/application/test_use_cases.py | 6 +++++- 9 files changed, 60 insertions(+), 28 deletions(-) diff --git a/nad_ch/application/dtos.py b/nad_ch/application/dtos.py index 05d5f8a..4e95405 100644 --- a/nad_ch/application/dtos.py +++ b/nad_ch/application/dtos.py @@ -1,7 +1,15 @@ -from dataclasses import dataclass +from dataclasses import dataclass, asdict @dataclass class DownloadResult: temp_dir: str extracted_dir: str + + +@dataclass +class DataSubmissionReport: + feature_count: int + + def to_dict(self): + return asdict(self) diff --git a/nad_ch/application/interfaces.py b/nad_ch/application/interfaces.py index e205d1e..8fd2de6 100644 --- a/nad_ch/application/interfaces.py +++ b/nad_ch/application/interfaces.py @@ -29,7 +29,9 @@ def cleanup_temp_dir(self, temp_dir: str) -> bool: class TaskQueue(Protocol): - def run_load_and_validate(self, path: str): + def run_load_and_validate( + self, submissions: DataSubmissionRepository, submission_id: int, path: str + ): ... diff --git a/nad_ch/application/use_cases.py b/nad_ch/application/use_cases.py index 171c9cd..0425c8a 100644 --- a/nad_ch/application/use_cases.py +++ b/nad_ch/application/use_cases.py @@ -83,8 +83,10 @@ def validate_data_submission(ctx: ApplicationContext, filename: str): ctx.logger.error("Data extration error") return - result = ctx.task_queue.run_load_and_validate(download_result.extracted_dir) + report = ctx.task_queue.run_load_and_validate( + ctx.submissions, submission.id, download_result.extracted_dir + ) - ctx.logger.info(f"Total number of features: {result.get()}") + ctx.logger.info(f"Total number of features: {report.feature_count}") ctx.storage.cleanup_temp_dir(download_result.temp_dir) diff --git a/nad_ch/config/development_local.py b/nad_ch/config/development_local.py index 43ac8e8..a669508 100644 --- a/nad_ch/config/development_local.py +++ b/nad_ch/config/development_local.py @@ -32,7 +32,7 @@ class DevLocalApplicationContext(ApplicationContext): def __init__(self): - self._session = create_session_factory(DATABASE_URL) + self._session_factory = create_session_factory(DATABASE_URL) self._providers = self.create_provider_repository() self._submissions = self.create_submission_repository() self._logger = self.create_logger() @@ -40,10 +40,10 @@ def __init__(self): self._task_queue = self.create_task_queue() def create_provider_repository(self): - return SqlAlchemyDataProviderRepository(self._session) + return SqlAlchemyDataProviderRepository(self._session_factory) def create_submission_repository(self): - return SqlAlchemyDataSubmissionRepository(self._session) + return SqlAlchemyDataSubmissionRepository(self._session_factory) def create_logger(self): return BasicLogger(__name__, logging.DEBUG) diff --git a/nad_ch/config/development_remote.py b/nad_ch/config/development_remote.py index 427e5b2..a95e123 100644 --- a/nad_ch/config/development_remote.py +++ b/nad_ch/config/development_remote.py @@ -39,7 +39,7 @@ def get_credentials(service_name, default={}): class DevRemoteApplicationContext(ApplicationContext): def __init__(self): - self._session = create_session_factory(DATABASE_URL) + self._session_factory = create_session_factory(DATABASE_URL) self._providers = self.create_provider_repository() self._submissions = self.create_submission_repository() self._logger = self.create_logger() @@ -47,10 +47,10 @@ def __init__(self): self._task_queue = self.create_task_queue() def create_provider_repository(self): - return SqlAlchemyDataProviderRepository(self._session) + return SqlAlchemyDataProviderRepository(self._session_factory) def create_submission_repository(self): - return SqlAlchemyDataSubmissionRepository(self._session) + return SqlAlchemyDataSubmissionRepository(self._session_factory) def create_logger(self): return BasicLogger(__name__) diff --git a/nad_ch/domain/repositories.py b/nad_ch/domain/repositories.py index ddc38ea..9884135 100644 --- a/nad_ch/domain/repositories.py +++ b/nad_ch/domain/repositories.py @@ -26,3 +26,6 @@ def get_by_provider(self, provider: DataProvider) -> Iterable[DataSubmission]: def get_by_filename() -> Optional[DataSubmission]: ... + + def update(self, submission: DataSubmission) -> DataSubmission: + ... diff --git a/nad_ch/infrastructure/database.py b/nad_ch/infrastructure/database.py index d3548e3..21bf370 100644 --- a/nad_ch/infrastructure/database.py +++ b/nad_ch/infrastructure/database.py @@ -17,7 +17,7 @@ def create_session_factory(connection_string: str): @contextlib.contextmanager def session_scope(session_factory): - session = session_factory + session = session_factory() try: yield session session.commit() @@ -106,11 +106,11 @@ def to_entity(self, provider: DataProvider): class SqlAlchemyDataProviderRepository(DataProviderRepository): - def __init__(self, session: Session): - self.session_factory = session + def __init__(self, session_factory): + self.session_factory = session_factory def add(self, provider: DataProvider) -> DataProvider: - with self.session_factory() as session: + with session_scope(self.session_factory) as session: provider_model = DataProviderModel.from_entity(provider) session.add(provider_model) session.commit() @@ -118,7 +118,7 @@ def add(self, provider: DataProvider) -> DataProvider: return provider_model.to_entity() def get_by_name(self, name: str) -> Optional[DataProvider]: - with self.session_factory() as session: + with session_scope(self.session_factory) as session: provider_model = ( session.query(DataProviderModel) .filter(DataProviderModel.name == name) @@ -127,18 +127,18 @@ def get_by_name(self, name: str) -> Optional[DataProvider]: return provider_model.to_entity() if provider_model else None def get_all(self) -> List[DataProvider]: - with self.session_factory() as session: + with session_scope(self.session_factory) as session: provider_models = session.query(DataProviderModel).all() providers_entities = [provider.to_entity() for provider in provider_models] return providers_entities class SqlAlchemyDataSubmissionRepository(DataSubmissionRepository): - def __init__(self, session: Session): - self.session_factory = session + def __init__(self, session_factory): + self.session_factory = session_factory def add(self, submission: DataSubmission) -> DataSubmission: - with self.session_factory() as session: + with session_scope(self.session_factory) as session: submission_model = DataSubmissionModel.from_entity(submission) session.add(submission_model) session.commit() @@ -151,7 +151,7 @@ def add(self, submission: DataSubmission) -> DataSubmission: return submission_model.to_entity(provider_model.to_entity()) def get_by_id(self, id: int) -> Optional[DataSubmission]: - with self.session_factory() as session: + with session_scope(self.session_factory) as session: result = ( session.query(DataSubmissionModel, DataProviderModel) .join( @@ -169,7 +169,7 @@ def get_by_id(self, id: int) -> Optional[DataSubmission]: return None def get_by_provider(self, provider: DataProvider) -> List[DataSubmission]: - with self.session_factory() as session: + with session_scope(self.session_factory) as session: submission_models = ( session.query(DataSubmissionModel) .filter(DataSubmissionModel.data_provider_id == provider.id) @@ -181,7 +181,7 @@ def get_by_provider(self, provider: DataProvider) -> List[DataSubmission]: return submission_entities def get_by_filename(self, filename: str) -> Optional[DataSubmission]: - with self.session_factory() as session: + with session_scope(self.session_factory) as session: result = ( session.query(DataSubmissionModel, DataProviderModel) .join( @@ -197,3 +197,10 @@ def get_by_filename(self, filename: str) -> Optional[DataSubmission]: return submission_model.to_entity(provider_model.to_entity()) else: return None + + def update_report(self, id: int, report) -> DataSubmission: + with session_scope(self.session_factory) as session: + model_instance = session.query(DataSubmissionModel).filter(DataSubmissionModel.id == id).first() + + if model_instance: + model_instance.report = report diff --git a/nad_ch/infrastructure/task_queue.py b/nad_ch/infrastructure/task_queue.py index fb94fb6..ecab482 100644 --- a/nad_ch/infrastructure/task_queue.py +++ b/nad_ch/infrastructure/task_queue.py @@ -1,9 +1,10 @@ -import os from celery import Celery import geopandas as gpd +from nad_ch.application.dtos import DataSubmissionReport from nad_ch.application.interfaces import TaskQueue from nad_ch.application.validation import get_feature_count from nad_ch.config import QUEUE_BROKER_URL, QUEUE_BACKEND_URL +from nad_ch.domain.repositories import DataSubmissionRepository celery_app = Celery( @@ -21,16 +22,21 @@ @celery_app.task -def load_and_validate(gdb_file_path: str) -> int: +def load_and_validate(gdb_file_path: str) -> dict: gdf = gpd.read_file(gdb_file_path) feature_count = get_feature_count(gdf) - return feature_count + report = DataSubmissionReport(feature_count) + return report.to_dict() class CeleryTaskQueue(TaskQueue): def __init__(self, app): self.app = app - def run_load_and_validate(self, path: str): + def run_load_and_validate( + self, submissions: DataSubmissionRepository, submission_id: int, path: str + ): task_result = load_and_validate.apply_async(args=[path]) - return task_result + report_dict = task_result.get() + submissions.update_report(submission_id, report_dict) + return DataSubmissionReport(**report_dict) diff --git a/tests/application/test_use_cases.py b/tests/application/test_use_cases.py index 7842564..8a596eb 100644 --- a/tests/application/test_use_cases.py +++ b/tests/application/test_use_cases.py @@ -2,6 +2,7 @@ import re from nad_ch.config import create_app_context from nad_ch.domain.entities import DataProvider, DataSubmission +from nad_ch.domain.repositories import DataSubmissionRepository from nad_ch.application.use_cases import ( add_data_provider, list_data_providers, @@ -93,9 +94,12 @@ def test_validate_data_submission(app_context, caplog): filename = "my_cool_file.zip" ingest_data_submission(app_context, filename, provider_name) submission = app_context.submissions.get_by_id(1) + submissions = app_context.submissions class CustomMockTestTaskQueue: - def run_load_and_validate(self, path: str): + def run_load_and_validate( + self, submissions: DataSubmissionRepository, submission_id: int, path: str + ): return MockCeleryTask(1) app_context._task_queue = CustomMockTestTaskQueue() From fb05c1a767ac551de65db9dc86541f15e3d63dd6 Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Tue, 6 Feb 2024 17:19:13 -0500 Subject: [PATCH 05/11] Tweaking new submission repo metho --- nad_ch/application/use_cases.py | 4 +++- nad_ch/domain/repositories.py | 2 +- nad_ch/infrastructure/database.py | 8 ++++++-- tests/application/test_use_cases.py | 3 ++- tests/fakes_and_mocks.py | 5 ++++- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/nad_ch/application/use_cases.py b/nad_ch/application/use_cases.py index 0425c8a..52939d6 100644 --- a/nad_ch/application/use_cases.py +++ b/nad_ch/application/use_cases.py @@ -83,10 +83,12 @@ def validate_data_submission(ctx: ApplicationContext, filename: str): ctx.logger.error("Data extration error") return - report = ctx.task_queue.run_load_and_validate( + task_result = ctx.task_queue.run_load_and_validate( ctx.submissions, submission.id, download_result.extracted_dir ) + report = task_result.get() + ctx.logger.info(f"Total number of features: {report.feature_count}") ctx.storage.cleanup_temp_dir(download_result.temp_dir) diff --git a/nad_ch/domain/repositories.py b/nad_ch/domain/repositories.py index 9884135..e0d8808 100644 --- a/nad_ch/domain/repositories.py +++ b/nad_ch/domain/repositories.py @@ -27,5 +27,5 @@ def get_by_provider(self, provider: DataProvider) -> Iterable[DataSubmission]: def get_by_filename() -> Optional[DataSubmission]: ... - def update(self, submission: DataSubmission) -> DataSubmission: + def update_report(self, submission_id: int, report) -> None: ... diff --git a/nad_ch/infrastructure/database.py b/nad_ch/infrastructure/database.py index 21bf370..79ef192 100644 --- a/nad_ch/infrastructure/database.py +++ b/nad_ch/infrastructure/database.py @@ -198,9 +198,13 @@ def get_by_filename(self, filename: str) -> Optional[DataSubmission]: else: return None - def update_report(self, id: int, report) -> DataSubmission: + def update_report(self, id: int, report) -> None: with session_scope(self.session_factory) as session: - model_instance = session.query(DataSubmissionModel).filter(DataSubmissionModel.id == id).first() + model_instance = ( + session.query(DataSubmissionModel) + .filter(DataSubmissionModel.id == id) + .first() + ) if model_instance: model_instance.report = report diff --git a/tests/application/test_use_cases.py b/tests/application/test_use_cases.py index 8a596eb..d2612c4 100644 --- a/tests/application/test_use_cases.py +++ b/tests/application/test_use_cases.py @@ -1,5 +1,6 @@ import pytest import re +from nad_ch.application.dtos import DataSubmissionReport from nad_ch.config import create_app_context from nad_ch.domain.entities import DataProvider, DataSubmission from nad_ch.domain.repositories import DataSubmissionRepository @@ -100,7 +101,7 @@ class CustomMockTestTaskQueue: def run_load_and_validate( self, submissions: DataSubmissionRepository, submission_id: int, path: str ): - return MockCeleryTask(1) + return MockCeleryTask(DataSubmissionReport(1)) app_context._task_queue = CustomMockTestTaskQueue() diff --git a/tests/fakes_and_mocks.py b/tests/fakes_and_mocks.py index 6b4e7cd..d71c103 100644 --- a/tests/fakes_and_mocks.py +++ b/tests/fakes_and_mocks.py @@ -40,7 +40,10 @@ def get_by_provider(self, provider: DataProvider) -> Optional[DataSubmission]: return [s for s in self._submissions if s.provider.name == provider.name] def get_by_filename(self, filename: str) -> Optional[DataSubmission]: - return [s for s in self._submissions if s.filename == filename] + return next((s for s in self._submissions if s.filename == filename), None) + + def update_report(self, submission_id: int, report) -> None: + return None class FakeStorage: From 9164467f6227a5a78361d922c01e81a9f554c1c1 Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Wed, 7 Feb 2024 11:08:09 -0500 Subject: [PATCH 06/11] Running through manual tests --- .gitignore | 7 +++++-- nad_ch/application/use_cases.py | 4 +--- tests/application/test_use_cases.py | 4 +--- tests/domain/test_entities.py | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 2b47213..e20844c 100644 --- a/.gitignore +++ b/.gitignore @@ -171,5 +171,8 @@ control/ # Zipped artifacts *.zip -# frontend dependencies -node_modules/ \ No newline at end of file +# Frontend dependencies +node_modules/ + +# Local notes +.notes \ No newline at end of file diff --git a/nad_ch/application/use_cases.py b/nad_ch/application/use_cases.py index 52939d6..0425c8a 100644 --- a/nad_ch/application/use_cases.py +++ b/nad_ch/application/use_cases.py @@ -83,12 +83,10 @@ def validate_data_submission(ctx: ApplicationContext, filename: str): ctx.logger.error("Data extration error") return - task_result = ctx.task_queue.run_load_and_validate( + report = ctx.task_queue.run_load_and_validate( ctx.submissions, submission.id, download_result.extracted_dir ) - report = task_result.get() - ctx.logger.info(f"Total number of features: {report.feature_count}") ctx.storage.cleanup_temp_dir(download_result.temp_dir) diff --git a/tests/application/test_use_cases.py b/tests/application/test_use_cases.py index d2612c4..f0d796e 100644 --- a/tests/application/test_use_cases.py +++ b/tests/application/test_use_cases.py @@ -10,7 +10,6 @@ ingest_data_submission, validate_data_submission, ) -from tests.fakes_and_mocks import MockCeleryTask @pytest.fixture(scope="function") @@ -95,13 +94,12 @@ def test_validate_data_submission(app_context, caplog): filename = "my_cool_file.zip" ingest_data_submission(app_context, filename, provider_name) submission = app_context.submissions.get_by_id(1) - submissions = app_context.submissions class CustomMockTestTaskQueue: def run_load_and_validate( self, submissions: DataSubmissionRepository, submission_id: int, path: str ): - return MockCeleryTask(DataSubmissionReport(1)) + return DataSubmissionReport(1) app_context._task_queue = CustomMockTestTaskQueue() diff --git a/tests/domain/test_entities.py b/tests/domain/test_entities.py index ce15653..8908486 100644 --- a/tests/domain/test_entities.py +++ b/tests/domain/test_entities.py @@ -16,9 +16,9 @@ def test_data_submission_knows_if_it_has_a_report(): submission = DataSubmission( "someupload.zip", DataProvider("Some provider"), report_data ) - assert submission.has_report() == True + assert submission.has_report() def test_data_submission_knows_if_it_does_not_have_a_report(): submission = DataSubmission("someupload.zip", DataProvider("Some provider")) - assert submission.has_report() == False + assert not submission.has_report() From 0e574c002b09ab06ad1ff0ef3f376be0dbcdf5d2 Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Wed, 7 Feb 2024 14:40:46 -0500 Subject: [PATCH 07/11] Flesh out report data structures --- Dockerfile | 2 +- nad_ch/application/dtos.py | 27 ++++++++++++++++++++++++--- nad_ch/application/use_cases.py | 2 +- nad_ch/infrastructure/task_queue.py | 10 +++++++--- tests/application/test_use_cases.py | 6 ++++-- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07329c3..0e00ef9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,6 @@ ENV PATH="${PATH}:/opt/poetry/bin" # Install dependencies and start app WORKDIR /app COPY pyproject.toml poetry.lock ./ -RUN poetry install --only main +RUN poetry install --without dev COPY . . CMD ["/bin/sh", "start_local.sh"] \ No newline at end of file diff --git a/nad_ch/application/dtos.py b/nad_ch/application/dtos.py index 4e95405..a10bd8a 100644 --- a/nad_ch/application/dtos.py +++ b/nad_ch/application/dtos.py @@ -1,4 +1,5 @@ -from dataclasses import dataclass, asdict +from dataclasses import dataclass, asdict, field +from typing import List @dataclass @@ -7,9 +8,29 @@ class DownloadResult: extracted_dir: str +@dataclass +class DataSubmissionReportOverview: + feature_count: int = 0 + features_flagged: int = 0 + etl_update_required: bool = False + data_update_required: bool = False + + +@dataclass +class DataSubmissionReportFeature: + provided_feature_name: str + nad_feature_name: str + populated_count: int + null_count: int + + @dataclass class DataSubmissionReport: - feature_count: int + overview: DataSubmissionReportOverview + features: List[DataSubmissionReportFeature] = field(default_factory=list) def to_dict(self): - return asdict(self) + return { + "overview": asdict(self.overview), + "features": [asdict(feature) for feature in self.features], + } diff --git a/nad_ch/application/use_cases.py b/nad_ch/application/use_cases.py index 0425c8a..565788d 100644 --- a/nad_ch/application/use_cases.py +++ b/nad_ch/application/use_cases.py @@ -87,6 +87,6 @@ def validate_data_submission(ctx: ApplicationContext, filename: str): ctx.submissions, submission.id, download_result.extracted_dir ) - ctx.logger.info(f"Total number of features: {report.feature_count}") + ctx.logger.info(f"Total number of features: {report.overview.feature_count}") ctx.storage.cleanup_temp_dir(download_result.temp_dir) diff --git a/nad_ch/infrastructure/task_queue.py b/nad_ch/infrastructure/task_queue.py index ecab482..c608716 100644 --- a/nad_ch/infrastructure/task_queue.py +++ b/nad_ch/infrastructure/task_queue.py @@ -1,6 +1,10 @@ from celery import Celery import geopandas as gpd -from nad_ch.application.dtos import DataSubmissionReport +from nad_ch.application.dtos import ( + DataSubmissionReport, + DataSubmissionReportOverview, + DataSubmissionReportFeature, +) from nad_ch.application.interfaces import TaskQueue from nad_ch.application.validation import get_feature_count from nad_ch.config import QUEUE_BROKER_URL, QUEUE_BACKEND_URL @@ -24,8 +28,8 @@ @celery_app.task def load_and_validate(gdb_file_path: str) -> dict: gdf = gpd.read_file(gdb_file_path) - feature_count = get_feature_count(gdf) - report = DataSubmissionReport(feature_count) + overview = DataSubmissionReportOverview(feature_count=get_feature_count(gdf)) + report = DataSubmissionReport(overview) return report.to_dict() diff --git a/tests/application/test_use_cases.py b/tests/application/test_use_cases.py index f0d796e..ab87313 100644 --- a/tests/application/test_use_cases.py +++ b/tests/application/test_use_cases.py @@ -1,6 +1,6 @@ import pytest import re -from nad_ch.application.dtos import DataSubmissionReport +from nad_ch.application.dtos import DataSubmissionReport, DataSubmissionReportOverview from nad_ch.config import create_app_context from nad_ch.domain.entities import DataProvider, DataSubmission from nad_ch.domain.repositories import DataSubmissionRepository @@ -99,7 +99,9 @@ class CustomMockTestTaskQueue: def run_load_and_validate( self, submissions: DataSubmissionRepository, submission_id: int, path: str ): - return DataSubmissionReport(1) + return DataSubmissionReport( + overview=DataSubmissionReportOverview(feature_count=1) + ) app_context._task_queue = CustomMockTestTaskQueue() From 205efdee459e9bf5e707c9664a7fb738930b36e7 Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Wed, 7 Feb 2024 16:16:10 -0500 Subject: [PATCH 08/11] Get feature details function working --- nad_ch/application/dtos.py | 40 ++++++++++++++--- nad_ch/application/use_cases.py | 2 +- nad_ch/application/validation.py | 21 +++++++++ nad_ch/infrastructure/task_queue.py | 8 ++-- start_local.sh | 2 +- tests/application/test_validation.py | 14 ++++-- tests/factories.py | 65 ++++++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 tests/factories.py diff --git a/nad_ch/application/dtos.py b/nad_ch/application/dtos.py index a10bd8a..268c107 100644 --- a/nad_ch/application/dtos.py +++ b/nad_ch/application/dtos.py @@ -1,5 +1,6 @@ -from dataclasses import dataclass, asdict, field +from dataclasses import dataclass, asdict, field, is_dataclass from typing import List +import numpy as np @dataclass @@ -30,7 +31,36 @@ class DataSubmissionReport: features: List[DataSubmissionReportFeature] = field(default_factory=list) def to_dict(self): - return { - "overview": asdict(self.overview), - "features": [asdict(feature) for feature in self.features], - } + # We need to make sure that every value in the DTO is JSON-serializable, so we + # check each item recursively to convert any non "int" numbers to int + def convert(item): + if isinstance(item, dict): + return {k: convert(v) for k, v in item.items()} + elif isinstance(item, list): + return [convert(i) for i in item] + elif isinstance(item, (np.int64, np.int32, np.float64, np.float32)): + return ( + int(item) + if item.dtype == np.int64 or item.dtype == np.int32 + else float(item) + ) + elif is_dataclass(item): + return convert(asdict(item)) + else: + return item + + return convert(asdict(self)) + + @classmethod + def from_dict(cls, data: dict): + overview_data = data.get("overview", {}) + features_data = data.get("features", []) + + overview = DataSubmissionReportOverview(**overview_data) + + features = [ + DataSubmissionReportFeature(**feature_data) + for feature_data in features_data + ] + + return cls(overview=overview, features=features) diff --git a/nad_ch/application/use_cases.py b/nad_ch/application/use_cases.py index 565788d..a21ccf0 100644 --- a/nad_ch/application/use_cases.py +++ b/nad_ch/application/use_cases.py @@ -86,7 +86,7 @@ def validate_data_submission(ctx: ApplicationContext, filename: str): report = ctx.task_queue.run_load_and_validate( ctx.submissions, submission.id, download_result.extracted_dir ) - + print(report) ctx.logger.info(f"Total number of features: {report.overview.feature_count}") ctx.storage.cleanup_temp_dir(download_result.temp_dir) diff --git a/nad_ch/application/validation.py b/nad_ch/application/validation.py index 14e1ec6..d3fc832 100644 --- a/nad_ch/application/validation.py +++ b/nad_ch/application/validation.py @@ -1,5 +1,26 @@ +from typing import List from geopandas import GeoDataFrame +from nad_ch.application.dtos import DataSubmissionReportFeature def get_feature_count(gdf: GeoDataFrame) -> int: return len(gdf) + + +def get_feature_details(gdf: GeoDataFrame) -> List[DataSubmissionReportFeature]: + report_features = [] + + for column in gdf.columns: + populated_count = gdf[column].notna().sum() + null_count = gdf[column].isna().sum() + + report_feature = DataSubmissionReportFeature( + provided_feature_name=column, + nad_feature_name=column, + populated_count=populated_count, + null_count=null_count, + ) + + report_features.append(report_feature) + + return report_features diff --git a/nad_ch/infrastructure/task_queue.py b/nad_ch/infrastructure/task_queue.py index c608716..c9dfe03 100644 --- a/nad_ch/infrastructure/task_queue.py +++ b/nad_ch/infrastructure/task_queue.py @@ -3,10 +3,9 @@ from nad_ch.application.dtos import ( DataSubmissionReport, DataSubmissionReportOverview, - DataSubmissionReportFeature, ) from nad_ch.application.interfaces import TaskQueue -from nad_ch.application.validation import get_feature_count +from nad_ch.application.validation import get_feature_count, get_feature_details from nad_ch.config import QUEUE_BROKER_URL, QUEUE_BACKEND_URL from nad_ch.domain.repositories import DataSubmissionRepository @@ -29,7 +28,8 @@ def load_and_validate(gdb_file_path: str) -> dict: gdf = gpd.read_file(gdb_file_path) overview = DataSubmissionReportOverview(feature_count=get_feature_count(gdf)) - report = DataSubmissionReport(overview) + feature_details = get_feature_details(gdf) + report = DataSubmissionReport(overview, feature_details) return report.to_dict() @@ -43,4 +43,4 @@ def run_load_and_validate( task_result = load_and_validate.apply_async(args=[path]) report_dict = task_result.get() submissions.update_report(submission_id, report_dict) - return DataSubmissionReport(**report_dict) + return DataSubmissionReport.from_dict(report_dict) diff --git a/start_local.sh b/start_local.sh index 9f4bb5f..de34278 100644 --- a/start_local.sh +++ b/start_local.sh @@ -1,3 +1,3 @@ #/bin/bash -poetry run celery -A nad_ch.infrastructure.task_queue worker --loglevel=info & poetry run start-web +poetry run celery -A nad_ch.infrastructure.task_queue worker --loglevel=INFO & poetry run start-web diff --git a/tests/application/test_validation.py b/tests/application/test_validation.py index 08ba362..6e9541a 100644 --- a/tests/application/test_validation.py +++ b/tests/application/test_validation.py @@ -1,11 +1,19 @@ -import pytest import geopandas as gpd from shapely.geometry import Polygon -from nad_ch.application.validation import get_feature_count +from nad_ch.application.dtos import DataSubmissionReportFeature +from nad_ch.application.validation import get_feature_count, get_feature_details +from tests.factories import create_fake_geopandas_dataframe -def test_get_feature_count(): +def test_get_feature_count_finds_the_length_of_a_geopandas_dataframe(): coordinates = [(0, 0), (0, 1), (1, 1), (1, 0)] polygon = Polygon(coordinates) gdf = gpd.GeoDataFrame(geometry=[polygon]) assert (get_feature_count(gdf)) == 1 + + +def test_get_feature_details_returns_instances_of_correct_dataclass(): + gdf = create_fake_geopandas_dataframe(num_rows=1) + feature_details = get_feature_details(gdf) + feature = feature_details[0] + assert isinstance(feature, DataSubmissionReportFeature) diff --git a/tests/factories.py b/tests/factories.py new file mode 100644 index 0000000..fc61abe --- /dev/null +++ b/tests/factories.py @@ -0,0 +1,65 @@ +import geopandas as gpd +import pandas as pd +import numpy as np +from uuid import uuid4 + + +def create_fake_geopandas_dataframe(num_rows=10): + # Generate random data + data = { + "AddNum_Pre": [None] * num_rows, + "Add_Number": np.random.randint(1, 100, size=num_rows), + "AddNum_Suf": [None] * num_rows, + "AddNo_Full": [str(np.random.randint(1, 100)) for _ in range(num_rows)], + "St_PreMod": [None] * num_rows, + "St_PreDir": ["South"] * num_rows, + "St_PreTyp": [None] * num_rows, + "St_PreSep": [None] * num_rows, + "St_Name": ["Street{}".format(i) for i in range(num_rows)], + "St_PosTyp": ["Street"] * num_rows, + "St_PosDir": [None] * num_rows, + "St_PosMod": [None] * num_rows, + "StNam_Full": ["South Street{}".format(i) for i in range(num_rows)], + "Building": [None] * num_rows, + "Floor": [None] * num_rows, + "Unit": [None] * num_rows, + "Room": [None] * num_rows, + "Seat": [None] * num_rows, + "Addtl_Loc": [None] * num_rows, + "SubAddress": [None] * num_rows, + "LandmkName": [None] * num_rows, + "County": ["Anycounty"] * num_rows, + "Inc_Muni": ["Anytown"] * num_rows, + "Post_City": ["Anytown"] * num_rows, + "State": ["IN"] * num_rows, + "Zip_Code": [str(np.random.randint(10000, 99999)) for _ in range(num_rows)], + "UUID": [str(uuid4()) for _ in range(num_rows)], + "AddAuth": ["Anycounty County"] * num_rows, + "Longitude": np.random.uniform(low=-180, high=180, size=num_rows), + "Latitude": np.random.uniform(low=-90, high=90, size=num_rows), + "Placement": ["Structure - Rooftop"] * num_rows, + "DateUpdate": [pd.Timestamp.now() for _ in range(num_rows)], + "Addr_Type": ["Commercial"] * num_rows, + "NAD_Source": ["Indiana State Library"] * num_rows, + "DataSet_ID": [str(uuid4()) for _ in range(num_rows)], + } + + # Create DataFrame + df = pd.DataFrame(data) + + # Create GeoDataFrame + gdf = gpd.GeoDataFrame( + df, geometry=gpd.points_from_xy(df["Longitude"], df["Latitude"]) + ) + + # Set GeoJSON metadata + gdf_metadata = { + "type": "FeatureCollection", + "crs": { + "type": "name", + "properties": {"name": "urn:ogc:def:crs:OGC:1.3:CRS84"}, + }, + } + gdf.__geo_interface__["metadata"] = gdf_metadata + + return gdf From 750461576a3901eb6f902de9f77889fb9912ade2 Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Wed, 7 Feb 2024 16:29:59 -0500 Subject: [PATCH 09/11] Factor conversion functions out to keep dto clean --- nad_ch/application/dtos.py | 62 +++++++++++++---------------- nad_ch/application/use_cases.py | 2 +- nad_ch/infrastructure/task_queue.py | 6 ++- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/nad_ch/application/dtos.py b/nad_ch/application/dtos.py index 268c107..29ec1d5 100644 --- a/nad_ch/application/dtos.py +++ b/nad_ch/application/dtos.py @@ -30,37 +30,31 @@ class DataSubmissionReport: overview: DataSubmissionReportOverview features: List[DataSubmissionReportFeature] = field(default_factory=list) - def to_dict(self): - # We need to make sure that every value in the DTO is JSON-serializable, so we - # check each item recursively to convert any non "int" numbers to int - def convert(item): - if isinstance(item, dict): - return {k: convert(v) for k, v in item.items()} - elif isinstance(item, list): - return [convert(i) for i in item] - elif isinstance(item, (np.int64, np.int32, np.float64, np.float32)): - return ( - int(item) - if item.dtype == np.int64 or item.dtype == np.int32 - else float(item) - ) - elif is_dataclass(item): - return convert(asdict(item)) - else: - return item - - return convert(asdict(self)) - - @classmethod - def from_dict(cls, data: dict): - overview_data = data.get("overview", {}) - features_data = data.get("features", []) - - overview = DataSubmissionReportOverview(**overview_data) - - features = [ - DataSubmissionReportFeature(**feature_data) - for feature_data in features_data - ] - - return cls(overview=overview, features=features) + +def report_to_dict(data_submission_report: DataSubmissionReport) -> dict: + return convert(asdict(data_submission_report)) + + +def report_from_dict(data: dict) -> DataSubmissionReport: + overview_data = data.get("overview", {}) + features_data = data.get("features", []) + + overview = DataSubmissionReportOverview(**overview_data) + features = [ + DataSubmissionReportFeature(**feature_data) for feature_data in features_data + ] + + return DataSubmissionReport(overview=overview, features=features) + + +def convert(item): + if isinstance(item, dict): + return {k: convert(v) for k, v in item.items()} + elif isinstance(item, list): + return [convert(i) for i in item] + elif isinstance(item, (np.int64, np.int32, np.float64, np.float32)): + return int(item) if isinstance(item, (np.int64, np.int32)) else float(item) + elif is_dataclass(item): + return convert(asdict(item)) + else: + return item diff --git a/nad_ch/application/use_cases.py b/nad_ch/application/use_cases.py index a21ccf0..565788d 100644 --- a/nad_ch/application/use_cases.py +++ b/nad_ch/application/use_cases.py @@ -86,7 +86,7 @@ def validate_data_submission(ctx: ApplicationContext, filename: str): report = ctx.task_queue.run_load_and_validate( ctx.submissions, submission.id, download_result.extracted_dir ) - print(report) + ctx.logger.info(f"Total number of features: {report.overview.feature_count}") ctx.storage.cleanup_temp_dir(download_result.temp_dir) diff --git a/nad_ch/infrastructure/task_queue.py b/nad_ch/infrastructure/task_queue.py index c9dfe03..8b93649 100644 --- a/nad_ch/infrastructure/task_queue.py +++ b/nad_ch/infrastructure/task_queue.py @@ -3,6 +3,8 @@ from nad_ch.application.dtos import ( DataSubmissionReport, DataSubmissionReportOverview, + report_to_dict, + report_from_dict, ) from nad_ch.application.interfaces import TaskQueue from nad_ch.application.validation import get_feature_count, get_feature_details @@ -30,7 +32,7 @@ def load_and_validate(gdb_file_path: str) -> dict: overview = DataSubmissionReportOverview(feature_count=get_feature_count(gdf)) feature_details = get_feature_details(gdf) report = DataSubmissionReport(overview, feature_details) - return report.to_dict() + return report_to_dict(report) class CeleryTaskQueue(TaskQueue): @@ -43,4 +45,4 @@ def run_load_and_validate( task_result = load_and_validate.apply_async(args=[path]) report_dict = task_result.get() submissions.update_report(submission_id, report_dict) - return DataSubmissionReport.from_dict(report_dict) + return report_from_dict(report_dict) From 20fe8d976a87fed903fff8233b0d1cce71b9fde0 Mon Sep 17 00:00:00 2001 From: Andy Kuny Date: Wed, 7 Feb 2024 16:37:09 -0500 Subject: [PATCH 10/11] Add tests for dto utility functions along with comments explaining their purpose --- .flake8 | 2 +- nad_ch/application/dtos.py | 13 +++++ tests/application/test_dto.py | 92 +++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 tests/application/test_dto.py diff --git a/.flake8 b/.flake8 index 07ab66a..db33bc3 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] max-line-length = 88 exclude = venv,.git,__pycache__,docs/source/conf.py,old,build,dist,node_modules -ignore = F403, F401 +ignore = F403, F401, W503 diff --git a/nad_ch/application/dtos.py b/nad_ch/application/dtos.py index 29ec1d5..734db59 100644 --- a/nad_ch/application/dtos.py +++ b/nad_ch/application/dtos.py @@ -32,10 +32,19 @@ class DataSubmissionReport: def report_to_dict(data_submission_report: DataSubmissionReport) -> dict: + """ + Converts a DataSubmissionReport instance into a dictionary because all data types + within the dictionary must be JSON-serializable. + """ return convert(asdict(data_submission_report)) def report_from_dict(data: dict) -> DataSubmissionReport: + """ + Creates a DataSubmissionReport instance from a dictionary, reconstructing the + overview and features properties from their respective dictionary representations. + """ + overview_data = data.get("overview", {}) features_data = data.get("features", []) @@ -48,6 +57,10 @@ def report_from_dict(data: dict) -> DataSubmissionReport: def convert(item): + """ + Recursively converts items within a data structure (including dictionaries, lists, + and dataclass instances) such that all numeric types are JSON-serializable. + """ if isinstance(item, dict): return {k: convert(v) for k, v in item.items()} elif isinstance(item, list): diff --git a/tests/application/test_dto.py b/tests/application/test_dto.py new file mode 100644 index 0000000..864f5e3 --- /dev/null +++ b/tests/application/test_dto.py @@ -0,0 +1,92 @@ +import numpy as np +from nad_ch.application.dtos import ( + DataSubmissionReport, + DataSubmissionReportOverview, + DataSubmissionReportFeature, + report_to_dict, + report_from_dict, +) + + +def test_to_dict_simple(): + overview = DataSubmissionReportOverview(feature_count=100, features_flagged=5) + + overview_dict = report_to_dict(overview) + + assert overview_dict == { + "feature_count": 100, + "features_flagged": 5, + "etl_update_required": False, + "data_update_required": False, + } + + +def test_to_dict_with_numpy_types(): + feature = DataSubmissionReportFeature( + provided_feature_name="id", + nad_feature_name="id", + populated_count=np.int64(100), + null_count=np.float32(0), + ) + + feature_dict = report_to_dict(feature) + + assert feature_dict == { + "provided_feature_name": "id", + "nad_feature_name": "id", + "populated_count": 100, + "null_count": 0, + } + assert isinstance(feature_dict["populated_count"], int) + assert isinstance(feature_dict["null_count"], float) + + +def test_from_dict_to_dataclass(): + report_dict = { + "overview": { + "feature_count": 100, + "features_flagged": 5, + "etl_update_required": False, + "data_update_required": False, + }, + "features": [ + { + "provided_feature_name": "id", + "nad_feature_name": "id", + "populated_count": 100, + "null_count": 0, + } + ], + } + + report = report_from_dict(report_dict) + + assert report.overview.feature_count == 100 + assert report.features[0].provided_feature_name == "id" + assert report.features[0].populated_count == 100 + + +def test_round_trip_conversion(): + original_report = DataSubmissionReport( + overview=DataSubmissionReportOverview(feature_count=100, features_flagged=5), + features=[ + DataSubmissionReportFeature( + provided_feature_name="id", + nad_feature_name="id", + populated_count=100, + null_count=0, + ) + ], + ) + + report_dict = report_to_dict(original_report) + converted_report = report_from_dict(report_dict) + + assert ( + converted_report.overview.feature_count + == original_report.overview.feature_count + ) + assert ( + converted_report.features[0].provided_feature_name + == original_report.features[0].provided_feature_name + ) From 8f3867fc8a3622a59ce94658070ef3d38daceebe Mon Sep 17 00:00:00 2001 From: andy kuny Date: Fri, 9 Feb 2024 13:36:19 -0500 Subject: [PATCH 11/11] Update nad_ch/domain/entities.py Co-authored-by: Daniel Naab --- nad_ch/domain/entities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nad_ch/domain/entities.py b/nad_ch/domain/entities.py index 94cccbb..a54eb50 100644 --- a/nad_ch/domain/entities.py +++ b/nad_ch/domain/entities.py @@ -59,4 +59,4 @@ def generate_filename(file_path: str, provider: DataProvider) -> str: return filename def has_report(self) -> bool: - return False if self.report is None or not self.report else True + return self.report is not None