diff --git a/.flake8 b/.flake8 index c6f753f..a3835cb 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] max-line-length = 88 exclude = venv,.git,__pycache__,docs/source/conf.py,old,build,dist,node_modules -ignore = F403, F401, F405, W503 +ignore = F403, F401, F405, W503, E704 diff --git a/alembic/versions/3ff8e29d705e_add_name_field_to_data_submissions_table.py b/alembic/versions/3ff8e29d705e_add_name_field_to_data_submissions_table.py new file mode 100644 index 0000000..ef79b35 --- /dev/null +++ b/alembic/versions/3ff8e29d705e_add_name_field_to_data_submissions_table.py @@ -0,0 +1,30 @@ +"""Add name field to data_submissions table + +Revision ID: 3ff8e29d705e +Revises: a511ca087149 +Create Date: 2024-04-12 12:50:48.903157 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "3ff8e29d705e" +down_revision: Union[str, None] = "a511ca087149" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "data_submissions", + sa.Column("name", sa.String, nullable=False), + ) + + +def downgrade() -> None: + op.drop_column("data_submissions", "name") diff --git a/alembic/versions/558f8d429963_rename_data_submissions_field.py b/alembic/versions/558f8d429963_rename_data_submissions_field.py new file mode 100644 index 0000000..7805002 --- /dev/null +++ b/alembic/versions/558f8d429963_rename_data_submissions_field.py @@ -0,0 +1,27 @@ +"""Rename data_submissions field + +Revision ID: 558f8d429963 +Revises: 3ff8e29d705e +Create Date: 2024-04-12 13:18:47.612471 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "558f8d429963" +down_revision: Union[str, None] = "3ff8e29d705e" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("data_submissions", "filename", new_column_name="file_path") + + +def downgrade() -> None: + op.alter_column("data_submissions", "file_path", new_column_name="filename") diff --git a/alembic/versions/a511ca087149_add_datasubmission_status_column.py b/alembic/versions/a511ca087149_add_datasubmission_status_column.py new file mode 100644 index 0000000..d986de6 --- /dev/null +++ b/alembic/versions/a511ca087149_add_datasubmission_status_column.py @@ -0,0 +1,55 @@ +"""Add DataSubmission status column + +Revision ID: a511ca087149 +Revises: 9bb8e29b98fa +Create Date: 2024-03-29 09:30:08.502456 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision: str = "a511ca087149" +down_revision: Union[str, None] = "9bb8e29b98fa" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade(): + data_submission_status = postgresql.ENUM( + "PENDING_SUBMISSION", + "CANCELED", + "PENDING_VALIDATION", + "FAILED", + "VALIDATED", + name="data_submission_status", + ) + data_submission_status.create(op.get_bind(), checkfirst=True) + + op.add_column( + "data_submissions", + sa.Column( + "status", + data_submission_status, + nullable=False, + server_default="PENDING_SUBMISSION", + ), + ) + + +def downgrade(): + op.drop_column("data_submissions", "status") + data_submission_status = postgresql.ENUM( + "PENDING_SUBMISSION", + "CANCELED", + "PENDING_VALIDATION", + "FAILED", + "VALIDATED", + name="data_submission_status", + ) + data_submission_status.drop(op.get_bind(), checkfirst=True) diff --git a/nad_ch/application/exceptions.py b/nad_ch/application/exceptions.py index dce2f97..7e91f77 100644 --- a/nad_ch/application/exceptions.py +++ b/nad_ch/application/exceptions.py @@ -26,3 +26,17 @@ class OAuth2TokenError(NadChError): def __init__(self, message="OAuth2 token retrieval failed."): super().__init__(message) + + +class InvalidDataSubmissionFileError(NadChError): + """Exception raised when a data submission file is invalid.""" + + def __init__(self, message="Invalid data submission file."): + super().__init__(message) + + +class InvalidSchemaError(NadChError): + """Exception raised when a schema is invalid.""" + + def __init__(self, message="Invalid schema."): + super().__init__(message) diff --git a/nad_ch/application/interfaces.py b/nad_ch/application/interfaces.py index c4f60f9..0d62362 100644 --- a/nad_ch/application/interfaces.py +++ b/nad_ch/application/interfaces.py @@ -9,28 +9,21 @@ class Logger(Protocol): - def info(self, message): - ... + def info(self, message): ... - def error(self, message): - ... + def error(self, message): ... - def warning(self, message): - ... + def warning(self, message): ... class Storage(Protocol): - def upload(self, source: str, destination: str) -> bool: - ... + def upload(self, source: str, destination: str) -> bool: ... - def delete(self, key: str) -> bool: - ... + def delete(self, key: str) -> bool: ... - def download_temp(self, key: str) -> Optional[DownloadResult]: - ... + def download_temp(self, key: str) -> Optional[DownloadResult]: ... - def cleanup_temp_dir(self, temp_dir: str) -> bool: - ... + def cleanup_temp_dir(self, temp_dir: str) -> bool: ... class TaskQueue(Protocol): @@ -40,30 +33,25 @@ def run_load_and_validate( submission_id: int, path: str, column_map: Dict[str, str], - ): - ... + ): ... class Authentication(Protocol): - def fetch_oauth2_token(self, provider_name: str, code: str) -> str | None: - ... + def fetch_oauth2_token(self, provider_name: str, code: str) -> str | None: ... def fetch_user_email_from_login_provider( self, provider_name: str, oauth2_token: str - ) -> str | list[str] | None: - ... + ) -> str | list[str] | None: ... - def get_logout_url(self, provider_name: str) -> str: - ... + def get_logout_url(self, provider_name: str) -> str: ... - def make_login_url(self, provider_name: str, state_token: str) -> str | None: - ... + def make_login_url(self, provider_name: str, state_token: str) -> str | None: ... - def make_logout_url(self, provider_name: str) -> str | None: - ... + def make_logout_url(self, provider_name: str) -> str | None: ... - def user_email_address_has_permitted_domain(self, email: str | list[str]) -> bool: - ... + def user_email_address_has_permitted_domain( + self, email: str | list[str] + ) -> bool: ... class ApplicationContext: diff --git a/nad_ch/application/use_cases/data_submissions.py b/nad_ch/application/use_cases/data_submissions.py index 413c785..657144f 100644 --- a/nad_ch/application/use_cases/data_submissions.py +++ b/nad_ch/application/use_cases/data_submissions.py @@ -1,49 +1,20 @@ import os -from typing import List +from typing import List, IO from nad_ch.application.dtos import DownloadResult +from nad_ch.application.exceptions import ( + InvalidDataSubmissionFileError, + InvalidSchemaError, +) from nad_ch.application.interfaces import ApplicationContext +from nad_ch.application.validation import FileValidator from nad_ch.application.view_models import ( get_view_model, DataSubmissionViewModel, ) -from nad_ch.core.entities import DataSubmission, ColumnMap +from nad_ch.core.entities import DataSubmissionStatus, DataSubmission, ColumnMap from nad_ch.config import LANDING_ZONE -def ingest_data_submission( - ctx: ApplicationContext, file_path: str, producer_name: str -) -> DataSubmissionViewModel: - if not file_path: - ctx.logger.error("File path required") - return - - _, file_extension = os.path.splitext(file_path) - if file_extension.lower() not in [".zip", ".csv"]: - ctx.logger.error("Invalid file format. Only ZIP or CSV files are accepted.") - return - - producer = ctx.producers.get_by_name(producer_name) - if not producer: - ctx.logger.error("Producer with that name does not exist") - return - - try: - filename = DataSubmission.generate_filename(file_path, producer) - ctx.storage.upload(file_path, filename) - - # TODO: Finish logic for obtaining column map from user - column_map = ColumnMap("placeholder", producer, 1) - - submission = DataSubmission(filename, producer, column_map) - saved_submission = ctx.submissions.add(submission) - ctx.logger.info(f"Submission added: {saved_submission.filename}") - - return get_view_model(saved_submission) - except Exception as e: - ctx.storage.delete(filename) - ctx.logger.error(f"Failed to process submission: {e}") - - def get_data_submission( ctx: ApplicationContext, submission_id: int ) -> DataSubmissionViewModel: @@ -55,7 +26,7 @@ def get_data_submission( return get_view_model(submission) -def list_data_submissions_by_producer( +def get_data_submissions_by_producer( ctx: ApplicationContext, producer_name: str ) -> List[DataSubmissionViewModel]: producer = ctx.producers.get_by_name(producer_name) @@ -66,28 +37,32 @@ def list_data_submissions_by_producer( submissions = ctx.submissions.get_by_producer(producer) ctx.logger.info(f"Data submissions for {producer.name}") for s in submissions: - ctx.logger.info(f"{s.producer.name}: {s.filename}") + ctx.logger.info(f"{s.producer.name}: {s.name}") return get_view_model(submissions) def validate_data_submission( - ctx: ApplicationContext, filename: str, column_map_name: str + ctx: ApplicationContext, file_path: str, column_map_name: str ): - submission = ctx.submissions.get_by_filename(filename) + submission = ctx.submissions.get_by_file_path(file_path) if not submission: ctx.logger.error("Data submission with that filename does not exist") return - download_result: DownloadResult = ctx.storage.download_temp(filename) + download_result: DownloadResult = ctx.storage.download_temp(file_path) if not download_result: ctx.logger.error("Data extration error") return + column_map = ctx.column_maps.get_by_name_and_version(column_map_name, 1) + if column_map is None: + ctx.logger.error("Column map not found") + return + # Using version 1 for column maps for now, may add feature for user to select # version later try: - column_map = ctx.column_maps.get_by_name_and_version(column_map_name, 1) mapped_data_local_dir = submission.get_mapped_data_dir( download_result.extracted_dir, LANDING_ZONE ) @@ -112,3 +87,71 @@ def validate_data_submission( finally: ctx.storage.cleanup_temp_dir(download_result.temp_dir) ctx.storage.cleanup_temp_dir(mapped_data_local_dir) + + +def validate_file_before_submission( + ctx: ApplicationContext, file: IO[bytes], column_map_id: int +) -> bool: + column_map = ctx.column_maps.get_by_id(column_map_id) + if column_map is None: + raise ValueError("Column map not found") + + _, file_extension = os.path.splitext(file.filename) + if file_extension.lower() != ".zip": + raise InvalidDataSubmissionFileError( + "Invalid file format. Only ZIP files are accepted." + ) + + file_validator = FileValidator(file, file.name) + if not file_validator.validate_file(): + raise InvalidDataSubmissionFileError( + "Invalid file format. Only Shapefiles and Geodatabase files are accepted." + ) + + if not file_validator.validate_schema(column_map): + raise InvalidSchemaError( + "Invalid schema. The schema of the file must align with the schema of the \ + selected mapping." + ) + + return True + + +def create_data_submission( + ctx: ApplicationContext, + user_id: int, + column_map_id: int, + submission_name: str, + file: IO[bytes], +): + user = ctx.users.get_by_id(user_id) + if user is None: + raise ValueError("User not found") + + producer = user.producer + if producer is None: + raise ValueError("Producer not found") + + column_map = ctx.column_maps.get_by_id(column_map_id) + if column_map is None: + raise ValueError("Column map not found") + + try: + file_path = DataSubmission.generate_zipped_file_path(submission_name, producer) + submission = DataSubmission( + submission_name, + file_path, + DataSubmissionStatus.PENDING_SUBMISSION, + producer, + column_map, + ) + saved_submission = ctx.submissions.add(submission) + + ctx.storage.upload(file, file_path) + + ctx.logger.info(f"Submission added: {saved_submission.file_path}") + + return get_view_model(saved_submission) + except Exception as e: + ctx.storage.delete(file_path) + ctx.logger.error(f"Failed to process submission: {e}") diff --git a/nad_ch/application/validation.py b/nad_ch/application/validation.py index 67ba68d..10b742d 100644 --- a/nad_ch/application/validation.py +++ b/nad_ch/application/validation.py @@ -1,10 +1,16 @@ -from typing import Dict, Optional +import os +from typing import Dict, List, Optional, IO +from zipfile import ZipFile +import fiona from geopandas import GeoDataFrame import pandas as pd +import shapefile +import tempfile from nad_ch.application.dtos import ( DataSubmissionReportFeature, DataSubmissionReportOverview, ) +from nad_ch.application.interfaces import Storage import glob from pathlib import Path from nad_ch.core.entities import ColumnMap @@ -162,3 +168,104 @@ def run(self, gdf_batch: GeoDataFrame): self.initialize_overview_details(gdf_batch, self.valid_mappings) self.update_feature_details(gdf_batch) self.update_overview_details(gdf_batch) + + +class FileValidator: + def __init__(self, file: IO[bytes], filename: str) -> None: + self.file = file + self.filename = filename + + def validate_file(self) -> bool: + """Confirm that the file is a valid shapefile or geodatabase.""" + if not self._is_zipped(): + return False + + with ZipFile(self.file) as zip_file: + file_names = zip_file.namelist() + if not ( + self._is_valid_shapefile(file_names) + or self._is_valid_geodatabase(file_names) + ): + return False + + return True + + def validate_schema(self, column_map: Dict[str, str]) -> bool: + """Confirm that the schema is accommodated by the selected mapping.""" + with ZipFile(self.file) as zip_file: + file_names = zip_file.namelist() + if self._is_valid_shapefile(file_names): + return self._validate_shapefile_schema(zip_file, column_map) + elif self._is_valid_geodatabase(file_names): + return self._validate_gdb_schema(zip_file, column_map) + else: + return False + + def _is_zipped(self) -> bool: + _, file_extension = os.path.splitext(self.filename) + if file_extension.lower() != ".zip": + return False + + return True + + def _is_valid_shapefile(self, file_names: List[str]) -> bool: + required_extensions = {".shp", ".shx", ".dbf"} + found_extensions = set(os.path.splitext(name)[1].lower() for name in file_names) + return required_extensions.issubset(found_extensions) + + def _is_valid_geodatabase(self, file_names: List[str]) -> bool: + return any( + (name.endswith(".gdb") or name.endswith(".gdb/")) and "/" in name + for name in file_names + ) + + def _validate_shapefile_schema( + self, zip_file: ZipFile, expected_schema: Dict[str, str] + ) -> bool: + file_names = zip_file.namelist() + + shp_file = next((name for name in file_names if name.endswith(".shp")), None) + if not shp_file: + return False + + temp_dir = tempfile.mkdtemp() + zip_file.extractall(temp_dir) + path = os.path.join(temp_dir, shp_file) + sf = shapefile.Reader(path) + + fields = [field[0] for field in sf.fields] + filtered_fields = [ + item for item in fields if (item != "DeletionFlag" and item != "index") + ] + filtered_expected_fields = [ + value for value in expected_schema.values() if value is not None + ] + + return all(value in filtered_fields for value in filtered_expected_fields) + + def _validate_gdb_schema( + self, zip_file: ZipFile, expected_schema: Dict[str, str] + ) -> bool: + file_names = zip_file.namelist() + gdb_dir = next((name for name in file_names if name.endswith(".gdb/")), None) + + if not gdb_dir: + return False + + temp_dir = tempfile.mkdtemp() + zip_file.extractall(temp_dir) + gdb_path = os.path.join(temp_dir, gdb_dir) + + layers = fiona.listlayers(gdb_path) + + for layer_name in layers: + with fiona.open(gdb_path, layer=layer_name) as layer: + fields = layer.schema["properties"].keys() + filtered_expected_fields = [ + value for value in expected_schema.values() if value is not None + ] + all_present = all(value in fields for value in filtered_expected_fields) + if all_present: + return True + + return False diff --git a/nad_ch/application/view_models.py b/nad_ch/application/view_models.py index 3fb036a..79bb276 100644 --- a/nad_ch/application/view_models.py +++ b/nad_ch/application/view_models.py @@ -3,7 +3,13 @@ import json import numpy as np from typing import Union, Dict, List, Tuple, Protocol -from nad_ch.core.entities import Entity, ColumnMap, DataProducer, DataSubmission +from nad_ch.core.entities import ( + Entity, + ColumnMap, + DataProducer, + DataSubmissionStatus, + DataSubmission, +) class ViewModel(Protocol): @@ -96,7 +102,9 @@ def create_data_producer_vm(producer: DataProducer) -> DataProducerViewModel: class DataSubmissionViewModel(ViewModel): id: int date_created: str - filename: str + name: str + status: str + status_tag_class: str producer_name: str report: str @@ -106,10 +114,28 @@ def create_data_submission_vm(submission: DataSubmission) -> DataSubmissionViewM if submission.report is not None: report_json = enrich_report(submission.report) + status_map = { + DataSubmissionStatus.PENDING_SUBMISSION: "Pending submission", + DataSubmissionStatus.CANCELED: "Canceled", + DataSubmissionStatus.PENDING_VALIDATION: "Pending validation", + DataSubmissionStatus.FAILED: "Validation failed", + DataSubmissionStatus.VALIDATED: "Validated", + } + + status_tag_class_map = { + DataSubmissionStatus.PENDING_SUBMISSION: "usa-tag__warning", + DataSubmissionStatus.CANCELED: "usa-tag__error", + DataSubmissionStatus.PENDING_VALIDATION: "usa-tag__warning", + DataSubmissionStatus.FAILED: "usa-tag__error", + DataSubmissionStatus.VALIDATED: "usa-tag__success", + } + return DataSubmissionViewModel( id=submission.id, date_created=present_date(submission.created_at), - filename=submission.filename, + name=submission.name, + status=status_map[submission.status], + status_tag_class=status_tag_class_map[submission.status], producer_name=submission.producer.name, report=report_json, ) diff --git a/nad_ch/controllers/cli.py b/nad_ch/controllers/cli.py index ad6b293..5ff488b 100644 --- a/nad_ch/controllers/cli.py +++ b/nad_ch/controllers/cli.py @@ -4,8 +4,7 @@ list_data_producers, ) from nad_ch.application.use_cases.data_submissions import ( - ingest_data_submission, - list_data_submissions_by_producer, + get_data_submissions_by_producer, validate_data_submission, ) @@ -31,21 +30,12 @@ def list_producers(ctx): list_data_producers(context) -@cli.command() -@click.pass_context -@click.argument("file_path") -@click.argument("producer") -def ingest(ctx, file_path, producer): - context = ctx.obj - ingest_data_submission(context, file_path, producer) - - @cli.command() @click.pass_context @click.argument("producer") def list_submissions_by_producer(ctx, producer): context = ctx.obj - list_data_submissions_by_producer(context, producer) + get_data_submissions_by_producer(context, producer) @cli.command() diff --git a/nad_ch/controllers/web/images/submission.png b/nad_ch/controllers/web/images/submission.png new file mode 100644 index 0000000..18b089c Binary files /dev/null and b/nad_ch/controllers/web/images/submission.png differ diff --git a/nad_ch/controllers/web/routes/data_submissions.py b/nad_ch/controllers/web/routes/data_submissions.py index 90676d3..c5d09d8 100644 --- a/nad_ch/controllers/web/routes/data_submissions.py +++ b/nad_ch/controllers/web/routes/data_submissions.py @@ -1,8 +1,26 @@ -from flask import Blueprint, current_app, render_template, g, jsonify +from flask import ( + Blueprint, + current_app, + render_template, + g, + jsonify, + request, + abort, + flash, + redirect, + url_for, +) from flask_login import login_required, current_user +from nad_ch.application.exceptions import ( + InvalidDataSubmissionFileError, + InvalidSchemaError, +) +from nad_ch.application.use_cases.column_maps import get_column_maps_by_producer from nad_ch.application.use_cases.data_submissions import ( get_data_submission, - list_data_submissions_by_producer, + get_data_submissions_by_producer, + create_data_submission, + validate_file_before_submission, ) @@ -17,28 +35,151 @@ def before_request(): @submissions_bp.route("/data-submissions") @login_required def index(): - pass + try: + view_models = get_data_submissions_by_producer( + g.ctx, current_user.producer.name + ) + return render_template("data_submissions/index.html", submissions=view_models) + except ValueError: + abort(404) -@submissions_bp.route("/data-submissions/") +@submissions_bp.route("/data-submissions/") @login_required -def show(submission_id): - pass +def show(id): + try: + view_model = get_data_submission(g.ctx, id) + return render_template("data_submissions/show.html", submission=view_model) + except Exception: + abort(404) -@submissions_bp.route("/reports") +@submissions_bp.route("/data-submissions/create") @login_required -def reports(): - # For demo purposes, hard-code the producer name - view_model = list_data_submissions_by_producer(g.ctx, current_user.producer.name) - return render_template("data_submissions/index.html", submissions=view_model) +def create(): + if "name" not in request.args: + abort(404) + name = request.args.get("name") + column_map_options = get_column_maps_by_producer(g.ctx, current_user.producer.name) + return render_template( + "data_submissions/create.html", name=name, column_map_options=column_map_options + ) -@submissions_bp.route("/reports/") + +@submissions_bp.route("/data-submissions", methods=["POST"]) @login_required -def view_report(submission_id): - view_model = get_data_submission(g.ctx, submission_id) - return render_template("data_submissions/show.html", submission=view_model) +def store(): + name = request.form.get("name") + column_map_id = request.form.get("column-map-id") + + if not name: + flash("Name is required") + return redirect(url_for("submissions.create")) + + if "mapping-csv-input" not in request.files: + flash("No file included") + return redirect(url_for("submissions.create", name=name)) + + file = request.files["mapping-csv-input"] + if file.filename == "": + flash("No selected file") + return redirect(url_for("submissions.create", name=name)) + + try: + validate_file_before_submission(g.ctx, file, column_map_id) + except InvalidDataSubmissionFileError as e: + flash(f"Error: {e}") + return redirect(url_for("submissions.create", name=name)) + except InvalidSchemaError as e: + flash(f"Error: {e}") + return redirect(url_for("submissions.create", name=name)) + + try: + view_model = create_data_submission( + g.ctx, current_user.id, column_map_id, name, file + ) + + return redirect(url_for("submissions.edit", id=view_model.id)) + except ValueError as e: + flash(f"Error: {e}") + return redirect(url_for("submissions.create", name=name)) + + +@submissions_bp.route("/data-submissions/edit/") +@login_required +def edit(id): + try: + view_model = get_data_submission(g.ctx, id) + + data = [ + { + "Add_Number": "34", + "St_Name": "Willow Creek Blvd", + "Latitude": "44.968046", + "Longitude": "-94.420307", + }, + { + "Add_Number": "654", + "St_Name": "Pinehurst Lane", + "Latitude": "33.755787", + "Longitude": "-89.132008", + }, + { + "Add_Number": "324", + "St_Name": "Cedarwood Drive", + "Latitude": "33.844843", + "Longitude": "-116.359998", + }, + { + "Add_Number": "43", + "St_Name": "Mapleview Court", + "Latitude": "44.92057", + "Longitude": "-116.54911", + }, + { + "Add_Number": "5", + "St_Name": "Elm Street", + "Latitude": "44.240309", + "Longitude": "-93.44786", + }, + { + "Add_Number": "545", + "St_Name": "Oak Ridge Way", + "Latitude": "44.968041", + "Longitude": "-91.493619", + }, + { + "Add_Number": "34", + "St_Name": "Sunnybrook Road", + "Latitude": "44.333304", + "Longitude": "-94.419696", + }, + { + "Add_Number": "4", + "St_Name": "Riverside Terrace", + "Latitude": "33.755783", + "Longitude": "-89.132027", + }, + { + "Add_Number": "34", + "St_Name": "Meadowbrook Lane", + "Latitude": "33.844847", + "Longitude": "-116.360066", + }, + { + "Add_Number": "34", + "St_Name": "Aspen Grove Circle", + "Latitude": "44.920474", + "Longitude": "-116.549069", + }, + ] + + return render_template( + "data_submissions/edit.html", submission=view_model, data=data + ) + except Exception: + abort(404) @submissions_bp.route("/api/reports/") diff --git a/nad_ch/controllers/web/src/components/CompletenessReport.ts b/nad_ch/controllers/web/src/components/CompletenessReport.ts index c286172..56400bb 100644 --- a/nad_ch/controllers/web/src/components/CompletenessReport.ts +++ b/nad_ch/controllers/web/src/components/CompletenessReport.ts @@ -40,6 +40,7 @@ export interface CompletenessReportComponent { init(): Promise; groupFeatures(features: Feature[]): GroupedFeature[]; getStatusTagClass(status: string): string; + getNadFieldText(feature: Feature): string; } export function CompletenessReport( @@ -108,5 +109,9 @@ export function CompletenessReport( return 'usa-tag__info'; } }, + getNadFieldText(feature: Feature): string { + const embellishment = feature.required ? '*' : ''; + return `${embellishment} ${feature.nad_feature_name}`; + }, }; } diff --git a/nad_ch/controllers/web/src/components/MappingForm.spec.ts b/nad_ch/controllers/web/src/components/MappingForm.spec.ts index f51f5f2..c30fd3e 100644 --- a/nad_ch/controllers/web/src/components/MappingForm.spec.ts +++ b/nad_ch/controllers/web/src/components/MappingForm.spec.ts @@ -27,7 +27,7 @@ describe('MappingForm', () => { it('should set error state and message when name is invalid', () => { mappingForm.name = 'Invalid Name'; - mappingForm.createMapping(); + mappingForm.create(); expect(mappingForm.hasError).toBe(true); expect(mappingForm.errorMessage).toBe( @@ -39,7 +39,7 @@ describe('MappingForm', () => { const validName = 'ValidName123'; mappingForm.name = validName; - mappingForm.createMapping(); + mappingForm.create(); expect(mappingForm.hasError).toBe(false); expect(navigateTo).toHaveBeenCalledWith( diff --git a/nad_ch/controllers/web/src/components/MappingForm.ts b/nad_ch/controllers/web/src/components/MappingForm.ts index 0a9f9dd..a66a741 100644 --- a/nad_ch/controllers/web/src/components/MappingForm.ts +++ b/nad_ch/controllers/web/src/components/MappingForm.ts @@ -7,7 +7,7 @@ export interface MappingFormComponent { hasError: boolean; errorMessage: string; name: string; - createMapping: () => void; + create: () => void; closeModal: () => void; } @@ -16,7 +16,7 @@ export function MappingForm(): AlpineComponent { hasError: false, errorMessage: '', name: '', - createMapping(): void { + create(): void { this.hasError = false; const validationError = getMappingNameValidationError(this.name); diff --git a/nad_ch/controllers/web/src/components/SubmissionForm.spec.ts b/nad_ch/controllers/web/src/components/SubmissionForm.spec.ts new file mode 100644 index 0000000..a545826 --- /dev/null +++ b/nad_ch/controllers/web/src/components/SubmissionForm.spec.ts @@ -0,0 +1,61 @@ +/** + * @jest-environment jsdom + */ + +import { SubmissionForm, SubmissionFormComponent } from './SubmissionForm'; +import { BASE_URL } from '../config'; +import { navigateTo } from '../utilities'; +import { AlpineComponent } from 'alpinejs'; + +jest.mock('../utilities', () => ({ + navigateTo: jest.fn(), +})); + +describe('MappingForm', () => { + let submissionForm: AlpineComponent; + + beforeEach(() => { + submissionForm = SubmissionForm(); + }); + + it('should initialize with correct initial state', () => { + expect(submissionForm.hasError).toBe(false); + expect(submissionForm.errorMessage).toBe(''); + expect(submissionForm.name).toBe(''); + }); + + it('should set error state and message when name is invalid', () => { + submissionForm.name = 'Invalid Name'; + + submissionForm.create(); + + expect(submissionForm.hasError).toBe(true); + expect(submissionForm.errorMessage).toBe( + 'Name can only contain letters and numbers', + ); + }); + + it('should navigate to create mapping page when name is valid', () => { + const validName = 'ValidName123'; + submissionForm.name = validName; + + submissionForm.create(); + + expect(submissionForm.hasError).toBe(false); + expect(navigateTo).toHaveBeenCalledWith( + `${BASE_URL}/data-submissions/create?name=${encodeURIComponent(validName)}`, + ); + }); + + it('should reset state and close modal', () => { + submissionForm.hasError = true; + submissionForm.errorMessage = 'Error message'; + submissionForm.name = 'Some name'; + + submissionForm.closeModal(); + + expect(submissionForm.hasError).toBe(false); + expect(submissionForm.errorMessage).toBe(''); + expect(submissionForm.name).toBe(''); + }); +}); diff --git a/nad_ch/controllers/web/src/components/SubmissionForm.ts b/nad_ch/controllers/web/src/components/SubmissionForm.ts new file mode 100644 index 0000000..70a52d3 --- /dev/null +++ b/nad_ch/controllers/web/src/components/SubmissionForm.ts @@ -0,0 +1,45 @@ +import { AlpineComponent } from 'alpinejs'; +import { BASE_URL } from '../config'; +import { getSubmissionNameValidationError } from '../formValidation'; +import { navigateTo } from '../utilities'; + +export interface SubmissionFormComponent { + hasError: boolean; + errorMessage: string; + name: string; + create: () => void; + closeModal: () => void; +} + +export function SubmissionForm(): AlpineComponent { + return { + hasError: false, + errorMessage: '', + name: '', + create(): void { + this.hasError = false; + + const validationError = getSubmissionNameValidationError(this.name); + + if (validationError) { + this.hasError = true; + this.errorMessage = validationError; + } else { + navigateTo( + `${BASE_URL}/data-submissions/create?name=${encodeURIComponent(this.name)}`, + ); + } + }, + closeModal(): void { + this.name = ''; + this.hasError = false; + this.errorMessage = ''; + const button: HTMLElement | null = + document.getElementById('cancel-button'); + if (button) { + button.setAttribute('data-close-modal', ''); + button.click(); + } + }, + }; +} diff --git a/nad_ch/controllers/web/src/formValidation.ts b/nad_ch/controllers/web/src/formValidation.ts index b494947..fa72431 100644 --- a/nad_ch/controllers/web/src/formValidation.ts +++ b/nad_ch/controllers/web/src/formValidation.ts @@ -8,3 +8,14 @@ export function getMappingNameValidationError(name: string): string | null { } return null; } + +export function getSubmissionNameValidationError(name: string): string | null { + if (name === '') { + return 'Submission name is required'; + } else if (name.length > 25) { + return 'Enter less than 25 letters or numbers'; + } else if (/[^a-zA-Z0-9]/.test(name)) { + return 'Name can only contain letters and numbers'; + } + return null; +} diff --git a/nad_ch/controllers/web/src/index.ts b/nad_ch/controllers/web/src/index.ts index c2210df..7f91a5f 100644 --- a/nad_ch/controllers/web/src/index.ts +++ b/nad_ch/controllers/web/src/index.ts @@ -2,6 +2,7 @@ import '@uswds/uswds/css/uswds.css'; import '@uswds/uswds'; import Alpine from 'alpinejs'; import { MappingForm } from './components/MappingForm'; +import { SubmissionForm } from './components/SubmissionForm'; import { CompletenessReport } from './components/CompletenessReport'; declare global { @@ -13,6 +14,7 @@ declare global { document.addEventListener('alpine:init', () => { Alpine.data('MappingForm', MappingForm); + Alpine.data('SubmissionForm', SubmissionForm); Alpine.data('CompletenessReport', CompletenessReport); }); diff --git a/nad_ch/controllers/web/templates/_layouts/sidebar.html b/nad_ch/controllers/web/templates/_layouts/sidebar.html index 7775101..b3cec60 100644 --- a/nad_ch/controllers/web/templates/_layouts/sidebar.html +++ b/nad_ch/controllers/web/templates/_layouts/sidebar.html @@ -25,9 +25,9 @@
  • ReportsSubmissions
    • {% endif %} diff --git a/nad_ch/controllers/web/templates/column_maps/index.html b/nad_ch/controllers/web/templates/column_maps/index.html index 492cfcd..3d8aed7 100644 --- a/nad_ch/controllers/web/templates/column_maps/index.html +++ b/nad_ch/controllers/web/templates/column_maps/index.html @@ -4,7 +4,7 @@
    -

    Create Your First Mapping

    +

    Create your first mapping

    @@ -41,7 +41,7 @@

    Create Your First Mapping

    New mapping

    -
    +

    diff --git a/nad_ch/controllers/web/templates/data_submissions/create.html b/nad_ch/controllers/web/templates/data_submissions/create.html new file mode 100644 index 0000000..9fedc34 --- /dev/null +++ b/nad_ch/controllers/web/templates/data_submissions/create.html @@ -0,0 +1,71 @@ +{% extends "_layouts/base.html" %} {% block title %}{{ "Submission " + name}}{% +endblock %} {% block content %} + + +
    +
    +
    +
    + +
    + Cancel + +
    +
    +
    +
    +
    + +
      +
    1. +

      Select a mapping

      +

      + Select a mapping that matches the schema of the addresses you are about + to submit. +

      + +
      +

      +
      2. + +
    2. +

      Upload addresses

      +

      Select a zipped .gdb or shapefile to start the submission process.

      +
      + + + + +
      + + {% with messages = get_flashed_messages() %} {% if messages %} +
      + {% for message in messages %} + + {% endfor %} +
      + {% endif %} {% endwith %} +
      3. +
    + + +{% endblock %} diff --git a/nad_ch/controllers/web/templates/data_submissions/edit.html b/nad_ch/controllers/web/templates/data_submissions/edit.html new file mode 100644 index 0000000..6563fb6 --- /dev/null +++ b/nad_ch/controllers/web/templates/data_submissions/edit.html @@ -0,0 +1,58 @@ +{% extends "_layouts/base.html" %} {% block title %}Submission{% endblock %} {% +block content %} + +
    +
    +
    +
    +
    + +
    + Cancel + +
    +
    +
    +
    +
    +

    Review submission

    +

    This data is a sample of what the NAD administrator will receive.

    +
    + + + + + + + + + + + + {% for row in data %} + + + + + + + {% endfor %} + + + + +
    Add_NumberSt_NameLatitudeLongitude
    + {{ row["Add_Number"] }} + {{ row["St_Name"] }}{{ row["Latitude"] }}>{{ row["Longitude"] }}
    ...
    +
    +
    + +{% endblock %} diff --git a/nad_ch/controllers/web/templates/data_submissions/index.html b/nad_ch/controllers/web/templates/data_submissions/index.html index a9b1c0b..f5b0cff 100644 --- a/nad_ch/controllers/web/templates/data_submissions/index.html +++ b/nad_ch/controllers/web/templates/data_submissions/index.html @@ -1,12 +1,36 @@ -{% extends "_layouts/base.html" %} {% block title %}Home Page{% endblock %} {% -block content %} {% from "components/page-header.html" import page_header %} {{ -page_header("Reports") }} {% if submissions %} +{% extends "_layouts/base.html" %} {% block title %}Submissions{% endblock %} {% +block content %} +
    +
    +
    +
    + +
    + {% if submissions %} + Create submission + {% endif %} +
    +
    +
    +
    +
    + +{% if submissions %}
    + @@ -14,14 +38,25 @@ {% for sub in submissions %} + {% endfor %} @@ -29,5 +64,88 @@
    Name CreatedStatus
    - {{ sub.filename }} + {{ sub.name }} {{ sub.date_created }} - {{ sub.status }} + + {% if sub.status == "Validated" %} + View Report + {% else %} + + View + {% endif %}
    {% else %} -

    You haven't uploaded any data submissions yet.

    -{% endif %} {% endblock %} +
    +
    +
    +

    Prepare your first submission

    +
    +
    +
    + A placeholder submission image +
    +
    +
    +

    + Upload and submit your address data to begin collaborating and + contributing to the National Address Database. +

    +
    +
    + Create submission +
    +
    +
    +{% endif %} +
    +
    +
    +

    + New submission +

    +
    +
    + +
    + +
    + No more than 25 letters or numbers +
    + +
    +
      +
    • + +
      • +
    • + +
      • +
    +
    +
    +
    +
    +
    +
    +{% endblock %} diff --git a/nad_ch/controllers/web/templates/data_submissions/show.html b/nad_ch/controllers/web/templates/data_submissions/show.html index e4b3124..009211a 100644 --- a/nad_ch/controllers/web/templates/data_submissions/show.html +++ b/nad_ch/controllers/web/templates/data_submissions/show.html @@ -1,13 +1,11 @@ -{% extends "_layouts/base.html" %} {% block title %}Report{% endblock %} {%block -content %} +{% extends "_layouts/base.html" %} {% block title %}Submission{% endblock %} +{%block content %}