Merge pull request #48 from GSA-TTS/mapping-ui

Mapping UI

.github/workflows/ci.yaml

@@ -13,27 +13,47 @@ jobs:
 
     steps:
       - name: Check out repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
+      - name: Read .python-version
+        run: echo "##[set-output name=PYTHON_VERSION;]$(cat .python-version)"
+        id: python-version
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: "${{ steps.python-version.outputs.PYTHON_VERSION }}"
 
       - name: Install poetry
         shell: bash
         run: |
           curl -sSL https://install.python-poetry.org | python3 -
           echo "/root/.local/bin" >> $GITHUB_PATH
 
-      - name: Install dependencies
+      - name: Install Python dependencies
         shell: bash
         run: poetry install
 
-      - name: Lint
+      - name: Lint backend code
         shell: bash
         run: poetry run flake8
 
-      - name: Test
+      - name: Test backend code
         shell: bash
         run: poetry run pytest
+
+      - name: Read .nvmrc
+        run: echo "##[set-output name=NVMRC;]$(cat .nvmrc)"
+        id: nvm
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "${{ steps.nvm.outputs.NVMRC }}"
+
+      - name: Install npm dependencies, lint, and test frontend code
+        run: |
+          cd nad_ch/controllers/web
+          npm install
+          npm run lint
+          npm test

.github/workflows/deploy-dev.yml

@@ -9,7 +9,7 @@ jobs:
 
     steps:
       - name: Check out repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4

.nvmrc

@@ -0,0 +1 @@
+18.17.1

nad_ch/application/interfaces.py

@@ -1,6 +1,6 @@
 from typing import Optional, Protocol, Dict
 from nad_ch.application.dtos import DownloadResult
-from nad_ch.domain.repositories import (
+from nad_ch.core.repositories import (
     DataProducerRepository,
     DataSubmissionRepository,
     UserRepository,
@@ -106,6 +106,8 @@ def __getitem__(self, key: str):
             return self.submissions
         elif key == "users":
             return self.users
+        elif key == "column_maps":
+            return self.column_maps
         elif key == "logger":
             return self.logger
         elif key == "storage":

nad_ch/application/use_cases/auth.py

@@ -5,7 +5,7 @@
     OAuth2TokenError,
 )
 from nad_ch.application.interfaces import ApplicationContext
-from nad_ch.domain.entities import User
+from nad_ch.core.entities import User
 
 
 def get_or_create_user(ctx: ApplicationContext, provider_name: str, email: str) -> User:

nad_ch/application/use_cases/column_maps.py

@@ -0,0 +1,120 @@
+import csv
+from io import StringIO
+from typing import Dict, List, IO
+from nad_ch.application.interfaces import ApplicationContext
+from nad_ch.application.view_models import (
+    get_view_model,
+    ColumnMapViewModel,
+)
+from nad_ch.core.entities import ColumnMap
+
+
+def add_column_map(
+    ctx: ApplicationContext, user_id: int, name: str, mapping: Dict[str, str]
+):
+    user = ctx.users.get_by_id(user_id)
+    if user is None:
+        raise ValueError("User not found")
+
+    # TODO get the producer name from the user's producer property
+    producer = ctx.producers.get_by_name("New Jersey")
+    if producer is None:
+        raise ValueError("Producer not found")
+
+    # Note: will need to account for admin permissions to update any DataProducer's
+    # column mapping, and for users associated with the DataProducer to update ONLY
+    # their own column mapping
+
+    column_map = ColumnMap(name, producer, mapping, 1)
+
+    if not column_map.is_valid():
+        raise ValueError("Invalid mapping")
+
+    saved_column_map = ctx.column_maps.add(column_map)
+    ctx.logger.info("Column Map added")
+
+    return get_view_model(saved_column_map)
+
+
+def get_column_map(ctx: ApplicationContext, id: int) -> ColumnMapViewModel:
+    column_map = ctx.column_maps.get_by_id(id)
+
+    if column_map is None:
+        raise ValueError("Column map not found")
+
+    return get_view_model(column_map)
+
+
+def get_column_maps_by_producer(
+    ctx: ApplicationContext, producer_name: str
+) -> List[ColumnMapViewModel]:
+    producer = ctx.producers.get_by_name(producer_name)
+    if not producer:
+        raise ValueError("Producer not found")
+    column_maps = ctx.column_maps.get_by_producer(producer)
+
+    return [get_view_model(column_map) for column_map in column_maps]
+
+
+def update_column_mapping(
+    ctx: ApplicationContext, id: int, new_mapping: Dict[str, str]
+):
+    column_map = ctx.column_maps.get_by_id(id)
+
+    if column_map is None:
+        raise ValueError("Column map not found")
+
+    column_map.mapping = {
+        key: new_mapping[key] for key in ColumnMap.all_fields if key in new_mapping
+    }
+
+    if not column_map.is_valid():
+        raise ValueError("Invalid mapping")
+
+    ctx.column_maps.update(column_map)
+
+    return get_view_model(column_map)
+
+
+def update_column_mapping_field(
+    ctx: ApplicationContext, id: int, user_field: str, nad_field: str
+):
+    column_map = ctx.column_maps.get_by_id(id)
+
+    if column_map is None:
+        raise ValueError("Column map not found")
+
+    column_map.mapping[nad_field] = user_field
+
+    column_map.mapping = {
+        key: column_map.mapping[key]
+        for key in ColumnMap.all_fields
+        if key in column_map.mapping
+    }
+
+    if not column_map.is_valid():
+        raise ValueError("Invalid mapping")
+
+    ctx.column_maps.update(column_map)
+
+    return get_view_model(column_map)
+
+
+def get_column_map_from_csv_file(file: IO[bytes]) -> Dict[str, str]:
+    file_content = file.read().decode("utf-8-sig")
+    stream = StringIO(file_content)
+    csv_reader = csv.reader(stream, dialect="excel")
+
+    headers = next(csv_reader)
+    if not headers:
+        raise Exception("CSV file is empty or invalid")
+
+    csv_dict = {}
+
+    for row in csv_reader:
+        if len(row) < 2:
+            continue
+        key, value = row[:2]
+        csv_dict[key] = value
+
+    return csv_dict

nad_ch/application/use_cases/data_producers.py

@@ -4,7 +4,7 @@
     get_view_model,
     DataProducerViewModel,
 )
-from nad_ch.domain.entities import DataProducer
+from nad_ch.core.entities import DataProducer
 
 
 def add_data_producer(

nad_ch/application/use_cases/data_submissions.py

@@ -6,7 +6,7 @@
     get_view_model,
     DataSubmissionViewModel,
 )
-from nad_ch.domain.entities import DataSubmission, ColumnMap
+from nad_ch.core.entities import DataSubmission, ColumnMap
 
 
 def ingest_data_submission(

nad_ch/application/validation.py

@@ -7,7 +7,7 @@
 )
 import glob
 from pathlib import Path
-from nad_ch.domain.entities import ColumnMap
+from nad_ch.core.entities import ColumnMap
 
 
 class DataValidator:

nad_ch/application/view_models.py

@@ -2,8 +2,8 @@
 from datetime import datetime
 import json
 import numpy as np
-from typing import Union, List, Tuple, TypeVar, Protocol
-from nad_ch.domain.entities import Entity, DataProducer, DataSubmission
+from typing import Union, Dict, List, Tuple, Protocol
+from nad_ch.core.entities import Entity, ColumnMap, DataProducer, DataSubmission
 
 
 class ViewModel(Protocol):
@@ -18,6 +18,7 @@ def get_view_model(
     get a static view model object that it can return to its caller.
     """
     entity_to_vm_function_map = {
+        ColumnMap: create_column_map_view_model,
         DataProducer: create_data_producer_vm,
         DataSubmission: create_data_submission_vm,
     }
@@ -36,6 +37,46 @@ def get_view_model(
         raise ValueError(f"No mapping function defined for entity type: {entity_type}")
 
 
+@dataclass
+class ColumnMapViewModel(ViewModel):
+    id: int
+    date_created: str
+    date_updated: str
+    name: str
+    mapping: Dict[str, str]
+    version: int
+    producer_name: str
+    available_nad_fields: List[str]
+    required_nad_fields: List[str]
+
+
+def create_column_map_view_model(column_map: ColumnMap) -> ColumnMapViewModel:
+    available_nad_fields = [
+        key
+        for key in ColumnMap.all_fields
+        if key not in column_map.mapping or column_map.mapping.get(key) in ["", None]
+    ]
+
+    date_updated = (
+        "-"
+        if column_map.updated_at == column_map.created_at
+        and column_map.updated_at is not None
+        else present_date(column_map.updated_at)
+    )
+
+    return ColumnMapViewModel(
+        id=column_map.id,
+        date_created=present_date(column_map.created_at),
+        date_updated=date_updated,
+        name=column_map.name,
+        mapping=column_map.mapping,
+        version=column_map.version_id,
+        producer_name=column_map.producer.name,
+        available_nad_fields=available_nad_fields,
+        required_nad_fields=ColumnMap.required_fields,
+    )
+
+
 @dataclass
 class DataProducerViewModel(ViewModel):
     id: int
@@ -61,8 +102,7 @@ class DataSubmissionViewModel(ViewModel):
 
 
 def create_data_submission_vm(submission: DataSubmission) -> DataSubmissionViewModel:
-    # TODO make this be an empty array so the frontend doesn't have to check for None
-    report_json = None
+    report_json = []
     if submission.report is not None:
         enriched_report = enrich_report(submission.report)
         report_json = json.dumps(enriched_report)

nad_ch/controllers/cli.py

@@ -51,6 +51,7 @@ def list_submissions_by_producer(ctx, producer):
 @cli.command()
 @click.pass_context
 @click.argument("filename")
-def validate_submission(ctx, filename):
+@click.argument("mapping_name")
+def validate_submission(ctx, filename, mapping_name):
     context = ctx.obj
-    validate_data_submission(context, filename)
+    validate_data_submission(context, filename, mapping_name)

nad_ch/controllers/web/.eslintrc.js

@@ -0,0 +1,28 @@
+module.exports = {
+  env: {
+    browser: true,
+    es2021: true,
+    node: true,
+  },
+  extends: [
+    'eslint:recommended',
+    'plugin:prettier/recommended',
+    'plugin:@typescript-eslint/recommended',
+  ],
+  overrides: [
+    {
+      env: {
+        node: true,
+      },
+      files: ['.eslintrc.{js,cjs}'],
+      parserOptions: {
+        sourceType: 'script',
+      },
+    },
+  ],
+  parserOptions: {
+    ecmaVersion: 'latest',
+    sourceType: 'module',
+  },
+  plugins: ['prettier'],
+};

nad_ch/controllers/web/.prettierrc

@@ -0,0 +1,3 @@
+{
+  "singleQuote": true
+}