Merge branch 'dev'

pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     'aind-data-access-api[docdb]',
     'aind-data-access-api[rds]',
     'aind-data-schema-models',
+    'aind-metadata-validator',
     'flask',
 ]
 

src/aind_metadata_viz/app.py

@@ -2,21 +2,13 @@
 import altair as alt
 import pandas as pd
 from aind_metadata_viz import database
+from aind_metadata_viz.utils import hd_style, AIND_COLORS
 from aind_data_schema import __version__ as ads_version
 
 pn.extension(design="material")
 pn.extension("vega")
 alt.themes.enable("ggplot2")
 
-AIND_COLORS = colors = {
-    "dark_blue": "#003057",
-    "light_blue": "#2A7DE1",
-    "green": "#1D8649",
-    "yellow": "#FFB71B",
-    "grey": "#7C7C7F",
-    "red": "#FF5733",
-}
-
 # Define CSS to set the background color
 background_color = AIND_COLORS[pn.state.location.query_params["background"] if "background" in pn.state.location.query_params else "dark_blue"]
 css = f"""
@@ -281,20 +273,16 @@ def update_selection(event):
     return pane
 
 
-def hd_style(text):
-    return (
-        f"<span style='font-weight: bold; color:{colors[text]}'>{text}</span>"
-    )
 
 
 header = (
     f"# Metadata Portal\n\n"
     "This app steps through all of the metadata stored in DocDB and determines whether every record's fields "
     "(and subfields) are "
-    f"{hd_style('valid')} for aind-data-schema v{ads_version}, "
-    f"{hd_style('present')} but invalid, {hd_style('optional')}, "
-    f"{hd_style('missing')}, or "
-    f"{hd_style('excluded')} for the record's modality."
+    f"{hd_style('valid', colors)} for aind-data-schema v{ads_version}, "
+    f"{hd_style('present', colors)} but invalid, {hd_style('optional', colors)}, "
+    f"{hd_style('missing', colors)}, or "
+    f"{hd_style('excluded', colors)} for the record's modality."
 )
 
 download_md = """
@@ -313,6 +301,11 @@ def hd_style(text):
 
 
 header_pane = pn.pane.Markdown(header, styles=outer_style, width=420)
+
+total_md = f"<p style=\"text-align:center\"><b>{db.get_overall_valid():1.2f}%</b> of all metadata records are fully {hd_style('valid', colors)}</p>"
+
+percent_total = pn.pane.Markdown(total_md, styles=outer_style, width=420)
+
 download_pane = pn.pane.Markdown(download_md)
 
 control_col = pn.Column(
@@ -330,6 +323,7 @@ def hd_style(text):
 # Left column (controls)
 left_col = pn.Column(
     header_pane,
+    percent_total,
     control_col,
     width=420,
 )
@@ -358,6 +352,26 @@ def build_row(selected_modality, derived_filter):
 # Put everything in a column and buffer it
 main_col = pn.Column(top_row, mid_plot, styles=outer_style, width=515)
 
-pn.Row(pn.HSpacer(), left_col, pn.Spacer(width=20), main_col, pn.HSpacer(), margin=20).servable(
+main_row = pn.Row(pn.HSpacer(), left_col, pn.Spacer(width=20), main_col, pn.HSpacer(), margin=20)
+
+# Add the validator section
+
+validator_name_selector = pn.widgets.TextInput(name="Enter asset name to validate:", value="single-plane-ophys_655019_2023-04-03_18-17-55", width=800)
+pn.state.location.sync(validator_name_selector, {"value": "validator_name"})
+
+validator = database.RecordValidator(validator_name_selector.value, colors)
+
+
+def build_validator(validator_name):
+    validator.update(validator_name)
+    col = pn.Column(validator_name_selector, validator.panel(), width=(515+20+420), styles=outer_style)
+    row = pn.Row(pn.HSpacer(), col, pn.HSpacer())
+    return row
+
+
+validator_row = pn.bind(build_validator,
+                        validator_name=validator_name_selector)
+
+pn.Column(main_row, validator_row).servable(
     title="Metadata Portal",
 )

src/aind_metadata_viz/database.py

@@ -1,12 +1,14 @@
 from aind_data_access_api.document_db import MetadataDbClient
 from aind_data_access_api.rds_tables import RDSCredentials
 from aind_data_access_api.rds_tables import Client
+from aind_metadata_validator.metadata_validator import validate_metadata
 import panel as pn
 import pandas as pd
 import param
 import os
 import numpy as np
-import time
+import io
+import logging
 from io import StringIO
 
 from aind_data_schema_models.modalities import (
@@ -18,7 +20,7 @@
     first_layer_field_mapping,
     second_layer_field_mappings,
 )
-from aind_metadata_viz.utils import METASTATE_MAP
+from aind_metadata_viz.utils import METASTATE_MAP, hd_style
 
 API_GATEWAY_HOST = os.getenv("API_GATEWAY_HOST", "api.allenneuraldynamics-test.org")
 DATABASE = os.getenv("DATABASE", "metadata_index")
@@ -79,12 +81,8 @@ def __init__(
     ):
         """Initialize"""
         # get data
-        start = time.time()
         self._file_data = _get_metadata(test_mode=test_mode)
-        print(time.time() - start)
-        start = time.time()
         self._status_data = _get_status()
-        print(time.time() - start)
 
         # inner join only keeps records that are in both dataframes
         self.data = pd.merge(self._file_data, self._status_data, on="_id", how="inner")
@@ -159,6 +157,10 @@ def get_expected_files(self) -> tuple[list[str], list[str]]:
 
         return (expected_files_by_modality, excluded_files_by_modality)
 
+    def get_overall_valid(self):
+        """Get the percentage of valid records"""
+        return np.sum(self.data['metadata'].values=='valid') / len(self.data) * 100
+
     def get_file_presence(self):
         """Get the presence of a list of files
 
@@ -350,15 +352,69 @@ def _get_metadata(test_mode=False) -> pd.DataFrame:
     )
 
 
-@pn.cache(ttl=CACHE_RESET_DAY)
-def _get_all(test_mode=False):
-    filter = {}
-    limit = 0 if not test_mode else 10
-    paginate_batch_size = 500
-    response = docdb_api_client.retrieve_docdb_records(
-        filter_query=filter,
-        limit=limit,
-        paginate_batch_size=paginate_batch_size,
-    )
+class RecordValidator():
 
-    return response
+    def __init__(self, id, colors):
+        """Populate the validator with a record and run validation
+
+        Parameters
+        ----------
+        id : _type_
+            _description_
+        """
+        self.update(id)
+        self.state = None
+        self.log = None
+        self.colors = colors
+
+    def update(self, name):
+
+        records = docdb_api_client.retrieve_docdb_records(filter_query={"name": name})
+        print(records)
+
+        if len(records) > 0:
+            self.record = records[0]
+        else:
+            self.state = None
+            self.log = None
+            return
+
+        # Create an in-memory buffer to capture log output
+        log_capture_string = io.StringIO()
+
+        # Set up a custom handler that writes to the buffer
+        ch = logging.StreamHandler(log_capture_string)
+        ch.setLevel(logging.INFO)  # Adjust level as needed
+
+        # Get the logger used in `validate_metadata`
+        logger = logging.getLogger()
+        logger.addHandler(ch)
+
+        # run the validator, capturing any errors
+        self.state = validate_metadata(self.record)
+
+        ch.flush()
+        self.log = log_capture_string.getvalue()
+        logger.removeHandler(ch)
+        log_capture_string.close()
+
+    def panel(self):
+        """Return a panel object with the validation results"""
+        if self.state is None:
+            return pn.pane.Markdown("No record was found.")
+        else:
+            print(self.state["metadata"].value)
+            state = pn.pane.Markdown(f"""
+Overall metadata: {hd_style(METASTATE_MAP[self.state["metadata"].value], self.colors)}
+""")
+            file_state = {}
+            for file in ALL_FILES:
+                file_state[file] = hd_style(METASTATE_MAP[self.state[file].value], self.colors)
+            print(file_state)
+            df = pd.DataFrame(file_state, index=[0])
+            file_state = pn.pane.DataFrame(df, width=920, escape=False)
+
+            log = pn.pane.Markdown(self.log, width=920)
+
+            return pn.Column(state, file_state, log, width=515)
+        # return (self.state, self.log)

src/aind_metadata_viz/utils.py

@@ -48,6 +48,21 @@ class MetadataState(int, Enum):
     "EXASPIM": "SPIM",
 }
 
+AIND_COLORS = {
+    "dark_blue": "#003057",
+    "light_blue": "#2A7DE1",
+    "green": "#1D8649",
+    "yellow": "#FFB71B",
+    "grey": "#7C7C7F",
+    "red": "#FF5733",
+}
+
+
+def hd_style(text, colors):
+    return (
+        f"<span style='font-weight: bold; color:{colors[text]}'>{text}</span>"
+    )
+
 
 def expected_files_from_modalities(
     modalities: list[str],