Merge pull request #445 from OHDSI/data_quality_issues_delta

Add data quality delta column to data network overview page

src/pages/reports/network/NetworkOverview/components/DataSourcesList.vue

@@ -3,7 +3,7 @@
     <DataTable
       size="small"
       showGridlines
-      :value="store.getters.getSources"
+      :value="augmentedData"
       v-model:filters="newFilters"
       :globalFilterFields="['cdm_source_name', 'releases[0].release_name', '']"
       paginator
@@ -67,7 +67,28 @@
         header="Data Quality Issues"
       >
         <template #body="slotProps">
-          {{ slotProps.data.releases[0].count_data_quality_issues }}
+          <div class="flex items-center justify-end">
+            {{ slotProps.data.releases[0].count_data_quality_issues }}
+
+            <svg-icon
+              v-if="slotProps.data.issue_delta === 1"
+              class="text-red-600 text-lg"
+              type="mdi"
+              :path="mdiArrowUp"
+            ></svg-icon>
+            <svg-icon
+              v-if="slotProps.data.issue_delta === -1"
+              class="text-green-600 text-lg"
+              type="mdi"
+              :path="mdiArrowDown"
+            ></svg-icon>
+            <svg-icon
+              v-if="slotProps.data.issue_delta === 0"
+              class="text-gray-600 text-lg"
+              type="mdi"
+              :path="mdiMinus"
+            ></svg-icon>
+          </div>
         </template>
       </Column>
       <Column
@@ -106,7 +127,7 @@
 </template>
 
 <script setup lang="ts">
-import { computed } from "vue";
+import { computed, ref } from "vue";
 import { useStore } from "vuex";
 import { debounce } from "lodash";
 import Panel from "primevue/panel";
@@ -118,11 +139,38 @@ import InputGroup from "primevue/inputgroup";
 import { useRoute, useRouter } from "vue-router";
 import { FilterMatchMode } from "primevue/api";
 import { helpers } from "@/shared/lib/mixins";
+import { mdiArrowDown, mdiArrowLeft, mdiArrowUp, mdiMinus } from "@mdi/js";
+import SvgIcon from "@jamescoyle/vue-icon";
 
 const store = useStore();
 const route = useRoute();
 const router = useRouter();
 
+const data = ref(store.getters.getSources);
+
+const augmentedData = computed(() => {
+  return data.value.map((source) => {
+    const releases = source.releases;
+    let status;
+    if (releases.length > 1) {
+      const moreIssues =
+        releases[0].count_data_quality_issues >
+        releases[1].count_data_quality_issues;
+
+      const noChanges =
+        releases[0].count_data_quality_issues ===
+        releases[1].count_data_quality_issues;
+      status = moreIssues ? 1 : noChanges ? 0 : -1;
+    } else {
+      status = 0;
+    }
+    return {
+      ...source,
+      issue_delta: status,
+    };
+  });
+});
+
 const newFilters = computed(() => ({
   global: { value: route.query.search, matchMode: FilterMatchMode.CONTAINS },
 }));

src/pages/reports/release/DataQualityResults/components/PivotDataTable.vue

@@ -13,6 +13,7 @@
         'subcategory',
         'cdmTableName',
         'failed',
+        'delta',
       ]"
       :aggregator-names-list="['Count']"
     >

src/pages/reports/release/DataQualityResults/components/ResultsTable.vue

@@ -395,6 +395,50 @@
       header="Execution Duration"
     >
     </Column>
+    <Column
+      v-if="checks[0].delta"
+      style="text-align: end; min-width: 12rem"
+      :pt="{ headerContent: 'justify-end' }"
+      sortable
+      filter-field="delta"
+      field="delta"
+      :show-clear-button="false"
+      :show-filter-menu="false"
+    >
+      <template #header>
+        <svg-icon type="mdi" :path="mdiDelta"></svg-icon
+      ></template>
+      <template #filter="{ filterModel, filterCallback }">
+        <MultiSelect
+          :maxSelectedLabels="2"
+          filter
+          v-model="filterModel.value"
+          @change="filterCallback()"
+          :options="deltaOptions"
+          placeholder="Select One"
+          class="p-column-filter w-full"
+          style="min-width: 12rem"
+        >
+          <template #option="slotProps">
+            {{ slotProps.option }}
+          </template>
+        </MultiSelect>
+      </template>
+      <template #body="slotProps">
+        <div v-if="slotProps.data.delta === 'NEW'" class="text-red-600">
+          {{ slotProps.data.delta }}
+        </div>
+        <div v-if="slotProps.data.delta === 'EXISTING'" class="text-yellow-600">
+          {{ slotProps.data.delta }}
+        </div>
+        <div v-if="slotProps.data.delta === 'RESOLVED'" class="text-green-600">
+          {{ slotProps.data.delta }}
+        </div>
+        <div v-if="slotProps.data.delta === 'STABLE'" class="text-blue-600">
+          {{ slotProps.data.delta }}
+        </div>
+      </template>
+    </Column>
 
     <Column expander style="width: 5rem" />
 
@@ -551,7 +595,7 @@ import Dropdown from "primevue/dropdown";
 import InputGroup from "primevue/inputgroup";
 import InputGroupAddon from "primevue/inputgroupaddon";
 import MultiSelect from "primevue/multiselect";
-import { mdiTable } from "@mdi/js";
+import { mdiDelta, mdiTable } from "@mdi/js";
 import { UPDATE_COLUMN_SELECTION } from "@/widgets/settings/model/store/actions.type";
 
 const store = useStore();
@@ -592,6 +636,9 @@ const checkLevelOptions = computed(() => {
 const notesExistOptions = computed(() => {
   return helpers.getValuesArray(checks.value, "notesExist", true);
 });
+const deltaOptions = computed(() => {
+  return helpers.getValuesArray(checks.value, "delta", true);
+});
 
 const filters = ref({
   global: { value: null, matchMode: FilterMatchMode.CONTAINS },
@@ -604,6 +651,7 @@ const filters = ref({
   context: { value: null, matchMode: FilterMatchMode.IN },
   checkLevel: { value: null, matchMode: FilterMatchMode.IN },
   notesExist: { value: null, matchMode: FilterMatchMode.IN },
+  delta: { value: null, matchMode: FilterMatchMode.IN },
 });
 
 const expanded = ref([]);
@@ -616,6 +664,13 @@ const headers = ref({
     show: true,
     default: true,
   },
+  delta: {
+    title: "Delta",
+    sortable: true,
+    key: "delta",
+    show: true,
+    default: true,
+  },
   cdmTableName: {
     title: "Table",
     sortable: true,

src/pages/reports/source/DataQualityHistory/DataQualityHistory.vue

@@ -3,13 +3,15 @@
     <HistoricalDataQuality />
     <HistoricalDataQualityByCategory />
     <HistoricalDataQualityByDomain />
+    <DataQualityDelta />
   </div>
 </template>
 
 <script setup lang="ts">
 import HistoricalDataQuality from "@/pages/reports/source/DataQualityHistory/charts/HistoricalDataQuality/HistoricalDataQuality.vue";
 import HistoricalDataQualityByCategory from "@/pages/reports/source/DataQualityHistory/charts/HistoricalDataQualityByCategory/HistoricalDataQualityByCategory.vue";
 import HistoricalDataQualityByDomain from "@/pages/reports/source/DataQualityHistory/charts/HistoricalDataQualityByDomain/HistoricalDataQualityByDomain.vue";
+import DataQualityDelta from "@/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue";
 </script>
 
 <style scoped></style>

src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue

@@ -0,0 +1,65 @@
+<template>
+  <Panel header="Data Quality Delta" elevation="2" class="ma-4">
+    <template #icons>
+      <ChartHeader
+        title="Historical Data Quality by Category"
+        table-toggle
+        @table-toggled="toggleTable"
+      />
+    </template>
+    <Chart :id="reportId" :data="data" :chartSpec="specDataQualityDelta" />
+    <div v-if="showTable" class="p-4">
+      <DataTable
+        size="small"
+        :value="data"
+        paginator
+        currentPageReportTemplate="{first} to {last} of {totalRecords}"
+        paginator-template="FirstPageLink PrevPageLink PageLinks NextPageLink LastPageLink RowsPerPageDropdown CurrentPageReport"
+        :rows="5"
+        :rowsPerPageOptions="[5, 10, 20, 50]"
+      >
+        <Column field="release" header="Release"> </Column>
+        <Column field="NEW" header="New"> </Column>
+        <Column field="EXISTING" header="Existing"> </Column>
+        <Column field="RESOLVED" header="Resolved"> </Column>
+        <Column field="STABLE" header="Stable"> </Column>
+      </DataTable>
+    </div>
+    <NotesPanel v-if="notesMode" :notes="notes" />
+  </Panel>
+</template>
+
+<script setup lang="ts">
+import { QUALITY_DELTA } from "@/shared/config/files";
+import { Chart } from "@/widgets/chart";
+import { specDataQualityDelta } from "./specDataQualityDelta";
+import { useStore } from "vuex";
+import { computed, ref } from "vue";
+import NotesPanel from "@/widgets/notesPanel/ui/NotesPanel.vue";
+import ChartHeader from "@/widgets/chart/ui/ChartHeader.vue";
+import Panel from "primevue/panel";
+import Column from "primevue/column";
+import DataTable from "primevue/datatable";
+import useAnnotations from "@/shared/lib/composables/useAnnotations";
+import useAnnotationControls from "@/shared/lib/composables/useAnnotationControls";
+
+const store = useStore();
+
+const reportId = "viz-sourcedataqualitydelta";
+
+const { notesMode } = useAnnotationControls();
+
+const { notes } = useAnnotations(reportId);
+
+const showTable = ref(false);
+
+function toggleTable(mode) {
+  showTable.value = mode;
+}
+
+const data = computed(() => {
+  return store.getters.getData[QUALITY_DELTA] || [];
+});
+</script>
+
+<style scoped></style>

src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts

@@ -0,0 +1,88 @@
+import { VEGA_SCHEMA } from "@/shared/config/links";
+import { TopLevelSpec } from "vega-lite";
+
+export function specDataQualityDelta(zeroBaseline = false) {
+  return {
+    $schema: VEGA_SCHEMA,
+    width: "container",
+    height: 150,
+    data: { name: "conceptData" },
+    encoding: {
+      x: { field: "Release", type: "nominal", title: "Release Name" },
+      y: { field: "value", type: "quantitative", title: "Count" },
+      color: {
+        field: "variable",
+        type: "nominal",
+        title: "Issue status",
+      },
+    },
+    transform: [
+      {
+        fold: ["NEW", "EXISTING", "RESOLVED", "STABLE"],
+        as: ["variable", "value"],
+      },
+      {
+        calculate: "datum.variable",
+        as: "copiedVariable",
+      },
+    ],
+    layer: [
+      {
+        mark: { type: "line", interpolate: "linear", point: true },
+        params: [
+          {
+            name: "source",
+            select: { type: "point", fields: ["variable"] },
+            bind: "legend",
+          },
+        ],
+        transform: [
+          {
+            filter: { param: "source" },
+          },
+        ],
+      },
+      {
+        selection: {
+          release: {
+            type: "multi",
+            fields: ["variable"],
+            bind: "legend",
+          },
+          x: {
+            type: "single",
+            on: "mousemove",
+            fields: ["Release"],
+            nearest: true,
+          },
+        },
+        transform: [
+          {
+            filter: { selection: "release" },
+          },
+        ],
+        mark: { type: "point", tooltip: true },
+      },
+      {
+        transform: [
+          {
+            filter: {
+              and: ["x.Release", { selection: "x" }],
+            },
+          },
+          { filter: { selection: "release" } },
+        ],
+        layer: [
+          {
+            mark: "rule",
+            encoding: {
+              y: {
+                height: 1,
+              },
+            },
+          },
+        ],
+      },
+    ],
+  };
+}

src/processes/exploreReports/config/dataLoadConfig.ts

@@ -23,6 +23,7 @@ import {
   OBSERVATION_PERIOD,
   PERSON,
   QUALITY_COMPLETENESS,
+  QUALITY_DELTA,
   QUALITY_INDEX,
   QUALITY_RESULTS,
   RECORDS_DOMAIN,
@@ -220,7 +221,10 @@ export default function getFilesByView(params = null) {
     dataQualityHistory: {
       loadMethod: FETCH_FILES,
       payload: {
-        files: [{ name: QUALITY_INDEX, required: true }],
+        files: [
+          { name: QUALITY_INDEX, required: true },
+          { name: QUALITY_DELTA, required: false },
+        ],
       },
     },
     domainContinuity: {

src/processes/exploreReports/model/store/preprocessing/index.ts

@@ -21,6 +21,7 @@ import {
   COHORT_INDEX_EVENT_BREAKDOWN,
   COHORT_TEMPORAL_COVARIATE_DISTRIBUTION,
   LOCATION,
+  QUALITY_DELTA,
 } from "@/shared/config/files";
 import csvParse from "@/processes/exploreReports/model/store/preprocessing/csvParse";
 
@@ -47,4 +48,5 @@ export default {
   [COHORT_INDEX_EVENT_BREAKDOWN]: csvParse,
   [COHORT_TEMPORAL_COVARIATE_DISTRIBUTION]: csvParse,
   [LOCATION]: csvParse,
+  [QUALITY_DELTA]: csvParse,
 };

src/shared/api/axios/files.ts

@@ -13,6 +13,7 @@ export default function getFilePath(params: Params) {
     [files.CONCEPT]: `data/${params.cdm}/${params.release}/concepts/${params.domain}/concept_${params.concept}.json`,
     [files.QUALITY_INDEX]: `data/${params.cdm}/data-quality-index.json`,
     [files.QUALITY_RESULTS]: `data/${params.cdm}/${params.release}/dq-result.json`,
+    [files.QUALITY_DELTA]: `data/${params.cdm}/data-quality-delta.csv`,
     [files.DEATH]: `data/${params.cdm}/${params.release}/death.json`,
     [files.SOURCE_HISTORY_INDEX]: `data/${params.cdm}/data-source-history-index.json`,
     [files.DENSITY_RECORDS_PERSON]: `data/${params.cdm}/${params.release}/datadensity-records-per-person.csv`,