Merge pull request #446 from OHDSI/source_data_strand

Add data strand visualization to the data source overview report

src/pages/reports/source/SourceOverview/SourceOverview.vue

@@ -9,6 +9,7 @@
       <PopulationHistory class="basis-full" />
       <DataQualityIssuesHistory class="basis-full" />
     </div>
+    <SourceDataStrand />
     <ReleaseListing />
   </div>
 </template>
@@ -19,6 +20,7 @@ import OverviewDisplay from "@/pages/reports/source/SourceOverview/charts/overvi
 import PopulationHistory from "@/pages/reports/source/SourceOverview/charts/populationHistory/PopulationHistory.vue";
 import DataQualityIssuesHistory from "@/pages/reports/source/SourceOverview/charts/dataQualityIssuesHistory/DataQualityIssuesHistory.vue";
 import ReleaseListing from "@/pages/reports/source/SourceOverview/charts/releaseListing/ReleaseListing.vue";
+import SourceDataStrand from "@/pages/reports/source/SourceOverview/charts/sourceDataStrand/SourceDataStrand.vue";
 
 const store = useStore();
 </script>

src/pages/reports/source/SourceOverview/charts/sourceDataStrand/SourceDataStrand.vue

@@ -0,0 +1,150 @@
+<template>
+  <Panel header="Data Strands">
+    <template #icons>
+      <ChartHeader table-toggle @table-toggled="toggleTable" />
+    </template>
+    <div
+      v-if="store.getters.getData"
+      id="viz-sourcedatastrand"
+      class="viz-container"
+    ></div>
+    <div v-if="showTable" class="p-4">
+      <DataTable
+        size="small"
+        :value="data.dataStrandReport"
+        paginator
+        currentPageReportTemplate="{first} to {last} of {totalRecords}"
+        paginator-template="FirstPageLink PrevPageLink PageLinks NextPageLink LastPageLink RowsPerPageDropdown CurrentPageReport"
+        :rows="5"
+        :rowsPerPageOptions="[5, 10, 20, 50]"
+      >
+        <Column field="domain" header="Domain"> </Column>
+        <Column
+          :pt="{ headerContent: 'justify-end' }"
+          sortable
+          header="Number of Records"
+          field="count_records"
+        >
+          <template #body="slotProps">
+            <div class="flex justify-end">
+              {{
+                slotProps.data.count_records
+                  ? helpers.formatComma(slotProps.data.count_records)
+                  : "No data"
+              }}
+            </div>
+          </template>
+        </Column>
+      </DataTable>
+    </div>
+
+    <template #footer>
+      <div class="flex flex-row gap-2">
+        <ChartActionIcon
+          :icon="mdiHelpCircle"
+          tooltip="Data strands are simple visualizations that describe the composition of
+            a data source across the various CDM domain tables. Each individual
+            strand shows the percentage of the data source comprised of data from a
+            particular domain table. Across the network, the strands can be visually
+            compared and contrasted."
+        />
+        <ChartActionIcon
+          v-if="store.getters.getQueryIndex"
+          :icon="mdiCodeBraces"
+          tooltip="View Export Query"
+          @iconClicked="
+            helpers.openNewTab(
+              links.getSqlQueryLink(
+                store.getters.getQueryIndex.DOMAIN_SUMMARY.RECORDS_BY_DOMAIN[0]
+              )
+            )
+          "
+        />
+      </div>
+    </template>
+  </Panel>
+</template>
+
+<script setup lang="ts">
+import { watch, computed, ref, onMounted } from "vue";
+import * as vega from "vega";
+import { useStore } from "vuex";
+import { helpers } from "@/shared/lib/mixins";
+import ChartActionIcon from "@/entities/toggleIcon/ToggleIcon.vue";
+import Panel from "primevue/panel";
+import { mdiCodeBraces, mdiHelpCircle } from "@mdi/js";
+import ChartHeader from "@/widgets/chart/ui/ChartHeader.vue";
+import Column from "primevue/column";
+import DataTable from "primevue/datatable";
+import { Handler } from "vega-tooltip";
+import { darkTheme, lightTheme } from "@/widgets/chart/model/themes";
+
+const store = useStore();
+const route = useRoute();
+const router = useRouter();
+
+import { specDatastrand } from "./specDatastrand";
+import { links } from "@/shared/config/links";
+import { useRoute, useRouter } from "vue-router";
+
+const specs = ref(specDatastrand);
+
+const parsedConfig = computed(() => {
+  return {
+    ...specs.value,
+    config: store.getters.getSettings.darkMode ? darkTheme : lightTheme,
+  };
+});
+
+const renderChart = function () {
+  const view = new vega.View(vega.parse(parsedConfig.value, {}), {
+    renderer: "svg",
+    container: `#viz-sourcedatastrand`,
+    hover: true,
+  }).tooltip(new Handler().call);
+  view.runAsync().then(() => {
+    view.addSignalListener("selectDomain", (name, value) => {
+      const domainKey = value.domain.toLowerCase().replace(" ", "_");
+      router.push({
+        name: "domainTable",
+        params: {
+          cdm: route.params.cdm_source_key,
+          release: value.cdm_release_key.split("-").join(""),
+          domain: domainKey,
+        },
+      });
+      document.getElementById("vg-tooltip-element").style.display = "none";
+    });
+  });
+};
+
+const data = computed(() => store.getters.getData);
+
+const darkMode = computed(() => store.getters.getSettings.darkMode);
+
+onMounted(() => {
+  if (data.value) {
+    const spec = specDatastrand;
+    spec.data[0].values = data.value.dataStrandReport;
+    specs.value = spec;
+
+    renderChart();
+  }
+});
+
+watch(darkMode, () => {
+  renderChart();
+});
+
+const showTable = ref(false);
+
+function toggleTable(mode) {
+  showTable.value = mode;
+}
+</script>
+
+<style scoped>
+.viz-container {
+  width: 95%;
+}
+</style>

src/pages/reports/source/SourceOverview/charts/sourceDataStrand/specDatastrand.ts

@@ -0,0 +1,135 @@
+export const specDatastrand = {
+  $schema: "https://vega.github.io/schema/vega/v5.json",
+  description: "Data Strand Visualization",
+  autosize: { type: "fit-x" },
+  data: [
+    {
+      name: "source_0",
+      values: [],
+    },
+    {
+      name: "data_0",
+      source: "source_0",
+      transform: [
+        {
+          type: "filter",
+          expr: "indexof(['condition occurrence','drug exposure','measurement','observation','procedure occurrence','visit occurrence','device exposure'], lower(datum.domain)) >= 0",
+        },
+        {
+          type: "joinaggregate",
+          as: ["total_records"],
+          ops: ["sum"],
+          fields: ["count_records"],
+          groupby: ["cdm_release_key"],
+        },
+        {
+          type: "formula",
+          expr: "datum.count_records/datum.total_records",
+          as: "percent",
+        },
+        {
+          type: "aggregate",
+          groupby: ["domain", "percent", "count_records", "cdm_release_key"],
+          ops: ["sum", "sum"],
+          fields: ["percent", "count_records"],
+          as: ["sum_percent", "sum_count_records"],
+        },
+        {
+          type: "stack",
+          groupby: ["cdm_release_key"],
+          field: "sum_percent",
+          sort: { field: ["sum_count_records"], order: ["descending"] },
+          as: ["sum_percent_start", "sum_percent_end"],
+          offset: "zero",
+        },
+        {
+          type: "filter",
+          expr: 'isValid(datum["sum_percent"]) && isFinite(+datum["sum_percent"])',
+        },
+      ],
+    },
+  ],
+  signals: [
+    {
+      name: "selectDomain",
+      on: [{ events: "mousedown", update: "datum" }],
+    },
+    {
+      name: "width",
+      init: "isFinite(containerSize()[0]) ? containerSize()[0] : 200",
+      on: [
+        {
+          update: "isFinite(containerSize()[0]) ? containerSize()[0] : 200",
+          events: "window:resize",
+        },
+      ],
+    },
+    { name: "y_step", value: 20 },
+    {
+      name: "height",
+      update: "bandspace(domain('y').length, 3, 3) * y_step",
+    },
+  ],
+  marks: [
+    {
+      name: "marks",
+      type: "rect",
+      style: ["bar"],
+      from: { data: "data_0" },
+      encode: {
+        enter: {
+          strokeWidth: { value: 7 },
+        },
+        update: {
+          cornerRadius: { value: 10 },
+          fill: { scale: "color", field: "domain" },
+          tooltip: {
+            signal:
+              '{"Data Source": datum["cdm_release_key"], "Domain": isValid(datum["domain"]) ? datum["domain"] : ""+datum["domain"], "Percent": format(datum["sum_percent"], "0.2%"), "Number of Records": format(datum["count_records"], ",")}',
+          },
+          x: { scale: "x", field: "sum_percent_end" },
+          x2: { scale: "x", field: "sum_percent_start" },
+          yc: { scale: "y", field: "cdm_release_key" },
+          height: { value: 20 },
+        },
+      },
+    },
+  ],
+  scales: [
+    {
+      name: "x",
+      type: "linear",
+      domain: {
+        data: "data_0",
+        fields: ["sum_percent_start", "sum_percent_end"],
+      },
+      range: [0, { signal: "width" }],
+      nice: true,
+      zero: true,
+    },
+    {
+      name: "y",
+      type: "band",
+      domain: { data: "data_0", field: "cdm_release_key", sort: true },
+      range: { step: { signal: "y_step" } },
+      padding: 3,
+    },
+    {
+      name: "color",
+      type: "ordinal",
+      domain: { data: "data_0", field: "domain", sort: true },
+      range: "category",
+    },
+  ],
+  axes: [{ scale: "y", orient: "left", zindex: 0 }],
+  legends: [
+    {
+      columns: 2,
+      columnPadding: 15,
+      rowPadding: 10,
+      orient: "right",
+      fill: "color",
+      symbolType: "circle",
+    },
+  ],
+};

src/processes/exploreReports/config/dataLoadConfig.ts

@@ -129,6 +129,12 @@ export default function getFilesByView(params = null) {
         files: [{ name: RECORDS_DOMAIN, instanceParams: [{}] }],
       },
     },
+    dataSourceOverview: {
+      loadMethod: FETCH_MULTIPLE_FILES_BY_RELEASE,
+      payload: {
+        files: [{ name: RECORDS_DOMAIN }],
+      },
+    },
     population: {
       loadMethod: FETCH_MULTIPLE_FILES_BY_SOURCE,
       payload: {

src/processes/exploreReports/model/store/data.module.ts

@@ -349,7 +349,7 @@ const actions = {
         .map((filtered) => {
           const { data, payload } = filtered.value;
           return {
-            data: isDuckDb ? convertTableToArray(data) : data,
+            data: processData(data, isDuckDb, file),
             release: payload,
           };
         });

src/processes/exploreReports/model/store/postprocessing/dataSourceOverview.ts

@@ -0,0 +1,24 @@
+import { RECORDS_DOMAIN } from "@/shared/config/files";
+import { RecordsDomain } from "@/processes/exploreReports/model/interfaces/files/RecordsDomain";
+import { MultipleFilesRawInterface } from "@/processes/exploreReports/model/interfaces/MultipleFilesRawInterface";
+
+export default function dataSourceOverview(data) {
+  const recordsDomain: MultipleFilesRawInterface<RecordsDomain[]>[] =
+    data[RECORDS_DOMAIN];
+  let processedData;
+
+  if (recordsDomain && recordsDomain.length) {
+    processedData = recordsDomain.reduce(
+      (prevValue, current) => [
+        ...prevValue,
+        ...current.data.map((value) => ({
+          ...value,
+          cdm_release_key: current.release,
+        })),
+      ],
+      []
+    );
+  }
+
+  return { dataStrandReport: processedData };
+}

src/processes/exploreReports/model/store/postprocessing/index.ts

@@ -18,6 +18,7 @@ import networkUnmappedSourceCodes from "@/processes/exploreReports/model/store/p
 import networkDiversityReport from "@/processes/exploreReports/model/store/postprocessing/networkDiversityReport";
 import cohortReport from "@/processes/exploreReports/model/store/postprocessing/cohortReport";
 import networkConceptDashboard from "@/processes/exploreReports/model/store/postprocessing/networkConceptDashboard";
+import dataSourceOverview from "@/processes/exploreReports/model/store/postprocessing/dataSourceOverview";
 
 export default {
   dataStrandReport,
@@ -40,4 +41,5 @@ export default {
   networkUnmappedSourceCodes,
   cohortReport,
   networkConceptDashboard,
+  dataSourceOverview,
 };