From e0b11e7c73c263ece36ea67c39ffd3b83822a5d7 Mon Sep 17 00:00:00 2001 From: Mikhail-iontsev Date: Thu, 8 Aug 2024 22:31:30 +0700 Subject: [PATCH 1/2] feat: added data quality delta --- .../components/DataSourcesList.vue | 54 ++++++++++- .../components/ResultsTable.vue | 60 +++++++++++- .../DataQualityHistory/DataQualityHistory.vue | 2 + .../DataQualityDelta/DataQualityDelta.vue | 82 ++++++++++++++++ .../DataQualityDelta/specDataQualityDelta.ts | 94 +++++++++++++++++++ .../exploreReports/config/dataLoadConfig.ts | 6 +- .../model/store/preprocessing/index.ts | 2 + src/shared/api/axios/files.ts | 3 +- src/shared/config/files.ts | 1 + .../lib/mixins/methods/getValuesArray.ts | 1 + 10 files changed, 299 insertions(+), 6 deletions(-) create mode 100644 src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue create mode 100644 src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts diff --git a/src/pages/reports/network/NetworkOverview/components/DataSourcesList.vue b/src/pages/reports/network/NetworkOverview/components/DataSourcesList.vue index 86e19d17..6a65fe7a 100644 --- a/src/pages/reports/network/NetworkOverview/components/DataSourcesList.vue +++ b/src/pages/reports/network/NetworkOverview/components/DataSourcesList.vue @@ -3,7 +3,7 @@ diff --git a/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue b/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue new file mode 100644 index 00000000..4671aaf4 --- /dev/null +++ b/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue @@ -0,0 +1,82 @@ + + + + + diff --git a/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts b/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts new file mode 100644 index 00000000..f45a7829 --- /dev/null +++ b/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts @@ -0,0 +1,94 @@ +import { VEGA_SCHEMA } from "@/shared/config/links"; +import { TopLevelSpec } from "vega-lite"; + +export function specDataQualityDelta(zeroBaseline = false) { + return { + $schema: VEGA_SCHEMA, + width: "container", + height: 150, + data: { name: "conceptData" }, + encoding: { + x: { field: "Release", type: "nominal", title: "Release Name" }, + y: { field: "value", type: "quantitative", title: "Count" }, + color: { + field: "variable", + type: "nominal", + title: "Issue status", + }, + }, + transform: [ + { + fold: ["NEW", "EXISTING", "RESOLVED", "STABLE"], + as: ["variable", "value"], + }, + { + calculate: "datum.variable", + as: "copiedVariable", + }, + ], + layer: [ + { + mark: { type: "line", interpolate: "linear", point: true }, + params: [ + { + name: "source", + select: { type: "point", fields: ["variable"] }, + bind: "legend", + }, + ], + // encoding: { + // opacity: { + // condition: { param: "source", value: 1 }, + // value: 0.2, + // }, + // }, + transform: [ + { + filter: { param: "source" }, + }, + ], + }, + // { + // selection: { + // release: { + // type: "multi", + // fields: ["copiedVariable"], + // bind: "legend", + // }, + // x: { + // type: "single", + // on: "mousemove", + // fields: ["copiedVariable"], + // nearest: true, + // }, + // }, + // transform: [ + // { + // filter: { selection: "release" }, + // }, + // ], + // mark: { type: "point", tooltip: true }, + // }, + // { + // transform: [ + // { + // filter: { + // and: ["x.Release", { selection: "x" }], + // }, + // }, + // { filter: { selection: "release" } }, + // ], + // layer: [ + // { + // mark: "rule", + // encoding: { + // y: { + // height: 1, + // }, + // }, + // }, + // ], + // }, + ], + }; +} diff --git a/src/processes/exploreReports/config/dataLoadConfig.ts b/src/processes/exploreReports/config/dataLoadConfig.ts index 71596529..9e5c55ab 100644 --- a/src/processes/exploreReports/config/dataLoadConfig.ts +++ b/src/processes/exploreReports/config/dataLoadConfig.ts @@ -23,6 +23,7 @@ import { OBSERVATION_PERIOD, PERSON, QUALITY_COMPLETENESS, + QUALITY_DELTA, QUALITY_INDEX, QUALITY_RESULTS, RECORDS_DOMAIN, @@ -220,7 +221,10 @@ export default function getFilesByView(params = null) { dataQualityHistory: { loadMethod: FETCH_FILES, payload: { - files: [{ name: QUALITY_INDEX, required: true }], + files: [ + { name: QUALITY_INDEX, required: true }, + { name: QUALITY_DELTA, required: false }, + ], }, }, domainContinuity: { diff --git a/src/processes/exploreReports/model/store/preprocessing/index.ts b/src/processes/exploreReports/model/store/preprocessing/index.ts index 065e6be6..de415df8 100644 --- a/src/processes/exploreReports/model/store/preprocessing/index.ts +++ b/src/processes/exploreReports/model/store/preprocessing/index.ts @@ -21,6 +21,7 @@ import { COHORT_INDEX_EVENT_BREAKDOWN, COHORT_TEMPORAL_COVARIATE_DISTRIBUTION, LOCATION, + QUALITY_DELTA, } from "@/shared/config/files"; import csvParse from "@/processes/exploreReports/model/store/preprocessing/csvParse"; @@ -47,4 +48,5 @@ export default { [COHORT_INDEX_EVENT_BREAKDOWN]: csvParse, [COHORT_TEMPORAL_COVARIATE_DISTRIBUTION]: csvParse, [LOCATION]: csvParse, + [QUALITY_DELTA]: csvParse, }; diff --git a/src/shared/api/axios/files.ts b/src/shared/api/axios/files.ts index 3373d340..7e94157b 100644 --- a/src/shared/api/axios/files.ts +++ b/src/shared/api/axios/files.ts @@ -12,7 +12,8 @@ export default function getFilePath(params: Params) { [files.INDEX]: "data/index.json", [files.CONCEPT]: `data/${params.cdm}/${params.release}/concepts/${params.domain}/concept_${params.concept}.json`, [files.QUALITY_INDEX]: `data/${params.cdm}/data-quality-index.json`, - [files.QUALITY_RESULTS]: `data/${params.cdm}/${params.release}/dq-result.json`, + [files.QUALITY_RESULTS]: `data/${params.cdm}/${params.release}/updated_dq-result.json`, + [files.QUALITY_DELTA]: `data/${params.cdm}/data-quality-delta.csv`, [files.DEATH]: `data/${params.cdm}/${params.release}/death.json`, [files.SOURCE_HISTORY_INDEX]: `data/${params.cdm}/data-source-history-index.json`, [files.DENSITY_RECORDS_PERSON]: `data/${params.cdm}/${params.release}/datadensity-records-per-person.csv`, diff --git a/src/shared/config/files.ts b/src/shared/config/files.ts index beb834ea..f43df288 100644 --- a/src/shared/config/files.ts +++ b/src/shared/config/files.ts @@ -2,6 +2,7 @@ export const INDEX = "index"; export const CONCEPT = "concept"; export const QUALITY_INDEX = "qualityIndex"; export const QUALITY_RESULTS = "qualityResults"; +export const QUALITY_DELTA = "qualityDelta"; export const NOTES = "notes"; export const DEATH = "death"; diff --git a/src/shared/lib/mixins/methods/getValuesArray.ts b/src/shared/lib/mixins/methods/getValuesArray.ts index 84d197f6..54ddc4f5 100644 --- a/src/shared/lib/mixins/methods/getValuesArray.ts +++ b/src/shared/lib/mixins/methods/getValuesArray.ts @@ -3,6 +3,7 @@ export const getValuesArray = function ( val: string, unique?: boolean ): never[] { + if (!array) return []; if (unique) { return [...new Set(array.map((d) => d[val]))]; } else { From 12ba4296bfa66b16ab930df117bceef3f2dfed10 Mon Sep 17 00:00:00 2001 From: Mikhail-iontsev Date: Tue, 13 Aug 2024 06:50:37 +0700 Subject: [PATCH 2/2] feat: added data quality delta --- .../components/PivotDataTable.vue | 1 + .../components/ResultsTable.vue | 31 +++---- .../DataQualityDelta/DataQualityDelta.vue | 25 +----- .../DataQualityDelta/specDataQualityDelta.ts | 88 +++++++++---------- src/shared/api/axios/files.ts | 2 +- 5 files changed, 61 insertions(+), 86 deletions(-) diff --git a/src/pages/reports/release/DataQualityResults/components/PivotDataTable.vue b/src/pages/reports/release/DataQualityResults/components/PivotDataTable.vue index e1b7bc6f..8cb5e9b2 100644 --- a/src/pages/reports/release/DataQualityResults/components/PivotDataTable.vue +++ b/src/pages/reports/release/DataQualityResults/components/PivotDataTable.vue @@ -13,6 +13,7 @@ 'subcategory', 'cdmTableName', 'failed', + 'delta', ]" :aggregator-names-list="['Count']" > diff --git a/src/pages/reports/release/DataQualityResults/components/ResultsTable.vue b/src/pages/reports/release/DataQualityResults/components/ResultsTable.vue index b3864e8f..ae261359 100644 --- a/src/pages/reports/release/DataQualityResults/components/ResultsTable.vue +++ b/src/pages/reports/release/DataQualityResults/components/ResultsTable.vue @@ -396,12 +396,12 @@ > @@ -425,20 +425,17 @@ @@ -640,7 +637,7 @@ const notesExistOptions = computed(() => { return helpers.getValuesArray(checks.value, "notesExist", true); }); const deltaOptions = computed(() => { - return helpers.getValuesArray(checks.value, "Status", true); + return helpers.getValuesArray(checks.value, "delta", true); }); const filters = ref({ @@ -654,7 +651,7 @@ const filters = ref({ context: { value: null, matchMode: FilterMatchMode.IN }, checkLevel: { value: null, matchMode: FilterMatchMode.IN }, notesExist: { value: null, matchMode: FilterMatchMode.IN }, - Status: { value: null, matchMode: FilterMatchMode.IN }, + delta: { value: null, matchMode: FilterMatchMode.IN }, }); const expanded = ref([]); @@ -670,7 +667,7 @@ const headers = ref({ delta: { title: "Delta", sortable: true, - key: "status", + key: "delta", show: true, default: true, }, diff --git a/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue b/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue index 4671aaf4..07424b68 100644 --- a/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue +++ b/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/DataQualityDelta.vue @@ -3,26 +3,11 @@ - +
- + @@ -51,7 +36,6 @@ import { specDataQualityDelta } from "./specDataQualityDelta"; import { useStore } from "vuex"; import { computed, ref } from "vue"; import NotesPanel from "@/widgets/notesPanel/ui/NotesPanel.vue"; -import { specDataQualityResultsByCategoryAnnotation } from "@/pages/reports/source/DataQualityHistory/charts/HistoricalDataQualityByCategory/specDataQualityResultsByCategoryAnnotation"; import ChartHeader from "@/widgets/chart/ui/ChartHeader.vue"; import Panel from "primevue/panel"; import Column from "primevue/column"; @@ -63,10 +47,9 @@ const store = useStore(); const reportId = "viz-sourcedataqualitydelta"; -const { notesMode, annotationsMode, toggleNotesMode, toggleAnnotationsMode } = - useAnnotationControls(); +const { notesMode } = useAnnotationControls(); -const { annotations, notes } = useAnnotations(reportId); +const { notes } = useAnnotations(reportId); const showTable = ref(false); diff --git a/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts b/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts index f45a7829..9e613a94 100644 --- a/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts +++ b/src/pages/reports/source/DataQualityHistory/charts/DataQualityDelta/specDataQualityDelta.ts @@ -36,59 +36,53 @@ export function specDataQualityDelta(zeroBaseline = false) { bind: "legend", }, ], - // encoding: { - // opacity: { - // condition: { param: "source", value: 1 }, - // value: 0.2, - // }, - // }, transform: [ { filter: { param: "source" }, }, ], }, - // { - // selection: { - // release: { - // type: "multi", - // fields: ["copiedVariable"], - // bind: "legend", - // }, - // x: { - // type: "single", - // on: "mousemove", - // fields: ["copiedVariable"], - // nearest: true, - // }, - // }, - // transform: [ - // { - // filter: { selection: "release" }, - // }, - // ], - // mark: { type: "point", tooltip: true }, - // }, - // { - // transform: [ - // { - // filter: { - // and: ["x.Release", { selection: "x" }], - // }, - // }, - // { filter: { selection: "release" } }, - // ], - // layer: [ - // { - // mark: "rule", - // encoding: { - // y: { - // height: 1, - // }, - // }, - // }, - // ], - // }, + { + selection: { + release: { + type: "multi", + fields: ["variable"], + bind: "legend", + }, + x: { + type: "single", + on: "mousemove", + fields: ["Release"], + nearest: true, + }, + }, + transform: [ + { + filter: { selection: "release" }, + }, + ], + mark: { type: "point", tooltip: true }, + }, + { + transform: [ + { + filter: { + and: ["x.Release", { selection: "x" }], + }, + }, + { filter: { selection: "release" } }, + ], + layer: [ + { + mark: "rule", + encoding: { + y: { + height: 1, + }, + }, + }, + ], + }, ], }; } diff --git a/src/shared/api/axios/files.ts b/src/shared/api/axios/files.ts index 7e94157b..56fc917e 100644 --- a/src/shared/api/axios/files.ts +++ b/src/shared/api/axios/files.ts @@ -12,7 +12,7 @@ export default function getFilePath(params: Params) { [files.INDEX]: "data/index.json", [files.CONCEPT]: `data/${params.cdm}/${params.release}/concepts/${params.domain}/concept_${params.concept}.json`, [files.QUALITY_INDEX]: `data/${params.cdm}/data-quality-index.json`, - [files.QUALITY_RESULTS]: `data/${params.cdm}/${params.release}/updated_dq-result.json`, + [files.QUALITY_RESULTS]: `data/${params.cdm}/${params.release}/dq-result.json`, [files.QUALITY_DELTA]: `data/${params.cdm}/data-quality-delta.csv`, [files.DEATH]: `data/${params.cdm}/${params.release}/death.json`, [files.SOURCE_HISTORY_INDEX]: `data/${params.cdm}/data-source-history-index.json`,