Skip to content

Commit

Permalink
ref
Browse files Browse the repository at this point in the history
  • Loading branch information
@xsalonx
xsalonx committed Dec 11, 2024
1 parent e34e920 commit db84fcc
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 51 deletions.
123 changes: 73 additions & 50 deletions lib/database/migrations/20241127123000-create-gaq-views.js
View file
Original file line number Diff line number Diff line change
@@ -1,69 +1,85 @@
'use strict';

const SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = `
SELECT gaqd.data_pass_id,
gaqd.run_number,
COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start), 0) AS ord_timestamp,
COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start)) AS timestamp
FROM global_aggregated_quality_detectors AS gaqd
INNER JOIN runs as r
ON gaqd.run_number = r.run_number
`;

const SELECT_RUNS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = `
SELECT gaqd.data_pass_id,
gaqd.run_number,
UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW(3))) AS ord_timestamp,
UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end)) AS timestamp
FROM global_aggregated_quality_detectors AS gaqd
INNER JOIN runs as r
ON gaqd.run_number = r.run_number
`;

const SELECT_QCF_EFFECTIVE_PERIODS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS = `
SELECT gaqd.data_pass_id,
gaqd.run_number,
COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS ord_timestamp,
UNIX_TIMESTAMP(qcfep.\`from\`) AS timestamp
FROM quality_control_flag_effective_periods AS qcfep
INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
-- Only flags of detectors which are defined in global_aggregated_quality_detectors
-- should be taken into account for calculation of gaq_effective_periods
INNER JOIN global_aggregated_quality_detectors AS gaqd
ON gaqd.data_pass_id = dpqcf.data_pass_id
AND gaqd.run_number = qcf.run_number
AND gaqd.detector_id = qcf.detector_id
`;

const SELECT_QCF_EFFECTIVE_PERIODS_TO_TIMESTAMPS_FOR_GAQ_PERIODS = `
SELECT gaqd.data_pass_id,
gaqd.run_number,
UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW(3))) AS ord_timestamp,
UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`)) AS timestamp
FROM quality_control_flag_effective_periods AS qcfep
INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
-- Only flags of detectors which are defined in global_aggregated_quality_detectors
-- should be taken into account for calculation of gaq_effective_periods
INNER JOIN global_aggregated_quality_detectors AS gaqd
ON gaqd.data_pass_id = dpqcf.data_pass_id
AND gaqd.run_number = qcf.run_number
AND gaqd.detector_id = qcf.detector_id
`;

const CREATE_GAQ_PERIODS_TIMESTAMPS_VIEW = `
CREATE OR REPLACE VIEW gaq_periods_timestamps AS
SELECT * FROM (
SELECT
data_pass_id,
run_number,
timestamp AS \`from\`,
NTH_VALUE(timestamp, 2) OVER (
PARTITION BY data_pass_id,
run_number
ORDER BY ap.timestamp
ORDER BY ap.ord_timestamp
ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING
) AS \`to\`
FROM (
-- Two selects for runs' timestamps (in case QC flag's eff. period doesn't start at run's start or end at run's end )
(
SELECT gaqd.data_pass_id,
gaqd.run_number,
COALESCE(UNIX_TIMESTAMP(first_tf_timestamp), UNIX_TIMESTAMP(time_start), 0) AS timestamp
FROM global_aggregated_quality_detectors AS gaqd
INNER JOIN runs as r
ON gaqd.run_number = r.run_number
)
( ${SELECT_RUNS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS} )
UNION
(
SELECT gaqd.data_pass_id,
gaqd.run_number,
UNIX_TIMESTAMP(COALESCE(last_tf_timestamp, time_end, NOW(3))) AS timestamp
FROM global_aggregated_quality_detectors AS gaqd
INNER JOIN runs as r
ON gaqd.run_number = r.run_number
)
( ${SELECT_RUNS_TO_TIMESTAMPS_FOR_GAQ_PERIODS} )
UNION
-- Two selectes for timestamps of QC flags' effective periods
(
SELECT gaqd.data_pass_id,
gaqd.run_number,
COALESCE(UNIX_TIMESTAMP(qcfep.\`from\`), 0) AS timestamp
FROM quality_control_flag_effective_periods AS qcfep
INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
-- Only flags of detectors which are defined in global_aggregated_quality_detectors
-- should be taken into account for calculation of gaq_effective_periods
INNER JOIN global_aggregated_quality_detectors AS gaqd
ON gaqd.data_pass_id = dpqcf.data_pass_id
AND gaqd.run_number = qcf.run_number
AND gaqd.detector_id = qcf.detector_id
)
( ${SELECT_QCF_EFFECTIVE_PERIODS_FROM_TIMESTAMPS_FOR_GAQ_PERIODS} )
UNION
(
SELECT gaqd.data_pass_id,
gaqd.run_number,
UNIX_TIMESTAMP(COALESCE(qcfep.\`to\`, NOW(3))) AS timestamp
FROM quality_control_flag_effective_periods AS qcfep
INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id
INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id
-- Only flags of detectors which are defined in global_aggregated_quality_detectors
-- should be taken into account for calculation of gaq_effective_periods
INNER JOIN global_aggregated_quality_detectors AS gaqd
ON gaqd.data_pass_id = dpqcf.data_pass_id
AND gaqd.run_number = qcf.run_number
AND gaqd.detector_id = qcf.detector_id
)
ORDER BY timestamp
( ${SELECT_QCF_EFFECTIVE_PERIODS_TO_TIMESTAMPS_FOR_GAQ_PERIODS} )
ORDER BY ord_timestamp
) AS ap
) AS gaq_periods_with_last_nullish_row
WHERE gaq_periods_with_last_nullish_row.\`to\` IS NOT NULL
`;

const DROP_GAQ_PERIODS_TIMESTAMPS_VIEW = 'DROP VIEW gaq_periods_timestamps';
Expand All @@ -73,8 +89,12 @@ CREATE OR REPLACE VIEW gaq_periods AS
SELECT
gaq_periods_timestamps.data_pass_id AS dataPassId,
gaq_periods_timestamps.run_number AS runNumber,
IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`) AS \`from\`,
IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) AS \`to\`,
-- IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`) AS \`from\`,
-- IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`) AS \`to\`,
gaq_periods_timestamps.\`from\` AS \`from\`,
gaq_periods_timestamps.\`to\` AS \`to\`,
IF(COUNT( DISTINCT gaqd.detector_id ) > COUNT( DISTINCT qcfep.flag_id ),
null,
SUM(qcft.bad) >= 1
Expand Down Expand Up @@ -115,8 +135,11 @@ WHERE gaq_periods_timestamps.\`to\` IS NOT null
GROUP BY
gaq_periods_timestamps.data_pass_id,
gaq_periods_timestamps.run_number,
IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`),
IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`)
-- IF(gaq_periods_timestamps.\`from\` = 0, null, gaq_periods_timestamps.\`from\`),
-- IF(gaq_periods_timestamps.\`to\` = UNIX_TIMESTAMP(NOW(3)), null, gaq_periods_timestamps.\`to\`)
gaq_periods_timestamps.\`from\`,
gaq_periods_timestamps.\`to\`
`;

const DROP_GAQ_PERIODS_VIEW = 'DROP VIEW gaq_periods';
Expand Down
2 changes: 1 addition & 1 deletion lib/server/services/qualityControlFlag/QcFlagService.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ class QcFlagService {
IF(
SUM(
COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`to\` ), 0)
COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`from\`), 0)
+ COALESCE(UNIX_TIMESTAMP(effectivePeriods.\`from\`), 0)
) = 0,
1,
null
Expand Down

0 comments on commit db84fcc

Please sign in to comment.