Merge pull request #879 from OHDSI/read_snomed_test

Read manual work fix
OHDSI · Aug 31, 2023 · 4928419 · 4928419
2 parents 98697ee + 9a481c1
commit 4928419
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 81 deletions.
diff --git a/Read/manual_work/create_manual_table.sql b/Read/manual_work/create_manual_table.sql
diff --git a/Read/manual_work/crm_changes.sql b/Read/manual_work/crm_changes.sql
@@ -16,42 +16,24 @@
 * Authors: Irina Zherko, Darina Ivakhnenko, Dmitry Dymshyts
 * Date: 2022
 **************************************************************************/
--- create current date backup of concept_relationship_manual table
-DO
-$body$
-    DECLARE
-        update text;
-    BEGIN
-        SELECT TO_CHAR(CURRENT_DATE, 'YYYY_MM_DD')
-        INTO update;
-        EXECUTE format('create table %I as select * from concept_relationship_manual',
-                       'concept_relationship_manual_backup_' || update);
 
-    END
-$body$;
-
---restore concept_relationship_manual table (run it only if something went wrong)
-TRUNCATE TABLE dev_read.concept_relationship_manual;
-INSERT INTO dev_read.concept_relationship_manual
-SELECT * FROM dev_read.concept_relationship_manual_backup_2022_05_26;
-
-DO
-$body$
-    DECLARE
-        update text;
-    BEGIN
-        SELECT TO_CHAR(CURRENT_DATE, 'YYYY_MM_DD')
-        INTO update;
-        EXECUTE FORMAT('create table %I as select * from concept_manual',
-                       'concept_manual_backup_' || update);
-
-    END
-$body$;
-
---restore concept_manual table (run it only if something went wrong)
-/*TRUNCATE TABLE dev_icd10cm.concept_manual;
-INSERT INTO dev_icd10cm.concept_manual
-SELECT * FROM dev_icd10cm.concept_manual_backup_2022_06_01;*/
+--DROP TABLE IF EXISTS refresh_lookup_done;
+--TRUNCATE TABLE refresh_lookup_done;
+CREATE TABLE refresh_lookup_done (
+id serial primary key,
+read_code VARCHAR,
+read_name VARCHAR,
+cr_invalid_reason varchar,
+repl_by_relationship VARCHAR,
+to_value varchar,
+repl_by_id INT,
+repl_by_code VARCHAR,
+repl_by_name VARCHAR,
+repl_by_domain VARCHAR,
+repl_by_vocabulary VARCHAR);
+
+SELECT *
+FROM refresh_lookup_done;
 
 
 -- deprecate previous inaccurate mapping
@@ -62,32 +44,33 @@ SET invalid_reason = 'D',
 --SELECT * FROM concept_relationship_manual crm --use this SELECT for QA
 WHERE invalid_reason IS NULL --deprecate only what's not yet deprecated in order to preserve the original deprecation date
 
-    AND concept_code_1 IN (SELECT read_code FROM refresh_lookup_done) --work only with the codes presented in the manual file of the current vocabulary refresh
-
-    AND NOT EXISTS (SELECT 1 --don't deprecate mapping if the same exists in the current manual file
-                    FROM refresh_lookup_done rl
-                    WHERE rl.read_code = crm.concept_code_1 --the same source_code is mapped
-                        AND rl.repl_by_code = crm.concept_code_2 --to the same concept_code
-                        AND rl.repl_by_vocabulary = crm.vocabulary_id_2 --of the same vocabulary
-                        AND rl.repl_by_relationship = crm.relationship_id --with the same relationship
+    AND (concept_code_1, concept_code_2) IN (SELECT read_code, repl_by_code FROM refresh_lookup_done) --work only with the codes presented in the manual file of the current vocabulary refresh, have the same target as before
+    AND NOT EXISTS(SELECT 1 --don't deprecate mapping if the same exists in the current manual file
+                   FROM refresh_lookup_done rl
+                   WHERE rl.read_code = crm.concept_code_1           --the same source_code is mapped
+                     AND rl.repl_by_code = crm.concept_code_2        --to the same concept_code
+                     AND rl.repl_by_vocabulary = crm.vocabulary_id_2 --of the same vocabulary
+                     AND rl.repl_by_relationship = crm.relationship_id --with the same relationship
+                     AND rl.cr_invalid_reason = crm.invalid_reason -- the same validity of links
         )
 ;
 
 -- activate mapping, that became valid again
 UPDATE concept_relationship_manual crm
 SET invalid_reason = null,
     valid_end_date = to_date('20991231','yyyymmdd'),
-    valid_start_date =current_date
+    valid_start_date = current_date
 
 --SELECT * FROM concept_relationship_manual crm --use this SELECT for QA
 WHERE invalid_reason = 'D' -- activate only deprecated mappings
 
     AND EXISTS (SELECT 1 -- activate mapping if the same exists in the current manual file
                     FROM refresh_lookup_done rl
-                    WHERE rl.icd_code = crm.concept_code_1 --the same source_code is mapped
+                    WHERE rl.read_code = crm.concept_code_1 --the same source_code is mapped
                         AND rl.repl_by_code = crm.concept_code_2 --to the same concept_code
                         AND rl.repl_by_vocabulary = crm.vocabulary_id_2 --of the same vocabulary
                         AND rl.repl_by_relationship = crm.relationship_id --with the same relationship
+                        AND (cr_invalid_reason IS NULL OR cr_invalid_reason = '') --the same validity of links
         )
 ;
 
@@ -99,9 +82,23 @@ with mapping AS -- select all new codes with their mappings from manual file
                'Read' AS vocabulary_id_1, -- set current vocabulary name as vocabulary_id_1
                repl_by_vocabulary AS vocabulary_id_2,
                repl_by_relationship AS relationship_id,
-               current_date AS valid_start_date, -- set the date of the refresh as valid_start_date
-               to_date('20991231','yyyymmdd') AS valid_end_date,
-               NULL AS invalid_reason -- make all new mappings valid
+               CASE WHEN cr_invalid_reason IN ('U', 'D') --for case when we want to deprecate mapping that doesn't exist in crm: taking valid start date from devv5.concept
+                   THEN (SELECT valid_start_date
+                         FROM devv5.concept_relationship
+                         WHERE concept_id_1 IN (SELECT concept_id
+                                                FROM devv5.concept
+                                                WHERE concept_code = read_code AND vocabulary_id = 'Read')
+                         AND concept_id_2 IN (SELECT concept_id
+                                              FROM devv5.concept
+                                              WHERE concept_code = repl_by_code AND vocabulary_id = repl_by_vocabulary)
+                         AND relationship_id = repl_by_relationship and invalid_reason IS NULL)
+                   ELSE current_date END AS valid_start_date, -- set the date of the refresh as valid_start_date
+               CASE WHEN (cr_invalid_reason NOT IN ('U', 'D') OR cr_invalid_reason IS NULL)
+                   THEN to_date('20991231','yyyymmdd')
+                   ELSE current_date END AS valid_end_date,
+               CASE WHEN (cr_invalid_reason NOT IN ('U', 'D') OR cr_invalid_reason IS NULL)
+                   THEN NULL
+                ELSE cr_invalid_reason END AS invalid_reason
         FROM refresh_lookup_done
         WHERE repl_by_id != 0 -- select only codes with mapping to standard concepts
     )