Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GEN-478] Fix BIRTH_YEAR redaction #529

Draft
wants to merge 4 commits into
base: develop
Choose a base branch
from
Draft

[GEN-478] Fix BIRTH_YEAR redaction #529

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a47286a
Fix year redaction
thomasyu888 Jul 7, 2023
8b501b9
Simplify code
thomasyu888 Jul 7, 2023
e2aa1bc
Add in redaction of birth year and current year when patient is alive
thomasyu888 Jul 7, 2023
43992ed
Merge branch 'develop' into fix-year-redaction
thomasyu888 Sep 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions genie/database_to_staging.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"""
Functions for releasing GENIE consortium releases
"""
import datetime
import logging
import math
import os
Expand Down Expand Up @@ -99,7 +100,9 @@ def _to_redact_difference(df_col_year1, df_col_year2):
year1 = pd.to_numeric(df_col_year1, errors="coerce")
year2 = pd.to_numeric(df_col_year2, errors="coerce")
to_redact = year2 - year1 > 89
return to_redact
to_redact_peds = year2 - year1 < 18

return to_redact, to_redact_peds


# TODO: Add to transform.py
Expand Down Expand Up @@ -131,15 +134,33 @@ def redact_phi(
# Redact BIRTH_YEAR values that have < or >
# Birth year has to be done separately because it is not an interval
clinicaldf["BIRTH_YEAR"] = _redact_year(clinicaldf["BIRTH_YEAR"])
to_redact = _to_redact_difference(
to_redact, to_redact_peds = _to_redact_difference(
clinicaldf["BIRTH_YEAR"], clinicaldf["YEAR_CONTACT"]
)
clinicaldf.loc[to_redact, "BIRTH_YEAR"] = "cannotReleaseHIPAA"
to_redact = _to_redact_difference(
clinicaldf.loc[to_redact_peds, "BIRTH_YEAR"] = "withheld"
clinicaldf.loc[to_redact, interval_cols_to_redact] = ">32485"
clinicaldf.loc[to_redact_peds, interval_cols_to_redact] = "<6570"

to_redact, to_redact_peds = _to_redact_difference(
clinicaldf["BIRTH_YEAR"], clinicaldf["YEAR_DEATH"]
)
clinicaldf.loc[to_redact, "BIRTH_YEAR"] = "cannotReleaseHIPAA"

clinicaldf.loc[to_redact_peds, "BIRTH_YEAR"] = "withheld"
clinicaldf.loc[to_redact, interval_cols_to_redact] = ">32485"
clinicaldf.loc[to_redact_peds, interval_cols_to_redact] = "<6570"

# this redaction only makes sense for patients that are alive
# Because if a patient dies at age 20, there is no need to calculate
# the current age.
to_redact, to_redact_peds = _to_redact_difference(
clinicaldf["BIRTH_YEAR"], datetime.datetime.utcnow().year
)
is_alive = clinicaldf["DEAD"].isin(["False", "FALSE"])
clinicaldf.loc[(to_redact & is_alive), "BIRTH_YEAR"] = "cannotReleaseHIPAA"
clinicaldf.loc[(to_redact_peds & is_alive), "BIRTH_YEAR"] = "withheld"
clinicaldf.loc[(to_redact & is_alive), interval_cols_to_redact] = ">32485"
clinicaldf.loc[(to_redact_peds & is_alive), interval_cols_to_redact] = "<6570"
return clinicaldf


Expand Down
Loading