Extract_Discharge_Diagnosis_Subsections.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from extract import append_addendums, mimic3_notes, mimic3_diagnosis_icd\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ADD YOUR MIMIC-III Database host, dbname, username and password\n",
    "username = ''\n",
    "password = ''\n",
    "host = '10.200.102.52' # hostname\n",
    "dbname = 'mimic'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# these should write somewhere ... \n",
    "conn = psycopg2.connect(f'host={host} dbname={dbname} user={username} password={password}')\n",
    "codes = mimic3_diagnosis_icd(conn) # if successful, will write to data/admission_codes.csv\n",
    "notes = mimic3_notes(conn) # if successful, will write to data/notes.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "notes = append_addendums(notes)\n",
    "codes = pd.read_csv('data/admission_codes.csv')\n",
    "notes = pd.read_csv('data/notes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "notes_out = notes.loc[:, ['hadm_id', 'text']]\n",
    "notes_out['name'] = notes_out['hadm_id'].apply(lambda i: f'Admission ID:{str(i)}')\n",
    "notes_out = notes_out.drop('hadm_id', axis=1).loc[:, ['name', 'text']]\n",
    "notes_out.to_csv('mimic3_full_notes.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_50_codes = pd.DataFrame(codes.groupby('icd9_code').count().sort_values('hadm_id', ascending=False)['hadm_id'][0:50])\n",
    "top_50_codes.reset_index(inplace=True)\n",
    "top_50_codes['desc'] = top_50_codes.apply(lambda x: codes[codes['icd9_code'] == x.icd9_code].short_title.iloc[0], axis=1)\n",
    "top_50_codes.columns = ['ICD9_Code', 'Count', 'Desc']\n",
    "top_50_codes.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "DISCHARGE_DIAG_SYNONYMS = [\n",
    "    'discharge diagnosis',\n",
    "    'discharge diagnoses',\n",
    "    'discharge diagnose',\n",
    "    'final diagnosis',\n",
    "    'final diagnoses',\n",
    "    'death diagnosis',\n",
    "    'death diagnoses',\n",
    "    'discharge diagnosis list',\n",
    "    'discharge diagnoses list',\n",
    "    'death diagnosis',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "srs = notes.text\n",
    "srs_hadmid = notes.hadm_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "diag_pattern = '|'.join([f'(?<={term}:)(?: ?\\n?\\n?.*)(?:.+\\n)+' for term in DISCHARGE_DIAG_SYNONYMS])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = re.compile(f'({diag_pattern})', re.M | re.I)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for non matches, that are excluded Series.str.extractAll\n",
    "single_extract = srs.str.extract(reg)\n",
    "no_matches = single_extract.loc[single_extract.isnull()[0], :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_matches = srs.str.extractall(reg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = list(all_matches[0:100].reset_index().groupby('level_0'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tuples of: index of hadm_id,  last match in the discharge_summary\n",
    "last_matches = [(g.iloc[-1, :].level_0, g.iloc[-1, :][0]) for k, g in all_matches.reset_index().groupby('level_0')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_match_tuples =[(d[0], d[1][0]) for d in single_extract[pd.isna(single_extract.reset_index()[0])].iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_matches.extend(no_match_tuples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_matches = sorted(last_matches, key=lambda x: x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches = pd.DataFrame(sorted_matches, columns=['idx', 'match'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches = matches.drop('idx', axis=1)\n",
    "matches['note'] = srs\n",
    "matches['hadm_id'] = srs_hadmid\n",
    "matches.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The matches for diagnosis: , diagnoses: can result in fale positives on sections that are not the final diagnosis. These are marked as false positives."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches['dd_line_len'] = matches.match.str.split('\\n').apply(lambda x: len(x) if type(x) == list else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# arbitrary cutoff, DD subsetions longer than 60 lines are excluded.\n",
    "no_dis_diag = matches[(pd.isna(matches.match)) | (matches['dd_line_len'] > 60)]\n",
    "dis_diag = matches[(~pd.isna(matches.match)) & (matches['dd_line_len'] <= 60)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- death of patient: 50\n",
    "- followed up with primary care dr: 51, 55, 58 (medications on discharge), \n",
    "- followed up with non-icu department in hospital: 56, 101960, 101967, 101969\n",
    "- patient is still in hospital at time of writing:  53\n",
    "- notes are incomplete: 54, 57 (addendum referred to but not provided in dataset), 61\n",
    "- addenudems are added, but no clearly defined list still: 101936"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# should be possible to further clean this to just include the hospital course section.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[print(f'{r.hadm_id}\\n{r.match}') for r in dis_diag.loc[181:190, ['match', 'hadm_id']].itertuples()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matches[matches.match.str.contains('GERD').fillna(False)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total Notes with a discharge diag section: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 475,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summaries w/ DD Sections:48898\n",
      "Summaries w/o DD Sections:3828\n",
      "92.74%\n"
     ]
    }
   ],
   "source": [
    "print(f'Summaries w/ DD Sections:{dis_diag.shape[0]}')\n",
    "print(f'Summaries w/o DD Sections:{no_dis_diag.shape[0]}')\n",
    "print(f'{round(dis_diag.shape[0] / (dis_diag.shape[0] + no_dis_diag.shape[0]) * 100, 2)}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert dis_diag to MedCATTrainer Upload Format\n",
    "This is a two column flat table, wit column headers, (name, text)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [],
   "source": [
    "medcat_df = dis_diag.loc[:, ['hadm_id', 'match']]\n",
    "medcat_df['name'] = medcat_df['hadm_id'].apply(lambda i: f'Admission ID:{str(i)}')\n",
    "medcat_df = medcat_df.drop('hadm_id', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {},
   "outputs": [],
   "source": [
    "medcat_df.columns = ['text', 'name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 493,
   "metadata": {},
   "outputs": [],
   "source": [
    "medcat_df = medcat_df.loc[:, ['name', 'text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 497,
   "metadata": {},
   "outputs": [],
   "source": [
    "medcat_df.to_csv('mimic_dd_icd9_sections.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:mimc3_dd]",
   "language": "python",
   "name": "conda-env-mimc3_dd-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}