Krystkowiak_Krystian_Project_7_Heart Disease Patients Classification with a Web App - data and model preparation .ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d9af0697",
   "metadata": {},
   "source": [
    "# Heart Disease Patients Classification with a Web App\n",
    "###### METIS Data Science and Machine Learning Bootcamp 2022 by Krystian Krystkowiak\n",
    "###### project/month(7/7) focus: ENGENEERING\n",
    "\n",
    "ABSTRACT\n",
    "\n",
    "- Developed a pipeline for data processing and deployment, using a SQL database and deploying the resulting model through Github and Streamlit to create a web application for exploring data and predicting heart disease risk."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "54555807",
   "metadata": {},
   "source": [
    "### moving data: csv -> SQL -> pandas -> pickles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "903205af",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, Float\n",
    "\n",
    "import imblearn.under_sampling\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import precision_score, recall_score, accuracy_score, roc_auc_score, f1_score, precision_recall_curve\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "389f4202",
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing data from Kaggle csv file to a dataframe\n",
    "df = pd.read_csv('heart_2020.csv')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "52340ce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#innitiate a sql database engine\n",
    "engine = create_engine('sqlite:///heart.db', echo=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c361ca53",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a table in the database\n",
    "\n",
    "meta = MetaData()\n",
    "\n",
    "my_table = Table(\n",
    "    'my_table', meta, \n",
    "    Column('HeartDisease', Integer, primary_key = True),\n",
    "    Column('BMI', Float),\n",
    "    Column('Smoking',String),\n",
    "    Column('AlcoholDrinking',String),\n",
    "    Column('Stroke',String),\n",
    "    Column('PhysicalHealth', Float),\n",
    "    Column('MentalHealth', Float),\n",
    "    Column('DiffWalking',String),\n",
    "    Column('Sex',String),\n",
    "    Column('AgeCategory',String),\n",
    "    Column('Race',String),\n",
    "    Column('Diabetic',String),\n",
    "    Column('PhysicalActivity',String),\n",
    "    Column('GenHealth',String),\n",
    "    Column('SleepTime', Float),\n",
    "    Column('Asthma',String),\n",
    "    Column('KidneyDisease',String),\n",
    "    Column('SkinCancer',String)\n",
    "#    Column('date', Date)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "82364294",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 'No', 16.6, 'Yes', 'No', 'No', 3.0, 30.0, 'No', 'Female', '55-59', 'White', 'Yes', 'Yes', 'Very good', 5.0, 'Yes', 'No', 'Yes'),\n",
       " (1, 'No', 20.34, 'No', 'No', 'Yes', 0.0, 0.0, 'No', 'Female', '80 or older', 'White', 'No', 'Yes', 'Very good', 7.0, 'No', 'No', 'No'),\n",
       " (2, 'No', 26.58, 'Yes', 'No', 'No', 20.0, 30.0, 'No', 'Male', '65-69', 'White', 'Yes', 'Yes', 'Fair', 8.0, 'Yes', 'No', 'No'),\n",
       " (3, 'No', 24.21, 'No', 'No', 'No', 0.0, 0.0, 'No', 'Female', '75-79', 'White', 'No', 'No', 'Good', 6.0, 'No', 'No', 'Yes'),\n",
       " (4, 'No', 23.71, 'No', 'No', 'No', 28.0, 0.0, 'Yes', 'Female', '40-44', 'White', 'No', 'Yes', 'Very good', 8.0, 'No', 'No', 'No')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create the table\n",
    "df.to_sql('my_table', con=engine, if_exists='append')\n",
    "engine.execute(\"SELECT * FROM my_table LIMIT(5)\").fetchall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a30c65f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#comment out - to drop the table\n",
    "#engine.execute(\"DROP TABLE my_table\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "64a42dd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HeartDisease</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Smoking</th>\n",
       "      <th>AlcoholDrinking</th>\n",
       "      <th>Stroke</th>\n",
       "      <th>PhysicalHealth</th>\n",
       "      <th>MentalHealth</th>\n",
       "      <th>DiffWalking</th>\n",
       "      <th>Sex</th>\n",
       "      <th>AgeCategory</th>\n",
       "      <th>Race</th>\n",
       "      <th>Diabetic</th>\n",
       "      <th>PhysicalActivity</th>\n",
       "      <th>GenHealth</th>\n",
       "      <th>SleepTime</th>\n",
       "      <th>Asthma</th>\n",
       "      <th>KidneyDisease</th>\n",
       "      <th>SkinCancer</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>index</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>No</td>\n",
       "      <td>16.60</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>3.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>No</td>\n",
       "      <td>Female</td>\n",
       "      <td>55-59</td>\n",
       "      <td>White</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Very good</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>No</td>\n",
       "      <td>20.34</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>No</td>\n",
       "      <td>Female</td>\n",
       "      <td>80 or older</td>\n",
       "      <td>White</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Very good</td>\n",
       "      <td>7.0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>No</td>\n",
       "      <td>26.58</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>20.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>No</td>\n",
       "      <td>Male</td>\n",
       "      <td>65-69</td>\n",
       "      <td>White</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Fair</td>\n",
       "      <td>8.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>No</td>\n",
       "      <td>24.21</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>No</td>\n",
       "      <td>Female</td>\n",
       "      <td>75-79</td>\n",
       "      <td>White</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Good</td>\n",
       "      <td>6.0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>No</td>\n",
       "      <td>23.71</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Female</td>\n",
       "      <td>40-44</td>\n",
       "      <td>White</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Very good</td>\n",
       "      <td>8.0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      HeartDisease    BMI Smoking AlcoholDrinking Stroke  PhysicalHealth  \\\n",
       "index                                                                      \n",
       "0               No  16.60     Yes              No     No             3.0   \n",
       "1               No  20.34      No              No    Yes             0.0   \n",
       "2               No  26.58     Yes              No     No            20.0   \n",
       "3               No  24.21      No              No     No             0.0   \n",
       "4               No  23.71      No              No     No            28.0   \n",
       "\n",
       "       MentalHealth DiffWalking     Sex  AgeCategory   Race Diabetic  \\\n",
       "index                                                                  \n",
       "0              30.0          No  Female        55-59  White      Yes   \n",
       "1               0.0          No  Female  80 or older  White       No   \n",
       "2              30.0          No    Male        65-69  White      Yes   \n",
       "3               0.0          No  Female        75-79  White       No   \n",
       "4               0.0         Yes  Female        40-44  White       No   \n",
       "\n",
       "      PhysicalActivity  GenHealth  SleepTime Asthma KidneyDisease SkinCancer  \n",
       "index                                                                         \n",
       "0                  Yes  Very good        5.0    Yes            No        Yes  \n",
       "1                  Yes  Very good        7.0     No            No         No  \n",
       "2                  Yes       Fair        8.0    Yes            No         No  \n",
       "3                   No       Good        6.0     No            No        Yes  \n",
       "4                  Yes  Very good        8.0     No            No         No  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_raw = pd.read_sql(\n",
    "    \"SELECT * FROM my_table\",\n",
    "    con=engine, index_col='index'\n",
    ")\n",
    "\n",
    "df_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f48aae7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_sql = df_raw.sample(frac = 0.7) - experiments with sampling\n",
    "df_raw.to_csv('out.csv', compression='gzip')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "2b6ae6ec",
   "metadata": {},
   "source": [
    "### data preparation, modelling and pickling"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a47cc1b6",
   "metadata": {},
   "source": [
    "Using Naive Bayes model from previous project (Heart Disease Patients Classification) with test set recall of 0.90 and accuracy of 0.81. Small size of the model was also taken into consideration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "56b41bd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#data preparation:\n",
    "\n",
    "#dummy variables\n",
    "df = pd.get_dummies(df_raw, columns=['AgeCategory','Race','Diabetic', 'GenHealth'])\n",
    "\n",
    "#replace Yes/No with 1/0\n",
    "df =  df[df.columns].replace({'Yes':1, 'No':0, 'Female':1,'Male':0 })\n",
    "\n",
    "df = df.drop(columns=['PhysicalHealth', 'MentalHealth'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4f09d4c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of training feature: (231051, 39)\n",
      "Shape of validation feature: (40774, 39)\n",
      "Shape of testing feature: (47970, 39)\n",
      "Shape of training label: (231051,)\n",
      "Shape of validation label: (40774,)\n",
      "Shape of training label: (47970,)\n"
     ]
    }
   ],
   "source": [
    "#Select Features\n",
    "X = df.drop(columns =['HeartDisease'], axis = 1)\n",
    "\n",
    "#Select Target \n",
    "y = df['HeartDisease']\n",
    "\n",
    "# Set Training and Testing Data\n",
    "X_trainval, X_test, y_trainval, y_test = train_test_split(X, y, shuffle = True, test_size = .15, random_state = 42)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_trainval, y_trainval, shuffle = True, test_size = .15, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8b9b1ac7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#to address class imbalance and since I have a lot of data, I undersample the majority class\n",
    "RUS = imblearn.under_sampling.RandomUnderSampler(sampling_strategy = 0.5, random_state=42)\n",
    "\n",
    "X_tr_us, y_tr_us = RUS.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2e20a7d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#scaling\n",
    "std_scale = StandardScaler()\n",
    "\n",
    "X_train_scaled = std_scale.fit_transform(X_train)\n",
    "X_val_scaled = std_scale.fit_transform(X_val)\n",
    "X_test_scaled = std_scale.fit_transform(X_test)\n",
    "X_tr_us_scaled = std_scale.fit_transform(X_tr_us)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "cb75560c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BernoulliNB()"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import BernoulliNB\n",
    "nb = BernoulliNB() \n",
    "nb.fit(X_tr_us_scaled, y_tr_us)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "aa59bc88",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pickling the model for use in the app\n",
    "pickle.dump(nb, open('model.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c2343bd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pickling the model for use in the app\n",
    "pickle.dump(randomforest, open('model.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8009d105",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pickling scaler for data transformation inside the app\n",
    "std_scale.fit(X_train)\n",
    "pickle.dump(std_scale, open('scaled.pkl', 'wb'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (classification)",
   "language": "python",
   "name": "classification"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}