Default_data_models .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "#loading metrics packages\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_curve, auc, accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "#loading feature selection packages\n",
    "from sklearn.feature_selection import RFECV\n",
    "\n",
    "#loading model parameter selection packages\n",
    "from sklearn.model_selection import validation_curve\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, cross_val_predict\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "\n",
    "# loading algorithm packages\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn import svm\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import BaggingClassifier \n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# loading data\n",
    "data = pd.read_csv('defaulter_full.csv', sep=',') \n",
    "#deleting unwanted rows\n",
    "del data['ID']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#split data into train and test and splitting featurs and class\n",
    "train = data.sample(frac=0.6, random_state=1)\n",
    "test = data.loc[~data.index.isin(train.index)] \n",
    "columns = data.columns.tolist()\n",
    "columns = [c for c in columns if c not in [\"default.payment.next.month\"]]\n",
    "def_n_train_X =train[columns]\n",
    "def_n_test_X =test[columns]\n",
    "def_n_train_Y=train['default.payment.next.month']\n",
    "def_n_test_Y=test['default.payment.next.month']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6 2 3 2 4 1 3 3 3 4 3 5 5 6 6 5 6 4 4 5 4 5 6]\n"
     ]
    }
   ],
   "source": [
    "#ranking features for random sampled data and extracting important features\n",
    "estimator= LogisticRegression()\n",
    "selector = RFECV(estimator, 5,cv=5)\n",
    "selector = selector.fit(def_n_train_X, def_n_train_Y)\n",
    "print \"fearure ranking for random sampled data= \", selector.ranking_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns = def_n_train_X.columns.tolist()\n",
    "columns = [c for c in columns if c in [\"SEX\",\"EDUCATION\",\"MARRIAGE\",'AGE','PAY_0','PAY_2','PAY_3','PAY_5','PAY_4']]\n",
    "def_f_n_train_X =def_n_train_X[columns]\n",
    "def_f_n_test_X=def_n_test_X[columns]\n",
    "def_f_n_train_Y=def_n_train_Y\n",
    "def_f_n_test_Y=def_n_test_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#split data into train and test and splitting featurs and class for under sampled data\n",
    "train = pd.read_csv('defaulter_undersample_train.csv', sep=',') \n",
    "test = pd.read_csv('defaulter_undersample_test.csv', sep=',') \n",
    "columns = train.columns.tolist()\n",
    "columns = [c for c in columns if c not in [\"default.payment.next.month\"]]\n",
    "def_u_train_X =train[columns]\n",
    "def_u_test_X =test[columns]\n",
    "def_u_train_Y=train['default.payment.next.month']\n",
    "def_u_test_Y=test['default.payment.next.month']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fearure ranking=  [6 4 3 2 4 1 3 2 3 3 3 5 5 6 6 6 5 4 4 5 4 6 5]\n"
     ]
    }
   ],
   "source": [
    "#ranking features for random sampled data and extracting important features for under sampled data\n",
    "estimator= LogisticRegression()\n",
    "selector = RFECV(estimator, 5,cv=5)\n",
    "selector = selector.fit(def_u_train_X, def_u_train_Y)\n",
    "print \"fearure ranking for under sampled data= \", selector.ranking_\n",
    "columns = def_u_train_X.columns.tolist()\n",
    "columns = [c for c in columns if c in [\"SEX\",\"EDUCATION\",\"MARRIAGE\",'AGE','PAY_0','PAY_2','PAY_3','PAY_5','PAY_4']]\n",
    "def_f_u_train_X =train[columns]\n",
    "def_f_u_test_X =test[columns]\n",
    "def_f_u_train_Y=def_u_train_Y\n",
    "def_f_u_test_Y=def_u_test_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# defining parameter dictionaries\n",
    "ID3_params={'max_depth':[i for i in range(1,15)], 'criterion':['gini', 'entropy']}\n",
    "SVM_params = {'loss':['squared_hinge'],'penalty':['l2', 'l1'], 'dual':[False]}\n",
    "ada_params= {'n_estimators':[20, 40, 50], 'learning_rate':[0.5,1.0]}\n",
    "knn_params={'n_neighbors':[5,10,15], 'weights':['uniform','distance'],'algorithm':['ball_tree', 'kd_tree', 'brute']}\n",
    "bag_params={'n_estimators':[10,15,20], 'bootstrap_features':[True,False],'max_features': [0.5,1.0],'max_samples':[1,0.5,0.75]}\n",
    "logit_params={'max_iter':[100,50,200],'C':[0.25,0.5,0.75,1.0],'solver':['newton-cg', 'lbfgs', 'liblinear', 'sag'],'warm_start':[True,False]}\n",
    "xg_params={'learning_rate':[0.01,0.1,0.3,0.5], 'n_estimators':[500,600,700,1000]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#function to calculate best parameters\n",
    "def bestparam (model, train_X, train_Y):\n",
    "    model.fit(train_X,train_Y)\n",
    "    print\"Best paramters\", model.best_params_\n",
    "    print\"\"\n",
    "    return model.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#function to print metrics\n",
    "def calcmetrics(predicted,actual,ModelName):\n",
    "    print\"************************** \"+ModelName+\" ****************************\"\n",
    "    print\"Confusion Matrix\"\n",
    "    print metrics.confusion_matrix(actual, predicted)\n",
    "    print\"\"\n",
    "    print metrics.classification_report(actual, predicted)\n",
    "    print\"\"\n",
    "    print\"Accuracy = \",\"{0:.2f}\".format(round(accuracy_score(actual, predicted)*100,2))\n",
    "    print\"\"\n",
    "    false_positive_rate, true_positive_rate, thresholds = roc_curve(actual, predicted)\n",
    "    roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "    print\"AUC =\",\"{0:.2f}\".format(round(roc_auc*100,2))\n",
    "    print\"\"\n",
    "    plt.title('Receiver Operating Characteristic '+ModelName)\n",
    "    plt.plot(false_positive_rate, true_positive_rate, 'b',\n",
    "    label='AUC = %0.2f'% roc_auc)\n",
    "    plt.legend(loc='lower right')\n",
    "    plt.plot([0,1],[0,1],'r--')\n",
    "    plt.xlim([-0.1,1.2])\n",
    "    plt.ylim([-0.1,1.2])\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## buliding Decision tree model from undersampled data and Random sampled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ID3=GridSearchCV(DecisionTreeClassifier(), param_grid = ID3_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (ID3, def_u_train_X, def_u_train_Y)\n",
    "ID3_best=DecisionTreeClassifier(criterion = b_param['criterion'],max_depth=b_param['max_depth'])\n",
    "ID3_best.fit(def_u_train_X,def_u_train_Y)\n",
    "ID3_predicted = ID3_best.predict(def_u_test_X)\n",
    "calcmetrics(ID3_predicted,def_u_test_Y,\"Decision Tree - Under sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ID3=GridSearchCV(DecisionTreeClassifier(), param_grid = ID3_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (ID3, def_f_u_train_X, def_f_u_train_Y)\n",
    "ID3_best=DecisionTreeClassifier(criterion = b_param['criterion'],max_depth=b_param['max_depth'])\n",
    "ID3_best.fit(def_f_u_train_X,def_f_u_train_Y)\n",
    "ID3_predicted = ID3_best.predict(def_f_u_test_X)\n",
    "calcmetrics(ID3_predicted,def_f_u_test_Y,\"Decision Tree - Under sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ID3=GridSearchCV(DecisionTreeClassifier(), param_grid = ID3_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (ID3, def_n_train_X, def_n_train_Y)\n",
    "ID3_best=DecisionTreeClassifier(criterion = b_param['criterion'],max_depth=b_param['max_depth'])\n",
    "ID3_best.fit(def_n_train_X,def_n_train_Y)\n",
    "ID3_predicted = ID3_best.predict(def_n_test_X)\n",
    "calcmetrics(ID3_predicted,def_n_test_Y,\"Decision Tree - Random sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ID3=GridSearchCV(DecisionTreeClassifier(), param_grid = ID3_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (ID3, def_f_n_train_X, def_f_n_train_Y)\n",
    "ID3_best=DecisionTreeClassifier(criterion = b_param['criterion'],max_depth=b_param['max_depth'])\n",
    "ID3_best.fit(def_f_n_train_X,def_f_n_train_Y)\n",
    "ID3_predicted = ID3_best.predict(def_f_n_test_X)\n",
    "calcmetrics(ID3_predicted,def_f_n_test_Y,\"Decision Tree - Under sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## buliding SVM model from undersampled data and Random sampled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SVM=GridSearchCV(svm.LinearSVC(), param_grid = SVM_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (SVM, def_u_train_X, def_u_train_Y)\n",
    "SVM_best=svm.LinearSVC(penalty= b_param['penalty'], loss= b_param['loss'], dual= b_param['dual'])\n",
    "SVM_best.fit(def_u_train_X,def_u_train_Y)\n",
    "SVM_predicted = SVM_best.predict(def_u_test_X)\n",
    "calcmetrics(SVM_predicted,def_u_test_Y,\"Linear SVM - Under sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SVM=GridSearchCV(svm.LinearSVC(), param_grid = SVM_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (SVM, def_f_u_train_X, def_f_u_train_Y)\n",
    "SVM_best=svm.LinearSVC(penalty= b_param['penalty'], loss= b_param['loss'], dual= b_param['dual'])\n",
    "SVM_best.fit(def_f_u_train_X,def_f_u_train_Y)\n",
    "SVM_predicted = SVM_best.predict(def_f_u_test_X)\n",
    "calcmetrics(SVM_predicted,def_f_u_test_Y,\"Linear SVM - Under sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SVM=GridSearchCV(svm.LinearSVC(), param_grid = SVM_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (SVM, def_n_train_X, def_n_train_Y)\n",
    "SVM_best=svm.LinearSVC(penalty= b_param['penalty'], loss= b_param['loss'], dual= b_param['dual'])\n",
    "SVM_best.fit(def_n_train_X,def_n_train_Y)\n",
    "SVM_predicted = SVM_best.predict(def_n_test_X)\n",
    "calcmetrics(SVM_predicted,def_n_test_Y,\"Linear SVM - Random sampling with out feature reduction \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SVM=GridSearchCV(svm.LinearSVC(), param_grid = SVM_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (SVM, def_f_n_train_X, def_f_n_train_Y)\n",
    "SVM_best=svm.LinearSVC(penalty= b_param['penalty'], loss= b_param['loss'], dual= b_param['dual'])\n",
    "SVM_best.fit(def_f_n_train_X,def_f_n_train_Y)\n",
    "SVM_predicted = SVM_best.predict(def_f_n_test_X)\n",
    "calcmetrics(SVM_predicted,def_f_n_test_Y,\"Linear SVM - Random sampling with feature reduction \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## buliding KNN model from undersampled data and Random sampled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "knn=GridSearchCV(KNeighborsClassifier(), param_grid = knn_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (knn, def_u_train_X, def_u_train_Y)\n",
    "knn_best=KNeighborsClassifier(n_neighbors= b_param['n_neighbors'], weights=b_param['weights'], algorithm= b_param['algorithm'])\n",
    "knn_best.fit(def_u_train_X,def_u_train_Y)\n",
    "knn_predicted = knn_best.predict(def_u_test_X)\n",
    "calcmetrics(knn_predicted,def_u_test_Y,\"KNN- Under sampling with out feature reduction \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "knn=GridSearchCV(KNeighborsClassifier(), param_grid = knn_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (knn, def_f_u_train_X, def_f_u_train_Y)\n",
    "knn_best=KNeighborsClassifier(n_neighbors= b_param['n_neighbors'], weights=b_param['weights'], algorithm= b_param['algorithm'])\n",
    "knn_best.fit(def_f_u_train_X,def_f_u_train_Y)\n",
    "knn_predicted = knn_best.predict(def_f_u_test_X)\n",
    "calcmetrics(knn_predicted,def_f_u_test_Y,\"KNN- Under sampling with feature reduction \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "knn=GridSearchCV(KNeighborsClassifier(), param_grid = knn_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (knn, def_n_train_X, def_n_train_Y)\n",
    "knn_best=KNeighborsClassifier(n_neighbors= b_param['n_neighbors'], weights=b_param['weights'], algorithm= b_param['algorithm'])\n",
    "knn_best.fit(def_n_train_X,def_n_train_Y)\n",
    "knn_predicted = knn_best.predict(def_n_test_X)\n",
    "calcmetrics(knn_predicted,def_n_test_Y,\"KNN- Random sampling with out feature reduction \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "knn=GridSearchCV(KNeighborsClassifier(), param_grid = knn_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (knn, def_f_n_train_X, def_f_n_train_Y)\n",
    "knn_best=KNeighborsClassifier(n_neighbors= b_param['n_neighbors'], weights=b_param['weights'], algorithm= b_param['algorithm'])\n",
    "knn_best.fit(def_f_n_train_X,def_f_n_train_Y)\n",
    "knn_predicted = knn_best.predict(def_f_n_test_X)\n",
    "calcmetrics(knn_predicted,def_f_n_test_Y,\"KNN- Random sampling with feature reduction \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## buliding adaboost model from undersampled data and Random sampled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ada=GridSearchCV(AdaBoostClassifier(ID3_best), param_grid = ada_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (ada, def_u_train_X, def_u_train_Y)\n",
    "ada_best=AdaBoostClassifier(ID3_best, n_estimators= b_param['n_estimators'], learning_rate=b_param['learning_rate'])\n",
    "ada_best.fit(def_u_train_X,def_u_train_Y)\n",
    "ada_predicted = ada_best.predict(def_u_test_X)\n",
    "calcmetrics(ada_predicted,def_u_test_Y,\"Adaboost- Under sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ada=GridSearchCV(AdaBoostClassifier(ID3_best), param_grid = ada_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (ada, def_f_u_train_X, def_f_u_train_Y)\n",
    "ada_best=AdaBoostClassifier(ID3_best, n_estimators= b_param['n_estimators'], learning_rate=b_param['learning_rate'])\n",
    "ada_best.fit(def_f_u_train_X,def_f_u_train_Y)\n",
    "ada_predicted = ada_best.predict(def_f_u_test_X)\n",
    "calcmetrics(ada_predicted,def_f_u_test_Y,\"Adaboost- Under sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ada=GridSearchCV(AdaBoostClassifier(ID3_best), param_grid = ada_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (ada, def_n_train_X, def_n_train_Y)\n",
    "ada_best=AdaBoostClassifier(ID3_best, n_estimators= b_param['n_estimators'], learning_rate=b_param['learning_rate'])\n",
    "ada_best.fit(def_n_train_X,def_n_train_Y)\n",
    "ada_predicted = ada_best.predict(def_n_test_X)\n",
    "calcmetrics(ada_predicted,def_n_test_Y,\"Adaboost- Random sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ada=GridSearchCV(AdaBoostClassifier(ID3_best), param_grid = ada_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (ada, def_f_n_train_X, def_f_n_train_Y)\n",
    "ada_best=AdaBoostClassifier(ID3_best, n_estimators= b_param['n_estimators'], learning_rate=b_param['learning_rate'])\n",
    "ada_best.fit(def_f_n_train_X,def_f_n_train_Y)\n",
    "ada_predicted = ada_best.predict(def_f_n_test_X)\n",
    "calcmetrics(ada_predicted,def_f_n_test_Y,\"Adaboost- Random sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## buliding bagging model from undersampled data and Random sampled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag=GridSearchCV(BaggingClassifier(ID3_best), param_grid = bag_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (bag, def_u_train_X, def_u_train_Y)\n",
    "bag_best=BaggingClassifier(ID3_best,max_features= b_param['max_features'], max_samples= b_param['max_samples'], n_estimators= b_param['n_estimators'], bootstrap_features= b_param['bootstrap_features'])\n",
    "bag_best.fit(def_u_train_X,def_u_train_Y)\n",
    "bag_predicted = bag_best.predict(def_u_test_X)\n",
    "calcmetrics(bag_predicted,def_u_test_Y,\"Bagging- Under sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag=GridSearchCV(BaggingClassifier(ID3_best), param_grid = bag_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (bag, def_f_u_train_X, def_f_u_train_Y)\n",
    "bag_best=BaggingClassifier(ID3_best,max_features= b_param['max_features'], max_samples= b_param['max_samples'], n_estimators= b_param['n_estimators'], bootstrap_features= b_param['bootstrap_features'])\n",
    "bag_best.fit(def_f_u_train_X,def_f_u_train_Y)\n",
    "bag_predicted = bag_best.predict(def_f_u_test_X)\n",
    "calcmetrics(bag_predicted,def_f_u_test_Y,\"Bagging- Under sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag=GridSearchCV(BaggingClassifier(ID3_best), param_grid = bag_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (bag, def_n_train_X, def_n_train_Y)\n",
    "bag_best=BaggingClassifier(ID3_best,max_features= b_param['max_features'], max_samples= b_param['max_samples'], n_estimators= b_param['n_estimators'], bootstrap_features= b_param['bootstrap_features'])\n",
    "bag_best.fit(def_n_train_X,def_n_train_Y)\n",
    "bag_predicted = bag_best.predict(def_n_test_X)\n",
    "calcmetrics(bag_predicted,def_n_test_Y,\"Bagging- Random sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bag=GridSearchCV(BaggingClassifier(ID3_best), param_grid = bag_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (bag, def_f_n_train_X, def_f_n_train_Y)\n",
    "bag_best=BaggingClassifier(ID3_best,max_features= b_param['max_features'], max_samples= b_param['max_samples'], n_estimators= b_param['n_estimators'], bootstrap_features= b_param['bootstrap_features'])\n",
    "bag_best.fit(def_f_n_train_X,def_f_n_train_Y)\n",
    "bag_predicted = bag_best.predict(def_f_n_test_X)\n",
    "calcmetrics(bag_predicted,def_f_n_test_Y,\"Bagging- Random sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## buliding Logistic regression model from undersampled data and Random sampled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logit=GridSearchCV(LogisticRegression(), param_grid = logit_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (logit, def_u_train_X, def_u_train_Y)\n",
    "logit_best=LogisticRegression(warm_start=b_param['warm_start'], C= b_param['C'], max_iter= b_param['max_iter'], solver= b_param['solver'])\n",
    "logit_best.fit(def_u_train_X,def_u_train_Y)\n",
    "logit_predicted = logit_best.predict(def_u_test_X)\n",
    "calcmetrics(logit_predicted,def_u_test_Y,\"logistic Regression- Under sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logit=GridSearchCV(LogisticRegression(), param_grid = logit_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (logit, def_f_u_train_X, def_f_u_train_Y)\n",
    "logit_best=LogisticRegression(warm_start=b_param['warm_start'], C= b_param['C'], max_iter= b_param['max_iter'], solver= b_param['solver'])\n",
    "logit_best.fit(def_f_u_train_X,def_f_u_train_Y)\n",
    "logit_predicted = logit_best.predict(def_f_u_test_X)\n",
    "calcmetrics(logit_predicted,def_f_u_test_Y,\"logistic Regression- Under sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logit=GridSearchCV(LogisticRegression(), param_grid = logit_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (logit, def_n_train_X, def_n_train_Y)\n",
    "logit_best=LogisticRegression(warm_start=b_param['warm_start'], C= b_param['C'], max_iter= b_param['max_iter'], solver= b_param['solver'])\n",
    "logit_best.fit(def_n_train_X,def_n_train_Y)\n",
    "logit_predicted = logit_best.predict(def_n_test_X)\n",
    "calcmetrics(logit_predicted,def_n_test_Y,\"logistic Regression-  Random sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logit=GridSearchCV(LogisticRegression(), param_grid = logit_params, n_jobs = -1, cv = 5)\n",
    "b_param=bestparam (logit, def_f_n_train_X, def_f_n_train_Y)\n",
    "logit_best=LogisticRegression(warm_start=b_param['warm_start'], C= b_param['C'], max_iter= b_param['max_iter'], solver= b_param['solver'])\n",
    "logit_best.fit(def_f_n_train_X,def_f_n_train_Y)\n",
    "logit_predicted = logit_best.predict(def_f_n_test_X)\n",
    "calcmetrics(logit_predicted,def_f_n_test_Y,\"logistic Regression-  Random sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## buliding XGBOOST model from undersampled data and Random sampled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Xgboost = xgb.XGBClassifier(max_depth=4, n_estimators=600, learning_rate=0.03).fit(def_u_train_X, def_u_train_Y) \n",
    "xgb_predicted = Xgboost.predict(def_u_test_X)\n",
    "calcmetrics(xgb_predicted,def_u_test_Y,\"XGBoost - Under sampling with out feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Xgboost = xgb.XGBClassifier(max_depth=4, n_estimators=600, learning_rate=0.03).fit(def_f_u_train_X, def_f_u_train_Y) \n",
    "xgb_predicted = Xgboost.predict(def_f_u_test_X)\n",
    "calcmetrics(xgb_predicted,def_f_u_test_Y,\"XGBoost - Under sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Xgboost = xgb.XGBClassifier(max_depth=4, n_estimators=600, learning_rate=0.03).fit(def_n_train_X, def_n_train_Y) \n",
    "xgb_predicted = Xgboost.predict(def_n_test_X)\n",
    "calcmetrics(xgb_predicted,def_n_test_Y,\"XGBoost - Random sampling with feature reduction\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Xgboost = xgb.XGBClassifier(max_depth=4, n_estimators=600, learning_rate=0.03).fit(def_f_n_train_X, def_f_n_train_Y) \n",
    "xgb_predicted = Xgboost.predict(def_f_n_test_X)\n",
    "calcmetrics(xgb_predicted,def_f_n_test_Y,\"XGBoost - Random sampling with out feature reduction\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}