fraud-stack-lgbm-26.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "import datetime\n",
    "import lightgbm as lgb\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import random\n",
    "import time\n",
    "\n",
    "from copy import deepcopy\n",
    "from scipy.stats import rankdata\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed=13):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    \n",
    "def read_from_disk(path, filename):\n",
    "    with open(os.path.join(path, filename), 'rb') as handle:\n",
    "        return pickle.load(handle)\n",
    "    \n",
    "    \n",
    "def save_to_disk(obj, filename):\n",
    "    with open(filename, 'wb') as handle:\n",
    "        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "def timedelta(seconds):\n",
    "    return str(datetime.timedelta(seconds=seconds)).split('.')[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['fraud-lgb-kfold-newfeatures-seed-44',\n",
       " 'fraud-fs-fe9477-no-day',\n",
       " 'fraud-dima-944193',\n",
       " 'fraud-artem-new-val-no-day',\n",
       " 'fraud-kirill-predictions-9438',\n",
       " 'fraud-dima-942747',\n",
       " 'ieee-fraud-detection',\n",
       " 'fraud-lgb-kfold-newfeatures-seed-111',\n",
       " 'fraud-dima-937684',\n",
       " 'fraud-artem-new-val-seed-99-no-day',\n",
       " 'fraud-dima-cb-hope-v1-944414',\n",
       " 'fraud-dima-944053',\n",
       " 'fraud-lgb-kfold-newfeatures',\n",
       " 'fraud-fs-fe9477-no-day-seed-144',\n",
       " 'fraud-misha-9436',\n",
       " 'fraud-fs-fe9477-no-day-seed-133',\n",
       " 'fraud-catboost-dima-943978',\n",
       " 'fraud-dima-cb-hope-v2-943496',\n",
       " 'fraud-dima-940490',\n",
       " 'fraud-mix-lgbm-seed-101-no-day',\n",
       " 'fraud-validation-new',\n",
       " 'fraud-dima-943354',\n",
       " 'fraud-mix-lgbm-no-day',\n",
       " 'fraud-lgb-kfold-newfeatures-seed-122',\n",
       " 'fraud-dima-943304',\n",
       " 'fraud-fs-fe9477-no-day-seed-88',\n",
       " 'fraud-kirill-predictions-9459',\n",
       " 'fraud-fs-fe9477-catboost-no-cat']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('../input')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0: (137321,)\n",
      "Fold 1: (92585,)\n",
      "Fold 2: (86021,)\n",
      "Fold 3: (101632,)\n",
      "Fold 4: (83655,)\n",
      "Fold 5: (89326,)\n"
     ]
    }
   ],
   "source": [
    "TARGET_PATH = '../input/fraud-validation-new'\n",
    "NFOLDS = 6\n",
    "\n",
    "y_val_array = []\n",
    "for i in range(NFOLDS):\n",
    "    y_val = read_from_disk(TARGET_PATH, 'y_val_fold{}.pkl'.format(i))\n",
    "    print('Fold {}:'.format(i), y_val.shape)\n",
    "    y_val_array.append(y_val.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0\n",
      "model_94450504 0.9192044150287948\n",
      "model_9444475 0.9229167125109011\n",
      "model_9444014 0.9174466202015826\n",
      "model_94436288 0.9182588617651293\n",
      "model_9441978 0.9182213496872573\n",
      "model_944193 0.9225717062615205\n",
      "model_94408984 0.921800849275633\n",
      "model_944053 0.9225550496691793\n",
      "model_9440478 0.9218366725308682\n",
      "model_943978 0.9181286371148675\n",
      "model_94397184 0.9220750829065565\n",
      "model_9436 0.9172470316889038\n",
      "model_943354 0.9194929762772012\n",
      "model_943304 0.918634742724612\n",
      "model_94317371 0.9184197947701459\n",
      "model_94314358 0.9172053775734671\n",
      "model_942747 0.9203007388809614\n",
      "model_9426038 0.921639236108881\n",
      "model_9424485 0.9217879409430095\n",
      "model_94049 0.9181943980152754\n",
      "model_937684 0.9181286371148675\n",
      "model_9325177 0.9133559064202742\n",
      "model_9459 0.9116326650726736\n",
      "model_944414 0.9241696672924647\n",
      "model_9438 0.9135601718345446\n",
      "model_943496 0.922756312264148\n",
      "Fold 1\n",
      "model_94450504 0.944781470049839\n",
      "model_9444475 0.9455159331722646\n",
      "model_9444014 0.9445456671173191\n",
      "model_94436288 0.9444888891723182\n",
      "model_9441978 0.9443139728952652\n",
      "model_944193 0.9470413397615065\n",
      "model_94408984 0.9453281745178567\n",
      "model_944053 0.9460520637083951\n",
      "model_9440478 0.9462608153527904\n",
      "model_943978 0.9472976346217616\n",
      "model_94397184 0.9457212411524553\n",
      "model_9436 0.9504421919924786\n",
      "model_943354 0.9465272195235717\n",
      "model_943304 0.9463812099833465\n",
      "model_94317371 0.9465845197880113\n",
      "model_94314358 0.9468091596609053\n",
      "model_942747 0.9455746515947142\n",
      "model_9426038 0.9448581234641682\n",
      "model_9424485 0.9450612358806731\n",
      "model_94049 0.9428784113229136\n",
      "model_937684 0.9472976346217616\n",
      "model_9325177 0.9323482753133613\n",
      "model_9459 0.9419667255187057\n",
      "model_944414 0.9446720502009717\n",
      "model_9438 0.9432031725439751\n",
      "model_943496 0.9456013050697782\n",
      "Fold 2\n",
      "model_94450504 0.9514585782938733\n",
      "model_9444475 0.9514273309923081\n",
      "model_9444014 0.9512758655476183\n",
      "model_94436288 0.951488747011946\n",
      "model_9441978 0.9508621040561281\n",
      "model_944193 0.9505778006901232\n",
      "model_94408984 0.950702125882442\n",
      "model_944053 0.9497965373450681\n",
      "model_9440478 0.9509404436480211\n",
      "model_943978 0.949013215205464\n",
      "model_94397184 0.9503649789519175\n",
      "model_9436 0.9505545742553756\n",
      "model_943354 0.9518310409288427\n",
      "model_943304 0.9519255592731203\n",
      "model_94317371 0.9515102870620804\n",
      "model_94314358 0.9514955417366173\n",
      "model_942747 0.9498418132586356\n",
      "model_9426038 0.9481145548982097\n",
      "model_9424485 0.9472188422507279\n",
      "model_94049 0.9473291669374543\n",
      "model_937684 0.949013215205464\n",
      "model_9325177 0.9363597725626474\n",
      "model_9459 0.9438987005205005\n",
      "model_944414 0.9502988832149776\n",
      "model_9438 0.9483279738976331\n",
      "model_943496 0.9499251241718365\n",
      "Fold 3\n",
      "model_94450504 0.9478957617559817\n",
      "model_9444475 0.9467549500888268\n",
      "model_9444014 0.9487925041453605\n",
      "model_94436288 0.9481148913594625\n",
      "model_9441978 0.9482709223634439\n",
      "model_944193 0.9456520628364473\n",
      "model_94408984 0.9466090565628372\n",
      "model_944053 0.9465843539479972\n",
      "model_9440478 0.9467334592472505\n",
      "model_943978 0.9474378444574251\n",
      "model_94397184 0.9462436357852483\n",
      "model_9436 0.9452989232680525\n",
      "model_943354 0.944123075233033\n",
      "model_943304 0.944740102759462\n",
      "model_94317371 0.9449089630133135\n",
      "model_94314358 0.9451823945261882\n",
      "model_942747 0.9438247703205498\n",
      "model_9426038 0.9448499874640155\n",
      "model_9424485 0.9446247937267298\n",
      "model_94049 0.941049339395975\n",
      "model_937684 0.9474378444574251\n",
      "model_9325177 0.937291416833934\n",
      "model_9459 0.936672290184265\n",
      "model_944414 0.9477688151432214\n",
      "model_9438 0.9447357260811337\n",
      "model_943496 0.9449566425521206\n",
      "Fold 4\n",
      "model_94450504 0.9599103645484884\n",
      "model_9444475 0.956994454900262\n",
      "model_9444014 0.960106266393365\n",
      "model_94436288 0.9603303248356115\n",
      "model_9441978 0.9603951821314489\n",
      "model_944193 0.9559958537879173\n",
      "model_94408984 0.9569537730794198\n",
      "model_944053 0.9622734756680539\n",
      "model_9440478 0.9567275578313148\n",
      "model_943978 0.9626849793512744\n",
      "model_94397184 0.9569471845406317\n",
      "model_9436 0.9593483613149071\n",
      "model_943354 0.9566695646904299\n",
      "model_943304 0.9569268764416583\n",
      "model_94317371 0.9590361494486389\n",
      "model_94314358 0.958725831896644\n",
      "model_942747 0.9557148215471082\n",
      "model_9426038 0.9565226350256275\n",
      "model_9424485 0.956330224318834\n",
      "model_94049 0.953767759590047\n",
      "model_937684 0.9626849793512744\n",
      "model_9325177 0.9453792853785589\n",
      "model_9459 0.9505408914728718\n",
      "model_944414 0.9626949496566513\n",
      "model_9438 0.957030068446081\n",
      "model_943496 0.9618361559374311\n",
      "Fold 5\n",
      "model_94450504 0.9437796369160149\n",
      "model_9444475 0.9430754498306083\n",
      "model_9444014 0.9442417023041972\n",
      "model_94436288 0.9434955915606587\n",
      "model_9441978 0.943123046499077\n",
      "model_944193 0.9433198064629615\n",
      "model_94408984 0.9431450680598834\n",
      "model_944053 0.9370562517676475\n",
      "model_9440478 0.9417879782005023\n",
      "model_943978 0.9393044698613187\n",
      "model_94397184 0.9424789344699438\n",
      "model_9436 0.9387351611109606\n",
      "model_943354 0.9414830809003903\n",
      "model_943304 0.9412145161956957\n",
      "model_94317371 0.9385825336660942\n",
      "model_94314358 0.9394431624856543\n",
      "model_942747 0.9412271994525542\n",
      "model_9426038 0.939638279772219\n",
      "model_9424485 0.9396680267587888\n",
      "model_94049 0.9397235350976149\n",
      "model_937684 0.9393044698613187\n",
      "model_9325177 0.9303716725429071\n",
      "model_9459 0.9317642438599106\n",
      "model_944414 0.9368805709669232\n",
      "model_9438 0.9413412034934956\n",
      "model_943496 0.9359028538959432\n"
     ]
    }
   ],
   "source": [
    "NUM_MODELS = 26\n",
    "MODEL_DIRS = [\n",
    "    '../input/fraud-lgb-kfold-newfeatures',\n",
    "    '../input/fraud-fs-fe9477-no-day-seed-144',\n",
    "    '../input/fraud-lgb-kfold-newfeatures-seed-122',\n",
    "    '../input/fraud-lgb-kfold-newfeatures-seed-44',\n",
    "    '../input/fraud-lgb-kfold-newfeatures-seed-111',\n",
    "    '../input/fraud-dima-944193',\n",
    "    '../input/fraud-fs-fe9477-no-day-seed-88',\n",
    "    '../input/fraud-dima-944053',\n",
    "    '../input/fraud-fs-fe9477-no-day-seed-133',\n",
    "    '../input/fraud-catboost-dima-943978',\n",
    "    '../input/fraud-fs-fe9477-no-day',\n",
    "    '../input/fraud-misha-9436',\n",
    "    '../input/fraud-dima-943354',\n",
    "    '../input/fraud-dima-943304',\n",
    "    '../input/fraud-artem-new-val-seed-99-no-day',\n",
    "    '../input/fraud-artem-new-val-no-day',\n",
    "    '../input/fraud-dima-942747',\n",
    "    '../input/fraud-mix-lgbm-no-day',\n",
    "    '../input/fraud-mix-lgbm-seed-101-no-day',\n",
    "    '../input/fraud-dima-940490',\n",
    "    '../input/fraud-catboost-dima-943978',\n",
    "    '../input/fraud-fs-fe9477-catboost-no-cat',\n",
    "    '../input/fraud-kirill-predictions-9459',\n",
    "    '../input/fraud-dima-cb-hope-v1-944414',\n",
    "    '../input/fraud-kirill-predictions-9438',\n",
    "    '../input/fraud-dima-cb-hope-v2-943496'\n",
    "]\n",
    "cv_scores = [\n",
    "    0.94450504,\n",
    "    0.9444475,\n",
    "    0.9444014,\n",
    "    0.94436288,\n",
    "    0.9441978,\n",
    "    0.944193,\n",
    "    0.94408984,\n",
    "    0.944053,\n",
    "    0.9440478,\n",
    "    0.943978,\n",
    "    0.94397184,\n",
    "    0.9436,\n",
    "    0.943354,\n",
    "    0.943304,\n",
    "    0.94317371,\n",
    "    0.94314358,\n",
    "    0.942747,\n",
    "    0.9426038,\n",
    "    0.9424485,\n",
    "    0.940490,\n",
    "    0.937684,\n",
    "    0.9325177,\n",
    "    0.9459,\n",
    "    0.944414,\n",
    "    0.9438,\n",
    "    0.943496\n",
    "]\n",
    "MODEL_NAMES = ['model_{}'.format(str(x).split('.')[1]) for x in cv_scores]\n",
    "\n",
    "val_preds = []\n",
    "test_preds = []\n",
    "for i in range(NFOLDS):\n",
    "    val_preds_fold = pd.DataFrame()\n",
    "    test_preds_fold = pd.DataFrame()\n",
    "    print('Fold', i)\n",
    "    for j in range(NUM_MODELS):\n",
    "        if 'misha' in MODEL_DIRS[j]:\n",
    "            y_pred_val = read_from_disk(MODEL_DIRS[j], 'y_val_pred_fold{}.pkl'.format(i))\n",
    "            y_pred_test = read_from_disk(MODEL_DIRS[j], 'y_test_pred_fold{}.pkl'.format(i))\n",
    "        else:\n",
    "            y_pred_val = read_from_disk(MODEL_DIRS[j], 'y_pred_valid_fold{}.pkl'.format(i))\n",
    "            y_pred_test = read_from_disk(MODEL_DIRS[j], 'y_pred_test_fold{}.pkl'.format(i))\n",
    "\n",
    "        val_preds_fold['{}_fold{}'.format(MODEL_NAMES[j], i)] = y_pred_val\n",
    "        test_preds_fold['{}_fold{}'.format(MODEL_NAMES[j], i)] = y_pred_test\n",
    "        \n",
    "        print(MODEL_NAMES[j], roc_auc_score(y_val_array[i], y_pred_val))\n",
    "    val_preds.append(val_preds_fold)\n",
    "    test_preds.append(test_preds_fold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_results(params, skf, val_preds_fold, y_val):\n",
    "    cv_scores = []\n",
    "    models = []\n",
    "    y_clf_pred = np.zeros_like(y_val)\n",
    "    for train_index, test_index in skf.split(np.array(val_preds_fold), y_val):\n",
    "        X_train, X_test = val_preds_fold.iloc[train_index], val_preds_fold.iloc[test_index]\n",
    "        y_train, y_test = y_val[train_index], y_val[test_index]\n",
    "        dtrain = lgb.Dataset(X_train, label=y_train)\n",
    "        dvalid = lgb.Dataset(X_test, label=y_test)\n",
    "        clf = lgb.train(params, dtrain, valid_sets = [dtrain, dvalid], verbose_eval=500)\n",
    "        y_test_pred = clf.predict(X_test)\n",
    "        y_clf_pred[test_index] = y_test_pred\n",
    "        cv_scores.append(roc_auc_score(y_test, y_test_pred))\n",
    "        models.append(clf)\n",
    "    cv_scores = np.array(cv_scores)\n",
    "#     model = model.fit(val_preds_fold, y_val)\n",
    "    return cv_scores, models, y_clf_pred\n",
    "\n",
    "params = {\n",
    "        'objective':'binary',\n",
    "        'boosting_type':'gbdt',\n",
    "        'metric':'auc',\n",
    "        'n_jobs':-1,\n",
    "        'max_depth':-1,\n",
    "        'tree_learner':'serial',\n",
    "        'min_data_in_leaf':30,\n",
    "        'n_estimators':1800,\n",
    "        'max_bin':255,\n",
    "        'verbose':-1,\n",
    "        'seed': 1229,\n",
    "        'learning_rate': 0.01,\n",
    "        'early_stopping_rounds':200,\n",
    "        'colsample_bytree': 0.5,          \n",
    "        'num_leaves': 256, \n",
    "        'reg_alpha': 0.35\n",
    "         }\n",
    "\n",
    "def models_tuning(\n",
    "    val_preds_fold,\n",
    "    y_val,\n",
    "    n_splits=5\n",
    "):\n",
    "    best = {'models': None, 'score_mean': 0, 'score_std': 0}\n",
    "    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=13)\n",
    "    cv_scores, models, y_clf_pred = train_results(params, skf, val_preds_fold, y_val)\n",
    "    best['models'] = models\n",
    "    best['score_mean'] = cv_scores.mean()\n",
    "    best['score_std'] = cv_scores.std()\n",
    "    return best, y_clf_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#################\n",
      "Fold 0\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999325\tvalid_1's auc: 0.92964\n",
      "Early stopping, best iteration is:\n",
      "[749]\ttraining's auc: 0.999721\tvalid_1's auc: 0.930214\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999248\tvalid_1's auc: 0.948785\n",
      "Early stopping, best iteration is:\n",
      "[475]\ttraining's auc: 0.999186\tvalid_1's auc: 0.949027\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999279\tvalid_1's auc: 0.952919\n",
      "Early stopping, best iteration is:\n",
      "[779]\ttraining's auc: 0.999725\tvalid_1's auc: 0.953069\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999263\tvalid_1's auc: 0.94754\n",
      "Early stopping, best iteration is:\n",
      "[711]\ttraining's auc: 0.99965\tvalid_1's auc: 0.947907\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999281\tvalid_1's auc: 0.938217\n",
      "Early stopping, best iteration is:\n",
      "[498]\ttraining's auc: 0.999276\tvalid_1's auc: 0.938217\n",
      "Best CV mean: 0.943686936356524\n",
      "Best CV std: 0.008316311958715317\n",
      "Time: fold 0:02:59 | total 0:02:59\n",
      "#################\n",
      "Fold 1\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999608\tvalid_1's auc: 0.956389\n",
      "Early stopping, best iteration is:\n",
      "[709]\ttraining's auc: 0.999885\tvalid_1's auc: 0.956752\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999588\tvalid_1's auc: 0.958786\n",
      "Early stopping, best iteration is:\n",
      "[343]\ttraining's auc: 0.999186\tvalid_1's auc: 0.958909\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999617\tvalid_1's auc: 0.968678\n",
      "Early stopping, best iteration is:\n",
      "[376]\ttraining's auc: 0.999341\tvalid_1's auc: 0.968913\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[98]\ttraining's auc: 0.990436\tvalid_1's auc: 0.964658\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[244]\ttraining's auc: 0.997652\tvalid_1's auc: 0.959666\n",
      "Best CV mean: 0.961779692645378\n",
      "Best CV std: 0.004407994295031419\n",
      "Time: fold 0:01:27 | total 0:04:27\n",
      "#################\n",
      "Fold 2\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.99969\tvalid_1's auc: 0.964653\n",
      "Early stopping, best iteration is:\n",
      "[359]\ttraining's auc: 0.99932\tvalid_1's auc: 0.965146\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[263]\ttraining's auc: 0.997154\tvalid_1's auc: 0.967201\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999714\tvalid_1's auc: 0.968515\n",
      "Early stopping, best iteration is:\n",
      "[346]\ttraining's auc: 0.999205\tvalid_1's auc: 0.968838\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999686\tvalid_1's auc: 0.968598\n",
      "Early stopping, best iteration is:\n",
      "[522]\ttraining's auc: 0.999716\tvalid_1's auc: 0.968668\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999665\tvalid_1's auc: 0.958683\n",
      "Early stopping, best iteration is:\n",
      "[346]\ttraining's auc: 0.998996\tvalid_1's auc: 0.959368\n",
      "Best CV mean: 0.9658441534356866\n",
      "Best CV std: 0.0034989148650607974\n",
      "Time: fold 0:01:27 | total 0:05:54\n",
      "#################\n",
      "Fold 3\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999404\tvalid_1's auc: 0.958766\n",
      "[1000]\ttraining's auc: 0.999974\tvalid_1's auc: 0.959194\n",
      "Early stopping, best iteration is:\n",
      "[882]\ttraining's auc: 0.999946\tvalid_1's auc: 0.959263\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[123]\ttraining's auc: 0.988568\tvalid_1's auc: 0.956368\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[102]\ttraining's auc: 0.986647\tvalid_1's auc: 0.958672\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[151]\ttraining's auc: 0.989619\tvalid_1's auc: 0.965025\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999476\tvalid_1's auc: 0.962369\n",
      "Early stopping, best iteration is:\n",
      "[572]\ttraining's auc: 0.999649\tvalid_1's auc: 0.962534\n",
      "Best CV mean: 0.9603724303880524\n",
      "Best CV std: 0.003048885504911314\n",
      "Time: fold 0:01:36 | total 0:07:31\n",
      "#################\n",
      "Fold 4\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999697\tvalid_1's auc: 0.969458\n",
      "Early stopping, best iteration is:\n",
      "[602]\ttraining's auc: 0.999833\tvalid_1's auc: 0.969683\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[32]\ttraining's auc: 0.991023\tvalid_1's auc: 0.967763\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999678\tvalid_1's auc: 0.974449\n",
      "Early stopping, best iteration is:\n",
      "[504]\ttraining's auc: 0.999684\tvalid_1's auc: 0.974457\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[44]\ttraining's auc: 0.9912\tvalid_1's auc: 0.975759\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999671\tvalid_1's auc: 0.965913\n",
      "Early stopping, best iteration is:\n",
      "[585]\ttraining's auc: 0.999791\tvalid_1's auc: 0.966246\n",
      "Best CV mean: 0.9707814928953603\n",
      "Best CV std: 0.0037192884323218782\n",
      "Time: fold 0:01:20 | total 0:08:52\n",
      "#################\n",
      "Fold 5\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999373\tvalid_1's auc: 0.962187\n",
      "Early stopping, best iteration is:\n",
      "[480]\ttraining's auc: 0.999317\tvalid_1's auc: 0.962305\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999527\tvalid_1's auc: 0.951223\n",
      "Early stopping, best iteration is:\n",
      "[473]\ttraining's auc: 0.999456\tvalid_1's auc: 0.951303\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999456\tvalid_1's auc: 0.958373\n",
      "Early stopping, best iteration is:\n",
      "[566]\ttraining's auc: 0.999597\tvalid_1's auc: 0.958557\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "[500]\ttraining's auc: 0.999486\tvalid_1's auc: 0.957344\n",
      "Early stopping, best iteration is:\n",
      "[433]\ttraining's auc: 0.999232\tvalid_1's auc: 0.95753\n",
      "Training until validation scores don't improve for 200 rounds.\n",
      "Early stopping, best iteration is:\n",
      "[201]\ttraining's auc: 0.99478\tvalid_1's auc: 0.958965\n",
      "Best CV mean: 0.9577321420733957\n",
      "Best CV std: 0.0035912124806067344\n",
      "Time: fold 0:01:41 | total 0:10:33\n"
     ]
    }
   ],
   "source": [
    "tuning_results = []\n",
    "cv_score = 0\n",
    "start_time = time.time()\n",
    "for i in range(NFOLDS):\n",
    "    start_time_fold = time.time()\n",
    "    print('#################')\n",
    "    print('Fold', i)\n",
    "    best, y_clf_pred = models_tuning(val_preds[i], y_val_array[i])\n",
    "    tuning_results.append(best)\n",
    "    cv_score += best['score_mean'] / NFOLDS\n",
    "    print('Best CV mean:', best['score_mean'])\n",
    "    print('Best CV std:', best['score_std'])\n",
    "    print('Time: fold {} | total {}'.format(timedelta(time.time() - start_time_fold), timedelta(time.time() - start_time)))\n",
    "    save_to_disk(y_clf_pred, 'y_clf_pred_fold{}.pkl'.format(i))\n",
    "save_to_disk(tuning_results, 'tuning_results.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final CV score:\n",
      "0.9600328079657329\n"
     ]
    }
   ],
   "source": [
    "print('Final CV score:')\n",
    "print(cv_score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/ieee-fraud-detection/sample_submission.csv')\n",
    "y_preds = np.zeros(len(sub))\n",
    "for i in range(NFOLDS):\n",
    "    models = tuning_results[i]['models']\n",
    "    for j in range(len(models)):\n",
    "        y_preds += models[j].predict(test_preds[i]) / NFOLDS / len(models)\n",
    "sub['isFraud'] = y_preds\n",
    "sub.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}