music_box_project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install --upgrade pandas\n",
    "#!pip install --upgrade scikit-learn\n",
    "#pip install --upgrade scipy\n",
    "#!pip install --upgrade matplotlib\n",
    "#!pip install --upgrade xgboost\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from pyspark.sql import functions as F\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import precision_score, accuracy_score, recall_score, f1_score, roc_auc_score,roc_curve\n",
    "\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load play data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "schema = StructType([StructField(\"uid\", StringType(), True), \n",
    "                     StructField('device', StringType(), True),\n",
    "                     StructField('song_id', StringType(), True),\n",
    "                     StructField('song_type', StringType(), True),\n",
    "                     StructField('song_name', StringType(), True),\n",
    "                     StructField('singer', StringType(), True),\n",
    "                     StructField('play_time', StringType(), True),\n",
    "                     StructField('song_length', StringType(), True),\n",
    "                     StructField('paid_flag', StringType(), True),\n",
    "                     StructField('file_name', StringType(), True)])\n",
    "# note: the data type should be the right type. First, I change the types and it doesn't work."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = sqlContext.read.format(\"com.databricks.spark.csv\")\\\n",
    "         .options(delimiter=\"\\t\").schema(schema)\\\n",
    "         .options(header=\"false\").load(\"gs://musix_box/all_play.log.fn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# delete the extra space in each column\n",
    "df = df.withColumn('uid', trim(df.uid))\n",
    "df = df.withColumn('device', trim(df.device))\n",
    "df = df.withColumn('song_id', trim(df.song_id))\n",
    "df = df.withColumn('song_type', trim(df.song_type))\n",
    "df = df.withColumn('song_name', trim(df.song_name))\n",
    "df = df.withColumn('singer', trim(df.singer))\n",
    "df = df.withColumn('play_time', trim(df.play_time))\n",
    "df = df.withColumn('song_length', trim(df.song_length))\n",
    "df = df.withColumn('paid_flag', trim(df.paid_flag))\n",
    "df = df.withColumn('file_name', trim(df.file_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema_down = StructType([StructField(\"uid\", StringType(), True), \n",
    "                     StructField('device', StringType(), True),\n",
    "                     StructField('song_id', StringType(), True),\n",
    "                     StructField('song_name', StringType(), True),\n",
    "                     StructField('singer', StringType(), True),\n",
    "                     StructField('paid_flag', StringType(), True),\n",
    "                     StructField('file_name', StringType(), True)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_down = sqlContext.read.format(\"com.databricks.spark.csv\")\\\n",
    "         .options(delimiter=\"\\t\").schema(schema_down)\\\n",
    "         .options(header=\"false\").load(\"gs://musix_box/all_down.log.fn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete the extra space in each column\n",
    "df_down = df_down.withColumn('uid', trim(df_down.uid))\n",
    "df_down = df_down.withColumn('device', trim(df_down.device))\n",
    "df_down = df_down.withColumn('song_id', trim(df_down.song_id))\n",
    "df_down = df_down.withColumn('song_name', trim(df_down.song_name))\n",
    "df_down = df_down.withColumn('singer', trim(df_down.singer))\n",
    "df_down = df_down.withColumn('file_name', trim(df_down.file_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------+-------+--------------------+-------------------+---------+-------------------+\n",
      "|      uid|device|song_id|           song_name|             singer|paid_flag|          file_name|\n",
      "+---------+------+-------+--------------------+-------------------+---------+-------------------+\n",
      "|168019810|    ar| 442554|                 小酒窝|            林俊杰&蔡卓妍|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|6334611|                 社会摇|                 萧全|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|9867382|                  台阶|                 武艺|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|6660691| 一次就好-(电影《夏洛特烦恼》暖水曲)|                杨宗纬|       0 |20170330_1_down.log|\n",
      "|168019810|    ar| 157606|               一路上有你|                张学友|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|3372481|        十年 (OT：明年今日)|                陈奕迅|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|3216525|       你不知道的事(Live版)|                王力宏|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|6427523|             Victory|Two Steps From Hell|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|6538686|                  野子|                流浪者|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|9327383|なんでもないや-[没什么大不了](...|           RADWIMPS|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|6716077|我们不该这样的-(电视剧《北上广不...|                张赫宣|       0 |20170330_1_down.log|\n",
      "|168019810|    ar| 350052|                爱情转移|                陈奕迅|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|3638735|               咱们结婚吧|                 齐晨|       0 |20170330_1_down.log|\n",
      "|168019810|    ar| 409466|        AlwaysOnline|                林俊杰|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|1076725|Change The World(...|                 V6|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|7130410|  Kyoto(Feat. Sirah)|           Skrillex|       0 |20170330_1_down.log|\n",
      "|168019810|    ar| 912614|               客官不可以|              徐良&小凌|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|3654821|                  犯贱|                 徐良|       0 |20170330_1_down.log|\n",
      "|168050395|    ar|6544560|Let's Not Fall In...|            BIGBANG|       0 |20170330_1_down.log|\n",
      "|168019810|    ar|6673573|   What Do You Mean？|      Justin Bieber|       0 |20170330_1_down.log|\n",
      "+---------+------+-------+--------------------+-------------------+---------+-------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_down.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Date Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define get_date function\n",
    "def get_date(file_name):\n",
    "    tmp_list = str(file_name).split('_')\n",
    "    return tmp_list[0]\n",
    "\n",
    "example_udf = F.udf(get_date)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add date to play dataset\n",
    "df_play = df.withColumn('date', example_udf(df.file_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('uid', 'string'),\n",
       " ('device', 'string'),\n",
       " ('song_id', 'string'),\n",
       " ('song_type', 'string'),\n",
       " ('song_name', 'string'),\n",
       " ('singer', 'string'),\n",
       " ('play_time', 'string'),\n",
       " ('song_length', 'string'),\n",
       " ('paid_flag', 'string'),\n",
       " ('file_name', 'string'),\n",
       " ('date', 'string')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check types of df_play\n",
    "df_play.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('uid', 'string'),\n",
       " ('device', 'string'),\n",
       " ('song_id', 'string'),\n",
       " ('song_type', 'string'),\n",
       " ('song_name', 'string'),\n",
       " ('singer', 'string'),\n",
       " ('play_time', 'float'),\n",
       " ('song_length', 'float'),\n",
       " ('paid_flag', 'string'),\n",
       " ('file_name', 'string'),\n",
       " ('date', 'int')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert features to right data types\n",
    "df_play = df_play.withColumn('play_time', df_play['play_time'].cast('float'))\n",
    "df_play = df_play.withColumn('song_length', df_play['song_length'].cast('float'))\n",
    "df_play = df_play.withColumn('date', df_play['date'].cast('integer'))\n",
    "df_play.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------+-------+--------------------+-------------------+---------+-------------------+--------+\n",
      "|      uid|device|song_id|           song_name|             singer|paid_flag|          file_name|    date|\n",
      "+---------+------+-------+--------------------+-------------------+---------+-------------------+--------+\n",
      "|168019810|    ar| 442554|                 小酒窝|            林俊杰&蔡卓妍|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|6334611|                 社会摇|                 萧全|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|9867382|                  台阶|                 武艺|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|6660691| 一次就好-(电影《夏洛特烦恼》暖水曲)|                杨宗纬|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar| 157606|               一路上有你|                张学友|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|3372481|        十年 (OT：明年今日)|                陈奕迅|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|3216525|       你不知道的事(Live版)|                王力宏|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|6427523|             Victory|Two Steps From Hell|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|6538686|                  野子|                流浪者|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|9327383|なんでもないや-[没什么大不了](...|           RADWIMPS|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|6716077|我们不该这样的-(电视剧《北上广不...|                张赫宣|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar| 350052|                爱情转移|                陈奕迅|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|3638735|               咱们结婚吧|                 齐晨|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar| 409466|        AlwaysOnline|                林俊杰|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|1076725|Change The World(...|                 V6|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|7130410|  Kyoto(Feat. Sirah)|           Skrillex|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar| 912614|               客官不可以|              徐良&小凌|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|3654821|                  犯贱|                 徐良|       0 |20170330_1_down.log|20170330|\n",
      "|168050395|    ar|6544560|Let's Not Fall In...|            BIGBANG|       0 |20170330_1_down.log|20170330|\n",
      "|168019810|    ar|6673573|   What Do You Mean？|      Justin Bieber|       0 |20170330_1_down.log|20170330|\n",
      "+---------+------+-------+--------------------+-------------------+---------+-------------------+--------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# add date to download dataset\n",
    "df_down = df_down.withColumn('date', example_udf(df_down.file_name))\n",
    "df_down.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean Data - exclude the extreme data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select the 95% quantile of play_time and 99% quantile of song_length with 0.005 relativeError\n",
    "extreme_play_time = df_play.approxQuantile(\"play_time\", [0.95], 0.005)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "extreme_song_length = df_play.approxQuantile(\"song_length\", [0.99], 0.005)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([62694.0], [1386.0])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extreme_play_time, extreme_song_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_play_valid = df_play.filter((col('play_time') < extreme_play_time[0]) &\\\n",
    "                               (col('song_length') < extreme_song_length[0]) &\\\n",
    "                               (col('song_length') != 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop missing values\n",
    "df_play_valid = df_play_valid.na.drop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------+--------+---------+--------------------+------+---------+-----------+---------+-------------------+--------+\n",
      "|      uid|device| song_id|song_type|           song_name|singer|play_time|song_length|paid_flag|          file_name|    date|\n",
      "+---------+------+--------+---------+--------------------+------+---------+-----------+---------+-------------------+--------+\n",
      "|154824972|    ar| 3300104|        0|     Not A Love Song|  Bean|     22.0|      178.0|        0|20170302_1_play.log|20170302|\n",
      "|154431666|    ar| 6651913|        0|             再见只是陌生人|   庄心妍|    252.0|      251.0|        0|20170302_1_play.log|20170302|\n",
      "|154439430|    ar|20870987|        0|                  哥哥|    张杰|    358.0|      358.0|        0|20170302_1_play.log|20170302|\n",
      "|154739280|    ar|  872687|        0|             苦酒一杯又一杯|   龙飘飘|    164.0|      165.0|        0|20170302_1_play.log|20170302|\n",
      "|154777138|    ip| 9950164|        0|               刚好遇见你|   李玉刚|      0.0|      200.0|        0|20170302_1_play.log|20170302|\n",
      "|154412619|    ar| 2268011|        0|               有一点动心|    群星|    235.0|      235.0|        0|20170302_1_play.log|20170302|\n",
      "|154830959|    ar|  238895|        0|                自由飞翔|  凤凰传奇|    256.0|      256.0|        0|20170302_1_play.log|20170302|\n",
      "|154412733|    ar|  135368|        0|       Big Big World|Emilia|    206.0|      206.0|        0|20170302_1_play.log|20170302|\n",
      "|154410884|    ip| 3433379|        0|                 就这样| 幼稚园杀手|    213.0|      213.0|        0|20170302_1_play.log|20170302|\n",
      "|154470406|    ar|20940997|        0|  向往-(电视剧《鸡毛飞上天》主题曲)|    孙楠|      5.0|      247.0|        0|20170302_1_play.log|20170302|\n",
      "|154707484|    ip|11950502|        0|                说散就散|    JC|    230.0|      230.0|        0|20170302_1_play.log|20170302|\n",
      "|154785481|    ip| 1092298|        0|         忘了你,忘了我(国语)|    王杰|    279.0|      279.0|        0|20170302_1_play.log|20170302|\n",
      "|154469823|    ar|20862199|        0|It's Love(Electro...|   张碧晨|    234.0|      235.0|        0|20170302_1_play.log|20170302|\n",
      "|154439516|    ar|20870987|        0|                  哥哥|    张杰|    358.0|      358.0|        0|20170302_1_play.log|20170302|\n",
      "|154436951|    ar|20870987|        0|                  哥哥|    张杰|     42.0|      358.0|        0|20170302_1_play.log|20170302|\n",
      "|154806770|    ar| 6710490|        0|            Be Happy| B.A.P|    216.0|      215.0|        0|20170302_1_play.log|20170302|\n",
      "|154470463|    ar|20866404|        0|那个静默的阳光午后-(电影《那个阳...|   范玮琪|      2.0|      170.0|        0|20170302_1_play.log|20170302|\n",
      "|154791407|    ar|20274975|        0|               枪硬马子稳|  MC千少|     94.0|       98.0|        0|20170302_1_play.log|20170302|\n",
      "|154439956|    ar|20870987|        0|                  哥哥|    张杰|    358.0|      358.0|        0|20170302_1_play.log|20170302|\n",
      "|154802726|    ar|  849802|        0|Kiss The Rain(钢琴曲...|    铃声|      6.0|       59.0|        0|20170302_1_play.log|20170302|\n",
      "+---------+------+--------+---------+--------------------+------+---------+-----------+---------+-------------------+--------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#df_play_valid.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features from Play dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# recency feature: 20170501(test time) - the latest time the user play before 20170501 \n",
    "df_recency = df_play_valid.filter((col('date') >= 20170329) & (col('date') <= 20170430)).groupBy(\"uid\").agg(20170501 -max('date'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split the data into three period p1: (20170329,20170409); p2: (20170410,20170419), p3: (20170420, 20170429)\n",
    "\n",
    "df_p1 = df_play_valid.filter((col('date') >= 20170329) & (col('date') < 20170409))\n",
    "df_p2 = df_play_valid.filter((col('date') >= 20170409) & (col('date') < 20170420))\n",
    "df_p3 = df_play_valid.filter((col('date') >= 20170420) & (col('date') <= 20170430))\n",
    "\n",
    "# note: df_play.select('date') is dataframe, df_play.date is column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_p1.count()\n",
    "# 50M +"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29139947"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_p2.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create the pipeline for data aggreation \n",
    "def pipeline(dataframe_list):\n",
    "    output = []\n",
    "    \n",
    "    for idx, df in enumerate(dataframe_list):\n",
    "        period_tag = \"p%s\" % (idx + 1)\n",
    "        group_uid = df.groupBy(\"uid\")\n",
    "        df_agg = group_uid.agg(countDistinct('uid').alias(period_tag + '_play_freq'), \n",
    "                               countDistinct(\"song_name\").alias(period_tag + '_song_num'), \n",
    "                               sum(\"play_time\").alias(period_tag + '_play_time'), \n",
    "                               mean(df.play_time/df.song_length).alias(period_tag + '_play_perc'),\n",
    "                               countDistinct(\"singer\").alias(period_tag + '_singer_num'))\n",
    "        output.append(df_agg)\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# extract features for three periods\n",
    "df_p1_whole, df_p2_whole, df_p3_whole = pipeline([df_p1,df_p2,df_p3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "519737"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_p1_whole.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "270507"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_p2_whole.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df_p3_whole.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('uid', 'string'),\n",
       " ('p1_play_freq', 'bigint'),\n",
       " ('p1_song_num', 'bigint'),\n",
       " ('p1_play_time', 'double'),\n",
       " ('p1_play_perc', 'double'),\n",
       " ('p1_singer_num', 'bigint')]"
      ]
     },
     "execution_count": 322,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_p1_whole.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# join all features together\n",
    "df_play_whole = df_p1_whole.join(df_p2_whole, 'uid', 'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_play_whole = df_play_whole.join(df_p3_whole, 'uid', 'outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_play_whole = df_play_whole.join(df_recency, 'uid', 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the feature name\n",
    "df_play_whole = df_play_whole.withColumnRenamed(\"colName\", \"newColName\").withColumnRenamed(\"(20170501 - max(date))\", \"recency\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('uid', 'string'),\n",
       " ('p1_play_freq', 'bigint'),\n",
       " ('p1_song_num', 'bigint'),\n",
       " ('p1_play_time', 'double'),\n",
       " ('p1_play_perc', 'double'),\n",
       " ('p1_singer_num', 'bigint'),\n",
       " ('p2_play_freq', 'bigint'),\n",
       " ('p2_song_num', 'bigint'),\n",
       " ('p2_play_time', 'double'),\n",
       " ('p2_play_perc', 'double'),\n",
       " ('p2_singer_num', 'bigint'),\n",
       " ('p3_play_freq', 'bigint'),\n",
       " ('p3_song_num', 'bigint'),\n",
       " ('p3_play_time', 'double'),\n",
       " ('p3_play_perc', 'double'),\n",
       " ('p3_singer_num', 'bigint'),\n",
       " ('recency', 'int')]"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_play_whole.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# convert bigint to integer\n",
    "df_play_whole = df_play_whole.withColumn('p1_play_freq', df_play_whole['p1_play_freq'].cast('integer'))\n",
    "df_play_whole = df_play_whole.withColumn('p1_song_num', df_play_whole['p1_song_num'].cast('integer'))\n",
    "df_play_whole = df_play_whole.withColumn('p1_singer_num', df_play_whole['p1_singer_num'].cast('integer'))\n",
    "df_play_whole = df_play_whole.withColumn('p2_play_freq', df_play_whole['p2_play_freq'].cast('integer'))\n",
    "df_play_whole = df_play_whole.withColumn('p2_song_num', df_play_whole['p2_song_num'].cast('integer'))\n",
    "df_play_whole = df_play_whole.withColumn('p2_singer_num', df_play_whole['p2_singer_num'].cast('integer'))\n",
    "df_play_whole = df_play_whole.withColumn('p3_play_freq', df_play_whole['p3_play_freq'].cast('integer'))\n",
    "df_play_whole = df_play_whole.withColumn('p3_song_num', df_play_whole['p3_song_num'].cast('integer'))\n",
    "df_play_whole = df_play_whole.withColumn('p3_singer_num', df_play_whole['p3_singer_num'].cast('integer'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- uid: string (nullable = true)\n",
      " |-- p1_play_freq: integer (nullable = true)\n",
      " |-- p1_song_num: integer (nullable = true)\n",
      " |-- p1_play_time: double (nullable = true)\n",
      " |-- p1_play_perc: double (nullable = true)\n",
      " |-- p1_singer_num: integer (nullable = true)\n",
      " |-- p2_play_freq: integer (nullable = true)\n",
      " |-- p2_song_num: integer (nullable = true)\n",
      " |-- p2_play_time: double (nullable = true)\n",
      " |-- p2_play_perc: double (nullable = true)\n",
      " |-- p2_singer_num: integer (nullable = true)\n",
      " |-- p3_play_freq: integer (nullable = true)\n",
      " |-- p3_song_num: integer (nullable = true)\n",
      " |-- p3_play_time: double (nullable = true)\n",
      " |-- p3_play_perc: double (nullable = true)\n",
      " |-- p3_singer_num: integer (nullable = true)\n",
      " |-- recency: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_play_whole.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df_play_whole.count() # 567K+"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Therefore, after dropping outliers and aggregation, we now have 567k rows of data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Features from Download Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split into three periods\n",
    "df_down_p1 = df_down.filter((col('date') >= 20170329) & (col('date') < 20170409))\n",
    "df_down_p2 = df_down.filter((col('date') >= 20170409) & (col('date') < 20170420))\n",
    "df_down_p3 = df_down.filter((col('date') >= 20170420) & (col('date') <= 20170430))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create features\n",
    "def down_pipeline(dataframe_list):\n",
    "    output = []\n",
    "    \n",
    "    for idx, df in enumerate(dataframe_list):\n",
    "        period_tag = \"p%s\" % (idx + 1)\n",
    "        group_uid = df.groupBy(\"uid\")\n",
    "        df_agg = group_uid.agg(countDistinct('uid').alias(period_tag + '_down_freq'))\n",
    "        output.append(df_agg)\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# extract features for three periods\n",
    "down_p1, down_p2, down_p3 = down_pipeline([df_down_p1, df_down_p1, df_down_p1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------------+\n",
      "|      uid|p1_down_freq|\n",
      "+---------+------------+\n",
      "|167586784|           1|\n",
      "|167654139|           1|\n",
      "|166454870|           1|\n",
      "|167706312|           1|\n",
      "|167890834|           1|\n",
      "|167841519|           1|\n",
      "|167903143|           1|\n",
      "|167953403|           1|\n",
      "|167723994|           1|\n",
      "|167950920|           1|\n",
      "|167701571|           1|\n",
      "|167754291|           1|\n",
      "|167736662|           1|\n",
      "|167723630|           1|\n",
      "|167864822|           1|\n",
      "|167818843|           1|\n",
      "|167708458|           1|\n",
      "|159375694|           1|\n",
      "|167720746|           1|\n",
      "|167985660|           1|\n",
      "+---------+------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "down_p1.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine features in play and download dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_play_whole = df_play_whole.join(down_p1, 'uid', 'left')\n",
    "df_play_whole = df_play_whole.join(down_p2, 'uid', 'left')\n",
    "df_play_whole = df_play_whole.join(down_p3, 'uid', 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Since we use outer and left joins for the datasets, Nulls means no record for this period, so fill Nulls to 0\n",
    "df_play_whole = df_play_whole.na.fill(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- uid: string (nullable = true)\n",
      " |-- p1_play_freq: integer (nullable = true)\n",
      " |-- p1_song_num: integer (nullable = true)\n",
      " |-- p1_play_time: double (nullable = false)\n",
      " |-- p1_play_perc: double (nullable = false)\n",
      " |-- p1_singer_num: integer (nullable = true)\n",
      " |-- p2_play_freq: integer (nullable = true)\n",
      " |-- p2_song_num: integer (nullable = true)\n",
      " |-- p2_play_time: double (nullable = false)\n",
      " |-- p2_play_perc: double (nullable = false)\n",
      " |-- p2_singer_num: integer (nullable = true)\n",
      " |-- p3_play_freq: integer (nullable = true)\n",
      " |-- p3_song_num: integer (nullable = true)\n",
      " |-- p3_play_time: double (nullable = false)\n",
      " |-- p3_play_perc: double (nullable = false)\n",
      " |-- p3_singer_num: integer (nullable = true)\n",
      " |-- recency: integer (nullable = true)\n",
      " |-- p1_down_freq: long (nullable = true)\n",
      " |-- p2_down_freq: long (nullable = true)\n",
      " |-- p3_down_freq: long (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_play_whole.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare target variable: Churn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We define that if the user does not play in the last window (2017 May 1st to 2017 May 12th), so the user churns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_target = df_play_valid.filter((col('date') >= 20170501) & (col('date') <= 20170512))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create ply_freq in the last windows\n",
    "df_target_agg = df_target\\\n",
    "                .groupBy('uid')\\\n",
    "                .agg(count('uid').alias('target_play_freq'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- uid: string (nullable = true)\n",
      " |-- target_play_freq: long (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_target_agg.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----------------+\n",
      "|      uid|target_play_freq|\n",
      "+---------+----------------+\n",
      "|167581568|             267|\n",
      "|167833570|             461|\n",
      "|167935507|             236|\n",
      "|167575253|             205|\n",
      "|167635050|             738|\n",
      "|167677985|               7|\n",
      "|167776719|             395|\n",
      "|167571223|             319|\n",
      "|167672887|             145|\n",
      "|167883821|             528|\n",
      "|167774834|              74|\n",
      "|167825503|             103|\n",
      "|167651431|             721|\n",
      "|167907641|              48|\n",
      "|167638754|             140|\n",
      "|167794834|             511|\n",
      "|167772496|             266|\n",
      "|168013238|               5|\n",
      "|167779083|              13|\n",
      "|167756692|              18|\n",
      "+---------+----------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#df_target_agg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get target variable churn (churn means play_frec = 0)\n",
    "# 1 means churn, 0 means not churn\n",
    "df_target_agg = df_target_agg.withColumn(\"churn\", when(col('target_play_freq')>0, 0).otherwise(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+----------------+-----+\n",
      "|      uid|target_play_freq|churn|\n",
      "+---------+----------------+-----+\n",
      "|168028443|             134|    0|\n",
      "|167759265|              57|    0|\n",
      "|167810312|             467|    0|\n",
      "|167963866|              48|    0|\n",
      "|168034652|             230|    0|\n",
      "|167935507|             236|    0|\n",
      "|167635050|             738|    0|\n",
      "|167795994|             191|    0|\n",
      "|167640944|              35|    0|\n",
      "|167699178|              77|    0|\n",
      "|167708318|             222|    0|\n",
      "|167781195|             217|    0|\n",
      "|167779083|              13|    0|\n",
      "|167963608|              48|    0|\n",
      "|167595851|              26|    0|\n",
      "|167571223|             319|    0|\n",
      "|167969162|             335|    0|\n",
      "|167637640|              19|    0|\n",
      "|167877570|              74|    0|\n",
      "|167936958|              31|    0|\n",
      "+---------+----------------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#df_target_agg.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_label = df_target_agg.drop('target_play_freq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- uid: string (nullable = true)\n",
      " |-- churn: integer (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_label.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine input features and target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add churn label to the df_play_whole dataset\n",
    "df_play_whole = df_play_whole.join(df_label, 'uid', 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# deal with Null: no records in df_target, which means the users churn, so fill all Nulls to be 1\n",
    "df_play_whole = df_play_whole.na.fill(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- uid: string (nullable = true)\n",
      " |-- p1_play_freq: integer (nullable = true)\n",
      " |-- p1_song_num: integer (nullable = true)\n",
      " |-- p1_play_time: double (nullable = false)\n",
      " |-- p1_play_perc: double (nullable = false)\n",
      " |-- p1_singer_num: integer (nullable = true)\n",
      " |-- p2_play_freq: integer (nullable = true)\n",
      " |-- p2_song_num: integer (nullable = true)\n",
      " |-- p2_play_time: double (nullable = false)\n",
      " |-- p2_play_perc: double (nullable = false)\n",
      " |-- p2_singer_num: integer (nullable = true)\n",
      " |-- p3_play_freq: integer (nullable = true)\n",
      " |-- p3_song_num: integer (nullable = true)\n",
      " |-- p3_play_time: double (nullable = false)\n",
      " |-- p3_play_perc: double (nullable = false)\n",
      " |-- p3_singer_num: integer (nullable = true)\n",
      " |-- recency: integer (nullable = true)\n",
      " |-- p1_down_freq: long (nullable = true)\n",
      " |-- p2_down_freq: long (nullable = true)\n",
      " |-- p3_down_freq: long (nullable = true)\n",
      " |-- churn: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_play_whole.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# transfer to pandas\n",
    "dataset = df_play_whole.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>p1_play_freq</th>\n",
       "      <th>p1_song_num</th>\n",
       "      <th>p1_play_time</th>\n",
       "      <th>p1_play_perc</th>\n",
       "      <th>p1_singer_num</th>\n",
       "      <th>p2_play_freq</th>\n",
       "      <th>p2_song_num</th>\n",
       "      <th>p2_play_time</th>\n",
       "      <th>p2_play_perc</th>\n",
       "      <th>...</th>\n",
       "      <th>p3_play_freq</th>\n",
       "      <th>p3_song_num</th>\n",
       "      <th>p3_play_time</th>\n",
       "      <th>p3_play_perc</th>\n",
       "      <th>p3_singer_num</th>\n",
       "      <th>recency</th>\n",
       "      <th>p1_down_freq</th>\n",
       "      <th>p2_down_freq</th>\n",
       "      <th>p3_down_freq</th>\n",
       "      <th>churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>104777734</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>37.0</td>\n",
       "      <td>0.037255</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>170</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110367079</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>160.0</td>\n",
       "      <td>0.563380</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11596711</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>1758.0</td>\n",
       "      <td>0.490999</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>3829.0</td>\n",
       "      <td>0.307587</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>5889.0</td>\n",
       "      <td>0.565025</td>\n",
       "      <td>7</td>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>117677098</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>20136.0</td>\n",
       "      <td>0.707115</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>33</td>\n",
       "      <td>10811.0</td>\n",
       "      <td>0.426957</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>8741.0</td>\n",
       "      <td>0.672038</td>\n",
       "      <td>23</td>\n",
       "      <td>71</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>118301183</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>611.0</td>\n",
       "      <td>0.225863</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>120180992</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>21067.0</td>\n",
       "      <td>0.657256</td>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>11546.0</td>\n",
       "      <td>0.686217</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>85</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>126941437</td>\n",
       "      <td>1</td>\n",
       "      <td>150</td>\n",
       "      <td>41488.0</td>\n",
       "      <td>0.549926</td>\n",
       "      <td>103</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1701.0</td>\n",
       "      <td>0.814799</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>2326.0</td>\n",
       "      <td>0.514127</td>\n",
       "      <td>13</td>\n",
       "      <td>76</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>128106175</td>\n",
       "      <td>1</td>\n",
       "      <td>59</td>\n",
       "      <td>43154.0</td>\n",
       "      <td>0.904663</td>\n",
       "      <td>44</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>170</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>128119706</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>94</td>\n",
       "      <td>29094.0</td>\n",
       "      <td>0.494247</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>49</td>\n",
       "      <td>6235.0</td>\n",
       "      <td>0.368115</td>\n",
       "      <td>39</td>\n",
       "      <td>73</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>129766230</td>\n",
       "      <td>1</td>\n",
       "      <td>125</td>\n",
       "      <td>74635.0</td>\n",
       "      <td>0.626447</td>\n",
       "      <td>95</td>\n",
       "      <td>1</td>\n",
       "      <td>113</td>\n",
       "      <td>94613.0</td>\n",
       "      <td>0.921745</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>200</td>\n",
       "      <td>66754.0</td>\n",
       "      <td>0.577038</td>\n",
       "      <td>153</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         uid  p1_play_freq  p1_song_num  p1_play_time  p1_play_perc  \\\n",
       "0  104777734             1            3          37.0      0.037255   \n",
       "1  110367079             1            1         160.0      0.563380   \n",
       "2   11596711             1           10        1758.0      0.490999   \n",
       "3  117677098             1           35       20136.0      0.707115   \n",
       "4  118301183             1            9         611.0      0.225863   \n",
       "5  120180992             1           25       21067.0      0.657256   \n",
       "6  126941437             1          150       41488.0      0.549926   \n",
       "7  128106175             1           59       43154.0      0.904663   \n",
       "8  128119706             0            0           0.0      0.000000   \n",
       "9  129766230             1          125       74635.0      0.626447   \n",
       "\n",
       "   p1_singer_num  p2_play_freq  p2_song_num  p2_play_time  p2_play_perc  \\\n",
       "0              3             0            0           0.0      0.000000   \n",
       "1              1             0            0           0.0      0.000000   \n",
       "2             10             1           52        3829.0      0.307587   \n",
       "3             30             1           33       10811.0      0.426957   \n",
       "4              9             0            0           0.0      0.000000   \n",
       "5             14             1           15       11546.0      0.686217   \n",
       "6            103             1            2        1701.0      0.814799   \n",
       "7             44             0            0           0.0      0.000000   \n",
       "8              0             1           94       29094.0      0.494247   \n",
       "9             95             1          113       94613.0      0.921745   \n",
       "\n",
       "   ...    p3_play_freq  p3_song_num  p3_play_time  p3_play_perc  \\\n",
       "0  ...               0            0           0.0      0.000000   \n",
       "1  ...               0            0           0.0      0.000000   \n",
       "2  ...               1           12        5889.0      0.565025   \n",
       "3  ...               1           23        8741.0      0.672038   \n",
       "4  ...               0            0           0.0      0.000000   \n",
       "5  ...               0            0           0.0      0.000000   \n",
       "6  ...               1           15        2326.0      0.514127   \n",
       "7  ...               0            0           0.0      0.000000   \n",
       "8  ...               1           49        6235.0      0.368115   \n",
       "9  ...               1          200       66754.0      0.577038   \n",
       "\n",
       "   p3_singer_num  recency  p1_down_freq  p2_down_freq  p3_down_freq  churn  \n",
       "0              0      170             0             0             0      1  \n",
       "1              0      172             0             0             0      0  \n",
       "2              7       73             0             0             0      0  \n",
       "3             23       71             1             1             1      0  \n",
       "4              0      100             0             0             0      1  \n",
       "5              0       85             0             0             0      1  \n",
       "6             13       76             0             0             0      1  \n",
       "7              0      170             0             0             0      1  \n",
       "8             39       73             0             0             0      0  \n",
       "9            153       71             0             0             0      0  \n",
       "\n",
       "[10 rows x 21 columns]"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the prepared dataset\n",
    "dataset.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(570101, 21)"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    373389\n",
       "0    196712\n",
       "Name: churn, dtype: int64"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['churn'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.65495236808916313"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# churn rate\n",
    "churn_rate = dataset['churn'].value_counts()[1]/(dataset['churn'].value_counts()[0]+dataset['churn'].value_counts()[1])\n",
    "churn_rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There, there are about 65% of the users churn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# write into csv\n",
    "# dataset.to_csv('pre_dataset.csv') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# read the data\n",
    "#dataset = pd.read_csv('pre_dataset.csv') # 1 means churn, 0 means not churn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0       570101\n",
       "uid              570101\n",
       "p1_play_freq     570101\n",
       "p1_song_num      570101\n",
       "p1_play_time     570101\n",
       "p1_play_perc     570101\n",
       "p1_singer_num    570101\n",
       "p2_play_freq     570101\n",
       "p2_song_num      570101\n",
       "p2_play_time     570101\n",
       "p2_play_perc     570101\n",
       "p2_singer_num    570101\n",
       "p3_play_freq     570101\n",
       "p3_song_num      570101\n",
       "p3_play_time     570101\n",
       "p3_play_perc     570101\n",
       "p3_singer_num    570101\n",
       "recency          570101\n",
       "p1_down_freq     570101\n",
       "p2_down_freq     570101\n",
       "p3_down_freq     570101\n",
       "churn            570101\n",
       "dtype: int64"
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define features and target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "selected_features =  [u'p1_play_freq', u'p1_song_num', u'p1_play_time', u'p1_play_perc',u'p1_down_freq',\n",
    "            u'p2_play_freq', u'p2_song_num', u'p2_play_time', u'p2_play_perc',u'p2_down_freq',\n",
    "            u'p3_play_freq', u'p3_song_num', u'p3_play_time', u'p3_play_perc',u'p3_down_freq',u'recency']\n",
    "target = u'churn'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = dataset[selected_features].values  #get values of these features\n",
    "y = dataset['churn'].values            #get target value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalize dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [],
   "source": [
    "# since features are in different scales, it is important to scale them\n",
    "X_scaled = preprocessing.scale(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  3.02599037e-01,  -1.65091417e-01,   1.47073233e-03, ...,\n",
       "          3.99437179e-03,  -6.93220692e-01,   1.60785894e+00],\n",
       "       [  3.02599037e-01,  -1.79057532e-01,   1.47163483e-03, ...,\n",
       "          3.99437179e-03,  -6.93220692e-01,   1.65800686e+00],\n",
       "       [  3.02599037e-01,  -1.16210014e-01,   1.48336000e-03, ...,\n",
       "          8.27531787e-03,  -6.93220692e-01,  -8.24315629e-01],\n",
       "       ..., \n",
       "       [  3.02599037e-01,  -1.79057532e-01,   1.47739469e-03, ...,\n",
       "          1.95891284e-02,  -6.93220692e-01,  -7.74167700e-01],\n",
       "       [  3.02599037e-01,   5.83664270e-02,   1.47238325e-03, ...,\n",
       "          1.10106896e-02,  -6.93220692e-01,  -7.74167700e-01],\n",
       "       [  3.02599037e-01,  -1.79057532e-01,   1.47200904e-03, ...,\n",
       "          9.02084399e-03,   1.44254205e+00,  -8.24315629e-01]])"
      ]
     },
     "execution_count": 379,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_scaled"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split train and test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((456080, 16), (114021, 16), (456080,), (114021,))"
      ]
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 1, ..., 1, 0, 1]), array([0, 0, 0, ..., 1, 0, 0]))"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train,y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initialize model by providing parameters\n",
    "lr = LogisticRegression(C=100000, fit_intercept=True)     #set C be very big to turn off regulization\n",
    "\n",
    "# Fit a model by providing X and y from training set\n",
    "lr.fit(X_train, y_train)\n",
    "\n",
    "# Make prediction on the training data\n",
    "y_train_log = lr.predict(X_train)\n",
    "\n",
    "# Make predictions on test data\n",
    "y_test_log = lr.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculate metric scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_results(y_true, y_pred):\n",
    "    print(\"Accuracy of the model is: {}\".format(accuracy_score(y_true, y_pred)))\n",
    "    print(\"Precision of the model is: {}\".format(precision_score(y_true, y_pred)))\n",
    "    print(\"Recall of the model is: {}\".format(recall_score(y_true, y_pred)))\n",
    "    print(\"f1-score of the model is: {}\".format(f1_score(y_true, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set scores:\n",
      "Accuracy of the Logistic Regression is: 0.8287515348184529\n",
      "Precision of the Logistic Regression is: 0.8695601204452051\n",
      "Recall of the Logistic Regression is: 0.8689343784934432\n",
      "f1-score of the Logistic Regression is: 0.8692471368566612\n"
     ]
    }
   ],
   "source": [
    "print(\"Training set scores:\")\n",
    "print_results(y_train, y_train_log)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set scores:\n",
      "Accuracy of the Logistic Regression is: 0.8281193815174398\n",
      "Precision of the Logistic Regression is: 0.8698621848739496\n",
      "Recall of the Logistic Regression is: 0.8670642632178517\n",
      "f1-score of the Logistic Regression is: 0.8684609705349352\n"
     ]
    }
   ],
   "source": [
    "print(\"Test set scores:\")\n",
    "print_results(y_test, y_test_log)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC for train set is: 0.8106829724972014\n"
     ]
    }
   ],
   "source": [
    "print(\"AUC for train set is: {}\".format(roc_auc_score(y_train, y_train_log)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC for test set is: 0.8107208845907052\n"
     ]
    }
   ],
   "source": [
    "print(\"AUC for test set is: {}\".format(roc_auc_score(y_test, y_test_log)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions of the probability to be \"positive\"/\"churn\"/\"1\", as compared to \"negative\"/\"not churn\"/\"0\"\n",
    "y_test_log_proba = lr.predict_proba(X_test)[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_test, y_test_log_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xu8VXWd//HXGxBEQJCbIoiAoEap\niKSWZRc1UcdLal4eatlopmVOOjXZ1DRmNr/fdJ0uVtqUplN5S4oaStFMzJSLpqAodATlJnKQq4gg\n8Jk/vuscDqdz2QfO2mufs9/Px2M/1l5rf/fen8Vlffb3sr5fRQRmZmYAXYoOwMzMKoeTgpmZ1XNS\nMDOzek4KZmZWz0nBzMzqOSmYmVk9JwUzM6vnpGDWAkkvStoo6TVJyyXdKql3g9ffKemPktZLWivp\nt5LGNvqMPSX9l6RF2ee8kO0PLP8ZmbXMScGsdadGRG9gHHA48HkASe8A7gd+A+wLjASeBh6VNCor\n0x14EHgrMBHYE3gH8CpwZHlPw6x18h3NZs2T9CJwaUQ8kO1/DXhrRJwi6RFgTkR8otF7fg/URsSH\nJV0KfBU4ICJeK3P4Zm3mmoJZiSQNA04CaiTtAbwTuLuJoncBJ2TPjwf+4IRgHYWTglnrfi1pPbAY\nWAH8O9Cf9P/n5SbKvwzU9RcMaKaMWUVyUjBr3RkR0Qd4L3Aw6YK/GtgGDGmi/BBgZfb81WbKmFUk\nJwWzEkXEw8CtwDciYgPwGPChJoqeQ+pcBngAOFFSr7IEabaLnBTM2ua/gBMkHQZcC3xE0lWS+kja\nS9INpNFFX87K305qdvqVpIMldZE0QNK/Sjq5mFMwa56TglkbREQtcBvwpYj4M3AicCap3+Al0pDV\nd0XE37Lym0idzc8DU4F1wAxSE9T0sp+AWSs8JNXMzOq5pmBmZvWcFMzMrJ6TgpmZ1XNSMDOzet2K\nDqCtBg4cGCNGjCg6DDOzDuWJJ55YGRGDWivX4ZLCiBEjmDVrVtFhmJl1KJJeKqWcm4/MzKyek4KZ\nmdVzUjAzs3pOCmZmVs9JwczM6uWWFCT9VNIKSc8087okfVdSjaTZksbnFYuZmZUmz5rCraSFyptz\nEjAme1wG/DDHWMzMrAS53acQEdMkjWihyOnAbZGmaX1cUj9JQyLCSxeaWeewbRu88cb2x8aNsGUL\nbN6cHmvXQpcu2/c3b4ZNm5p+bN4Mp54Kb397riEXefPaUNLiI3WWZMf+LilIuoxUm2D48OFlCc7M\nqsDmzbB+Pbz2GmzYsH27bh2sWpXK1B3buBEWLID+/Xe8yK9fn8qvX5/2a2qgX7/0fNOm9o136NBO\nnRRKFhE3AzcDTJgwwQtAmFWjN99MF966i/eaNbB6ddrWXZTrXlu/fvtFes0aqK2Fnj23J4Da2pQQ\n2qpbt/RLf8gQ2H136NED+vRJj0GD0ne8+90prjFjtpcfOjSVf+012Hdf2G036N49PTZvhgED0mfV\nHevRY8fH7rtD164gtf+fa+NTzP0bmrcU2K/B/rDsmJl1Ftu2pQv22rXbL9avv54u3mvXptfWrEnH\n165Nj9WrU9m68q+9lo5v3Fjad+6xB+y5J/TqlS7SPXumi3CPHjByZLqA9+qVLtb9+8PgwdC7dzrW\nq1d6vsce6T39+28/3qNHaurp5IpMCpOBKyXdARwFrHV/glmF2roVVqxIv7BXroRXXoF589JFf968\n7b/CV61KF/lVq7Zf5EvRrVu6kO+1V2p66dMn/brec890ke7TB/r23X6x79Urlasr37dvKrPHHlVx\n4c5TbklB0i+B9wIDJS0B/h3YDSAifgRMAU4GaoDXgY/mFYuZNbJxY7qwr1iRLuCrVsHy5enYunXw\n6quwdGlKAqtWpV/v27Y1/Vm9eqVf/m99a7pADx0Khx6aLtSNL9p1v8h79tx+ke/XL/0KL0PTiLUu\nz9FH57fyegCfzOv7zarOtm3por5oESxenC7ytbVpu2xZem316rT/2mtNf0aPHuli3a9favsePz41\noQwYkNrR99477e+9d3r07Zt+5Vun4b9Ns45k1Sp44gmYPx9eeCGNdHnllfSrfvny1MzT2ODB2y/o\no0en/bqL+uDB6SLfvz/ss0+6yPsXe1VzUjCrJBGp+eall9Lwx3nz0oW/piYlgmXLtpft2TNd5PfZ\nB97yltRss+++MHw47L9/uugPGOBf8tYm/tdiVoQ33oBnn02PmhqYOzclgRdeSEmhoUGDYNQoOP74\n1G5/xBEpCQwZ4l/11u6cFMzysmXL9l/78+en58uWpefz56fXIY2WOeCANK79ne+EESNgv/1SIhgz\nJrXvm5WJk4LZrti6FV5+ObXpz58Pzz8Pzz2Xfv3Pm7fjDVJ1I3NGj4YzzoBx4+CQQ9LFv3v34s7B\nrAEnBbPmbNy4fdRO3WPRotTe/+KLsGRJer3uFz+ku05Hj06/8CdOhLFj4aCD0qN//8JOxaxUTgpW\nndasSRf45cth4cJ0oV++PN2YtXRpuujXzX3TUNeuMGxYujP2Pe9Jv/xHjkwdvKNGpYTgX/3WgTkp\nWOezcWO64NfUpI7b5cvTuP1Fi7YP3dywYcf3dO2aRusMGpQu9EcdlS7+++6bOnQHDdo+tLNr12LO\ny6wMnBSsY9q0KbXZP/98uvgvXJgec+emNv6GunVLF/f9908zTO69d7rgDx+eLvL7758SgS/2Zk4K\n1gEsXQqzZ6fhm7Nnw5NPpoTQsC1/8OA0aueEE1J7/v77pxE9o0fDwIGeD8esRE4KVlneeAOmTYMZ\nM9Kv/kceSR26dYYMSVMvnH46vO1t2ztxe/UqLmazTsRJwYq1bBnMmpUejz8Of/7z9imShw+Hd7wD\njjkGDj88jeQZOLDYeM06OScFK6+IdPH/zW/gvvvgqafS8S5d0i//Sy+FD3wAjj02zaBpZmXlpGD5\nW7ECpk5NjwcfTM1BXbumTt9vfCNtjzjCTUBmFcBJwdrXmjXpzt4ZM9JsntOmpekdIDX9HHssfPnL\ncPbZrgmYVSAnBds1a9fC734Hv/89zJyZEkKdAQPSDV6XXpomcxs/3sM+zSqck4K13apVcM896TFt\nWrpnYNCg1CH84Q+nmTzHj0+TunkWT7MOxUnBSrNyJdx9N0yZkmoFW7emoaCXXw7nnANHH+17Acw6\nAScFa9ns2fCd78Avf5mGiu6/P1x9NZx7buocdk3ArFNxUrAdRaTJ4f74R/jv/4bHHkvr9n7kI/CJ\nT6QF2Z0IzDotJwVLE8Xdf3+6e3jq1O1zB40cCV//euonGDy42BjNrCycFKpVBNx7L3zve/Dww+nY\nwIFw3HHwrneloaOHHOJagVmVcVKoNps2pf6BG25I00qPGgXXXw9nnZXW/XUSMKtqTgrVYuvWVCu4\n/npYvTotBfk//wPnn+9RQ2ZWz0mhs9u2LfUXXHcdTJ+ebiL77GfT1snAzBpxUujM7r8fPvWpdJfx\n0KFw221w4YVuIjKzZvmnYme0bRt86Utw4olp/5Zb0vxDF13khGBmLXJNobPZsgUuuywlggsugB//\nGHr2LDoqM+sgXFPoTJYvh4kTU0K49lq4/XYnBDNrE9cUOotp0+DMM2HdOrj5ZvjYx4qOyMw6INcU\nOrq1a+GKK9IU1X36pHUMnBDMbCflmhQkTZQ0T1KNpGubeH24pIck/VXSbEkn5xlPpxIBkyalJSxv\nugkuuSQtbTluXNGRmVkHlltSkNQVuBE4CRgLnC9pbKNiXwTuiojDgfOAH+QVT6eyYUNKAmeemVYv\ne/jhNHld375FR2ZmHVyeNYUjgZqIWBARm4E7gNMblQmgbk3GvsCyHOPpHGpr08L2t9wCn/kMPP00\nvPvdRUdlZp1Enh3NQ4HFDfaXAEc1KnMdcL+kTwG9gOOb+iBJlwGXAQwfPrzdA+0wFiyAU06BhQvT\n/EXnnVd0RGbWyRTd0Xw+cGtEDANOBm6X9HcxRcTNETEhIiYMGjSo7EFWhMWL4YQT0rDTP/zBCcHM\ncpFnTWEpsF+D/WHZsYYuASYCRMRjknYHBgIrcoyr49m6FT70odR0NHUqHNW4wmVm1j7yrCnMBMZI\nGimpO6kjeXKjMouA4wAkvQXYHajNMaaO6dOfTpPZ3XSTE4KZ5Sq3pBARW4ArgfuA50ijjJ6VdL2k\n07Ji/wx8TNLTwC+BiyMi8oqpw4lI01R8//vwyU+6ycjMcqeOdg2eMGFCzJo1q+gw8rd2bRp2+qtf\nwTHHpDWTu3cvOioz66AkPRERE1orV3RHszXlmWdg/Hj49a/hP/4DHnrICcHMysJzH1Wahx5Kax5s\n2gR/+lNaL9nMrExcU6gUL7+clsZ8//uhRw944AEnBDMrOyeFSvDww/D2t8O998IXvgBz5ngOIzMr\nhJNCkbZtgy9+Ed773tRnMGMG3HAD9OpVdGRmVqXcp1CUdevg4ovTTKcXXpiGnXpCOzMrmJNCEeru\nUJ46Fb72tTSxnddONrMK4KRQhG9/G+6/H374Q7j88qKjMTOr5z6FcvvDH9L6yaeeCh//eNHRmJnt\noKSkIKm7pNF5B9PprVgBF10EBx0Et9/uJiMzqzitJgVJpwBzgKnZ/jhJk/IOrNPZsgXOPhteew1u\nu82dymZWkUqpKVxPWhxnDUBEPAW41tBW3/wmPPII/OAHcMQRRUdjZtakUpLCmxGxptGxjjWLXtGW\nLoXrr4fTTkvDUM3MKlQpo4+ek3QO0EXSSOAq4PF8w+pkbrgB3nwzDT91P4KZVbBSagpXAkcA24B7\ngU3AP+UZVKeyZQvccw+cdVbqYDYzq2Cl1BROjIjPAZ+rOyDpTFKCsNbMmAErV6YhqGZmFa6UmsIX\nmzj2hfYOpNP67W+hSxf4wAeKjsTMrFXN1hQknQhMBIZK+laDl/YkNSVZayZNgm98A844AwYOLDoa\nM7NWtdR8tAJ4BngDeLbB8fXAtXkG1SncfXea6G7cOPjJT4qOxsysJM0mhYj4K/BXST+PiDfKGFPH\nN2MGfPSjcNhhcN990K9f0RGZmZWklI7moZK+CowFdq87GBEH5hZVR/bkk3DccTB4cGo+2muvoiMy\nMytZKR3NtwK3AAJOAu4C7swxpo5rzZo0JXavXvDggzB0aNERmZm1SSlJYY+IuA8gIl6IiC+SkoM1\n9pnPwMKF6b6EkSOLjsbMrM1KaT7aJKkL8IKky4GlQJ98w+qA/vjH1KF89dXwrncVHY2Z2U4pJSlc\nDfQiTW/xVaAv8I95BtXhvPwyXHABjBoFX/lK0dGYme20VpNCREzPnq4HLgKQ5Mbyhj76UVi7Nq2m\n1qtX0dGYme20FvsUJL1d0hmSBmb7b5V0GzC9pfdVlTvuSMNOr7sODjmk6GjMzHZJs0lB0v8Dfg5c\nAPxB0nXAQ8DTgIejAmzaBP/yL+l+hGuuKToaM7Nd1lLz0enAYRGxUVJ/YDFwSEQsKE9oHcC998Li\nxWnhnG6ldM+YmVW2lpqP3oiIjQARsQqY74TQyK23wrBhcJJH6JpZ59BSUhgl6d7sMQkY2WC/pGmz\nJU2UNE9SjaQm50uSdI6kuZKelfSLnTmJQixbBg88kOY36tq16GjMzNpFS20eZzXa/35bPlhSV+BG\n4ARgCTBT0uSImNugzBjg88AxEbFa0uC2fEehbr0Vtm2Df/ToXDPrPFqaEO/BXfzsI4GauiYnSXeQ\n+inmNijzMeDGiFidfeeKXfzO8ti6FW65BY4+GsaMKToaM7N2U8o0FztrKKlzus6S7FhDBwIHSnpU\n0uOSJjb1QZIukzRL0qza2tqcwm2Dhx+Gmhq48sqiIzEza1d5JoVSdAPGAO8Fzgd+LOnv5pmOiJsj\nYkJETBg0aFCZQ2zCXXfB7rvDmWcWHYmZWbsqOSlI6tHGz14K7Ndgf1h2rKElwOSIeDMiFgLzSUmi\nsv3+92l5zZ49i47EzKxdtZoUJB0paQ7wt2z/MEnfK+GzZwJjJI2U1B04D5jcqMyvSbUEsrumDwQq\ne9jrvHmwaBGceGLRkZiZtbtSagrfBf4BeBUgIp4G3tfamyJiC3AlcB/wHHBXRDwr6XpJp2XF7gNe\nlTSXdLf0ZyPi1bafRhk98EDannBCsXGYmeWglNtwu0TES5IaHttayodHxBRgSqNjX2rwPIBrskfl\nW7cOvvMdOPBAjzoys06plKSwWNKRQGT3HnyK1PZffa66Cl54AaZOLToSM7NclNJ8dAXpl/xw4BXg\n6OxYdXnsMfjZz9Lqau9/f9HRmJnlopSawpaIOC/3SCrd5z8PQ4bAv/1b0ZGYmeWmlJrCTElTJH1E\nUnUuw/nnP6cb1q6+Gnr3LjoaM7PctJoUIuIA4AbgCGCOpF9Lqq6aw403Qt++cEX1tZqZWXUp6ea1\niPhLRFwFjAfWkRbfqQ6vvw6TJsH557uWYGadXik3r/WWdIGk3wIzgFrgnblHVimmT08rrJ1yStGR\nmJnlrpSO5meA3wJfi4hHco6n8jzwAEhw5JFFR2JmlrtSksKoiNiWeySVKALuvBOOPRYGd5ylHszM\ndlazSUHSNyPin4FfSYrGr0dE558i9Mkn081qn/tc0ZGYmZVFSzWFO7Ntm1Zc61QmTUpLbXqKbDOr\nEi2tvDYje/qWiNghMUi6EtjVldkq34wZMG4cDBhQdCRmZmVRypDUphYhvqS9A6k4mzenkUfjxxcd\niZlZ2bTUp3AuaQ2EkZLubfBSH2BN3oEV7i9/SbOiHn980ZGYmZVNS30KM0hrKAwDbmxwfD3w1zyD\nqgi/+13aet0EM6siLfUpLAQWAg+UL5wKsm5d2u61V7FxmJmVUUvNRw9HxHskrQYaDkkVaX2c/rlH\nV6SFC+GII4qOwsysrFpqPqpbcnNgOQKpKCtXpllRPQGemVWZZkcfNbiLeT+ga0RsBd4BfBzoVYbY\nijN1Krz5Jnzwg0VHYmZWVqUMSf01aSnOA4BbgDHAL3KNqmjzs9VGDzmk2DjMzMqslKSwLSLeBM4E\nvhcRVwND8w2rYHPmwAEH+KY1M6s6pSSFLZI+BFwEZOM02S2/kCrAk0/C4YcXHYWZWdmVekfz+0hT\nZy+QNBL4Zb5hFWjLljTy6MADi47EzKzsWp06OyKekXQVMFrSwUBNRHw1/9AKMnt22o4ZU2wcZmYF\naDUpSHo3cDuwlHSPwj6SLoqIR/MOrhDTp6ftMccUG4eZWQFKWWTn28DJETEXQNJbSEliQp6BFWb2\nbOjXD0aPLjoSM7OyK6VPoXtdQgCIiOeA7vmFVLDZs2Hs2LQEp5lZlSklKTwp6UeS3pU9fkhnnhDv\n2WfTGgpmZlWolOajy4GrgH/J9h8BvpdbREVauhTWroWDDio6EjOzQrSYFCQdAhwATIqIr5UnpALN\nm5e2b3tbsXGYmRWk2eYjSf9KmuLiAmCqpKZWYOtcli1L22HDio3DzKwgLfUpXAAcGhEfAt4OtHnK\nUEkTJc2TVCPp2hbKnSUpJBU7omn16rTt16/QMMzMitJSUtgUERsAIqK2lbJ/R1JX0optJwFjgfMl\njW2iXB/gn4Dpbfn8XNTWplFHnvPIzKpUS30KoxqszSzggIZrNUfEma189pGku58XAEi6AzgdmNuo\n3FeA/wQ+25bAc/HKKzBwIHTtWnQkZmaFaCkpnNVo//tt/OyhwOIG+0uAoxoWkDQe2C8i/ldSs0lB\n0mXAZQDDhw9vYxht8OKLMLRzTwBrZtaSltZofjDPL5bUBfgWcHFrZSPiZuBmgAkTJkQrxXfO66+n\nKS7OapwLzcyqR5v6CdpoKWnVtjrDsmN1+gBvA/4k6UXgaGByYZ3NjzyS7lE455xCvt7MrBLkmRRm\nAmMkjZTUHTgPmFz3YkSsjYiBETEiIkYAjwOnRcSsHGNq3rRpqS/BE+GZWRUrOSlI6tGWD46ILcCV\nwH3Ac8BdEfGspOslnda2MMvgiSfg4IOhd++iIzEzK0wpU2cfCfwE6AsMl3QYcGlEfKq190bEFGBK\no2Nfaqbse0sJOBcRKSmcemphIZiZVYJSagrfBf4BeBUgIp4mrcTWedTWwsqVngjPzKpeKUmhS0S8\n1OjY1jyCKczzz6et11AwsypXyiypi7MmpMjuUv4UMD/fsMpsbnY/3cEHFxuHmVnBSqkpXAFcAwwH\nXiENHW3zPEgV7dFHoVs3GDmy6EjMzArVak0hIlaQhpN2Xps2wW67ebU1M6t6pYw++jHwd3cRR8Rl\nuURUhDlz4IQTio7CzKxwpfQpPNDg+e7AB9lxTqOObe3atLjOuecWHYmZWeFKaT66s+G+pNuBP+cW\nUbk9/ni6T+HYY4uOxMyscDszzcVIYO/2DqQwTz6ZtoceWmwcZmYVoJQ+hdVs71PoAqwCml1FrcOZ\nPh0OOCCto2BmVuVaTAqSBBzG9tlNt0VEPlNXF+Xpp+Goo1ovZ2ZWBVpsPsoSwJSI2Jo9OldCqK1N\nC+t4egszM6C0PoWnJB2eeyRFmD07bScUs4SDmVmlabb5SFK3bPrrw4GZkl4ANpDWa46IGF+mGPPz\nzDNpO3ZssXGYmVWIlvoUZgDjgcpb+6C9PPMM9O8PQ4YUHYmZWUVoKSkIICJeKFMs5feXv6SZUT29\nhZkZ0HJSGCTpmuZejIhv5RBPea1fD/vsU3QUZmYVo6Wk0BXoTVZj6HTefBOWLIGLLy46EjOzitFS\nUng5Iq4vWyTltnhxmt5ixIiiIzEzqxgtDUntnDWEOgsXpu1++xUbh5lZBWkpKRxXtiiKsGxZ2u61\nV7FxmJlVkGaTQkSsKmcgZbdhQ9ruu2+xcZiZVZCdmSW1c1i5Mm0HDSo2DjOzClK9SeGpp9ISnLvt\nVnQkZmYVo3qTQs+e0K2UhefMzKpH9SaFV17xwjpmZo1Ub1JYtsx3M5uZNVKdSSECampg5MiiIzEz\nqyjVmRRWr4ZNm2D48KIjMTOrKNWbFAAGDCg2DjOzCpNrUpA0UdI8STWSrm3i9WskzZU0W9KDkvbP\nM556W7akrUcfmZntILekIKkrcCNwEjAWOF9S4yXO/gpMiIhDgXuAr+UVzw62bk1bJwUzsx3kWVM4\nEqiJiAURsRm4Azi9YYGIeCgiXs92HweG5RjPdnU1ha5dy/J1ZmYdRZ5JYSiwuMH+kuxYcy4Bft/U\nC5IukzRL0qza2tpdj6yupuCkYGa2g4roaJZ0ITAB+HpTr0fEzRExISImDGqPuYrcfGRm1qQ8r4pL\ngYaLFQzLju1A0vHAF4D3RMSmHOPZzs1HZmZNyrOmMBMYI2mkpO7AecDkhgUkHQ7cBJwWEStyjGVH\nbj4yM2tSbkkhIrYAVwL3Ac8Bd0XEs5Kul3RaVuzrpHWg75b0lKTJzXxc+1q/Pm179y7L15mZdRS5\nNqpHxBRgSqNjX2rw/Pg8v79ZdauuDRlSyNebmVWqiuhoLrvly9PWScHMbAfVmRRefhn69oXddy86\nEjOzilKdSWHmTNh776KjMDOrONWZFLZsgV69io7CzKziVGdSePJJGDeu6CjMzCpO9SWFCJBg27ai\nIzEzqzjVlxRWrkwJYfz4oiMxM6s41ZcUXnopbffbr+VyZmZVqPqSwgsvpO0BBxQbh5lZBaq+pLAi\nm2LJQ1LNzP5O9SWFJUtgt91g8OCiIzEzqzjVlxQWLUr9CVLRkZiZVZzqSwovveROZjOzZlRfUli0\nCEaMKDoKM7OKVF1JISLNkLrvvkVHYmZWkaorKbz+elp1rV+/oiMxM6tI1ZUU1q1L2z59io3DzKxC\nVVdSqFuGc889i43DzKxCVVdS2LgxbXv2LDYOM7MKVV1J4fXX03aPPYqNw8ysQlVXUqirKTgpmJk1\nqbqSwubNadu9e7FxmJlVKCcFMzOrV11JYdOmtHVSMDNrUnUlhdWr07Z372LjMDOrUNWVFOpGH/Xt\nW2wcZmYVqrqSwoYNaev7FMzMmlRdSWHhQujfH3r0KDoSM7OKVF1JYc4cOPRQL7BjZtaM6kkKETB/\nPhx8cNGRmJlVrOpJCq++CmvWwIEHFh2JmVnFyjUpSJooaZ6kGknXNvF6D0l3Zq9PlzQit2CWL0/b\nffbJ7SvMzDq63JKCpK7AjcBJwFjgfEljGxW7BFgdEaOBbwP/mVc8LF6ctl6f2cysWXnWFI4EaiJi\nQURsBu4ATm9U5nTgZ9nze4DjpJx6gRcsSNtRo3L5eDOzziDPpDAUWNxgf0l2rMkyEbEFWAsMaPxB\nki6TNEvSrNra2p2LZtgwOOMMNx+ZmbWgQ3Q0R8TNETEhIiYMGjRo5z7k9NNh0iTo0iFO2cysEHle\nIZcCDRvwh2XHmiwjqRvQF3g1x5jMzKwFeSaFmcAYSSMldQfOAyY3KjMZ+Ej2/GzgjxEROcZkZmYt\n6JbXB0fEFklXAvcBXYGfRsSzkq4HZkXEZOAnwO2SaoBVpMRhZmYFyS0pAETEFGBKo2NfavD8DeBD\necZgZmalc6+rmZnVc1IwM7N6TgpmZlbPScHMzOqpo40AlVQLvLSTbx8IrGzHcDoCn3N18DlXh105\n5/0jotW7fztcUtgVkmZFxISi4ygnn3N18DlXh3Kcs5uPzMysnpOCmZnVq7akcHPRARTA51wdfM7V\nIfdzrqo+BTMza1m11RTMzKwFTgpmZlavUyYFSRMlzZNUI+naJl7vIenO7PXpkkaUP8r2VcI5XyNp\nrqTZkh6UtH8Rcban1s65QbmzJIWkDj98sZRzlnRO9nf9rKRflDvG9lbCv+3hkh6S9Nfs3/fJRcTZ\nXiT9VNIKSc8087okfTf785gtaXy7BhARnepBmqb7BWAU0B14GhjbqMwngB9lz88D7iw67jKc8/uA\nPbLnV1TDOWfl+gDTgMeBCUXHXYa/5zHAX4G9sv3BRcddhnO+Gbgiez4WeLHouHfxnI8FxgPPNPP6\nycDvAQFHA9Pb8/s7Y03hSKAmIhZExGbgDuD0RmVOB36WPb8HOE6Syhhje2v1nCPioYh4Pdt9nLQS\nXkdWyt8zwFeA/wTeKGdwOSnlnD8G3BgRqwEiYkWZY2xvpZxzAHtmz/sCy8oYX7uLiGmk9WWaczpw\nWySPA/0kDWmv7++MSWEosLjB/pLsWJNlImILsBYYUJbo8lHKOTd0CemXRkfW6jln1er9IuJ/yxlY\njkr5ez4QOFDSo5IelzSxbNEoIojDAAAFgklEQVTlo5Rzvg64UNIS0votnypPaIVp6//3Nsl1kR2r\nPJIuBCYA7yk6ljxJ6gJ8C7i44FDKrRupCem9pNrgNEmHRMSaQqPK1/nArRHxTUnvIK3m+LaI2FZ0\nYB1RZ6wpLAX2a7A/LDvWZBlJ3UhVzlfLEl0+SjlnJB0PfAE4LSI2lSm2vLR2zn2AtwF/kvQiqe11\ncgfvbC7l73kJMDki3oyIhcB8UpLoqEo550uAuwAi4jFgd9LEcZ1VSf/fd1ZnTAozgTGSRkrqTupI\nntyozGTgI9nzs4E/RtaD00G1es6SDgduIiWEjt7ODK2cc0SsjYiBETEiIkaQ+lFOi4hZxYTbLkr5\nt/1rUi0BSQNJzUkLyhlkOyvlnBcBxwFIegspKdSWNcrymgx8OBuFdDSwNiJebq8P73TNRxGxRdKV\nwH2kkQs/jYhnJV0PzIqIycBPSFXMGlKHznnFRbzrSjznrwO9gbuzPvVFEXFaYUHvohLPuVMp8Zzv\nAz4gaS6wFfhsRHTYWnCJ5/zPwI8lXU3qdL64I//Ik/RLUmIfmPWT/DuwG0BE/IjUb3IyUAO8Dny0\nXb+/A//ZmZlZO+uMzUdmZraTnBTMzKyek4KZmdVzUjAzs3pOCmZmVs9JwSqOpK2SnmrwGNFC2RHN\nzSbZxu/8UzYT59PZFBEH7cRnXC7pw9nziyXt2+C1/5Y0tp3jnClpXAnv+bSkPXb1u606OClYJdoY\nEeMaPF4s0/deEBGHkSZL/Hpb3xwRP4qI27Ldi4F9G7x2aUTMbZcot8f5A0qL89OAk4KVxEnBOoSs\nRvCIpCezxzubKPNWSTOy2sVsSWOy4xc2OH6TpK6tfN00YHT23uOyefrnZPPc98iO/39tX5/iG9mx\n6yR9RtLZpPmlfp59Z8/sF/6ErDZRfyHPahTf38k4H6PBRGiSfihpltI6Cl/Ojl1FSk4PSXooO/YB\nSY9lf453S+rdyvdYFXFSsErUs0HT0aTs2ArghIgYD5wLfLeJ910OfCcixpEuykuyaQ/OBY7Jjm8F\nLmjl+08F5kjaHbgVODciDiHNAHCFpAHAB4G3RsShwA0N3xwR9wCzSL/ox0XExgYv/yp7b51zgTt2\nMs6JpGkt6nwhIiYAhwLvkXRoRHyXNJX0+yLifdnUF18Ejs/+LGcB17TyPVZFOt00F9YpbMwujA3t\nBnw/a0PfSprTp7HHgC9IGgbcGxF/k3QccAQwM5veoycpwTTl55I2Ai+Spl8+CFgYEfOz138GfBL4\nPml9hp9I+h3wu1JPLCJqJS3I5qz5G3Aw8Gj2uW2Jsztp2pKGf07nSLqM9P96CGnBmdmN3nt0dvzR\n7Hu6k/7czAAnBes4rgZeAQ4j1XD/btGciPiFpOnAKcAUSR8nrU71s4j4fAnfcUHDCfMk9W+qUDYf\nz5GkSdjOBq4E3t+Gc7kDOAd4HpgUEaF0hS45TuAJUn/C94AzJY0EPgO8PSJWS7qVNDFcYwKmRsT5\nbYjXqoibj6yj6Au8nM2RfxFpcrQdSBoFLMiaTH5DakZ5EDhb0uCsTH+Vvj71PGCEpNHZ/kXAw1kb\nfN+ImEJKVoc18d71pOm7mzKJtHrW+aQEQVvjzCZ8+zfgaEkHk1Ye2wCslbQ3cFIzsTwOHFN3TpJ6\nSWqq1mVVyknBOoofAB+R9DSpyWVDE2XOAZ6R9BRpLYXbshE/XwTulzQbmEpqWmlVRLxBmoHybklz\ngG3Aj0gX2N9ln/dnmm6TvxX4UV1Hc6PPXQ08B+wfETOyY22OM+ur+CZpJtSnSWszPw/8gtQkVedm\n4A+SHoqIWtLIqF9m3/MY6c/TDPAsqWZm1oBrCmZmVs9JwczM6jkpmJlZPScFMzOr56RgZmb1nBTM\nzKyek4KZmdX7P4W2Rcvm+A5dAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6119ebf320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot ROC\n",
    "plt.plot(fpr, tpr, color='red')\n",
    "\n",
    "plt.xlabel(\"False Positive Rate\")\n",
    "plt.ylabel(\"True Positive Rate\")\n",
    "plt.title(\"ROC\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoosting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb = XGBClassifier()\n",
    "\n",
    "# Choose some parameter combinations to try\n",
    "parameters = {'n_estimators': [100,200,300],\n",
    "              'learning_rate':[0.1, 0.3, 0.5],\n",
    "              'max_depth': [3, 10, 15]\n",
    "             }\n",
    "\n",
    "# Run the grid search\n",
    "grid_xgb = GridSearchCV(xgb, parameters,\n",
    "                        cv=5, #Determines the cross-validation splitting strategy /to specify the number of folds in a (Stratified)KFold\n",
    "                        n_jobs=-1, #Number of jobs to run in parallel\n",
    "                        verbose=1)\n",
    "grid_xgb = grid_xgb.fit(X_train, y_train)\n",
    "\n",
    "# Set the xgb to the best combination of parameters\n",
    "xgb = grid_xgb.best_estimator_\n",
    "\n",
    "# Fit the best algorithm to the data. \n",
    "xgb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make predictions\n",
    "y_train_xgb = xgb.predict(X_train)\n",
    "y_test_xgb = xgb.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set scores:\n",
      "Accuracy of the model is: 0.8612874934222067\n",
      "Precision of the model is: 0.875240589907333\n",
      "Recall of the model is: 0.9192935128224009\n",
      "f1-score of the model is: 0.8967263372000013\n"
     ]
    }
   ],
   "source": [
    "print(\"Training set scores:\")\n",
    "print_results(y_train, y_train_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing set scores:\n",
      "Accuracy of the model is: 0.8436516080371159\n",
      "Precision of the model is: 0.8630946291560102\n",
      "Recall of the model is: 0.904563425584668\n",
      "f1-score of the model is: 0.8833426038019828\n"
     ]
    }
   ],
   "source": [
    "print(\"Testing set scores:\")\n",
    "print_results(y_test, y_test_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC for train set is: 0.8352045863731854\n"
     ]
    }
   ],
   "source": [
    "print(\"AUC for train set is: {}\".format(roc_auc_score(y_train, y_train_xgb)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC for test set is: 0.8164394552680991\n"
     ]
    }
   ],
   "source": [
    "print(\"AUC for test set is: {}\".format(roc_auc_score(y_test, y_test_xgb)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xm8VXW9//HXmyMIiKICpjIIKKRQ\n4oBztzSzHLqaZg5XG72Zlpna7Zf3Zz8ts251m645ZWVmN+dbXjJyTEVRFJxAxAFBFAQ5IoMiIMPn\n98d3n8P2cIYN56y99vB+Ph7rsfZa+7v3/qyDrs/6fr9rfb+KCMzMzAC65R2AmZlVDicFMzNr5qRg\nZmbNnBTMzKyZk4KZmTVzUjAzs2ZOCmZm1sxJwawdkl6WtELS25IWSLpWUp+i9w+U9A9Jb0laKumv\nkka1+I6tJP1S0iuF73mpsN2//Edk1j4nBbOO/XNE9AH2APYE/h1A0gHAXcD/AjsCw4CngYmShhfK\n9ADuBUYDhwNbAQcAi4B9y3sYZh2Tn2g2a5ukl4F/jYh7Cts/AUZHxFGSHgSmRcRXW3zm70BjRHxO\n0r8CPwB2joi3yxy+2UZzTcGsRJIGAUcAMyX1Bg4Ebmml6M3AYYXXHwPucEKwauGkYNax2yS9BbwK\nLAQuArYl/f8zv5Xy84Gm/oJ+bZQxq0hOCmYd+1REbAkcDOxKOuEvBtYBO7RSfgfgjcLrRW2UMatI\nTgpmJYqIB4BrgZ9GxHLgEeAzrRQ9gdS5DHAP8AlJW5QlSLNOclIw2zi/BA6TNAY4H/i8pLMlbSlp\nG0mXkO4u+l6h/B9JzU7/I2lXSd0k9ZP0fyUdmc8hmLXNScFsI0REI3AdcGFEPAR8AjiO1G8wh3TL\n6oci4sVC+VWkzubngLuBZcBjpCaoR8t+AGYd8C2pZmbWzDUFMzNr5qRgZmbNnBTMzKyZk4KZmTXb\nLO8ANlb//v1j6NCheYdhZlZVHn/88TciYkBH5aouKQwdOpQpU6bkHYaZWVWRNKeUcm4+MjOzZk4K\nZmbWzEnBzMyaOSmYmVkzJwUzM2uWWVKQdI2khZKeaeN9SbpU0kxJUyXtlVUsZmZWmixrCteSJipv\nyxHAiMJyOnBlhrGYmVkJMntOISImSBraTpFjgOsiDdM6SdLWknaICE9daGYbJwLWrIHVq9OyZg2s\nWwdr1264NO1fsQIaGtJ209L0fstl7VpYuTJ9d48erZdpa1m4ELbddv12xIavO9q3bFla/8u/wD77\nZPqnzPPhtYGkyUeazC3s2yApSDqdVJtgyJAhZQnOzNoQAatWpZPqihXw9tvpZLlqVVpWrkz7m9Yr\nVsC7765fmsqtXr1+3+zZMGBA2t9Uprj8ypXw8svQv/+G39WUCOrB8OE1nRRKFhFXA1cDjB071hNA\nmJWi6eT99tuweHFaL1uW1q++mq6S33orba9YAcuXwzvvwJIl8Npr0L07zJoFffrAnDmpfNMVc2fn\nYZFg883TVXf37ukq+J13YKed0r7NN1+/bL11KjNwYEoEgwatL9Ojx/rvaLl065Zibmh47+um7bff\nhm22gc02S9vFS1OZlvtWrUp/j5bvtbZI619Diql4X9PrUvY1LWWQZ1KYBwwu2h5U2GdmTd59N52k\nFy9OJ/Rly2Dp0rQsWgRvvAHPPgu9e6cT/KJFqbli6dJUds2a0n6nR4/0HVtsAb16pZN0r16w887p\nxLTffimGXXZJ+3v3TkuvXilB9OqVTt49e6aTde/e6XXx0vIEbhUpz6QwDjhL0o3AfsBS9ydYTYtI\nV6cLF6aT+aJFaVm8OC0LFqQr9IULobER5s9PV8/t6d4d+vZNV/mjRqW26513TlfAW24JW22Vrmy3\n3jptN+3r2XN9mS22SFfLZmSYFCTdABwM9Jc0F7gI6A4QEVcB44EjgZnAO8AXs4rFLDMR6Yp83jyY\nOxdefz1dpc+bl07wr7+eTvJPPpmuuNeta/u7+vVLTSQDBqS243790gm9b9/UZNK3bzqJN53M+/VL\n6zI1K1h9yPLuo5M7eD+Ar2X1+2adtmbN+iv2F19MHZ0LFsArr6zfP39+ukpvabPNYIcdYPvt03rP\nPVN7/OjRsN12qcO0X790Zb/ttulK3lfrVgH8X6HVt3ffTSf7F16AmTPTXTAvvpiWmTM3LN+7NwwZ\nAu97H+y9dzrhDxy4ftlxx9Q8069f6pg0qzJOClb7Vq9OTTsvvphO/s8/D889By+9lO6qKW7S6dMH\nRoxIV/YnnJA6TUePTh2sw4en991cYzXMScFqx9tvpxP+9Onp5P/UUzBtWkoILU/8u+0G++4Lp56a\nOmZHjICRI9MVvk/6VsecFKx6zZsH990HDz2UlunT17/X0JBO8v/0TzBsWLr/venEv/32PvGbtcFJ\nwarD0qUweTJMmZJqAE88kZqDIN2Bc+CBcOKJ6bbM0aNTU0+PHvnGbFaFnBSs8kTA00/DxInw6KPw\nyCPv7fTdaSfYfXf4ylfgkENgzBh36pp1EScFy9/atenqf/JkePhhuPvudOsnpNs3DzoIPv/51Aew\nzz7pPn0zy4STgpXfypUwaRJMmJCagx56KD3RC+lWzw9/GI44Ag49FAYPdvu/WRk5KVj21q1LT/RO\nmAB/+xvce2/aL8Guu8KnPgUf/SgccEDqC3ASMMuNk4Jl4/XX4R//gHvugTvuSEM+QEoC55yTmoGO\nOMJNQWYVxknBukZEeiDsb3+DG2+Exx9P+7fZJnUGf+pTcPDBqTnIzCqWk4J1zqxZcMUVKRHMK4x8\nvuee8P3vwyc+AXvt5TuDzKqIk4JtvKVL4frr4ZprUkfxZpulpqCLLoLDDoOhQ/OO0Mw2kZOClWbV\nKrjrLvjNb1I/wYoV6fmA//iPNFTEoEF5R2hmXcBJwdr3xBPwpz/Bf/93mhdg++3hS19KE4gfcIDv\nFDKrMU4KtqE33oA//CEt06al5qGjjkoPkB15ZBo51MxqkpOCrTdlCvziF3DTTekp4332gcsug1NO\nSZPAmFnNc1Kod+vWpYfJfvhDuP/+NInM17+eagV77JF3dGZWZk4K9eqtt1I/waWXpucLdtgBfvxj\nOOOMNHOYmdUlJ4V6s3hxaiK66qo0z/Duu6eO5OOOg549847OzHLmpFAvVq2CX/4SfvQjWLIEDj8c\nLrwQ9t/fdxCZWTMnhXrw5JPwuc/BM8+ku4cuuSQ9dWxm1kK3vAOwDL3zDpx7bpqHoLER/vrXNDaR\nE4KZtcFJoRatXg1XXpkmpP/lL9PDZtOmwSc/mXdkZlbh3HxUayZNStNUTp2aZiy7+eY0eb2ZWQlc\nU6gVM2fCMcekoScaG+GWW+DBB50QzGyjOClUu7Vr09DVe+yRHkK7+GJ4/nk4/njfVWRmG83NR9Xs\n5ZfTEBQPP5zmM/7tbz1stZl1imsK1SgizWXwwQ/CU0+l13ff7YRgZp3mpFBtZs2CY4+F005LTUYz\nZsAXv+imIjPrEk4K1eTKK2HUqDTZzU9/Cg88AEOG5B2VmdUQ9ylUgxUr4JvfTEnh4x+HX//aTUVm\nlolMawqSDpf0vKSZks5v5f0hku6T9KSkqZKOzDKeqjRjRnre4Mor4bzzYPx4JwQzy0xmSUFSA3A5\ncAQwCjhZ0qgWxb4D3BwRewInAVdkFU9Vuu22NNHN3Lnp9c9+Bg0NeUdlZjUsy5rCvsDMiJgVEe8C\nNwLHtCgTQNPg/X2B1zKMp3pEpOGtjz0WRo5MM6Id0/JPZ2bW9bJMCgOBV4u25xb2FfsucKqkucB4\n4OutfZGk0yVNkTSlsbExi1grx7p1aeaz885L8yJPmODOZDMrm7zvPjoZuDYiBgFHAn+UtEFMEXF1\nRIyNiLEDBgwoe5Bls2pVGrTu8svhnHPSqKZ9+uQdlZnVkSzvPpoHDC7aHlTYV+w04HCAiHhEUk+g\nP7Aww7gq05IlqYnooYfSyKZnn+1nD8ys7LKsKUwGRkgaJqkHqSN5XIsyrwCHAkjaDegJ1Hj7UCtm\nz4b99oOJE9PTyd/4hhOCmeUis5pCRKyRdBZwJ9AAXBMR0yVdDEyJiHHAN4HfSDqX1On8hYiIrGKq\nSLNnw0c+AkuXpqEqDjkk74jMrI5l+vBaRIwndSAX77uw6PWzwEFZxlDR5s2DD30Ili1LHcqeEc3M\ncpZ3R3P9amyEj340JYS77nJCMLOK4GEu8rB4cbrddM6c1GR0wAF5R2RmBjgplN+6dXDyyfD44/Dn\nP3tmNDOrKE4K5RQB3/oW3Hkn/OpXfkrZzCqO+xTK6ec/T8tXvwpf+1re0ZiZbcBJoVwmTEi1hCOP\nTLUEP4dgZhXISaEc5s2Dk06C4cPhhhugm//sZlaZ3KeQtVWr4DOfSQ+n/f3vsNVWHX/GzCwnJV2y\nSuohaZesg6lJZ58NjzwCv/sdjBmTdzRmZu3qMClIOgqYBtxd2N5D0l+yDqwm3HUXXH116ks46aS8\nozEz61ApNYWLgf2AJQAR8RTgWkNHVqyAM89Mk+RcfHHe0ZiZlaSUPoXVEbFE771bpr4GrdsUF1wA\ns2al2kLPnnlHY2ZWklKSwgxJJwDdJA0DzgYmZRtWlbv33jSd5hlnwGGH5R2NmVnJSmk+OgvYG1gH\n/BlYBXwjy6Cq2rJl8MUvws47pwfVzMyqSCk1hU9ExLeBbzftkHQcKUFYS9/+Nsydm2ZQ69Ur72jM\nzDZKKTWF77Sy74KuDqQmPPdcutvoq1+FAw/MOxozs43WZk1B0idI8ycPlFTcDrIVqSnJikXAV74C\nffvCd1rLo2Zmla+95qOFwDPASmB60f63gPOzDKoq/f73aXyjyy+H7bfPOxozs02ijqZEltQzIlaW\nKZ4OjR07NqZMmZJ3GO/1xhswbBjssQfcfz80NOQdkZnZe0h6PCLGdlSulI7mgZJ+AIwCmm+4j4iR\nnYivtlx0ESxfDldd5YRgZlWtlI7ma4HfAwKOAG4GbsowpuoydWpKBqefDqNH5x2NmVmnlJIUekfE\nnQAR8VJEfIeUHCwiDXi39dbwwx/mHY2ZWaeV0ny0SlI34CVJZwDzgC2zDatKXH89PPAAXHEFbLtt\n3tGYmXVaKUnhXGAL0vAWPwD6Al/KMqiqEAEXXgh77pmajszMakCHSSEiHi28fAv4LICkgVkGVRXu\nvz8NePf737tz2cxqRrt9CpL2kfQpSf0L26MlXQc82t7n6sKll6YmoxNPzDsSM7Mu02ZSkPQfwJ+A\nU4A7JH0XuA94Gqjv21HnzoXbboMvf9njG5lZTWmv+egYYExErJC0LfAq8MGImFWe0CrY978PUkoK\nZmY1pL3mo5URsQIgIt4EXnBCAObMgWuvhdNOS8Njm5nVkPZqCsMlNQ2PLWBY0TYRcVymkVWq666D\nd9+F8z38k5nVnvaSwqdbbF+WZSBVIQJuuAEOOsi1BDOrSW0mhYi4t7NfLulw4L+ABuC3EfGjVsqc\nAHyXNO/z0xHxL5393cxMnAgzZqSH1czMalApD69tEkkNwOXAYcBcYLKkcRHxbFGZEcC/AwdFxGJJ\n22UVT5f43vfSfAmnnJJ3JGZmmShl7KNNtS8wMyJmRcS7wI2kO5qKfRm4PCIWA0TEwgzj6Zzp0+Ge\ne9J0m1ttlXc0ZmaZKDkpSNp8I797IOk21iZzC/uKjQRGSpooaVKhuam13z5d0hRJUxobGzcyjC5y\n2WXQo0e668jMrEZ1mBQk7StpGvBiYXuMpF910e9vBowADgZOBn4jaeuWhSLi6ogYGxFjBwwY0EU/\nvRHWrIGbb4Zjj4XtKruFy8ysM0qpKVwKfBJYBBARTwOHlPC5ecDgou1BhX3F5gLjImJ1RMwGXiAl\nicoycSK8+SZ8uuUNWWZmtaWUpNAtIua02Le2hM9NBkZIGiapB3ASMK5FmdtItQQK4yuNBCrvAbnb\nb4fu3eEITyNhZrWtlKTwqqR9gZDUIOkc0hV9uyJiDXAWcCcwA7g5IqZLuljS0YVidwKLJD1LGlfp\nWxGxaJOOJEt33pmeTejTJ+9IzMwyVcotqWeSmpCGAK8D9xT2dSgixgPjW+y7sOh1AOcVlsrU2AjT\npsEll+QdiZlZ5kpJCmsi4qTMI6lU992X1ocemm8cZmZlUErz0WRJ4yV9XlL9TcM5blx6LmH33fOO\nxMwscx0mhYjYGbgE2BuYJuk2SfVRc1i7Fu64A44+Gnr3zjsaM7PMlfTwWkQ8HBFnA3sBy0iT79S+\nxx+HRYvg4x/POxIzs7Io5eG1PpJOkfRX4DGgETgw88gqwT/+kdbuTzCzOlFKR/MzwF+Bn0TEgxnH\nU1luvx323ht23DHvSMzMyqKUpDA8ItZlHkmlWbIEHn0UvvGNvCMxMyubNpOCpJ9FxDeB/5EULd+v\n+ZnX7rorjXl0XG0fpplZsfZqCjcV1vU549pjj8Hmm8Nee+UdiZlZ2bQ389pjhZe7RcR7EoOks4BO\nz8xW0SZMgH32gZ49847EzKxsSrkl9Uut7KvtSQWWLk23o37oQ3lHYmZWVu31KZxIGtl0mKQ/F721\nJbAk68ByNW4crFsHRx2VdyRmZmXVXp/CY6Q5FAaR5lpu8hbwZJZB5e7WW2HwYDiwPh7HMDNr0l6f\nwmxgNmlU1Pqxbh08+GCaZa1bllNYm5lVnvaajx6IiI9IWgwU35Iq0qjX22YeXR6mT4fFi+HDH847\nEjOzsmuv+ahpys3+5QikYjz8cFq7k9nM6lCb7SNFTzEPBhoiYi1wAPAVYIsyxJaPxx9PM6wNH553\nJGZmZVdKo/ltpKk4dwZ+D4wArs80qjzdeisccghIeUdiZlZ2pSSFdRGxGjgO+FVEnAsMzDasnCxZ\nkvoTPvCBvCMxM8tFKUlhjaTPAJ8Fbi/s655dSDmaNi2tDzgg3zjMzHJS6hPNh5CGzp4laRhwQ7Zh\n5WT8+LQeOzbfOMzMctLh0NkR8Yyks4FdJO0KzIyIH2QfWg5mzUrzMe+wQ96RmJnlosOkIOmfgD8C\n80jPKGwv6bMRMTHr4Mpu6lQ/n2Bmda2USXZ+ARwZEc8CSNqNlCRqq41l5Up44QX4zGfyjsTMLDel\n9Cn0aEoIABExA+iRXUg5mTEjDXExenTekZiZ5aaUmsITkq4C/ruwfQq1OCDeY4XpI9zJbGZ1rJSk\ncAZwNvB/CtsPAr/KLKK8PPwwbLedn2Q2s7rWblKQ9EFgZ+AvEfGT8oSUk6efhr339pPMZlbX2uxT\nkPR/SUNcnALcLam1GdhqQwS89BKMGJF3JGZmuWqvpnAKsHtELJc0ABgPXFOesMrslVfg7bfh/e/P\nOxIzs1y1d/fRqohYDhARjR2UrW5Nw1uMGZNvHGZmOWuvpjC8aG5mATsXz9UcEcdlGlk5PfVUWjsp\nmFmday8pfLrF9mUb++WSDgf+C2gAfhsRP2qj3KeBW4F9ImLKxv5Opz36aLrrqE+fsv+0mVklaW+O\n5ns788WSGoDLgcOAucBkSeOKH4QrlNsS+AbwaGd+r1OmTfPIqGZmZNtPsC9p8LxZEfEucCNwTCvl\nvg/8GFiZYSxtW7oU5szxHApmZmSbFAYCrxZtz6XF5DyS9gIGR8Tf2vsiSadLmiJpSmNjY9dGOWNG\nWu++e9d+r5lZFSo5KUjavCt/WFI34OfANzsqGxFXR8TYiBg7YMCArgwDXnwxrf2MgplZx0lB0r6S\npgEvFrbHSCplmIt5wOCi7UGFfU22BD4A3C/pZWB/YJyk8g4+9Oyz0L27h7cwM6O0msKlwCeBRQAR\n8TRpJraOTAZGSBomqQdwEjCu6c2IWBoR/SNiaEQMBSYBR5f97qOpU1MtoUftDfxqZraxSkkK3SJi\nTot9azv6UESsAc4C7gRmADdHxHRJF0s6euNDzUAETJoE++2XdyRmZhWhlFFSX5W0LxCF20y/DrxQ\nypdHxHjS8BjF+y5so+zBpXxnl5o/H95808Nlm5kVlFJTOBM4DxgCvE5q+z8zy6DKZubMtN5pp3zj\nMDOrEB3WFCJiIak/oPY891xa+xkFMzOghKQg6TdAtNwfEadnElE5zZ4NDQ0wcGDHZc3M6kApfQr3\nFL3uCRzLex9Kq17Tp8Ouu8JmpfwZzMxqXynNRzcVb0v6I/BQZhGV08svw7BheUdhZlYxNmWYi2HA\n+7o6kLJrmm1t553zjsTMrGKU0qewmPV9Ct2AN4HzswyqLBob4Z13YOjQvCMxM6sY7SYFSQLGsH54\ninURsUGnc1VqGghvt93yjcPMrIK023xUSADjI2JtYamNhACwYEFa+84jM7NmpfQpPCVpz8wjKbfZ\ns9PaD66ZmTVrs/lI0maF8Yv2JM2a9hKwnDRfc0TEXmWKMRuvvQZbbw1bbpl3JGZmFaO9PoXHgL2A\nyhi8rqvNmwfbb593FGZmFaW9pCCAiHipTLGU1yuvuOnIzKyF9pLCAEnntfVmRPw8g3jKZ84c2LP2\nukrMzDqjvaTQAPShUGOoKStWpOcUXFMwM3uP9pLC/Ii4uGyRlNMrr6T1kCH5xmFmVmHauyW19moI\nTZqSgmsKZmbv0V5SOLRsUZTbC4WJ41xTMDN7jzaTQkS8Wc5AyuqJJ9J68OB84zAzqzCbMkpq9Vu9\nGrbZJk2wY2ZmzeozKSxYAO9/f95RmJlVnPpMCkuWpCEuzMzsPeozKSxYAAMG5B2FmVnFqb+ksHZt\nGgzPdx6ZmW2g/pLC/PkpMfjOIzOzDdRfUli4MK09QqqZ2QbqNyn0759vHGZmFaj+ksLcuWk9aFC+\ncZiZVaD6Swrz56e1m4/MzDZQf0nhtdegXz/YfPO8IzEzqzj1lxTmzYOBA/OOwsysImWaFCQdLul5\nSTMlnd/K++dJelbSVEn3Ssp+LOtHHnHTkZlZGzJLCpIagMuBI4BRwMmSRrUo9iQwNiJ2B24FfpJV\nPM0aGkC1O1WEmVlnZFlT2BeYGRGzIuJd4EbgmOICEXFfRLxT2JwEZHtLUAS8/jqMHJnpz5iZVass\nk8JA4NWi7bmFfW05Dfh7hvGkgfAgDZttZmYbaG+O5rKRdCowFvhIG++fDpwOMKQzYxY1PbjmmoKZ\nWauyrCnMA4oHGBpU2Pcekj4GXAAcHRGrWvuiiLg6IsZGxNgBnRnddMGCtPYIqWZmrcoyKUwGRkga\nJqkHcBIwrriApD2BX5MSwsIMY0maHlzbccfMf8rMrBpllhQiYg1wFnAnMAO4OSKmS7pY0tGFYv8J\n9AFukfSUpHFtfF3XaOpT6Ncv058xM6tWmfYpRMR4YHyLfRcWvf5Ylr+/gcWL09qzrpmZtaq+nmhe\nvBh69UqLmZltoL6SwptvupZgZtaO+koKy5bBVlvlHYWZWcWqr6SwfDn06ZN3FGZmFau+ksKSJdC3\nb95RmJlVrPpKCkuXuk/BzKwd9ZUUlixxn4KZWTvqKym8/rqbj8zM2lE/SSEC1qyBtWvzjsTMrGLV\nT1JYVRhrz8Nmm5m1qX6SwooVab3ttvnGYWZWweonKSxfnta9e+cbh5lZBaufpPDWW2ntu4/MzNpU\nP0nhjTfS2k80m5m1qX6Swpo1ae27j8zM2lQ/SaEpGbij2cysTfWTFFavTuvNMp1XyMysqtVPUmhq\nPurePd84zMwqWP0khaaagpOCmVmb6icpLFiQ1lK+cZiZVbD6SQpNzyf07JlvHGZmFax+kkLT3Ufu\naDYza1P9JIV169K6oSHfOMzMKlj9JIWmmkK3+jlkM7ONVT9nyKak4JqCmVmbnBTMzKxZ/SQF9ymY\nmXWofpKC+xTMzDpUP2dINx+ZmXXIScHMzJrVT1IYORKOP95jH5mZtaN+Hu895pi0mJlZmzKtKUg6\nXNLzkmZKOr+V9zeXdFPh/UclDc0yHjMza19mSUFSA3A5cAQwCjhZ0qgWxU4DFkfELsAvgB9nFY+Z\nmXUsy5rCvsDMiJgVEe8CNwIt22+OAf5QeH0rcKjksa3NzPKSZVIYCLxatD23sK/VMhGxBlgK9Gv5\nRZJOlzRF0pTGxsaMwjUzs6q4+ygiro6IsRExdsCAAXmHY2ZWs7JMCvOAwUXbgwr7Wi0jaTOgL7Ao\nw5jMzKwdWSaFycAIScMk9QBOAsa1KDMO+Hzh9fHAPyIiMozJzMzakdlzChGxRtJZwJ1AA3BNREyX\ndDEwJSLGAb8D/ihpJvAmKXGYmVlOVG0X5pIagTmb+PH+wBtdGE418DHXBx9zfejMMe8UER12ylZd\nUugMSVMiYmzecZSTj7k++JjrQzmOuSruPjIzs/JwUjAzs2b1lhSuzjuAHPiY64OPuT5kfsx11adg\nZmbtq7eagpmZtcNJwczMmtVkUqjHeRxKOObzJD0raaqkeyXtlEecXamjYy4q92lJIanqb18s5Zgl\nnVD4t54u6fpyx9jVSvhve4ik+yQ9Wfjv+8g84uwqkq6RtFDSM228L0mXFv4eUyXt1aUBRERNLaSn\np18ChgM9gKeBUS3KfBW4qvD6JOCmvOMuwzEfAvQuvD6zHo65UG5LYAIwCRibd9xl+HceATwJbFPY\n3i7vuMtwzFcDZxZejwJezjvuTh7zh4G9gGfaeP9I4O+AgP2BR7vy92uxplCP8zh0eMwRcV9EvFPY\nnEQaoLCalfLvDPB90uRNK8sZXEZKOeYvA5dHxGKAiFhY5hi7WinHHMBWhdd9gdfKGF+Xi4gJpGF/\n2nIMcF0kk4CtJe3QVb9fi0mhy+ZxqCKlHHOx00hXGtWsw2MuVKsHR8TfyhlYhkr5dx4JjJQ0UdIk\nSYeXLbpslHLM3wVOlTQXGA98vTyh5WZj/3/fKJkNiGeVSdKpwFjgI3nHkiVJ3YCfA1/IOZRy24zU\nhHQwqTY4QdIHI2JJrlFl62Tg2oj4maQDSINsfiAi1uUdWDWqxZpCPc7jUMoxI+ljwAXA0RGxqkyx\nZaWjY94S+ABwv6SXSW2v46q8s7mUf+e5wLiIWB0Rs4EXSEmiWpVyzKcBNwNExCNAT9LAcbWqpP/f\nN1UtJoV6nMehw2OWtCfwa1JCqPZ2ZujgmCNiaUT0j4ihETGU1I9ydERMySfcLlHKf9u3kWoJSOpP\nak6aVc4gu1gpx/wKcCiApN1ISaGW5+0dB3yucBfS/sDSiJjfVV9ec81HUYfzOJR4zP8J9AFuKfSp\nvxIRR+cWdCeVeMw1pcRjvhO/+yegAAAEEklEQVT4uKRngbXAtyKiamvBJR7zN4HfSDqX1On8hWq+\nyJN0Aymx9y/0k1wEdAeIiKtI/SZHAjOBd4AvdunvV/HfzszMulgtNh+ZmdkmclIwM7NmTgpmZtbM\nScHMzJo5KZiZWTMnBas4ktZKeqpoGdpO2aFtjSa5kb95f2EkzqcLQ0S8fxO+4wxJnyu8/oKkHYve\n+62kUV0c52RJe5TwmXMk9e7sb1t9cFKwSrQiIvYoWl4u0++eEhFjSIMl/ufGfjgiroqI6wqbXwB2\nLHrvXyPi2S6Jcn2cV1BanOcATgpWEicFqwqFGsGDkp4oLAe2Uma0pMcKtYupkkYU9p9atP/Xkho6\n+LkJwC6Fzx5aGKd/WmGc+80L+3+k9fNT/LSw77uS/k3S8aTxpf5U+M1ehSv8sYXaRPOJvFCjuGwT\n43yEooHQJF0paYrSPArfK+w7m5Sc7pN0X2HfxyU9Uvg73iKpTwe/Y3XEScEqUa+ipqO/FPYtBA6L\niL2AE4FLW/ncGcB/RcQepJPy3MKwBycCBxX2rwVO6eD3/xmYJqkncC1wYkR8kDQCwJmS+gHHAqMj\nYnfgkuIPR8StwBTSFf0eEbGi6O3/KXy2yYnAjZsY5+GkYS2aXBARY4HdgY9I2j0iLiUNJX1IRBxS\nGPriO8DHCn/LKcB5HfyO1ZGaG+bCasKKwomxWHfgskIb+lrSmD4tPQJcIGkQ8OeIeFHSocDewOTC\n8B69SAmmNX+StAJ4mTT88vuB2RHxQuH9PwBfAy4jzc/wO0m3A7eXemAR0ShpVmHMmheBXYGJhe/d\nmDh7kIYtKf47nSDpdNL/1zuQJpyZ2uKz+xf2Tyz8Tg/S380McFKw6nEu8DowhlTD3WDSnIi4XtKj\nwFHAeElfIc1O9YeI+PcSfuOU4gHzJG3bWqHCeDz7kgZhOx44C/joRhzLjcAJwHPAXyIilM7QJccJ\nPE7qT/gVcJykYcC/AftExGJJ15IGhmtJwN0RcfJGxGt1xM1HVi36AvMLY+R/ljQ42ntIGg7MKjSZ\n/C+pGeVe4HhJ2xXKbKvS56d+HhgqaZfC9meBBwpt8H0jYjwpWY1p5bNvkYbvbs1fSLNnnUxKEGxs\nnIUB3/4fsL+kXUkzjy0Hlkp6H3BEG7FMAg5qOiZJW0hqrdZldcpJwarFFcDnJT1NanJZ3kqZE4Bn\nJD1FmkvhusIdP98B7pI0Fbib1LTSoYhYSRqB8hZJ04B1wFWkE+zthe97iNbb5K8FrmrqaG7xvYuB\nGcBOEfFYYd9Gx1noq/gZaSTUp0lzMz8HXE9qkmpyNXCHpPsiopF0Z9QNhd95hPT3NAM8SqqZmRVx\nTcHMzJo5KZiZWTMnBTMza+akYGZmzZwUzMysmZOCmZk1c1IwM7Nm/x8dC2ht19F+1gAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f611f533390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot ROC\n",
    "y_test_xgb_proba = xgb.predict_proba(X_test)[:,1]\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_test_xgb_proba)\n",
    "# plot TPR against FPR\n",
    "plt.plot(fpr, tpr, color='red')\n",
    "\n",
    "plt.xlabel(\"False Positive Rate\")\n",
    "plt.ylabel(\"True Positive Rate\")\n",
    "plt.title(\"ROC\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['p1_play_freq',\n",
       " 'p1_song_num',\n",
       " 'p1_play_time',\n",
       " 'p1_play_perc',\n",
       " 'p1_down_freq',\n",
       " 'p2_play_freq',\n",
       " 'p2_song_num',\n",
       " 'p2_play_time',\n",
       " 'p2_play_perc',\n",
       " 'p2_down_freq',\n",
       " 'p3_play_freq',\n",
       " 'p3_song_num',\n",
       " 'p3_play_time',\n",
       " 'p3_play_perc',\n",
       " 'p3_down_freq',\n",
       " 'recency']"
      ]
     },
     "execution_count": 404,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature ranking:\n",
      "1. feature 2 (0.203237)\n",
      "2. feature 3 (0.195899)\n",
      "3. feature 1 (0.110498)\n",
      "4. feature 8 (0.087719)\n",
      "5. feature 7 (0.081349)\n",
      "6. feature 12 (0.076802)\n",
      "7. feature 13 (0.074979)\n",
      "8. feature 15 (0.056431)\n",
      "9. feature 6 (0.050849)\n",
      "10. feature 11 (0.041485)\n",
      "11. feature 4 (0.016972)\n",
      "12. feature 0 (0.001643)\n",
      "13. feature 5 (0.001598)\n",
      "14. feature 10 (0.000540)\n",
      "15. feature 14 (0.000000)\n",
      "16. feature 9 (0.000000)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEICAYAAABfz4NwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAHVhJREFUeJzt3XuUXWWd5vHvY0IiF7mmVEgiiZKm\njZcOegjMqLQtggExQRs0DAr0MNKOnWU7tpeo46WjriXddjPjGkRAbiIQMLZSrTKBbtSZsRvMCQaS\ngJEiRFLhkhICoigQ8swf+y3clBXqVJ2Tqgp5PmvtlX159++8+yQ5z9mXs7dsExER8byx7kBERIwP\nCYSIiAASCBERUSQQIiICSCBERESRQIiICCCBELFdkr4q6VNj3Y+I0aL8DiE6TdIG4EXAU7XZf2T7\n3jZqvhH4hu1p7fVu5yTpUqDX9n8f677Ec1f2EGJHeZvtvWrDiMOgEyRNHMvXb4ekCWPdh9g1JBBi\nVEk6UtK/SXpY0q3lm3//sr+QdIekRyWtl/SXZf6ewHXAQZJ+XYaDJF0q6fO19d8oqbc2vUHSxyTd\nBvxG0sSy3rck9Um6W9IHnqWvT9fvry3po5I2S7pP0omSjpf0c0kPSfpEbd3PSlom6eqyPbdI+pPa\n8pdL+mF5H9ZKmj/gdc+T9H1JvwHOBE4FPlq2/Z9Lu8WS7ir1b5f09lqNMyT9P0lfkrSlbOtxteX7\nS7pE0r1l+Xdqy06QtKr07d8kvbq27GOSNpXXXCfp6Bb+2mNnYTtDho4OwAbgzYPMnwo8CBxP9WXk\nmDLdVZa/FXgZIOBPgceA15Rlb6Q6ZFKvdynw+dr0M9qUfqwCpgO7l9dcCXwamAS8FFgPvGU72/F0\n/VJ7a1l3N+C9QB9wJfAC4BXAb4GZpf1ngSeBk0r7DwN3l/HdgB7gE6UfbwIeBQ6tve4jwOtKn58/\ncFtLu5OBg0qbdwG/AQ4sy84or/9eYALwX4F7+f1h4u8BVwP7lf78aZl/GLAZOKKsd3p5HycDhwIb\ngYNK2xnAy8b631uGzg3ZQ4gd5TvlG+bDtW+f7wa+b/v7trfZvgFoUgUEtr9n+y5XfgRcD7yhzX58\n2fZG278FDqcKnyW2n7C9HrgQWNhirSeBL9h+ElgKTAH+p+1Hba8Fbgf+pNZ+pe1lpf0/Un2wH1mG\nvYAvln7cCHwXOKW27rW2f1zep98N1hnb37R9b2lzNXAnMLfW5Be2L7T9FHAZcCDwIkkHAscB77O9\nxfaT5f0GOAs43/bNtp+yfRnweOnzU1TBMFvSbrY32L6rxfcudgIJhNhRTrS9bxlOLPMOBk6uBcXD\nwOupPqiQdJykm8rhl4epgmJKm/3YWBs/mOqwU/31P0F1ArwVD5YPV6j2BgAeqC3/LdUH/R+8tu1t\nQC/VN/qDgI1lXr9fUO1BDdbvQUk6rXZo52HglTzz/bq/9vqPldG9qPaYHrK9ZZCyBwN/M+A9mk61\nV9ADfJBq72ezpKWSDhqqn7HzSCDEaNoIXF4Lin1t72n7i5ImA98CvgS8yPa+wPepDh8BDHY53G+A\nPWrTLx6kTX29jcDdA17/BbaPb3vLBje9f0TS84BpVIdt7gWml3n9XgJs2k6//2Ba0sFUezeLgAPK\n+7WG379fz2YjsL+kfbez7AsD3qM9bF8FYPtK26+nCg4DZ7fwerGTSCDEaPoG8DZJb5E0QdLzy8na\naVTH0idTHZffWk6AHltb9wHgAEn71OatAo4vJ0hfTPXt9dn8BHi0nBjdvfThlZIO79gWPtNrJb1D\n1RVOH6Q69HITcDPV+ZGPStqtnFh/G9VhqO15gOqcR789qT6Q+6A6IU+1hzAk2/dRnaT/iqT9Sh+O\nKosvBN4n6QhV9pT0VkkvkHSopDeV8P4d1R7Rtu28TOyEEggxamxvBBZQHabpo/o2+hHgebYfBT4A\nXANsAf4T0F1b92fAVcD6cijjIOBy4Faqk57XU50kfbbXfwo4AZhDdYL3l8DXgH2ebb02XEt1sncL\n8B7gHeV4/RNUAXBc6cNXgNPKNm7PRVTH7h+W9B3btwP/APw7VVi8CvjxMPr2HqpzIj+jOon8QQDb\nTaoT0f+r9LuH6gQ1VIH9xdLn+4EXAh8fxmvGOJcfpkXsAJI+Cxxi+91j3ZeIVmUPISIigARCREQU\nOWQUERFA9hAiIqLYqW74NWXKFM+YMWOsuxERsVNZuXLlL213DdVupwqEGTNm0Gw2x7obERE7FUm/\naKVdDhlFRASQQIiIiCKBEBERQAIhIiKKBEJERAAJhIiIKBIIEREBJBAiIqJIIEREBNBiIEiaJ2md\npB5JiwdZ/iFJt0u6TdK/lsf79S87XdKdZTi9Nv+1klaXml+W1Mqj/4ZHGvkQEbGLGTIQJE0AzqV6\nutNs4BRJswc0+ynQsP1qYBnwd2Xd/YHPAEcAc4HPSNqvrHMe1ZOZZpVhXttbExERI9bKHsJcoMf2\n+vLov6VUj0F8mu0f2H6sTN5E9TBxgLcAN9h+yPYW4AZgnqQDgb1t3+Tq/ttfB07swPZERMQItRII\nU6mefduvt8zbnjOpHuD9bOtOLeND1pR0lqSmpGZfX18L3Y2IiJHo6EllSe8GGsDfd6qm7QtsN2w3\nurqGvHtrRESMUCuBsAmYXpueVuY9g6Q3A58E5tt+fIh1N/H7w0rbrRkREaOnlUBYAcySNFPSJGAh\n0F1vIOkw4HyqMNhcW7QcOFbSfuVk8rHActv3Ab+SdGS5uug04NoObE9ERIzQkA/Isb1V0iKqD/cJ\nwMW210paAjRtd1MdItoL+Ga5evQe2/NtPyTpc1ShArDE9kNl/P3ApcDuVOccrmM8a/dS1Dy7OiLG\nOXkn+qBqNBoe1hPT2vkQH/i+JBAiYiclaaXtxlDt8kvliIgAEggREVEkECIiAkggREREkUCIiAgg\ngRAREUUCISIigARCREQUCYSIiAASCBERUSQQIiICSCBERESRQIiICCCBEBERRQIhIiKABEJERBQt\nBYKkeZLWSeqRtHiQ5UdJukXSVkkn1eb/maRVteF3kk4syy6VdHdt2ZzObVZERAzXkI/QlDQBOBc4\nBugFVkjqtn17rdk9wBnAh+vr2v4BMKfU2R/oAa6vNfmI7WXtbEBERHTGkIEAzAV6bK8HkLQUWAA8\nHQi2N5Rl256lzknAdbYfG3FvIyJih2nlkNFUYGNturfMG66FwFUD5n1B0m2SzpE0ebCVJJ0lqSmp\n2dfXN4KXjYiIVozKSWVJBwKvApbXZn8c+GPgcGB/4GODrWv7AtsN242urq4d3teIiF1VK4GwCZhe\nm55W5g3HO4Fv236yf4bt+1x5HLiE6tBURESMkVYCYQUwS9JMSZOoDv10D/N1TmHA4aKy14AkAScC\na4ZZMyIiOmjIQLC9FVhEdbjnDuAa22slLZE0H0DS4ZJ6gZOB8yWt7V9f0gyqPYwfDSh9haTVwGpg\nCvD59jcnIiJGSrbHug8tazQabjabra8gjfzFBr4v7dQarF5ExCiRtNJ2Y6h2+aVyREQACYSIiCgS\nCBERASQQIiKiSCBERASQQIiIiCKBEBERQAIhIiKKBEJERAAJhIiIKBIIEREBJBAiIqJIIEREBJBA\niIiIIoEQERFAAiEiIooEQkREAC0GgqR5ktZJ6pG0eJDlR0m6RdJWSScNWPaUpFVl6K7Nnynp5lLz\n6vK85oiIGCNDBoKkCcC5wHHAbOAUSbMHNLsHOAO4cpASv7U9pwzza/PPBs6xfQiwBThzBP2PiIgO\naWUPYS7QY3u97SeApcCCegPbG2zfBmxr5UUlCXgTsKzMugw4seVeR0REx7USCFOBjbXp3jKvVc+X\n1JR0k6T+D/0DgIdtbx2qpqSzyvrNvr6+YbxsREQMx8RReI2DbW+S9FLgRkmrgUdaXdn2BcAFAI1G\nwzuojxERu7xW9hA2AdNr09PKvJbY3lT+XA/8EDgMeBDYV1J/IA2rZkREdF4rgbACmFWuCpoELAS6\nh1gHAEn7SZpcxqcArwNut23gB0D/FUmnA9cOt/MREdE5QwZCOc6/CFgO3AFcY3utpCWS5gNIOlxS\nL3AycL6ktWX1lwNNSbdSBcAXbd9eln0M+JCkHqpzChd1csMiImJ4VH1Z3zk0Gg03m83WV5BG/mID\n35d2ag1WLyJilEhaabsxVLv8UjkiIoAEQkREFAmEiIgAEggREVEkECIiAkggREREkUCIiAgggRAR\nEUUCISIigARCREQUCYSIiAASCBERUSQQIiICSCBERESRQIiICCCBEBERRUuBIGmepHWSeiQtHmT5\nUZJukbRV0km1+XMk/buktZJuk/Su2rJLJd0taVUZ5nRmkyIiYiQmDtVA0gTgXOAYoBdYIam79ihM\ngHuAM4APD1j9MeA023dKOghYKWm57YfL8o/YXtbuRkRERPuGDARgLtBjez2ApKXAAuDpQLC9oSzb\nVl/R9s9r4/dK2gx0AQ8TERHjSiuHjKYCG2vTvWXesEiaC0wC7qrN/kI5lHSOpMnbWe8sSU1Jzb6+\nvuG+bEREtGhUTipLOhC4HPgL2/17ER8H/hg4HNgf+Nhg69q+wHbDdqOrq2s0uhsRsUtqJRA2AdNr\n09PKvJZI2hv4HvBJ2zf1z7d9nyuPA5dQHZqKiIgx0kogrABmSZopaRKwEOhupXhp/23g6wNPHpe9\nBiQJOBFYM5yOR0REZw0ZCLa3AouA5cAdwDW210paImk+gKTDJfUCJwPnS1pbVn8ncBRwxiCXl14h\naTWwGpgCfL6jWxYREcMi22Pdh5Y1Gg03m83WV5BG/mID35d2ag1WLyJilEhaabsxVLv8UjkiIoAE\nQkREFAmEiIgAEggREVEkECIiAkggREREkUCIiAgggRAREUUCISIigNaehxA7Qn75HBHjTPYQIiIC\nSCBERESRQIiICCCBEBERRQIhIiKABEJERBQJhIiIAFoMBEnzJK2T1CNp8SDLj5J0i6Stkk4asOx0\nSXeW4fTa/NdKWl1qfrk8WzkiIsbIkIEgaQJwLnAcMBs4RdLsAc3uAc4Arhyw7v7AZ4AjgLnAZyTt\nVxafB7wXmFWGeSPeioiIaFsrewhzgR7b620/ASwFFtQb2N5g+zZg24B13wLcYPsh21uAG4B5kg4E\n9rZ9k6uHOn8dOLHdjYmIiJFrJRCmAhtr071lXiu2t+7UMj5kTUlnSWpKavb19bX4shERMVzj/qSy\n7QtsN2w3urq6xro7ERHPWa0EwiZgem16WpnXiu2tu6mMj6RmRETsAK0EwgpglqSZkiYBC4HuFusv\nB46VtF85mXwssNz2fcCvJB1Zri46Dbh2BP2PiIgOGTIQbG8FFlF9uN8BXGN7raQlkuYDSDpcUi9w\nMnC+pLVl3YeAz1GFygpgSZkH8H7ga0APcBdwXUe3LCIihkXeie6r32g03Gw2W1+hnZ82DHxfOv38\ngjwPISJGiaSVthtDtRv3J5UjImJ0JBAiIgJIIERERJFAiIgIIIEQERFFAiEiIgCYONYdiA7p5CW2\nEbFLyh5CREQACYSIiCgSCBERASQQIiKiSCBERASQQIiIiCKXncYfyp1YI3ZJ2UOIiAgggRAREUVL\ngSBpnqR1knokLR5k+WRJV5flN0uaUeafKmlVbdgmaU5Z9sNSs3/ZCzu5YRERMTxDBoKkCcC5wHHA\nbOAUSbMHNDsT2GL7EOAc4GwA21fYnmN7DvAe4G7bq2rrndq/3PbmDmxPRESMUCt7CHOBHtvrbT8B\nLAUWDGizALisjC8Djpb+4MzkKWXd2NVI7Q0RMSpaCYSpwMbadG+ZN2gb21uBR4ADBrR5F3DVgHmX\nlMNFnxokQACQdJakpqRmX19fC92N57wETMQOMSonlSUdATxme01t9qm2XwW8oQzvGWxd2xfYbthu\ndHV1jUJvIyJ2Ta0EwiZgem16Wpk3aBtJE4F9gAdryxcyYO/A9qby56PAlVSHpiIiYoy0EggrgFmS\nZkqaRPXh3j2gTTdwehk/CbjRrn6dJOl5wDupnT+QNFHSlDK+G3ACsIaIiBgzQ/5S2fZWSYuA5cAE\n4GLbayUtAZq2u4GLgMsl9QAPUYVGv6OAjbbX1+ZNBpaXMJgA/AtwYUe2KCIiRkTeiW4z0Gg03Gw2\nW1+hk08R6/TtHMZTvfHct9GoF/EcJ2ml7cZQ7fJL5YiIABIIERFRJBAiIgJIIERERJFAiIgIIIEQ\nERFFAiEiIoAEQkREFAmEiIgAEggREVEkECIiAkggREREkUCIiAighdtfRzzndfJOsRE7sewhREQE\nkECIiIgigRAREUCLgSBpnqR1knokLR5k+WRJV5flN0uaUebPkPRbSavK8NXaOq+VtLqs82Wp3cdg\nRUREO4YMBEkTgHOB44DZwCmSZg9odiawxfYhwDnA2bVld9meU4b31eafB7wXmFWGeSPfjIiIaFcr\newhzgR7b620/ASwFFgxoswC4rIwvA45+tm/8kg4E9rZ9k6uHOn8dOHHYvY8Yb6T2hogx1EogTAU2\n1qZ7y7xB29jeCjwCHFCWzZT0U0k/kvSGWvveIWoCIOksSU1Jzb6+vha6GxERI7GjTyrfB7zE9mHA\nh4ArJe09nAK2L7DdsN3o6uraIZ2MiIjWAmETML02Pa3MG7SNpInAPsCDth+3/SCA7ZXAXcAflfbT\nhqgZERGjqJVAWAHMkjRT0iRgIdA9oE03cHoZPwm40bYldZWT0kh6KdXJ4/W27wN+JenIcq7hNODa\nDmxPRESM0JC3rrC9VdIiYDkwAbjY9lpJS4Cm7W7gIuByST3AQ1ShAXAUsETSk8A24H22HyrL3g9c\nCuwOXFeGiIgYI/JOdC+WRqPhZrPZ+gqdvEdNu1eAjOd647lv473eju5bRAdIWmm7MVS7/FI5IiKA\nBEJERBQJhIiIAPI8hIjxLeckYhRlDyEiIoAEQkREFAmEiIgAEggREVEkECIiAkggREREkUCIiAgg\ngRAREUUCISIigARCREQUCYSIiAASCBERUbQUCJLmSVonqUfS4kGWT5Z0dVl+s6QZZf4xklZKWl3+\nfFNtnR+WmqvK8MJObVRERAzfkHc7Lc9EPhc4BugFVkjqtn17rdmZwBbbh0haCJwNvAv4JfA22/dK\neiXVYzin1tY71fYwHoEWERE7Sit7CHOBHtvrbT8BLAUWDGizALisjC8DjpYk2z+1fW+ZvxbYXdLk\nTnQ8IiI6q5VAmApsrE338sxv+c9oY3sr8AhwwIA2fw7cYvvx2rxLyuGiT0mD3/hd0lmSmpKafX19\nLXQ3IiJGYlROKkt6BdVhpL+szT7V9quAN5ThPYOta/sC2w3bja6urh3f2YiIXVQrgbAJmF6bnlbm\nDdpG0kRgH+DBMj0N+DZwmu27+lewvan8+ShwJdWhqYiIGCOtBMIKYJakmZImAQuB7gFtuoHTy/hJ\nwI22LWlf4HvAYts/7m8saaKkKWV8N+AEYE17mxIREe0YMhDKOYFFVFcI3QFcY3utpCWS5pdmFwEH\nSOoBPgT0X5q6CDgE+PSAy0snA8sl3QasotrDuLCTGxYREcMj70QP4W40Gm42h3GVajsPKB/4vnT6\nYefjqd547tt4rzee+zZYvdglSVppuzFUu/xSOSIigARCREQUCYSIiABauHVFRDyHdPJ8STznZA8h\nIiKABEJERBQJhIiIABIIERFRJBAiIgJIIERERJFAiIgIIIEQERFFAiEiIoAEQkREFAmEiIgAEggR\nEVEkECIiAmgxECTNk7ROUo+kxYMsnyzp6rL8Zkkzass+Xuavk/SWVmtGRMToGjIQJE0AzgWOA2YD\np0iaPaDZmcAW24cA5wBnl3VnAwuBVwDzgK9ImtBizYgYz6T2hh1dL4atlT2EuUCP7fW2nwCWAgsG\ntFkAXFbGlwFHS1KZv9T247bvBnpKvVZqRkSMXMJl2Fp5QM5UYGNtuhc4YnttbG+V9AhwQJl/04B1\np5bxoWoCIOks4Kwy+WtJ61roc6umAL8cdMnw/1Fsv9Z4rzee+zb29cZz3zpdbzz3bXTrdbpvw9fp\nege30mjcPzHN9gXABTuitqSm7cZ4qzXe643nvnW63njuW6frjee+jfd647lvw9HKIaNNwPTa9LQy\nb9A2kiYC+wAPPsu6rdSMiIhR1EogrABmSZopaRLVSeLuAW26gdPL+EnAjbZd5i8sVyHNBGYBP2mx\nZkREjKIhDxmVcwKLgOXABOBi22slLQGatruBi4DLJfUAD1F9wFPaXQPcDmwF/sr2UwCD1ez85g2p\nk4eiOn1YazzXG89963S98dy3Ttcbz30b7/XGc99apuqLfERE7OryS+WIiAASCBERUexygSBpuqQf\nSLpd0lpJf91mvedL+omkW0u9v22z3sWSNkta006dWr3/Vvq1RtJVkp7fRq1DJa2qDb+S9MFh1viD\n7ZP095J+Juk2Sd+WtG8btT5X6qySdL2kg9rs22clbapt8/Gt1huk/r6SlpVtvUPSfxjm+oP17+Ty\n97tNUluXKZa7CPxU0nfbqVNqdezWNJI2SFpd3v/mCNYf7H3bX9INku4sf+7XTr3asr+RZElThtvP\nsv5fl/+ra4f7f6sjbO9SA3Ag8Joy/gLg58DsNuoJ2KuM7wbcDBzZRr2jgNcAazqwrVOBu4Hdy/Q1\nwBkdeh8nAPcDB7e7fcCxwMQyfjZwdhu19q6NfwD4apt9+yzw4Q69Z5cB/6WMTwL27cB793LgUOCH\nQKPN/n0IuBL4bgf+bdwFvLRs561t/h/bAExpY/3B3re/AxaX8cWt/pvbXr0yfzrVhTK/GEl/gVcC\na4A9qC74+RfgkE7822t12OX2EGzfZ/uWMv4ocAe///X0SOrZ9q/L5G5lGPGZetv/h+pKrU6ZCOxe\nfh+yB3Bvh+oeDdxl+xfDWWmw7bN9ve2tZfImqt+ljLTWr2qTezKMv4sd8N4/TdI+VB8kF5XXesL2\nw8OpsZ3tvcN227/elzQNeCvwtXZrMc5uTbOdv9f67XYuA05ssx5U93H7KCP///9y4Gbbj5X/Dz8C\n3jHCWiOyywVCnaq7sh5G9a2+nToTJK0CNgM32G6rXqfY3gR8CbgHuA94xPb1HSq/ELiqQ7Xq/jNw\nXTsFJH1B0kbgVODTHejTonIY6uLhHFoYYCbQB1xSDst8TdKeHehbp/wPqg+zbR2oNdjtbkb8pYvq\nA/Z6SStV3cqmE15k+74yfj/wonaKSVoAbLJ9axtl1gBvkHSApD2A43nmD3h3uF02ECTtBXwL+OCA\nb5XDZvsp23OovtnOlfTKTvSxXeXDawHVh9FBwJ6S3t2BupOA+cA32601oO4nqX6vckU7dWx/0vb0\nUmdRm906D3gZMIcqVP9hhHUmUh1mOM/2YcBvqA5VjDlJJwCbba8c675sx+ttv4bq7sh/JemoThZ3\ndbxmxHv15cP7E7T55cP2HVSHTK8H/jewCniqnZrDtUsGgqTdqMLgCtv/1Km65RDAD6hu9T0evBm4\n23af7SeBfwL+YwfqHgfcYvuBDtQCQNIZwAnAqeU/aCdcAfx5OwVsP1ACfxtwIdXhkJHoBXpre4/L\nqAJiPHgdMF/SBqrDO2+S9I026nX01jRlTxfbm4FvM/K/g7oHJB0IUP7c3Eatl1F96bq1vIfTgFsk\nvXi4hWxfZPu1to8CtlCd4xw1u1wgSBLVcdw7bP9jB+p19V8VI2l34BjgZ+3W7ZB7gCMl7VG2+2iq\ncybtOoUOHi6SNI/qcMV824+1WWtWbXIBbf5d9H9oFG+n2q0fNtv3AxslHVpmHU31C/4xZ/vjtqfZ\nnkF1KPBG2+3sSXbs1jSS9pT0gv5xqgsQOnEFXv12O6cD1460kO3Vtl9oe0Z5D3upLly5f7i1JL2w\n/PkSqvMHV460XyMymmewx8MAvJ5q9/A2ql2yVcDxbdR7NfDTUm8N8Ok2+3cV1aGJJ6n+YZ3ZZr2/\npfpQXANcDkxus96eVDcu3KdT20f1nIyNtb+Plq4M2k6tb5VtvQ34Z2Bqm327HFhd6nUDB7bx3s0B\nmqXWd4D9OvDevb2MPw48ACxv8+/3jbR5lVGpczzVt9u7gE+2UeelVFcp3QqsHUmt7bxvBwD/CtxJ\ndTXP/u3UG7B8AyO8Kgr4v1RfFG4Fjm7372G4Q25dERERwC54yCgiIgaXQIiICCCBEBERRQIhIiKA\nBEJERBQJhIiIABIIERFR/H8HWjpMf4AxYwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6119ebf2b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "importances_xgb =xgb.feature_importances_\n",
    "\n",
    "indices_xgb = np.argsort(importances_xgb)[::-1]\n",
    "\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices_xgb[f], importances_xgb[indices_xgb[f]]))\n",
    "\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X.shape[1]), importances_xgb[indices_xgb],\n",
    "       color=\"r\")\n",
    "plt.xticks(range(X.shape[1]), indices_xgb)\n",
    "plt.xlim([-1, X.shape[1]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see for the xgboost model, the top three important features are p1_play_time,p1_play_perc and p1_song_num."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  45 out of  45 | elapsed: 38.5min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=8, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 408,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Choose the type of classifier. \n",
    "RFR = RandomForestClassifier()\n",
    "\n",
    "# Choose some parameter combinations to try\n",
    "parameters = {'n_estimators': [100,150,200],\n",
    "              'min_samples_leaf': [3,5,8]\n",
    "             }\n",
    "\n",
    "# Run the grid search\n",
    "grid_obj = GridSearchCV(RFR, parameters,\n",
    "                        cv=5, #Determines the cross-validation splitting strategy /to specify the number of folds in a (Stratified)KFold\n",
    "                        n_jobs=-1, #Number of jobs to run in parallel\n",
    "                        verbose=1)\n",
    "grid_obj = grid_obj.fit(X_train, y_train)\n",
    "\n",
    "# Set the clf to the best combination of parameters\n",
    "RFR = grid_obj.best_estimator_\n",
    "\n",
    "# Fit the best algorithm to the data. \n",
    "RFR.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make predictions\n",
    "y_train_rf = RFR.predict(X_train)\n",
    "y_test_rf = RFR.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set scores:\n",
      "Accuracy of the model is: 0.8801920715663919\n",
      "Precision of the model is: 0.8876218602140294\n",
      "Recall of the model is: 0.9355599884862806\n",
      "f1-score of the model is: 0.910960689866446\n"
     ]
    }
   ],
   "source": [
    "print(\"Training set scores:\")\n",
    "print_results(y_train, y_train_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing set scores:\n",
      "Accuracy of the model is: 0.8438884065216057\n",
      "Precision of the model is: 0.8640446990375866\n",
      "Recall of the model is: 0.9036252764189506\n",
      "f1-score of the model is: 0.8833918557727582\n"
     ]
    }
   ],
   "source": [
    "print(\"Testing set scores:\")\n",
    "print_results(y_test, y_test_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC for train set is: 0.8552954100569046\n"
     ]
    }
   ],
   "source": [
    "print(\"AUC for train set is: {}\".format(roc_auc_score(y_train, y_train_rf)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC for test set is: 0.8172011577242699\n"
     ]
    }
   ],
   "source": [
    "print(\"AUC for test set is: {}\".format(roc_auc_score(y_test, y_test_rf)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xu8VVW5//HPFxAUQVDBvCCCiinm\njVApyzLvllre0tS0NMuym+XJc+qo2fF0jmWnTEuxUqOjeCkLjY6lWZoKisnFSyogCuRPEOQiyG3z\n/P4Yc28Wm31ZG/Zcc629vu/Xa77mZY211jM3Op81xphzDEUEZmZmAN2KDsDMzKqHk4KZmTVxUjAz\nsyZOCmZm1sRJwczMmjgpmJlZEycFMzNr4qRg1gZJsyS9LektSf9P0i2S+pS8/l5Jf5a0VNJiSfdK\nGt7sM7aS9ENJr2afMyPbH1D5MzJrm5OCWfuOj4g+wP7AAcC/Akh6D/BH4HfAjsBQYArwqKRdszI9\ngQeBvYFjgK2A9wALgIMqexpm7ZOfaDZrnaRZwPkR8UC2fzWwd0R8WNIjwLSI+Hyz9/wBmB8Rn5R0\nPnAVsFtEvFXh8M06zDUFszJJGgQcC0yX1Bt4L3BXC0XvBI7Mto8A/s8JwWqFk4JZ+34raSkwG5gH\nXA5sQ/r/57UWyr8GNPYXbNtKGbOq5KRg1r6PRkRf4IPAnqQL/pvAWmCHFsrvALyRbS9opYxZVXJS\nMCtTRPwVuAX4fkQsAx4HTm2h6GmkzmWAB4CjJW1ZkSDNNpGTglnH/BA4UtJ+wKXAOZK+JKmvpK0l\n/Qfp7qJvZ+XHkJqdfi1pT0ndJG0r6d8kHVfMKZi1zknBrAMiYj7wS+CyiPgbcDRwEqnf4BXSLavv\ni4iXsvIrSZ3N/wD+BCwBniA1QU2s+AmYtcO3pJqZWRPXFMzMrImTgpmZNXFSMDOzJk4KZmbWpEfR\nAXTUgAEDYsiQIUWHYWZWU5566qk3ImJge+VqLikMGTKESZMmFR2GmVlNkfRKOeXcfGRmZk2cFMzM\nrImTgpmZNXFSMDOzJk4KZmbWJLekIOkXkuZJeqaV1yXpWknTJU2VNCKvWMzMrDx51hRuIU1U3ppj\ngWHZcgHw0xxjMTOzMuT2nEJEPCxpSBtFTgR+GWmY1gmS+kvaISI8daGZbZy1a2H16vWXhobWlzVr\n4O23oXv39N7G443bpccWLYItt4SIdLxxab7f0vGVK2HJEujXb/3XylmXbh9/PBx4YK5/wiIfXtuJ\nNPlIoznZsQ2SgqQLSLUJBg8eXJHgzKwVEekit2xZWhYvThfXlSvXLW+/ndYrVqw7tmLFuv1Vq9Zd\ntBu3G4+vWgXLl8OcOTBgwIYX+cbyjevly9PndnUS7LRTl04KZYuI0cBogJEjR3oCCLNyrFmTLpjL\nl8PSpemX6pIl67bffjsty5alX8GLFqWyjceWL1934W+8oC9fDm+9lX65biwJevaEzTZbt95sM+jV\nKy2Nx7bYIsW5/fbryjQuPXuu/96VK6FHD9h22w3Ldu/e9vLWW+l93btDt27rjjdulx5btQp6907H\nunVL59K4Xbq0dBxSjI2vlbMu3a6QIpPCXGDnkv1B2TEzg3ThXbw4XRgXLUrrhQvhjTfg+efThWLR\nInjzzXXlli5NxxYvThfwcvXqlZo2ttwyXYx794Y+feAd70jbW2yxbunTJx1btgx23TVdtHr3hq22\nShfqXr3SfuNFvlev9L5evWDzzdddGK0qFZkUxgEXSRoLHAwsdn+CdVkR6UK9YAG8/jq89lpav/FG\nWhYsSBf8xov+ggXpIt/ezIjbbw/bbJMuyFtvDYMHp4t7//5p3bv3ugt2375p6ddv3YV9iy1SIth8\n88r8Hazq5ZYUJN0OfBAYIGkOcDmwGUBE3ACMB44DpgPLgU/lFYtZLhoa1r/Av/IKvPoqzJuXloUL\n113s589v/XP69UsX9gED0nr33dN6m23WXdwbl/79U7mBA9NF3b+4rZPleffRGe28HsAX8vp+s03S\n0JAu8P/8J8ydmzo9Z81K69deS8fnzNmwbb17d9huu9Tssu22sN9+6Rf8ttum5pN3vCP9mt9hh/Qr\nv7EN3KxK1ERHs1kuFi6Ef/wj/cJ/+WV46aV0wZ8xIx1bvXr98n37ws47pwv6Bz4Au+wCO+6YLu7b\nb59e23HHdZ2KZjXIScG6tiVL4MUXU8fszJnp1/6zz6btBQvWL7vjjmk54AA4+eTUjDNoULoNcNCg\n1HTj5hrr4pwUrGtYvTpd/P/xD5g8GZ58EqZOTb/8S+2wA+y5J5xySrro77UXDBmSli23LCJys6ri\npGC1Ze3a9Cv/+efh6afhhRfS8swz6V51SM03e+8NRx2VEsAee6SL/667pnZ9M2uVk4JVr4aGdPGf\nMCH9+p88GaZNS01CkJpyBg9OF/0vfCE1+7zznSkh9O5dbOxmNcpJwarHkiXw17/CI4+kRPDUU+se\nwOrbF971LjjrrHTxHz483dnjJh+zTuWkYMVZuRImToSHH07J4OGH0zACm20GI0bA+efDu98No0al\n9n/f1WOWOycFq5y1a2HKFLj3XvjTn1JncGM/wD77wEUXwUc+kpLAFlsUG6tZnXJSsHxNn55qAw8+\nCOPHpyd/JRg5Ej7/+XS///velx7iMrPCOSlY55s9G26+GW6/Pd0iCuke/6OOgmOPheOOS0/9mlnV\ncVKwTbd6NfzlL6k28MgjqWbQ0ADvf3+6K+jQQ9MdQd27Fx2pmbXDScE2TkR6MvjGG2HMmDQCaI8e\nqVno0ktTJ/GQIUVHaWYd5KRg5YuAxx+Hu+5K/QMvvpgSwWmnwamnpuYhPx9gVtOcFKx9M2fCTTel\nPoJXXklPBR96KHz5y/Cxj6WhI8ysS3BSsJatWgV33506jB94ID0jcMwxcPnladygvn2LjtDMcuCk\nYOt74YVUK7jlljSK6M47w5VXwqc+lUYKNbMuzUnBUl/BI4/AFVfAQw+lWsHHPgYXXABHHOEnic3q\niJNCPYuAP/wB/uVf0p1E228P3/0unHtu2jazuuOkUI8aGtJtpN//fkoGu+wCo0fDmWf67iGzOud2\ngXqyahX86lew//6pj6BHj5QMXngBPvMZJwQzc02hLixZkjqPf/SjNATFHnvA2LHp+QJPL2lmJZwU\nurI1a+Caa1I/weLF6dmCn/wkjT3kzmMza4GTQlf1xz/CxRenPoPjjoPLLoODDy46KjOrcv652NXM\nmAEf/SgcfTQsWwa//jXcd58TgpmVxUmhq1i+PNUG9torTWBz1VWpA/mkk9xvYGZlc/NRV/DEE3D6\n6fDyy+m20quvhh13LDoqM6tBrinUsqVL4Wtfg/e+N81p8OCD6ZZTJwQz20iuKdSqSZPgk5+E55+H\n885LD6L17190VGZW41xTqDWrVqUxig4+GBYtSv0HP/uZE4KZdQrXFGrJ00+nPoPnn4czzkjPHDgZ\nmFknck2hFqxYAV/5CowYkR5Cu/deuO02JwQz63S5JgVJx0h6QdJ0SZe28PpgSQ9JelrSVEnH5RlP\nTXrxRfjgB9MQFZ/5DEydCh/5SNFRmVkXlVvzkaTuwPXAkcAc4ElJ4yLiuZJi3wLujIifShoOjAeG\n5BVTzRk3Dj7xCdhsM7jjjjRWkZlZjvKsKRwETI+ImRGxChgLnNisTABbZdv9gH/mGE9tGTsWTj4Z\nhg2DKVOcEMysIvJMCjsBs0v252THSl0BnCVpDqmW8MWWPkjSBZImSZo0f/78PGKtLqNHpxrCqFHw\n5z/D4MFFR2RmdaLojuYzgFsiYhBwHDBG0gYxRcToiBgZESMHDhxY8SAr6ppr4LOfhWOPhfvvh623\nLjoiM6sjeSaFucDOJfuDsmOlzgPuBIiIx4HNgQE5xlS9Fi+Gs8+Gr38dTj01DWTnSW/MrMLyTApP\nAsMkDZXUEzgdGNeszKvA4QCS9iIlhTpoH2pm1qz0MNptt6WkcNttsPnmRUdlZnUot7uPImKNpIuA\n+4HuwC8i4llJVwKTImIc8DXgJklfJXU6nxsRkVdMVWnWLDjiCHjjDXjgATjssKIjMrM6lusTzREx\nntSBXHrsspLt54BD8oyhqi1YkBLC/Pnw+9/D+95XdERmVuc8zEVRVqyAE06AV19Ndxg5IZhZFXBS\nKMLKlWl2tMcegzFjnBDMrGoUfUtq/Vm9Gk45Jd1uesMNcNZZRUdkZtbESaGSItL4RffdB9ddl55H\nMDOrIk4KlfSd78Ctt8I3vgFf+ELR0ZiZbcBJoVLuvhsuvzzNpfzd7xYdjZlZi5wUKmHWLPj0p9MD\najffDFLREZmZtchJIW/Ll8NJJ6XtsWP9pLKZVTXfkpq3r3wFJk+Ge+6BIUOKjsbMrE1l1RQk9ZS0\ne97BdDljxsBNN8GXvwwnNp9Kwsys+rSbFCR9GJgG/Cnb31/SPXkHVvOmTYPzz4dDDoGrry46GjOz\nspRTU7gSOBhYBBARkwHXGtqyYgWccQb07w+/+U2aTtPMrAaU06ewOiIWaf07ZuprJNOO+sY34Nln\n00Nq221XdDRmZmUrJyk8L+k0oJukocCXgAn5hlXDHn4Yrr0WLrwQPvzhoqMxM+uQcpqPLgLeDawF\nfgOsBL6cZ1A1a82adLfRoEHuRzCzmlROTeHoiPgG8I3GA5JOIiUIK3X99fD00/CrX0GfPkVHY2bW\nYeXUFL7VwrFvdnYgNW/uXPj3f4ejjoJPfKLoaMzMNkqrNQVJRwPHADtJ+kHJS1uRmpKsUUMDnHxy\nGgX1Rz/yMBZmVrPaaj6aBzwDrACeLTm+FLg0z6Bqzs9+BhMnwo9/DHvuWXQ0ZmYbTRFt310qafOI\nWFGheNo1cuTImDRpUtFhrLNmDQwfnvoQnnrKtQQzq0qSnoqIke2VK6ejeSdJVwHDgabR3CJij02I\nr+u49VZ46SX43e+cEMys5pXT0XwLcDMg4FjgTuCOHGOqHStXwre/nYbEPv74oqMxM9tk5SSF3hFx\nP0BEzIiIb5GSg914I8yeDVdd5VqCmXUJ5TQfrZTUDZgh6XPAXKBvvmHVgLfeSsngQx+Cww8vOhoz\ns05RTlL4KrAlaXiLq4B+wKfzDKomXHcdzJuXEoOZWRfRblKIiInZ5lLgbABJO+UZVNVbsQJ++EM4\n5hgYNaroaMzMOk2bfQqSDpT0UUkDsv29Jf0SmNjW+7q8X/0KXn8dLrmk6EjMzDpVq0lB0neB/wXO\nBP5P0hXAQ8AUoH5vR21ogO9/H0aMgMMOKzoaM7NO1Vbz0YnAfhHxtqRtgNnAPhExszKhVanf/hZe\neAFuv913HJlZl9NW89GKiHgbICIWAi/WfUKIgP/8T9hhBzj11KKjMTPrdG3VFHaV1Dg8toChJftE\nxEntfbikY4AfAd2Bn0XEf7VQ5jTgCtJsblMionqHGL33Xvj73+Gmm6B796KjMTPrdG0lhZOb7V/X\nkQ+W1B24HjgSmAM8KWlcRDxXUmYY8K/AIRHxpqTqnrvy5pvT9JrnnFN0JGZmuWg1KUTEg5v42QcB\n0xubnCSNJfVTPFdS5jPA9RHxZvad8zbxO/Mzd24a3+iSS2CzzYqOxswsF+UMc7GxdiJ1Tjeakx0r\ntQewh6RHJU3Imps2IOkCSZMkTZo/f35O4bbjlltSn8IFFxTz/WZmFZBnUihHD2AY8EHgDOAmSf2b\nF4qI0RExMiJGDhw4sMIhkpLBmDFw6KGw226V/34zswopOylI6tXBz54L7FyyPyg7VmoOMC4iVkfE\ny8CLpCRRXZ56Kt2GevbZRUdiZpardpOCpIMkTQNeyvb3k/TjMj77SWCYpKGSegKnA+OalfktqZZA\n9tT0HkD13fY6Zgz07AmnnFJ0JGZmuSqnpnAt8BFgAUBETAHafZQ3ItYAFwH3A88Dd0bEs5KulHRC\nVux+YIGk50hPS18SEQs6fho5Wr06DWtxwgnQf4OWLTOzLqWcUVK7RcQrWv/p3YZyPjwixgPjmx27\nrGQ7gIuzpTpNmAALF8JppxUdiZlZ7spJCrMlHQRE9uzBF0lt//XhjjvSLahHHVV0JGZmuSun+ehC\n0i/5wcDrwKjsWNe3dm16NuG446Bfv6KjMTPLXTk1hTURcXrukVSjCRNgzhz4rw1G5zAz65LKqSk8\nKWm8pHMk1dc0nHfeCb16wfHHFx2JmVlFtJsUImI34D+AdwPTJP1WUtevOUSkpqMjj4Sttio6GjOz\niijr4bWIeCwivgSMAJaQJt/p2qZMgVmzXEsws7pSzsNrfSSdKele4AlgPvDe3CMr2oPZeIBHH11s\nHGZmFVROR/MzwL3A1RHxSM7xVI/f/x6GDoVddik6EjOziiknKewaEWtzj6TazJjhhGBmdafVpCDp\nmoj4GvBrSdH89XJmXqtZCxfCq696mGwzqztt1RTuyNYdmnGtS3jssbR+//uLjcPMrMLamnntiWxz\nr4hYLzFIugjY1JnZqtff/paGtjjwwKIjMTOrqHJuSf10C8fO6+xAqsqjj8K73w1bbFF0JGZmFdVW\nn8LHSXMgDJX0m5KX+gKL8g6sMGvXwuTJcO65RUdiZlZxbfUpPEGaQ2EQcH3J8aXA03kGVagXX4S3\n3oIDDig6EjOzimurT+Fl4GXggcqFUwX+8Ie0/tCHio3DzKwAbTUf/TUiPiDpTaD0llSR5sfZJvfo\nivDII+mhtSFDio7EzKzi2mo+apxyc0AlAqkKEamT2UNbmFmdavXuo5KnmHcGukdEA/Ae4LPAlhWI\nrfJmzIB582DUqKIjMTMrRDm3pP6WNBXnbsDNwDDgtlyjKsq4cWl97LHFxmFmVpByksLaiFgNnAT8\nOCK+CuyUb1gFeewx2HXX1KdgZlaHykkKaySdCpwN3Jcd2yy/kAr09NPpoTUzszpV7hPNh5GGzp4p\naShwe75hFWDJEpg5E/bbr+hIzMwK0+7Q2RHxjKQvAbtL2hOYHhFX5R9ahU2bltZOCmZWx9pNCpLe\nD4wB5pKeUdhe0tkR8WjewVXUlClp7aRgZnWsnEl2/gc4LiKeA5C0FylJjMwzsIqbOBEGDIBBg4qO\nxMysMOX0KfRsTAgAEfE80DO/kAryxBNwyCEgFR2JmVlhykkKf5d0g6T3ZctP6WoD4i1fDi+9BO96\nV9GRmJkVqpzmo88BXwL+Jdt/BPhxbhEVYfJkaGjwpDpmVvfaTAqS9gF2A+6JiKsrE1IBns4qPn5G\nwczqXKvNR5L+jTTExZnAnyS1NANb1zBlCmy7LezUNR/UNjMrV1t9CmcC+0bEqcCBwIUd/XBJx0h6\nQdJ0SZe2Ue5kSSGpmDuapk6Fffd1J7OZ1b22ksLKiFgGEBHz2ym7AUndSTO2HQsMB86QNLyFcn2B\nLwMTO/L5nWbtWnjmGdhnn0K+3sysmrTVp7BrydzMAnYrnas5Ik5q57MPIj39PBNA0ljgROC5ZuW+\nA/w3cElHAu80L78My5almoKZWZ1rKymc3Gz/ug5+9k7A7JL9OcDBpQUkjQB2jojfS2o1KUi6ALgA\nYPDgwR0Mox1Tp6a1k4KZWZtzND+Y5xdL6gb8ADi3vbIRMRoYDTBy5Mhop3jHTJ2a+hL23rtTP9bM\nrBZ1qJ+gg+aSZm1rNCg71qgv8C7gL5JmAaOAcRXvbJ42DXbfHXr3rujXmplVozyTwpPAMElDJfUE\nTgfGNb4YEYsjYkBEDImIIcAE4ISImJRjTBtqvPPIzMzKTwqSenXkgyNiDXARcD/wPHBnRDwr6UpJ\nJ3QszJwsXZqGt/CdR2ZmQHlDZx8E/BzoBwyWtB9wfkR8sb33RsR4YHyzY5e1UvaD5QTcqaZPT2s/\ntGZmBpRXU7gW+AiwACAippBmYqt9TzyR1iNGFBuHmVmVKCcpdIuIV5oda8gjmIpbuDCtd9212DjM\nzKpEOaOkzs6akCJ7SvmLwIv5hlUhc+bA1ltD//5FR2JmVhXKqSlcCFwMDAZeJ9062uFxkKrS7Nmw\n887tlzMzqxPt1hQiYh7pdtKux0nBzGw95dx9dBOwwVPEEXFBLhFV0uzZMGpU0VGYmVWNcvoUHijZ\n3hz4GOuPaVSbli+HBQtcUzAzK1FO89EdpfuSxgB/yy2iSpkzJ62dFMzMmmzMMBdDgXd0diAVNzur\n7DgpmJk1KadP4U3W9Sl0AxYCrc6iVjOcFMzMNtBmUpAkYD/WjW66NiI6d+jqojQmhUGDio3DzKyK\ntNl8lCWA8RHRkC1dIyFASgrbbQe9OjTOn5lZl1ZOn8JkSQfkHkml+RkFM7MNtNp8JKlHNvz1AcCT\nkmYAy0jzNUdE1PYocrNnw7BhRUdhZlZV2upTeAIYAVTH3AedbfZs+NCHio7CzKyqtJUUBBARMyoU\nS+UsWZIWNx+Zma2nraQwUNLFrb0YET/IIZ7K8O2oZmYtaispdAf6kNUYuhQnBTOzFrWVFF6LiCsr\nFkklOSmYmbWorVtSu14NodHs2dCtG+y4Y9GRmJlVlbaSwuEVi6LSZs+GHXaAHuUMEmtmVj9aTQoR\nsbCSgVTUa6+5lmBm1oKNGSW19j31VJqb2czM1lOfSWGrrWD16qKjMDOrOvWZFF5/Hfbdt+gozMyq\nTv0lhVWrYNkyGDiw6EjMzKpO/SWFxYvTul+/YuMwM6tC9ZcUFi1K6/79i43DzKwKOSmYmVmT+k0K\nbj4yM9tArklB0jGSXpA0XdKlLbx+saTnJE2V9KCkXfKMB1jXp+CagpnZBnJLCpK6A9cDxwLDgTMk\nDW9W7GlgZETsC9wNXJ1XPE3cfGRm1qo8awoHAdMjYmZErALGAieWFoiIhyJiebY7ARiUYzyJm4/M\nzFqVZ1LYCZhdsj8nO9aa84A/tPSCpAskTZI0af78+ZsW1eLFaYTUPn027XPMzLqgquholnQWMBL4\nXkuvR8ToiBgZESMHbupDZ4sWpVpCt6o4dTOzqpLn2NFzgdJZbAZlx9Yj6Qjgm8AHImJljvEkixa5\nP8HMrBV5/lx+EhgmaaiknsDpwLjSApIOAG4EToiIeTnGss7ixe5PMDNrRW5JISLWABcB9wPPA3dG\nxLOSrpR0Qlbse6R5oO+SNFnSuFY+rvO4pmBm1qpcpx6LiPHA+GbHLivZPiLP72/RokWw224V/1oz\ns1pQf72tbj4yM2tV/SUFNx+ZmbWqvpJCQwMsWeKkYGbWivpKCkuXprWbj8zMWlRfScHjHpmZtclJ\nwczMmtRXUvBUnGZmbaqvpOCagplZm5wUzMysSX0lBc+6ZmbWpvpKCo01ha22KjYOM7MqVX9JoU8f\n6JHrkE9mZjWrvpLC4sVuOjIza0N9JYXGWdfMzKxF9ZcUXFMwM2tVfSUFNx+ZmbWpvpKCm4/MzNpU\nf0nBNQUzs1bVT1KI8KxrZmbtqJ+ksGoVrFkDffsWHYmZWdWqn6TQOMFOnz7FxmFmVsXqJym89VZa\nOymYmbWq/pKCm4/MzFpVf0nBNQUzs1bVT1Jwn4KZWbvqJym4pmBm1q76SwpbbllsHGZmVax+ksLq\n1Wnds2excZiZVbH6SQoNDWndvXuxcZiZVTEnBTMza+KkYGZmTXJNCpKOkfSCpOmSLm3h9V6S7she\nnyhpSG7BOCmYmbUrt6QgqTtwPXAsMBw4Q9LwZsXOA96MiN2B/wH+O694nBTMzNqXZ03hIGB6RMyM\niFXAWODEZmVOBG7Ntu8GDpekXKJxUjAza1eeSWEnYHbJ/pzsWItlImINsBjYtvkHSbpA0iRJk+bP\nn79x0eyxB5xyim9JNTNrQ010NEfE6IgYGREjBw4cuHEfcsIJcNdd0KtX5wZnZtaF5JkU5gI7l+wP\nyo61WEZSD6AfsCDHmMzMrA15JoUngWGShkrqCZwOjGtWZhxwTrZ9CvDniIgcYzIzszb0yOuDI2KN\npIuA+4HuwC8i4llJVwKTImIc8HNgjKTpwEJS4jAzs4LklhQAImI8ML7ZsctKtlcAp+YZg5mZla8m\nOprNzKwynBTMzKyJk4KZmTVxUjAzsyaqtTtAJc0HXtnItw8A3ujEcGqBz7k++Jzrw6ac8y4R0e7T\nvzWXFDaFpEkRMbLoOCrJ51wffM71oRLn7OYjMzNr4qRgZmZN6i0pjC46gAL4nOuDz7k+5H7OddWn\nYGZmbau3moKZmbXBScHMzJp0yaQg6RhJL0iaLunSFl7vJemO7PWJkoZUPsrOVcY5XyzpOUlTJT0o\naZci4uxM7Z1zSbmTJYWkmr99sZxzlnRa9m/9rKTbKh1jZyvjv+3Bkh6S9HT23/dxRcTZWST9QtI8\nSc+08rokXZv9PaZKGtGpAUREl1pIw3TPAHYFegJTgOHNynweuCHbPh24o+i4K3DOhwG9s+0L6+Gc\ns3J9gYeBCcDIouOuwL/zMOBpYOtsf7ui467AOY8GLsy2hwOzio57E8/5UGAE8Ewrrx8H/AEQMAqY\n2Jnf3xVrCgcB0yNiZkSsAsYCJzYrcyJwa7Z9N3C4JFUwxs7W7jlHxEMRsTzbnUCaCa+WlfPvDPAd\n4L+BFZUMLiflnPNngOsj4k2AiJhX4Rg7WznnHMBW2XY/4J8VjK/TRcTDpPllWnMi8MtIJgD9Je3Q\nWd/fFZPCTsDskv052bEWy0TEGmAxsG1FostHOedc6jzSL41a1u45Z9XqnSPi95UMLEfl/DvvAewh\n6VFJEyQdU7Ho8lHOOV8BnCVpDmn+li9WJrTCdPT/9w7JdZIdqz6SzgJGAh8oOpY8SeoG/AA4t+BQ\nKq0HqQnpg6Ta4MOS9omIRYVGla8zgFsi4hpJ7yHN5viuiFhbdGC1qCvWFOYCO5fsD8qOtVhGUg9S\nlXNBRaLLRznnjKQjgG8CJ0TEygrFlpf2zrkv8C7gL5Jmkdpex9V4Z3M5/85zgHERsToiXgZeJCWJ\nWlXOOZ8H3AkQEY8Dm5MGjuuqyvr/fWN1xaTwJDBM0lBJPUkdyeOalRkHnJNtnwL8ObIenBrV7jlL\nOgC4kZQQar2dGdo554hYHBEDImJIRAwh9aOcEBGTigm3U5Tz3/ZvSbUEJA0gNSfNrGSQnaycc34V\nOBxA0l6kpDC/olFW1jjgk9ldSKOAxRHxWmd9eJdrPoqINZIuAu4n3bnwi4h4VtKVwKSIGAf8nFTF\nnE7q0Dm9uIg3XZnn/D2gD3At1tANAAAEIklEQVRX1qf+akScUFjQm6jMc+5Syjzn+4GjJD0HNACX\nRETN1oLLPOevATdJ+iqp0/ncWv6RJ+l2UmIfkPWTXA5sBhARN5D6TY4DpgPLgU916vfX8N/OzMw6\nWVdsPjIzs43kpGBmZk2cFMzMrImTgpmZNXFSMDOzJk4KVnUkNUiaXLIMaaPskNZGk+zgd/4lG4lz\nSjZExDs34jM+J+mT2fa5knYsee1nkoZ3cpxPStq/jPd8RVLvTf1uqw9OClaN3o6I/UuWWRX63jMj\nYj/SYInf6+ibI+KGiPhltnsusGPJa+dHxHOdEuW6OH9CeXF+BXBSsLI4KVhNyGoEj0j6e7a8t4Uy\ne0t6IqtdTJU0LDt+VsnxGyV1b+frHgZ2z957eDZO/7RsnPte2fH/0rr5Kb6fHbtC0tclnUIaX+p/\ns+/cIvuFPzKrTTRdyLMaxXUbGefjlAyEJumnkiYpzaPw7ezYl0jJ6SFJD2XHjpL0ePZ3vEtSn3a+\nx+qIk4JVoy1Kmo7uyY7NA46MiBHAx4FrW3jf54AfRcT+pIvynGzYg48Dh2THG4Az2/n+44FpkjYH\nbgE+HhH7kEYAuFDStsDHgL0jYl/gP0rfHBF3A5NIv+j3j4i3S17+dfbeRh8Hxm5knMeQhrVo9M2I\nGAnsC3xA0r4RcS1pKOnDIuKwbOiLbwFHZH/LScDF7XyP1ZEuN8yFdQlvZxfGUpsB12Vt6A2kMX2a\nexz4pqRBwG8i4iVJhwPvBp7MhvfYgpRgWvK/kt4GZpGGX34n8HJEvJi9fivwBeA60vwMP5d0H3Bf\nuScWEfMlzczGrHkJ2BN4NPvcjsTZkzRsSenf6TRJF5D+v96BNOHM1GbvHZUdfzT7np6kv5sZ4KRg\nteOrwOvAfqQa7gaT5kTEbZImAh8Gxkv6LGl2qlsj4l/L+I4zSwfMk7RNS4Wy8XgOIg3CdgpwEfCh\nDpzLWOA04B/APRERSlfosuMEniL1J/wYOEnSUODrwIER8aakW0gDwzUn4E8RcUYH4rU64uYjqxX9\ngNeyMfLPJg2Oth5JuwIzsyaT35GaUR4ETpG0XVZmG5U/P/ULwBBJu2f7ZwN/zdrg+0XEeFKy2q+F\n9y4lDd/dkntIs2edQUoQdDTObMC3fwdGSdqTNPPYMmCxpHcAx7YSywTgkMZzkrSlpJZqXVannBSs\nVvwEOEfSFFKTy7IWypwGPCNpMmkuhV9md/x8C/ijpKnAn0hNK+2KiBWkESjvkjQNWAvcQLrA3pd9\n3t9ouU3+FuCGxo7mZp/7JvA8sEtEPJEd63CcWV/FNaSRUKeQ5mb+B3AbqUmq0Wjg/yQ9FBHzSXdG\n3Z59z+Okv6cZ4FFSzcyshGsKZmbWxEnBzMyaOCmYmVkTJwUzM2vipGBmZk2cFMzMrImTgpmZNfn/\nSqpjw6BVWjAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6119dde630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_test_rf_proba = RFR.predict_proba(X_test)[:,1]\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_test_rf_proba)\n",
    "# plot TPR against FPR\n",
    "plt.plot(fpr, tpr, color='red')\n",
    "\n",
    "plt.xlabel(\"False Positive Rate\")\n",
    "plt.ylabel(\"True Positive Rate\")\n",
    "plt.title(\"ROC\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature ranking:\n",
      "1. feature 15 (0.272664)\n",
      "2. feature 10 (0.164425)\n",
      "3. feature 13 (0.135531)\n",
      "4. feature 11 (0.115516)\n",
      "5. feature 12 (0.085874)\n",
      "6. feature 6 (0.060335)\n",
      "7. feature 5 (0.035846)\n",
      "8. feature 8 (0.030464)\n",
      "9. feature 2 (0.029928)\n",
      "10. feature 1 (0.028435)\n",
      "11. feature 7 (0.022557)\n",
      "12. feature 3 (0.004937)\n",
      "13. feature 9 (0.003945)\n",
      "14. feature 4 (0.003332)\n",
      "15. feature 14 (0.003318)\n",
      "16. feature 0 (0.002895)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAF8tJREFUeJzt3X2wXHWd5/H3x/Ak4PAYFUgkqAwl\n4zrgXNFdFS1RBFSCrpY4OgO77LBODeW4juug7qqDY5VP87BbiwoqI6UiKs5oxtEF1qfdnVmQGwxI\nQDQEJAlPUcBBYYDAd/84B22uN0nf7k5yk9/7VdWVPk/f/p6+nU+fPuf06VQVkqQ2PGZbNyBJ2noM\nfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6alqSjyX5r9u6D2lriefpaxRJbgKeADw0MPo3q+qWMWq+\nEPhMVS0ar7vtU5JPAWur6r9s616043JLX+N4RVXtOXAbOfAnIclO2/Lxx5FkwbbuQW0w9DVxSZ6T\n5J+S3J3kqn4L/pFp/y7JdUnuSbI6yX/sx+8BfB04MMnP+9uBST6V5M8Hln9hkrUDwzcl+dMkVwO/\nSLJTv9yXkqxPcmOSN22i11/Wf6R2krcluSPJrUlOSnJCkh8muTPJOwaWfU+Si5J8vl+fK5P89sD0\npyX5dv88rExy4ozH/WiSryX5BXAa8Hrgbf26/30/35lJbujrX5vklQM1Tk3yf5N8OMld/boePzB9\n3yR/k+SWfvqXB6a9PMmKvrd/SvKMgWl/mmRd/5jXJzlmiD+7thdV5c3bnG/ATcCLZxl/EPBT4AS6\njYqX9MML++kvA54CBHgBcC/wzH7aC+l2bwzW+xTw5wPDj5qn72MFsBh4bP+Yy4F3AbsATwZWAy/d\nyHr8sn5fe0O/7M7AHwDrgQuAxwG/BdwHHNLP/x7gQeDV/fxvBW7s7+8MrALe0ffxIuAe4LCBx/0Z\n8Ny+591mrms/32uAA/t5Xgv8Ajign3Zq//h/ACwA/hC4hV/ttv0H4PPAPn0/L+jHHwncATy7X+6U\n/nncFTgMWAMc2M+7BHjKtn69eZvczS19jePL/Zbi3QNbkW8AvlZVX6uqh6vqUmCa7k2AqvqHqrqh\nOt8BLgGeP2Yf/72q1lTVfcCz6N5gzqqqB6pqNfBx4OQhaz0IvK+qHgQuBPYH/ltV3VNVK4Frgd8e\nmH95VV3Uz/+XdOH9nP62J/D+vo9vAl8FXjew7Feq6h/75+lfZmumqr5YVbf083we+BFw1MAsP66q\nj1fVQ8D5wAHAE5IcABwPvLGq7qqqB/vnG+B04JyquryqHqqq84H7+54fogv/w5PsXFU3VdUNQz53\n2g4Y+hrHSVW1d387qR93MPCagTeDu4Hn0YURSY5Pclm/q+RuujeD/cfsY83A/YPpdhENPv476A46\nD+OnfYBCt1UPcPvA9PvowvzXHruqHgbW0m2ZHwis6cc94sd0n4Rm63tWSX5/YDfM3cDTefTzddvA\n49/b392T7pPPnVV11yxlDwb+ZMZztJhu634V8Ga6TzF3JLkwyYGb61PbD0Nfk7YG+PTAm8HeVbVH\nVb0/ya7Al4APA0+oqr2Br9Ht6gGY7VSyXwC7Dww/cZZ5BpdbA9w44/EfV1UnjL1ms1v8yJ0kjwEW\n0e1iuQVY3I97xJOAdRvp+9eGkxxM9ynlDGC//vm6hl89X5uyBtg3yd4bmfa+Gc/R7lX1OYCquqCq\nnkf35lDAB4Z4PG0nDH1N2meAVyR5aZIFSXbrD5Auotu3vSvdfvIN/UHHYweWvR3YL8leA+NWACf0\nByWfSLcVuinfBe7pD0Y+tu/h6UmeNbE1fLTfSfKqdGcOvZluN8llwOV0xyvelmTn/mD2K+h2GW3M\n7XTHIB6xB13orofuIDjdlv5mVdWtdAfGP5Jkn76Ho/vJHwfemOTZ6eyR5GVJHpfksCQv6t+g/4Xu\nk83DG3kYbYcMfU1UVa0BltLtUllPt1X5n4HHVNU9wJuALwB3Ab8LLBtY9gfA54DV/W6HA4FPA1fR\nHWi8hO7A5KYe/yHg5cARdAdVfwJ8AthrU8uN4St0B1jvAn4PeFW///wBupA/vu/hI8Dv9+u4MZ+k\n25d+d5IvV9W1wF8A/4/uDeFfAf84h95+j+4YxQ/oDty+GaCqpukO/v6Pvu9VdAeFoXtTfn/f823A\n44G3z+ExNc/55SxpREneAzy1qt6wrXuRhuWWviQ1xNCXpIa4e0eSGuKWviQ1ZN5doGr//fevJUuW\nbOs2JGm7snz58p9U1cLNzTfvQn/JkiVMT09v6zYkabuS5MfDzOfuHUlqiKEvSQ0x9CWpIYa+JDXE\n0Jekhhj6ktQQQ1+SGmLoS1JDDH1Jasi8+0bunGWYX47bBC84J6khbulLUkMMfUlqiKEvSQ0x9CWp\nIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1ZKjQT3JckuuTrEpy5izT\n35Lk2iRXJ/lGkoMHpj2UZEV/WzbJ5iVJc7PZSysnWQCcDbwEWAtckWRZVV07MNv3gKmqujfJHwIf\nBF7bT7uvqo6YcN+SpBEMs6V/FLCqqlZX1QPAhcDSwRmq6ltVdW8/eBmwaLJtSpImYZjQPwhYMzC8\nth+3MacBXx8Y3i3JdJLLkpw02wJJTu/nmV6/fv0QLUmSRjHRX85K8gZgCnjBwOiDq2pdkicD30zy\n/aq6YXC5qjoXOBdgamrKn7KSpC1kmC39dcDigeFF/bhHSfJi4J3AiVV1/yPjq2pd/+9q4NvAkWP0\nK0kawzChfwVwaJJDkuwCnAw86iycJEcC59AF/h0D4/dJsmt/f3/gucDgAWBJ0la02d07VbUhyRnA\nxcAC4LyqWpnkLGC6qpYBHwL2BL6Y7ofKb66qE4GnAeckeZjuDeb9M876kSRtRamaX7vQp6amanp6\nevgFujeZ0c2z9ZekUSRZXlVTm5vPb+RKUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+S\nGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakh\nhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDVkqNBPclyS65OsSnLmLNPf\nkuTaJFcn+UaSgwemnZLkR/3tlEk2L0mam82GfpIFwNnA8cDhwOuSHD5jtu8BU1X1DOAi4IP9svsC\n7waeDRwFvDvJPpNrX5I0F8Ns6R8FrKqq1VX1AHAhsHRwhqr6VlXd2w9eBizq778UuLSq7qyqu4BL\ngeMm07okaa6GCf2DgDUDw2v7cRtzGvD1uSyb5PQk00mm169fP0RLkqRRTPRAbpI3AFPAh+ayXFWd\nW1VTVTW1cOHCSbYkSRowTOivAxYPDC/qxz1KkhcD7wROrKr757KsJGnrGCb0rwAOTXJIkl2Ak4Fl\ngzMkORI4hy7w7xiYdDFwbJJ9+gO4x/bjJEnbwE6bm6GqNiQ5gy6sFwDnVdXKJGcB01W1jG53zp7A\nF5MA3FxVJ1bVnUneS/fGAXBWVd25RdZEkrRZqapt3cOjTE1N1fT09PALdG8yo5tn6y9Jo0iyvKqm\nNjef38iVpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhL\nUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JashO27qBeScZb/mqyfQh\nSVuAW/qS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDVkqNBPclyS65OsSnLmLNOPTnJlkg1J\nXj1j2kNJVvS3ZZNqXJI0d5v9claSBcDZwEuAtcAVSZZV1bUDs90MnAq8dZYS91XVERPoVZI0pmG+\nkXsUsKqqVgMkuRBYCvwy9Kvqpn7aw1ugR0nShAyze+cgYM3A8Np+3LB2SzKd5LIkJ802Q5LT+3mm\n169fP4fSkqS52BoHcg+uqingd4G/TvKUmTNU1blVNVVVUwsXLtwKLUlSm4YJ/XXA4oHhRf24oVTV\nuv7f1cC3gSPn0J8kaYKGCf0rgEOTHJJkF+BkYKizcJLsk2TX/v7+wHMZOBYgSdq6Nhv6VbUBOAO4\nGLgO+EJVrUxyVpITAZI8K8la4DXAOUlW9os/DZhOchXwLeD9M876kSRtRal5dv33qampmp6eHn6B\nSV//3uvpS9oOJVneHz/dJL+RK0kNMfQlqSGGviQ1xNCXpIYY+pLUkGGuvaNxjHM2kGcCSZowt/Ql\nqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5Ia\nYuhLUkO8tPL2xB9tlzQmt/QlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDRkq9JMc\nl+T6JKuSnDnL9KOTXJlkQ5JXz5h2SpIf9bdTJtW4JGnuNhv6SRYAZwPHA4cDr0ty+IzZbgZOBS6Y\nsey+wLuBZwNHAe9Oss/4bUuSRjHMlv5RwKqqWl1VDwAXAksHZ6iqm6rqauDhGcu+FLi0qu6sqruA\nS4HjJtC3JGkEw4T+QcCageG1/bhhDLVsktOTTCeZXr9+/ZClJUlzNS8O5FbVuVU1VVVTCxcu3Nbt\nSNIOa5jQXwcsHhhe1I8bxjjLSpImbJjQvwI4NMkhSXYBTgaWDVn/YuDYJPv0B3CP7cdJkraBzYZ+\nVW0AzqAL6+uAL1TVyiRnJTkRIMmzkqwFXgOck2Rlv+ydwHvp3jiuAM7qx0mStoHUPPthjampqZqe\nnh5+gUn/sMh8qrele5O0w0iyvKqmNjffvDiQK0naOgx9SWqIoS9JDTH0Jakhhr4kNWSnbd2AtiHP\nBpKa45a+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENf\nkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SG+HOJmpxxfn7Rn16Utgq39CWpIYa+JDXE0Jek\nhhj6ktSQoUI/yXFJrk+yKsmZs0zfNcnn++mXJ1nSj1+S5L4kK/rbxybbviRpLjZ79k6SBcDZwEuA\ntcAVSZZV1bUDs50G3FVVT01yMvAB4LX9tBuq6ogJ9y1JGsEwW/pHAauqanVVPQBcCCydMc9S4Pz+\n/kXAMck45+9JkraEYUL/IGDNwPDaftys81TVBuBnwH79tEOSfC/Jd5I8f7YHSHJ6kukk0+vXr5/T\nCkiShrelD+TeCjypqo4E3gJckOQ3Zs5UVedW1VRVTS1cuHALtyRJ7Rom9NcBiweGF/XjZp0nyU7A\nXsBPq+r+qvopQFUtB24AfnPcpiVJoxkm9K8ADk1ySJJdgJOBZTPmWQac0t9/NfDNqqokC/sDwSR5\nMnAosHoyrUuS5mqzZ+9U1YYkZwAXAwuA86pqZZKzgOmqWgZ8Evh0klXAnXRvDABHA2cleRB4GHhj\nVd25JVZEkrR5qXl2oaupqamanp4efoFxTxKauf7zqd587m3S9ebZ61Da3iRZXlVTm5vPb+RKUkMM\nfUlqiKEvSQ0x9CWpIYa+JDXEn0vU/DTpM4skAW7pS1JTDH1JaoihL0kNcZ++2jCfvn08Wz1pKzH0\npfnAS1hoK3H3jiQ1xNCXpIYY+pLUEENfkhrigVxpR+OZRdoEt/QlqSGGviQ1xNCXpIYY+pLUEENf\nkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BAvuCZp07yA2w5lqC39JMcluT7JqiRn\nzjJ91ySf76dfnmTJwLS39+OvT/LSybUuabuUjH6bZK2tUW8e2mzoJ1kAnA0cDxwOvC7J4TNmOw24\nq6qeCvwV8IF+2cOBk4HfAo4DPtLXk6Qd3zx8AxlmS/8oYFVVra6qB4ALgaUz5lkKnN/fvwg4Jkn6\n8RdW1f1VdSOwqq8nSdoGhtmnfxCwZmB4LfDsjc1TVRuS/AzYrx9/2YxlD5r5AElOB07vB3+e5Pqh\nuh/O/sBPNjp17u+oW6/efO5t0vXmc2/bvt587m3S9eZzb1u33txrHTzMTPPiQG5VnQucuyVqJ5mu\nqqkW6s3n3iZdbz73Nul687m3Sdebz71tD/WGMczunXXA4oHhRf24WedJshOwF/DTIZeVJG0lw4T+\nFcChSQ5JsgvdgdllM+ZZBpzS33818M2qqn78yf3ZPYcAhwLfnUzrkqS52uzunX4f/RnAxcAC4Lyq\nWpnkLGC6qpYBnwQ+nWQVcCfdGwP9fF8ArgU2AH9UVQ9toXXZmEnvNprP9eZzb5OuN597m3S9+dzb\npOvN5962h3qblfKLE5LUDC/DIEkNMfQlqSE7VOgnOS/JHUmuGRj3niTrkqzobyeMWW/fJJcm+VH/\n7z5j1Hpvkqv7vi5JcuCYvb0mycokDyeZ02lgG6n3oSQ/6Hv8uyR7z6XmQJ29k1zU17ouyb8epc5A\nvZuSfL9/3qbHqdXX+0/983ZNks8l2W3EOouTfCvJtX29Px6zr1/7m4xZ77CB/wcrkvxzkjePWGu3\nJN9NclW/rn82gf7+uP8brBy1rxn1FiT5XpKvjrj8Rp//JH+SpJLsP2LtTV7aZouqqh3mBhwNPBO4\nZmDce4C3TrDeB4Ez+/tnAh8Yo9ZvDNx/E/CxMXt7GnAY8G1gagLreiywU3//A8Ou6yy1zwf+Q39/\nF2DvMf/ONwH7T+g1cxBwI/DYfvgLwKkj1joAeGZ//3HAD4HDx+jt1/4mk7rRnZRxG3DwiMsH2LO/\nvzNwOfCcMfp5OnANsDvdCSb/C3jqmOv4FuAC4KuTfP7pTkO/GPjxKK/D/rm/AXhy///hqnFeJ3O9\n7VBb+lX1v+nOHtqS9QYvOXE+cNKotarqnwcG9wCGPqq+kXrXVdVI32beSL1LqmpDP3gZ3fcs5iTJ\nXnT/eT7Z13ygqu4epcctaCfgsf13THYHbhmlSFXdWlVX9vfvAa5jlm+gz6HeRF/PMxwD3FBVPx5l\n4er8vB/cub+Nc1bI04DLq+re/jX3HeBVoxZLsgh4GfCJUWts4vn/K+BtjL6+w1zaZovZoUJ/E87o\nd1GcN+zumE14QlXd2t+/DXjCOMWSvC/JGuD1wLvG7G1L+vfA10dY7hBgPfA3/UftTyTZY8xeCrgk\nyfJ0l/AYvVDVOuDDwM3ArcDPquqSMfsj3ZVmj6TbAp6PTgY+N06BfvfJCuAO4NKqGmddrwGen2S/\nJLsDJ/DoL3bO1V/TBfPDY9T4NUmWAuuq6qoxysx2aZuRNw7mqoXQ/yjwFOAIuv/UfzGpwtV9Vhvr\nnNeqemdVLQY+C5wxkcYmLMk76b5n8dkRFt+J7iPyR6vqSOAXdLvFxvG8qnom3ZVf/yjJ0aMW6jcC\nltK9OR0I7JHkDeM0l2RP4EvAm2d8mpsX+i9Zngh8cZw6VfVQVR1B9wnwqCRPH6PWdXS7EC8B/iew\nAhjpOz1JXg7cUVXLR+1nI3V3B97B/N4426wdPvSr6vb+xfkw8HHGv8rn7UkOAOj/vWPcHnufBf7t\nhGpNTJJTgZcDr+/f5OZqLbB2YCvwIro3gZH1W+dU1R3A3zHe3/TFwI1Vtb6qHgT+Fvg3oxZLsjNd\n4H+2qv52jL62pOOBK6vq9kkU63fXfYvu8unj1PlkVf1OVR0N3EV3TGQUzwVOTHIT3a6TFyX5zDi9\n9Z5Ct3FwVV97EXBlkifOsc42vTzNDh/6jwR075V0HyPHMXjJiVOAr4xaKMmhA4NLgR+M0dfEJTmO\n7iPyiVV17yg1quo2YE2Sw/pRx9B9Q3vUnvZI8rhH7tMdbB7nb3oz8JwkuydJ3991I/YWumMX11XV\nX47R05b2OsbftbPwkbO5kjwWeAljvn6TPL7/90l0+/MvGKVOVb29qhZV1RK63VjfrKqxPr31db9f\nVY+vqiV97bV0B+5vm2OpYS5ts+VsrSPGW+NG90K+FXiQ7g9yGvBp4PvA1f0Te8CY9fYDvgH8iO4M\ng33HqPUlusC6Gvh74KAxe3tlf/9+4Hbg4jHrraLb97iivw19dtGM2kcA0/16fhnYZ4y/8ZPpzna4\nClgJvHMCr5s/owusa/rXy64j1nke3e6+qweesxMm+XqewLruQXcxxL3GrPMM4Hv9ul4DvGsCvf0f\nug2Cq4Bjxq3X13who5+9s8nnnzHOIqM7ZvFDurN4xn4Nz+XmZRgkqSE7/O4dSdKvGPqS1BBDX5Ia\nYuhLUkMMfUlqiKEvSQ0x9CWpIf8fsGuWLupJAhUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6119cee438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "importances_rf = RFR.feature_importances_\n",
    "\n",
    "indices_rf = np.argsort(importances_rf)[::-1]\n",
    "\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices_rf[f], importances_rf[indices_rf[f]]))\n",
    "\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X.shape[1]), importances_rf[indices_rf],\n",
    "       color=\"r\")\n",
    "plt.xticks(range(X.shape[1]), indices_rf)\n",
    "plt.xlim([-1, X.shape[1]])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see for the random forest model, the top three important features are recency, p3_play_freq and p3_play_perc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "python",
   "name": "pyspark"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}