1_Preprocess_FeatureEngineering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json \n",
    "import numpy as np\n",
    "import csv\n",
    "from pathlib import Path\n",
    "import ast"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Flatten data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten_all(inputfile, outputfile, \n",
    "                fcol_to_flat, scol_to_flat, col_to_del):\n",
    "    \"\"\"\n",
    "    Flatten all the columns with dictionaries in.\n",
    "    Append all useful column at the end of data frame\n",
    "    Return a data frame and write into csv\n",
    "    \"\"\"\n",
    "    raw = pd.read_csv(inputfile, delimiter=',', skipinitialspace=True)\n",
    "    # deal the float column\n",
    "    f_col = raw[fcol_to_flat].apply(json.loads)\n",
    "    df = raw.join(pd.DataFrame(f_col.tolist()).astype('float'))\n",
    "    # deal with the string column\n",
    "    for col in scol_to_flat:\n",
    "        s_col = raw[col].apply(json.loads)\n",
    "        df = df.join(pd.DataFrame(s_col.tolist()))\n",
    "    \n",
    "    df = df.drop(col_to_del, axis=1)\n",
    "    df = df.drop(scol_to_flat, axis=1)\n",
    "    df = df.drop(fcol_to_flat, axis=1)\n",
    "    df.to_csv(outputfile, sep=',')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "fcol_to_flat = 'totals'\n",
    "scol_to_flat = ['device', 'trafficSource', 'geoNetwork']\n",
    "col_to_del = ['campaignCode', 'cityId', 'latitude', 'longitude', 'networkLocation', \n",
    "              'browserSize', 'browserVersion', 'flashVersion', 'language',\n",
    "              'mobileDeviceBranding', 'mobileDeviceInfo', \n",
    "              'mobileDeviceMarketingName', 'mobileDeviceModel',\n",
    "              'mobileInputSelector','operatingSystemVersion',\n",
    "              'screenColors', 'screenResolution']\n",
    "inputfile = './data/train.csv'\n",
    "outputfile = './data/train_flatten.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['channelGrouping', 'date', 'fullVisitorId', 'sessionId',\n",
       "       'socialEngagementType', 'visitId', 'visitNumber', 'visitStartTime',\n",
       "       'bounces', 'hits', 'newVisits', 'pageviews', 'transactionRevenue',\n",
       "       'visits', 'browser', 'deviceCategory', 'isMobile', 'operatingSystem',\n",
       "       'adContent', 'adwordsClickInfo', 'campaign', 'isTrueDirect', 'keyword',\n",
       "       'medium', 'referralPath', 'source', 'city', 'continent', 'country',\n",
       "       'metro', 'networkDomain', 'region', 'subContinent'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = flatten_all(inputfile, outputfile, \n",
    "                 fcol_to_flat, scol_to_flat, col_to_del)\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transaction Level Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_traffic_source_columns(df):\n",
    "    df['isTrueDirect']=df['isTrueDirect'].fillna(False)\n",
    "    df['isTrueDirect']=df['isTrueDirect'].astype(int)\n",
    "    \n",
    "    is_source_mall_googleplex = (df['source']=='mall.googleplex.com').astype(int)\n",
    "    is_source_mall_googleplex.name = 'is_source_googleplex'    \n",
    "    \n",
    "    is_source_direct = (df['source']=='(direct)').astype(int)\n",
    "    is_source_direct.name = 'is_source_direct'\n",
    "    \n",
    "    is_source_google = (df['source']=='(direct)').astype(int)\n",
    "    is_source_google.name = 'is_source_google'\n",
    "    \n",
    "    is_medium_referral = (df['medium']=='referral').astype(int)\n",
    "    is_medium_referral.name = 'is_medium_referral'\n",
    "    is_medium_none = (df['medium']=='(none)').astype(int)\n",
    "    is_medium_none.name = 'is_medium_none'\n",
    "    is_medium_organic = (df['medium']=='organic').astype(int)\n",
    "    is_medium_organic.name=\"is_medium_organic\"\n",
    "    \n",
    "    df = df.drop(columns = ['adContent','adwordsClickInfo','campaign', 'keyword', 'medium', 'referralPath',\n",
    "                      'source'])\n",
    "    \n",
    "    return pd.concat([df,is_source_mall_googleplex,is_source_direct,is_source_google,is_medium_referral,\n",
    "                    is_medium_none,is_medium_organic],axis=1)\n",
    "\n",
    "\n",
    "def get_region_columns(df):\n",
    "    df['is_bayarea'] = (df['metro'] =='San Francisco-Oakland-San Jose CA').astype(int)\n",
    "    dummies = pd.get_dummies(df['continent'])\n",
    "    dummies = dummies.drop(['(not set)'],axis=1)\n",
    "    df = df.drop(['continent','city','subContinent','region','country','metro','networkDomain'],axis=1) \n",
    "    new_df = pd.concat([df,dummies],axis=1)\n",
    "    return new_df\n",
    "\n",
    "\n",
    "def extract_activities(df):\n",
    "    df['is_transaction'] = 1 - df['transactionRevenue'].isna()\n",
    "    df[['bounces', 'hits', 'pageviews', 'visitNumber']] = df[['bounces', 'hits', 'pageviews', 'visitNumber']].fillna(0)\n",
    "    df['transactionRevenue'] = df['transactionRevenue'].fillna(0)\n",
    "    return df\n",
    "\n",
    "\n",
    "def time_value(df):\n",
    "    import datetime as dt\n",
    "    time = pd.to_datetime(df.loc[:,'visitStartTime'], unit='s')\n",
    "    year = time.dt.year\n",
    "    year = pd.get_dummies(year)\n",
    "    month = time.dt.month\n",
    "    q1 = (month <= 3) & (month >= 1)\n",
    "    q2 = (month <= 6) & (month >= 4)\n",
    "    q3 = (month <= 9) & (month >= 7)\n",
    "    q4 = (month <= 12) & (month >= 10)\n",
    "    dow = time.dt.weekday\n",
    "    weekday = (dow <= 5) & (dow >=1)\n",
    "    hour = time.dt.hour\n",
    "    h_0_6 = (hour <= 6) & (hour >= 0)\n",
    "    h_7_12 = (hour <= 12) & (hour >= 7)\n",
    "    h_13_17 = (hour <= 17) & (hour >= 13)\n",
    "    h_18_23 = (hour <= 23) & (hour >= 18)\n",
    "    new_df = pd.DataFrame()\n",
    "    new_df['q1'] = q1.astype(int)\n",
    "    new_df['q2'] = q2.astype(int)\n",
    "    new_df['q3'] = q3.astype(int)\n",
    "    new_df['q4'] = q4.astype(int)\n",
    "    new_df['weekday'] = weekday.astype(int)\n",
    "    new_df['h_0_6'] = h_0_6.astype(int)\n",
    "    new_df['h_7_12'] = h_7_12.astype(int)\n",
    "    new_df['h_13_17'] = h_13_17.astype(int)\n",
    "    new_df['h_18_23'] = h_18_23.astype(int)\n",
    "    return new_df\n",
    "\n",
    "\n",
    "def device(df):\n",
    "    browser_chrome = (df['browser']=='Chrome').astype(int)\n",
    "    is_mobile = df['isMobile'].astype(int)\n",
    "    system_google = ((df['operatingSystem']=='Android')|(df['operatingSystem']=='Chrome OS')).astype(int)\n",
    "    new_df = pd.DataFrame()\n",
    "    new_df['browser_chrome'] = browser_chrome\n",
    "    new_df['is_mobile'] = is_mobile\n",
    "    new_df['system_google'] = system_google\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing(inputfile):\n",
    "    df = pd.read_csv(inputfile, index_col=0)\n",
    "    \n",
    "    df.sort_values(['fullVisitorId', 'visitStartTime'], ascending=True, inplace=True)\n",
    "    df['last_session_1'] = (\n",
    "    df['visitStartTime'] - df[['fullVisitorId', 'visitStartTime']].groupby('fullVisitorId')['visitStartTime'].shift(1)\n",
    ")\n",
    "    df['next_session_1']=(\n",
    "    -1*(df['visitStartTime'] - df[['fullVisitorId', 'visitStartTime']].groupby('fullVisitorId')['visitStartTime'].shift(-1))\n",
    ")\n",
    "\n",
    "    #df[['next_session_1','last_session_1']] = df[['next_session_1','last_session_1']]/86400\n",
    "    df[['next_session_1','last_session_1']] = df[['next_session_1','last_session_1']].fillna(0)\n",
    "    print(df.columns)\n",
    "    df = extract_traffic_source_columns(df)\n",
    "    # print(\"Congrats Brian\\n\"+df.columns)\n",
    "    df = get_region_columns(df)\n",
    "    # print(\"Congrats Ziyu\\n\"+df.columns)\n",
    "    df = extract_activities(df)\n",
    "    # print(\"Congrats Yixin\\n\"+df.columns)\n",
    "    col_time = time_value(df)\n",
    "    col_device = device(df)\n",
    "    \n",
    "    df_base_model = df[['fullVisitorId', 'is_transaction', 'transactionRevenue', # add three columns\n",
    "                        'bounces', 'hits', 'pageviews', 'visitNumber',\n",
    "                       'is_source_googleplex','is_source_direct','is_source_google','is_medium_referral',\n",
    "                    'is_medium_none','is_medium_organic',\n",
    "                        'is_bayarea','Africa','Americas','Asia','Europe','Oceania',\n",
    "                        'next_session_1','last_session_1'\n",
    "                       ]]\n",
    "    col_time = col_time.reset_index(drop=True)\n",
    "    col_device = col_device.reset_index(drop=True)\n",
    "    df_base_model = df_base_model.reset_index(drop=True)\n",
    "    df_base_model = pd.concat([df_base_model,col_time],axis=1)\n",
    "    df_base_model = pd.concat([df_base_model,col_device],axis=1)\n",
    "    # print(\"Congrats Xi\\n\"+df_base_model.columns)\n",
    "    return df_base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['channelGrouping', 'date', 'fullVisitorId', 'sessionId',\n",
      "       'socialEngagementType', 'visitId', 'visitNumber', 'visitStartTime',\n",
      "       'bounces', 'hits', 'newVisits', 'pageviews', 'transactionRevenue',\n",
      "       'visits', 'browser', 'deviceCategory', 'isMobile', 'operatingSystem',\n",
      "       'adContent', 'adwordsClickInfo', 'campaign', 'isTrueDirect', 'keyword',\n",
      "       'medium', 'referralPath', 'source', 'city', 'continent', 'country',\n",
      "       'metro', 'networkDomain', 'region', 'subContinent', 'last_session_1',\n",
      "       'next_session_1'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "inputfile = './data/train_flatten.csv'\n",
    "df_trans = preprocessing(inputfile)\n",
    "df_trans.transactionRevenue = df_trans.transactionRevenue.fillna(0)\n",
    "df_trans.to_csv('./data/train_trans.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fullVisitorId</th>\n",
       "      <th>is_transaction</th>\n",
       "      <th>transactionRevenue</th>\n",
       "      <th>bounces</th>\n",
       "      <th>hits</th>\n",
       "      <th>pageviews</th>\n",
       "      <th>visitNumber</th>\n",
       "      <th>is_source_googleplex</th>\n",
       "      <th>is_source_direct</th>\n",
       "      <th>is_source_google</th>\n",
       "      <th>...</th>\n",
       "      <th>q3</th>\n",
       "      <th>q4</th>\n",
       "      <th>weekday</th>\n",
       "      <th>h_0_6</th>\n",
       "      <th>h_7_12</th>\n",
       "      <th>h_13_17</th>\n",
       "      <th>h_18_23</th>\n",
       "      <th>browser_chrome</th>\n",
       "      <th>is_mobile</th>\n",
       "      <th>system_google</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4823595352351</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5103959234087</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10278554503158</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20424342248747</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>26722803385797</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    fullVisitorId  is_transaction  transactionRevenue  bounces  hits  \\\n",
       "0   4823595352351               0                 0.0      1.0   1.0   \n",
       "1   5103959234087               0                 0.0      0.0  10.0   \n",
       "2  10278554503158               0                 0.0      0.0  11.0   \n",
       "3  20424342248747               0                 0.0      0.0  17.0   \n",
       "4  26722803385797               0                 0.0      0.0   3.0   \n",
       "\n",
       "   pageviews  visitNumber  is_source_googleplex  is_source_direct  \\\n",
       "0        1.0            1                     0                 0   \n",
       "1        8.0            1                     0                 0   \n",
       "2        8.0            1                     0                 0   \n",
       "3       13.0            1                     0                 0   \n",
       "4        2.0            1                     0                 0   \n",
       "\n",
       "   is_source_google      ...        q3  q4  weekday  h_0_6  h_7_12  h_13_17  \\\n",
       "0                 0      ...         0   1        1      0       0        1   \n",
       "1                 0      ...         1   0        0      0       0        0   \n",
       "2                 0      ...         0   1        1      1       0        0   \n",
       "3                 0      ...         0   1        1      0       1        0   \n",
       "4                 0      ...         0   0        0      0       1        0   \n",
       "\n",
       "   h_18_23  browser_chrome  is_mobile  system_google  \n",
       "0        0               1          0              0  \n",
       "1        1               1          1              1  \n",
       "2        0               1          0              0  \n",
       "3        0               1          0              0  \n",
       "4        0               0          0              0  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trans.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User Level Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def agg_time(x):\n",
    "    if len(x)==1:\n",
    "        return 0\n",
    "    else:\n",
    "        return np.log(1+sum(x)/(len(x)-1))\n",
    "\n",
    "    \n",
    "def make_agg(v):\n",
    "    if v in y_c_variables:\n",
    "        return {\n",
    "            v: {v + '_' + 'total': lambda x: np.log(1 + sum(x))}\n",
    "               }\n",
    "    if v in y_d_variables:\n",
    "        return {\n",
    "            v: {v: 'max'}\n",
    "        }\n",
    "    if v in c_variables:\n",
    "        return {\n",
    "            v: {v + '_' + 'mean': 'mean'}\n",
    "        }\n",
    "    if v in i_variables:\n",
    "        return {\n",
    "            v: {v + '_' + 'total': 'max'}\n",
    "        }\n",
    "    if v in t_variables:\n",
    "        return {\n",
    "            v: {v + '_' + 'adjmean': lambda x: agg_time(x)}\n",
    "        }\n",
    "    if v in d_variables:\n",
    "        return {\n",
    "            v: {\n",
    "                v + '_' + 'freq': 'sum',\n",
    "                v + '_' + 'rate': 'mean'\n",
    "            }\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/ml/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3018: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "inputfile = './data/train_trans.csv'\n",
    "df_trans = pd.read_csv(inputfile, index_col=0)\n",
    "outputfile = './data/train_user.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_variables = ['fullVisitorId']\n",
    "y_c_variables = ['transactionRevenue']\n",
    "y_d_variables = ['is_transaction'] # classification\n",
    "c_variables = ['bounces', 'hits', 'pageviews'] # continuous\n",
    "i_variables = ['visitNumber']\n",
    "t_variables = ['next_session_1', 'last_session_1']\n",
    "d_variables = ['is_source_googleplex',\n",
    "       'is_source_direct', 'is_source_google', 'is_medium_referral',\n",
    "       'is_medium_none', 'is_medium_organic', 'is_bayarea', 'Africa',\n",
    "       'Americas', 'Asia', 'Europe', 'Oceania', 'q1', 'q2', 'q3', 'q4', 'weekday', 'h_0_6', 'h_7_12',\n",
    "       'h_13_17', 'h_18_23', 'browser_chrome',\n",
    "       'is_mobile', 'system_google'] # dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "aggregation = dict()\n",
    "for v in df_trans.columns:\n",
    "    if v not in id_variables:\n",
    "        agg_new = make_agg(v)\n",
    "        aggregation = {**aggregation, **agg_new}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/envs/ml/lib/python3.6/site-packages/pandas/core/groupby/groupby.py:4656: FutureWarning: using a dict with renaming is deprecated and will be removed in a future version\n",
      "  return super(DataFrameGroupBy, self).aggregate(arg, *args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "df_trans = pd.read_csv(inputfile, index_col=0)\n",
    "df_user = df_trans.groupby('fullVisitorId', as_index=False).agg(aggregation)\n",
    "df_user.columns = df_user.columns.droplevel(0)\n",
    "new_col = ['fullVisitorId']\n",
    "new_col.extend(df_user.columns[1:])\n",
    "df_user.columns = new_col\n",
    "df_user.to_csv(outputfile, sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fullVisitorId</th>\n",
       "      <th>is_transaction</th>\n",
       "      <th>transactionRevenue_total</th>\n",
       "      <th>bounces_mean</th>\n",
       "      <th>hits_mean</th>\n",
       "      <th>pageviews_mean</th>\n",
       "      <th>visitNumber_total</th>\n",
       "      <th>is_source_googleplex_freq</th>\n",
       "      <th>is_source_googleplex_rate</th>\n",
       "      <th>is_source_direct_freq</th>\n",
       "      <th>...</th>\n",
       "      <th>h_13_17_freq</th>\n",
       "      <th>h_13_17_rate</th>\n",
       "      <th>h_18_23_freq</th>\n",
       "      <th>h_18_23_rate</th>\n",
       "      <th>browser_chrome_freq</th>\n",
       "      <th>browser_chrome_rate</th>\n",
       "      <th>is_mobile_freq</th>\n",
       "      <th>is_mobile_rate</th>\n",
       "      <th>system_google_freq</th>\n",
       "      <th>system_google_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4823595352351</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5103959234087</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10278554503158</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20424342248747</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>26722803385797</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    fullVisitorId  is_transaction  transactionRevenue_total  bounces_mean  \\\n",
       "0   4823595352351               0                       0.0           1.0   \n",
       "1   5103959234087               0                       0.0           0.0   \n",
       "2  10278554503158               0                       0.0           0.0   \n",
       "3  20424342248747               0                       0.0           0.0   \n",
       "4  26722803385797               0                       0.0           0.0   \n",
       "\n",
       "   hits_mean  pageviews_mean  visitNumber_total  is_source_googleplex_freq  \\\n",
       "0        1.0             1.0                  1                          0   \n",
       "1       10.0             8.0                  1                          0   \n",
       "2       11.0             8.0                  1                          0   \n",
       "3       17.0            13.0                  1                          0   \n",
       "4        3.0             2.0                  1                          0   \n",
       "\n",
       "   is_source_googleplex_rate  is_source_direct_freq         ...          \\\n",
       "0                        0.0                      0         ...           \n",
       "1                        0.0                      0         ...           \n",
       "2                        0.0                      0         ...           \n",
       "3                        0.0                      0         ...           \n",
       "4                        0.0                      0         ...           \n",
       "\n",
       "   h_13_17_freq  h_13_17_rate  h_18_23_freq  h_18_23_rate  \\\n",
       "0             1           1.0             0           0.0   \n",
       "1             0           0.0             1           1.0   \n",
       "2             0           0.0             0           0.0   \n",
       "3             0           0.0             0           0.0   \n",
       "4             0           0.0             0           0.0   \n",
       "\n",
       "   browser_chrome_freq  browser_chrome_rate  is_mobile_freq  is_mobile_rate  \\\n",
       "0                    1                  1.0               0             0.0   \n",
       "1                    1                  1.0               1             1.0   \n",
       "2                    1                  1.0               0             0.0   \n",
       "3                    1                  1.0               0             0.0   \n",
       "4                    0                  0.0               0             0.0   \n",
       "\n",
       "   system_google_freq  system_google_rate  \n",
       "0                   0                 0.0  \n",
       "1                   1                 1.0  \n",
       "2                   0                 0.0  \n",
       "3                   0                 0.0  \n",
       "4                   0                 0.0  \n",
       "\n",
       "[5 rows x 57 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_user.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}